From 351bf85d9bf56453b6015bafeef52874634dfc8b Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Tue, 1 Oct 2019 15:14:37 -0400 Subject: [PATCH 001/168] remove dead read request processing code also, make client use default log level --- client/src/unifyfs-internal.h | 1 - client/src/unifyfs.c | 6 +- common/src/unifyfs_const.h | 8 +- common/src/unifyfs_log.c | 2 +- server/src/unifyfs_global.h | 64 -- server/src/unifyfs_metadata.c | 18 - server/src/unifyfs_metadata.h | 2 - server/src/unifyfs_request_manager.c | 539 +--------- server/src/unifyfs_request_manager.h | 30 +- server/src/unifyfs_service_manager.c | 1461 +------------------------- server/src/unifyfs_service_manager.h | 3 - server/src/unifyfs_sock.c | 71 +- 12 files changed, 66 insertions(+), 2139 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index f31504f18..f67361cfe 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -335,7 +335,6 @@ extern int client_sockfd; extern struct pollfd cmd_fd; extern void* shm_req_buf; extern void* shm_recv_buf; -extern char cmd_buf[CMD_BUF_SIZE]; extern unifyfs_fattr_buf_t unifyfs_fattrs; extern int app_id; diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 1626916d7..0776fcc2d 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -98,8 +98,6 @@ static char shm_recv_name[GEN_STR_LEN] = {0}; static size_t shm_recv_size = UNIFYFS_SHMEM_RECV_SIZE; void* shm_recv_buf; -char cmd_buf[CMD_BUF_SIZE] = {0}; - int client_rank; int app_id; size_t unifyfs_key_slice_range; @@ -1779,8 +1777,7 @@ static int unifyfs_init(int rank) char* cfgval; if (!unifyfs_initialized) { - /* unifyfs debug level default is zero */ - unifyfs_log_level = 0; + /* unifyfs default log level is LOG_ERR */ cfgval = client_cfg.log_verbosity; if (cfgval != NULL) { rc = configurator_int_val(cfgval, &l); @@ -1802,6 +1799,7 @@ static int unifyfs_init(int rank) if (*(void**)(wrap_unifyfs_list[i].function_address_pointer) == 0) { LOGERR("This function name failed to be wrapped: %s", wrap_unifyfs_list[i].name); + } } #endif diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h index 1474b35c6..31a223739 100644 --- a/common/src/unifyfs_const.h +++ b/common/src/unifyfs_const.h @@ -57,18 +57,16 @@ #define RECV_BUF_CNT 4 /* number of remote read buffers */ #define SENDRECV_BUF_LEN (8 * MIB) /* remote read buffer size */ #define MAX_META_PER_SEND (4 * KIB) /* max read request count per server */ -#define REQ_BUF_LEN (MAX_META_PER_SEND * 128) /* read requests (send_msg_t) */ +#define REQ_BUF_LEN (MAX_META_PER_SEND * 64) /* chunk read reqs buffer size */ #define SHM_WAIT_INTERVAL 1000 /* unit: ns */ #define RM_MAX_ACTIVE_REQUESTS 64 /* number of concurrent read requests */ // Service Manager #define LARGE_BURSTY_DATA (512 * MIB) #define MAX_BURSTY_INTERVAL 10000 /* unit: us */ -#define MIN_SLEEP_INTERVAL 10 /* unit: us */ +#define MIN_SLEEP_INTERVAL 100 /* unit: us */ #define SLEEP_INTERVAL 500 /* unit: us */ #define SLEEP_SLICE_PER_UNIT 50 /* unit: us */ -#define READ_BLOCK_SIZE MIB -#define READ_BUF_SZ GIB // Request and Service Managers, Command Handler #define MAX_NUM_CLIENTS 64 /* app processes per server */ @@ -90,7 +88,7 @@ #define UNIFYFS_FATTR_BUF_SIZE MIB #define UNIFYFS_MAX_READ_CNT KIB -/* max read size = UNIFYFS_MAX_SPLIT_CNT * META_DEFAULT_RANGE_SZ */ +/* NOTE: max read size = UNIFYFS_MAX_SPLIT_CNT * META_DEFAULT_RANGE_SZ */ #define UNIFYFS_MAX_SPLIT_CNT (4 * KIB) // Metadata/MDHIM Default Values diff --git a/common/src/unifyfs_log.c b/common/src/unifyfs_log.c index 294409e36..6c6d68b47 100644 --- a/common/src/unifyfs_log.c +++ b/common/src/unifyfs_log.c @@ -33,7 +33,7 @@ #include "unifyfs_const.h" /* one of the loglevel values */ -unifyfs_log_level_t unifyfs_log_level = 5; +unifyfs_log_level_t unifyfs_log_level = LOG_ERR; /* pointer to log file stream */ FILE* unifyfs_log_stream; // = NULL diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index 9e5f9c01e..db8f10527 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -63,73 +63,9 @@ extern size_t max_recs_per_slice; /* defines commands for messages sent to service manager threads */ typedef enum { SVC_CMD_INVALID = 0, - SVC_CMD_RDREQ_MSG = 1, /* read requests (send_msg_t) */ SVC_CMD_RDREQ_CHK, /* read requests (chunk_read_req_t) */ - SVC_CMD_EXIT, /* service manager thread should exit */ } service_cmd_e; -typedef enum { - READ_REQUEST_TAG = 5001, - READ_RESPONSE_TAG = 6001, - CHUNK_REQUEST_TAG = 7001, - CHUNK_RESPONSE_TAG = 8001 -} service_tag_e; - -/* this defines a read request as sent from the request manager to the - * service manager, it contains info about the physical location of - * the data: - * - * dest_delegator_rank - rank of delegator hosting data log file - * dest_app_id, dest_client_id - defines file on host delegator - * dest_offset - phyiscal offset of data in log file - * length - number of bytes to be read - * - * it also contains a return address to use in the read reply that - * the service manager sends back to the request manager: - * - * src_delegator_rank - rank of requesting delegator process - * src_thrd - thread id of request manager (used to compute MPI tag) - * src_app_id, src_cli_id - * src_fid - global file id - * src_offset - starting offset in logical file - * length - number of bytes - * src_dbg_rank - rank of application process making the request - * - * the arrival_time field is included but not set by the request - * manager, it is used to tag the time the request reaches the - * service manager for prioritizing read replies */ -typedef struct { - int dest_app_id; /* app id of log file */ - int dest_client_id; /* client id of log file */ - size_t dest_offset; /* data offset within log file */ - int dest_delegator_rank; /* delegator rank of service manager */ - size_t length; /* length of data to be read */ - int src_delegator_rank; /* delegator rank of request manager */ - int src_cli_id; /* client id of requesting client process */ - int src_app_id; /* app id of requesting client process */ - int src_fid; /* global file id */ - size_t src_offset; /* logical file offset */ - int src_thrd; /* thread id of request manager */ - int src_dbg_rank; /* MPI rank of client process */ - int arrival_time; /* records time reaches service mgr */ -} send_msg_t; - -/* defines header for read reply messages sent from service manager - * back to request manager, data payload of length bytes immediately - * follows the header */ -typedef struct { - size_t src_offset; /* file offset */ - size_t length; /* number of bytes */ - int src_fid; /* global file id */ - int errcode; /* indicates whether read was successful */ -} recv_msg_t; - -/* defines a fixed-length list of read requests */ -typedef struct { - int num; /* number of active read requests */ - send_msg_t msg_meta[MAX_META_PER_SEND]; /* list of requests */ -} msg_meta_t; - // NEW READ REQUEST STRUCTURES typedef enum { READREQ_INIT = 0, diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index 0f4436a65..059e235cc 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -218,24 +218,6 @@ int meta_init_indices(void) return 0; } -void print_bget_indices(int app_id, int cli_id, - send_msg_t* msgs, int tot_num) -{ - int i; - for (i = 0; i < tot_num; i++) { - LOGDBG("index:dbg_rank:%d, dest_offset:%zu, " - "dest_del_rank:%d, dest_cli_id:%d, dest_app_id:%d, " - "length:%zu, src_app_id:%d, src_cli_id:%d, src_offset:%zu, " - "src_del_rank:%d, src_fid:%d, num:%d", - msgs[i].src_dbg_rank, msgs[i].dest_offset, - msgs[i].dest_delegator_rank, msgs[i].dest_client_id, - msgs[i].dest_app_id, msgs[i].length, - msgs[i].src_app_id, msgs[i].src_cli_id, - msgs[i].src_offset, msgs[i].src_delegator_rank, - msgs[i].src_fid, tot_num); - } -} - void print_fsync_indices(unifyfs_key_t** keys, unifyfs_val_t** vals, size_t num_entries) diff --git a/server/src/unifyfs_metadata.h b/server/src/unifyfs_metadata.h index 200c78742..8e76102ca 100644 --- a/server/src/unifyfs_metadata.h +++ b/server/src/unifyfs_metadata.h @@ -79,8 +79,6 @@ void debug_log_key_val(const char* ctx, int meta_sanitize(void); int meta_init_store(unifyfs_cfg_t* cfg); -void print_bget_indices(int app_id, int client_id, - send_msg_t* index_set, int tot_num); int meta_init_indices(void); void meta_free_indices(void); diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index c3baa1e7e..301ac832b 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -108,43 +108,6 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) return NULL; } - /* allocate an array for listing read requests from client */ - thrd_ctrl->del_req_set = (msg_meta_t*)calloc(1, sizeof(msg_meta_t)); - if (thrd_ctrl->del_req_set == NULL) { - LOGERR("Failed to allocate read request structure for request " - "manager thread for app_id=%d client_id=%d", - app_id, client_id); - free(thrd_ctrl); - return NULL; - } - - /* allocate structure for tracking outstanding read requests - * this delegator has with service managers on other nodes */ - thrd_ctrl->del_req_stat = (del_req_stat_t*) - calloc(1, sizeof(del_req_stat_t)); - if (thrd_ctrl->del_req_stat == NULL) { - LOGERR("Failed to allocate delegator structure for request " - "manager thread for app_id=%d client_id=%d", - app_id, client_id); - free(thrd_ctrl->del_req_set); - free(thrd_ctrl); - return NULL; - } - - /* allocate a structure to track requests we have on each - * remote service manager */ - thrd_ctrl->del_req_stat->req_stat = (per_del_stat_t*) - calloc(glb_mpi_size, sizeof(per_del_stat_t)); - if (thrd_ctrl->del_req_stat->req_stat == NULL) { - LOGERR("Failed to allocate per-delegator structure for request " - "manager thread for app_id=%d client_id=%d", - app_id, client_id); - free(thrd_ctrl->del_req_stat); - free(thrd_ctrl->del_req_set); - free(thrd_ctrl); - return NULL; - } - /* initialize lock for shared data structures between * main thread and request delegator thread */ pthread_mutexattr_t attr; @@ -155,9 +118,6 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) LOGERR("pthread_mutex_init failed for request " "manager thread app_id=%d client_id=%d rc=%d (%s)", app_id, client_id, rc, strerror(rc)); - free(thrd_ctrl->del_req_stat->req_stat); - free(thrd_ctrl->del_req_stat); - free(thrd_ctrl->del_req_set); free(thrd_ctrl); return NULL; } @@ -170,9 +130,6 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) "manager thread app_id=%d client_id=%d rc=%d (%s)", app_id, client_id, rc, strerror(rc)); pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); - free(thrd_ctrl->del_req_stat->req_stat); - free(thrd_ctrl->del_req_stat); - free(thrd_ctrl->del_req_set); free(thrd_ctrl); return NULL; } @@ -194,9 +151,6 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) if (rc != 0) { pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); - free(thrd_ctrl->del_req_stat->req_stat); - free(thrd_ctrl->del_req_stat); - free(thrd_ctrl->del_req_set); free(thrd_ctrl); return NULL; } @@ -211,9 +165,6 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) app_id, client_id, rc, strerror(rc)); pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); - free(thrd_ctrl->del_req_stat->req_stat); - free(thrd_ctrl->del_req_stat); - free(thrd_ctrl->del_req_set); free(thrd_ctrl); return NULL; } @@ -227,44 +178,6 @@ reqmgr_thrd_t* rm_get_thread(int thrd_id) return (reqmgr_thrd_t*) arraylist_get(rm_thrd_list, thrd_id); } -static void print_send_msgs(send_msg_t* send_metas, - int msg_cnt) -{ - int i; - send_msg_t* msg; - for (i = 0; i < msg_cnt; i++) { - msg = send_metas + i; - LOGDBG("msg[%d] gfid:%d length:%zu file_offset:%zu " - "dest_offset:%zu dest_app:%d dest_clid:%d", - i, msg->src_fid, msg->length, msg->src_offset, - msg->dest_offset, msg->dest_app_id, msg->dest_client_id); - } -} - -static void print_remote_del_reqs(int app_id, int cli_id, - del_req_stat_t* del_req_stat) -{ - int i; - for (i = 0; i < del_req_stat->del_cnt; i++) { - LOGDBG("remote_delegator:%d, req_cnt:%d", - del_req_stat->req_stat[i].del_id, - del_req_stat->req_stat[i].req_cnt); - } -} - -#if 0 // NOT CURRENTLY USED -static void print_recv_msg(int app_id, int cli_id, - int thrd_id, - shm_meta_t* msg) -{ - LOGDBG("recv_msg: app_id:%d, cli_id:%d, thrd_id:%d, " - "fid:%d, offset:%ld, len:%ld", - app_id, cli_id, thrd_id, msg->src_fid, - msg->offset, msg->length); -} -#endif - - /* order keyvals by gfid, then host delegator rank */ static int compare_kv_gfid_rank(const void* a, const void* b) { @@ -290,23 +203,6 @@ static int compare_kv_gfid_rank(const void* a, const void* b) } } -/* order read requests by destination delegator rank */ -static int compare_msg_delegators(const void* a, const void* b) -{ - const send_msg_t* msg_a = a; - const send_msg_t* msg_b = b; - int rank_a = msg_a->dest_delegator_rank; - int rank_b = msg_b->dest_delegator_rank; - - if (rank_a == rank_b) { - return 0; - } else if (rank_a < rank_b) { - return -1; - } else { - return 1; - } -} - unifyfs_key_t** alloc_key_array(int elems) { int size = elems * (sizeof(unifyfs_key_t*) + sizeof(unifyfs_key_t)); @@ -471,110 +367,6 @@ static void signal_new_responses(reqmgr_thrd_t* thrd_ctrl) } } -/* issue remote chunk read requests for extent chunks - * contained within keyvals - */ -int create_request_messages(reqmgr_thrd_t* thrd_ctrl, - int client_rank, - int num_vals, - unifyfs_keyval_t* keyvals) -{ - int thrd_id = thrd_ctrl->thrd_ndx; - int app_id = thrd_ctrl->app_id; - int client_id = thrd_ctrl->client_id; - - /* wait for lock for shared data structures holding requests - * and condition variable */ - RM_LOCK(thrd_ctrl); - - // set up the thread_control delegator request set - // TODO: make this a function - int i; - for (i = 0; i < num_vals; i++) { - send_msg_t* meta = &(thrd_ctrl->del_req_set->msg_meta[i]); - memset(meta, 0, sizeof(send_msg_t)); - - debug_log_key_val(__func__, &keyvals[i].key, &keyvals[i].val); - - /* physical offset of the requested file segment on the log file */ - meta->dest_offset = keyvals[i].val.addr; - - /* rank of the remote delegator */ - meta->dest_delegator_rank = keyvals[i].val.delegator_rank; - - /* dest_client_id and dest_app_id uniquely identify the remote - * physical log file that contains the requested segments */ - meta->dest_app_id = keyvals[i].val.app_id; - meta->dest_client_id = keyvals[i].val.rank; - meta->length = (size_t)keyvals[i].val.len; - - /* src_app_id and src_cli_id identifies the requested client */ - meta->src_app_id = app_id; - meta->src_cli_id = client_id; - - /* src_offset is the logical offset of the shared file */ - meta->src_offset = keyvals[i].key.offset; - meta->src_delegator_rank = glb_mpi_rank; - meta->src_fid = keyvals[i].key.fid; - meta->src_dbg_rank = client_rank; - meta->src_thrd = thrd_id; - } - - thrd_ctrl->del_req_set->num = num_vals; - - if (num_vals > 1) { - /* sort read requests to be sent to the same delegators. */ - qsort(thrd_ctrl->del_req_set->msg_meta, - thrd_ctrl->del_req_set->num, - sizeof(send_msg_t), compare_msg_delegators); - } - - /* debug print */ - print_send_msgs(thrd_ctrl->del_req_set->msg_meta, - thrd_ctrl->del_req_set->num); - - /* get pointer to list of delegator stat objects to record - * delegator rank and count of requests for each delegator */ - per_del_stat_t* req_stat = thrd_ctrl->del_req_stat->req_stat; - - /* get pointer to send message structures, one for each request */ - send_msg_t* msg_meta = thrd_ctrl->del_req_set->msg_meta; - - /* iterate over read requests and count number of requests - * to be sent to each delegator */ - int del_ndx = 0; - req_stat[del_ndx].del_id = msg_meta[0].dest_delegator_rank; - req_stat[del_ndx].req_cnt = 1; - - for (i = 1; i < thrd_ctrl->del_req_set->num; i++) { - int cur_rank = msg_meta[i].dest_delegator_rank; - int prev_rank = msg_meta[i-1].dest_delegator_rank; - if (cur_rank == prev_rank) { - /* another message for the current delegator */ - req_stat[del_ndx].req_cnt++; - } else { - /* new delegator */ - del_ndx++; - req_stat[del_ndx].del_id = msg_meta[i].dest_delegator_rank; - req_stat[del_ndx].req_cnt = 1; - } - } - - /* record total number of delegators we'll send requests to */ - thrd_ctrl->del_req_stat->del_cnt = del_ndx + 1; - - /* debug print */ - print_remote_del_reqs(app_id, thrd_id, thrd_ctrl->del_req_stat); - - /* wake up the request manager thread for the requesting client */ - signal_new_requests(thrd_ctrl); - - /* done updating shared variables, release the lock */ - RM_UNLOCK(thrd_ctrl); - - return UNIFYFS_SUCCESS; -} - /* issue remote chunk read requests for extent chunks * listed within keyvals */ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, @@ -1005,11 +797,6 @@ int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl) pthread_join(thrd_ctrl->thrd, &status); thrd_ctrl->exited = 1; - /* free storage holding shared data structures */ - free(thrd_ctrl->del_req_set); - free(thrd_ctrl->del_req_stat->req_stat); - free(thrd_ctrl->del_req_stat); - return UNIFYFS_SUCCESS; } @@ -1179,65 +966,6 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) * These functions define the logic of the request manager thread ***********************/ -/* pack the the requests to be sent to the same - * delegator. - * ToDo: pack and send multiple rounds if the - * total request sizes is larger than REQ_BUF_LEN - * @param rank: source rank that sends the requests - * @param req_msg_buf: request buffer - * @param req_num: number of read requests - * @param *tot_sz: the total data size to read in these - * packed read requests - * @return success/error code */ -static int rm_pack_send_requests( - char* req_msg_buf, /* pointer to buffer to pack requests into */ - send_msg_t* send_metas, /* request objects to be packed */ - int req_cnt, /* number of requests */ - size_t* tot_sz) /* total data payload size we're requesting */ -{ - /* tot_sz records the aggregate data size - * requested in this transfer */ - - /* send format: - * (int) cmd - specifies type of message (SVC_CMD_RDREQ_MSG) - * (int) req_num - number of requests in message - * {sequence of send_meta_t requests} */ - size_t packed_size = (2 * sizeof(int)) + (req_cnt * sizeof(send_msg_t)); - - /* get pointer to start of send buffer */ - char* ptr = req_msg_buf; - memset(ptr, 0, packed_size); - - /* pack command */ - int cmd = (int)SVC_CMD_RDREQ_MSG; - *((int*)ptr) = cmd; - ptr += sizeof(int); - - /* pack request count */ - *((int*)ptr) = req_cnt; - ptr += sizeof(int); - - /* pack each request into the send buffer, - * total up incoming bytes as we go */ - int i; - size_t bytes = 0; - for (i = 0; i < req_cnt; i++) { - /* accumulate data size of this request */ - bytes += send_metas[i].length; - } - - /* copy requests into buffer */ - memcpy(ptr, send_metas, (req_cnt * sizeof(send_msg_t))); - ptr += (req_cnt * sizeof(send_msg_t)); - - /* increment running total size of data bytes */ - (*tot_sz) += bytes; - - /* return number of bytes used to pack requests */ - assert(packed_size == (ptr - req_msg_buf)); - return (int)packed_size; -} - /* pack the chunk read requests for a single remote delegator. * * @param req_msg_buf: request buffer used for packing @@ -1255,6 +983,8 @@ static size_t rm_pack_chunk_requests(char* req_msg_buf, size_t reqs_sz = req_cnt * sizeof(chunk_read_req_t); size_t packed_size = (2 * sizeof(int)) + sizeof(size_t) + reqs_sz; + assert(req_cnt < MAX_META_PER_SEND); + /* get pointer to start of send buffer */ char* ptr = req_msg_buf; memset(ptr, 0, packed_size); @@ -1280,67 +1010,6 @@ static size_t rm_pack_chunk_requests(char* req_msg_buf, return packed_size; } -/* send the read requests to the remote delegator service managers - * @param thrd_ctrl : reqmgr thread control structure - * @param tot_sz : output parameter for total size of data to read - * @return success/error code */ -static int rm_send_remote_requests(reqmgr_thrd_t* thrd_ctrl, - size_t* tot_sz) -{ - // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked - - int rc; - int i = 0; - - /* ToDo: Transfer the message in multiple - * rounds when total size > the size of - * send_msg_buf - * */ - - /* use this variable to total up number of incoming data bytes */ - *tot_sz = 0; - - /* get pointer to send buffer */ - char* sendbuf = thrd_ctrl->del_req_msg_buf; - - /* get pointer to start of read request array, - * and initialize index to point to first element */ - send_msg_t* msgs = thrd_ctrl->del_req_set->msg_meta; - int msg_cursor = 0; - - /* iterate over each delegator we need to send requests to */ - for (i = 0; i < thrd_ctrl->del_req_stat->del_cnt; i++) { - /* pointer to start of requests for this delegator */ - send_msg_t* reqs = msgs + msg_cursor; - - /* number of requests for this delegator */ - int req_num = thrd_ctrl->del_req_stat->req_stat[i].req_cnt; - - /* pack requests into send buffer, get size of packed data, - * increase total number of data payload we will get back */ - int packed_size = rm_pack_send_requests(sendbuf, reqs, - req_num, tot_sz); - - /* get rank of target delegator */ - int del_rank = thrd_ctrl->del_req_stat->req_stat[i].del_id; - - /* send requests */ - //MPI_Send(sendbuf, packed_size, MPI_BYTE, - // del_rank, (int)READ_REQUEST_TAG, MPI_COMM_WORLD); - rc = invoke_server_request_rpc(del_rank, 0, (int)READ_REQUEST_TAG, - (void*)sendbuf, (size_t)packed_size); - if (rc != (int)UNIFYFS_SUCCESS) { - LOGERR("server request rpc to %d failed - %s", del_rank, - unifyfs_error_enum_str((unifyfs_error_e)rc)); - } - - /* advance to requests for next delegator */ - msg_cursor += req_num; - } - - return UNIFYFS_SUCCESS; -} - /* send the chunk read requests to remote delegators * * @param thrd_ctrl : reqmgr thread control structure @@ -1524,172 +1193,6 @@ static shm_meta_t* reserve_shmem_meta(app_config_t* app_config, return meta; } -/* parse the read replies from message received from service manager, - * deliver replies back to client - * - * @param app_id : application id - * @param client_id : local client rank within app - * @param recv_msg_buf : message buffer containing packed read requests - * @param ptr_tot_sz : pointer to total processed data size - * @return success/error code */ -static int rm_process_received_msg(int app_id, - int client_id, - char* recv_msg_buf, - size_t* ptr_tot_sz) -{ - /* assume we'll succeed in processing the message */ - int rc = UNIFYFS_SUCCESS; - - /* look up client app config based on client id */ - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - /* format of read replies in shared memory - * shm_header_t - shared memory region header - * {sequence of shm_meta_t + data payload} */ - - /* get pointer to shared memory buffer for this client */ - size_t header_size = sizeof(shm_header_t); - shm_header_t* hdr = (shm_header_t*)app_config->shm_recv_bufs[client_id]; - - /* format of recv_msg_buf: - * (int) num - number of read replies packed in message - * {sequence of recv_msg_t containing read replies} */ - - /* get pointer to start of receive buffer */ - char* msgptr = recv_msg_buf; - - /* extract number of read requests in this message */ - int num = *(int*)msgptr; - msgptr += sizeof(int); - - /* unpack each read reply */ - int j; - for (j = 0; j < num; j++) { - /* point to first read reply in message */ - recv_msg_t* msg = (recv_msg_t*)msgptr; - msgptr += sizeof(recv_msg_t); - - /* get pointer in shared memory for next read reply */ - shm_meta_t* meta = reserve_shmem_meta(app_config, hdr, msg->length); - if (NULL == meta) { - LOGERR("failed to reserve space for read reply"); - rc = UNIFYFS_FAILURE; - break; - } - char* shmbuf = ((char*)meta) + sizeof(shm_meta_t); - - /* copy in header for this read request */ - meta->gfid = msg->src_fid; - meta->offset = msg->src_offset; - meta->length = msg->length; - meta->errcode = msg->errcode; - - /* copy data for this read request */ - memcpy(shmbuf, msgptr, msg->length); - - /* advance to next read reply in message buffer */ - msgptr += msg->length; - - /* decrement number of bytes processed from total */ - if (NULL != ptr_tot_sz) { - *ptr_tot_sz -= msg->length; - LOGDBG("processed message of size %zu, %zu left to receive", - msg->length, *ptr_tot_sz); - } - } - - return rc; -} - -/* receive the requested data returned from service managers - * as a result of the read requests we sent to them - * - * @param thrd_ctrl: request manager thread state - * @param tot_sz: total data size to receive (excludes header bytes) - * @return success/error code */ -static int rm_receive_remote_message(reqmgr_thrd_t* thrd_ctrl, - size_t tot_sz) -{ - // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked - - /* assume we'll succeed */ - int rc = UNIFYFS_SUCCESS; - - /* get app id and client id that we'll be serving, - * app id associates thread with a namespace (mountpoint) - * the client id associates the thread with a particular - * client process id */ - int app_id = thrd_ctrl->app_id; - int client_id = thrd_ctrl->client_id; - - /* lookup our data structure for this app id */ - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - /* get thread id for this client (used for MPI tags) */ - int thrd_id = app_config->thrd_idxs[client_id]; - - /* service manager will incorporate our thread id in tag, - * to distinguish between target request manager threads */ - int tag = (int)READ_RESPONSE_TAG + thrd_id; - - /* array of MPI_Request objects for window of posted receives */ - MPI_Request recv_req[RECV_BUF_CNT] = {MPI_REQUEST_NULL}; - - /* get number of receives to post and size of each buffer */ - int recv_buf_cnt = RECV_BUF_CNT; - int recv_buf_len = (int) SENDRECV_BUF_LEN; - - /* post a window of receive buffers for incoming data */ - int i; - for (i = 0; i < recv_buf_cnt; i++) { - /* post buffer for incoming receive */ - MPI_Irecv(thrd_ctrl->del_recv_msg_buf[i], recv_buf_len, MPI_BYTE, - MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &recv_req[i]); - } - - /* spin until we have received all incoming data */ - while (tot_sz > 0) { - /* wait for any receive to come in */ - int index; - MPI_Status status; - MPI_Waitany(recv_buf_cnt, recv_req, &index, &status); - - /* got a new message, get pointer to message buffer */ - char* buf = thrd_ctrl->del_recv_msg_buf[index]; - - /* unpack the data into client shared memory, - * this will internally signal client and wait - * for data to be processed if shared memory - * buffer is filled */ - int tmp_rc = rm_process_received_msg(app_id, client_id, buf, &tot_sz); - if (tmp_rc != UNIFYFS_SUCCESS) { - rc = tmp_rc; - } - - /* done processing, repost this receive buffer */ - MPI_Irecv(thrd_ctrl->del_recv_msg_buf[index], recv_buf_len, MPI_BYTE, - MPI_ANY_SOURCE, tag, MPI_COMM_WORLD, &recv_req[index]); - } - - /* cancel posted MPI receives */ - for (i = 0; i < recv_buf_cnt; i++) { - MPI_Status status; - MPI_Cancel(&recv_req[i]); - MPI_Wait(&recv_req[i], &status); - } - - /* signal client that we're now done writing data */ - shm_header_t* hdr = (shm_header_t*)app_config->shm_recv_bufs[client_id]; - client_signal(hdr, SHMEM_REGION_DATA_COMPLETE); - - /* wait for client to read data */ - client_wait(hdr); - - return rc; -} - int rm_post_chunk_read_responses(int app_id, int client_id, int src_rank, @@ -1859,16 +1362,11 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, return ret; } -/* entry point for request manager thread, one thread is created - * for each client process, client informs thread of a set of read - * requests, thread retrieves data for client and notifies client - * when data is ready - * - * delegate the read requests for the delegator thread's client. Each - * delegator thread handles one connection to one client-side rank. - * - * @param arg: pointer to control structure for the delegator thread +/* Entry point for request manager thread. One thread is created + * for each client process to retrieve remote data and notify the + * client when data is ready. * + * @param arg: pointer to RM thread control structure * @return NULL */ void* rm_delegate_request_thread(void* arg) { @@ -1926,25 +1424,6 @@ void* rm_delegate_request_thread(void* arg) LOGERR("failed to request remote chunks"); } - /* tot_sz tracks the total bytes we expect to receive. - * size is computed during send, decremented during receive */ - size_t tot_sz = 0; - rc = rm_send_remote_requests(thrd_ctrl, &tot_sz); - if (rc != UNIFYFS_SUCCESS) { - /* release lock and exit if we hit an error */ - RM_UNLOCK(thrd_ctrl); - return NULL; - } - if (tot_sz > 0) { - /* wait for data to come back from servers */ - rc = rm_receive_remote_message(thrd_ctrl, tot_sz); - if (rc != UNIFYFS_SUCCESS) { - /* release lock and exit if we hit an error */ - RM_UNLOCK(thrd_ctrl); - return NULL; - } - } - /* release lock */ RM_UNLOCK(thrd_ctrl); } @@ -1956,6 +1435,7 @@ void* rm_delegate_request_thread(void* arg) /* BEGIN MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ +#if 0 // DISABLE UNUSED RPCS /* invokes the server_hello rpc */ int invoke_server_hello_rpc(int dst_srvr_rank) { @@ -2016,9 +1496,7 @@ int invoke_server_request_rpc(int dst_srvr_rank, int req_id, int tag, if (dst_srvr_rank == glb_mpi_rank) { // short-circuit for local requests - if (tag == (int)READ_REQUEST_TAG) { - return sm_decode_msg((char*)data_buf); - } + return rc; } assert(dst_srvr_rank < (int)glb_num_servers); @@ -2062,6 +1540,7 @@ int invoke_server_request_rpc(int dst_srvr_rank, int req_id, int tag, return rc; } +#endif // DISABLE UNUSED RPCS /* invokes the server_request rpc */ int invoke_chunk_read_request_rpc(int dst_srvr_rank, diff --git a/server/src/unifyfs_request_manager.h b/server/src/unifyfs_request_manager.h index 464a4a18c..6ce140b84 100644 --- a/server/src/unifyfs_request_manager.h +++ b/server/src/unifyfs_request_manager.h @@ -32,22 +32,6 @@ #include "unifyfs_global.h" -/* one entry per delegator for which we have active read requests, - * records rank of delegator and request count */ -typedef struct { - int req_cnt; /* number of requests to this delegator */ - int del_id; /* rank of delegator */ -} per_del_stat_t; - -/* records list of delegator information (rank, req count) for - * set of delegators we have active read requests for */ -typedef struct { - per_del_stat_t* req_stat; /* delegator rank and request count */ - int del_cnt; /* number of delegators we have read requests for */ -} del_req_stat_t; - -// NEW READ REQUEST STRUCTURES - typedef struct { readreq_status_e status; /* aggregate request status */ int req_ndx; /* index in reqmgr read_reqs array */ @@ -86,21 +70,9 @@ typedef struct { int next_rdreq_ndx; server_read_req_t read_reqs[RM_MAX_ACTIVE_REQUESTS]; - /* a list of read requests to be sent to each delegator, - * main thread adds items to this list, request manager - * processes them */ - msg_meta_t* del_req_set; - - /* statistics of read requests to be sent to each delegator */ - del_req_stat_t* del_req_stat; - /* buffer to build read request messages */ char del_req_msg_buf[REQ_BUF_LEN]; - /* memory for posting receives for incoming read reply messages - * from the service threads */ - char del_recv_msg_buf[RECV_BUF_CNT][SENDRECV_BUF_LEN]; - /* flag set to indicate request manager thread should exit */ int exit_flag; @@ -167,12 +139,14 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, /* MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ +#if 0 // DISABLE UNUSED RPCS int invoke_server_hello_rpc(int dst_srvr_rank); int invoke_server_request_rpc(int dst_srvr_rank, int req_id, int tag, void* data_buf, size_t buf_sz); +#endif // DISABLE UNUSED RPCS int invoke_chunk_read_request_rpc(int dst_srvr_rank, server_read_req_t* rdreq, diff --git a/server/src/unifyfs_service_manager.c b/server/src/unifyfs_service_manager.c index c3ed6689c..83ae8b24d 100644 --- a/server/src/unifyfs_service_manager.c +++ b/server/src/unifyfs_service_manager.c @@ -37,119 +37,8 @@ #include "unifyfs_server_rpcs.h" #include "margo_server.h" -/* The service manager thread runs in a loop waiting for incoming - * requests. When it receives a message, it unpacks all read - * requests and appends them to the sm->service_msgs list. It delays - * for some time in acting on requests in the hopes of buffering - * read bursts to make I/O more efficient. If no read request - * has come in, and the delay time out has expired, and there are - * pening read requests to service, then it services all requests. - * - * It first creates a set of read tasks based on the set of read - * requests. The read requests are sorted by source file and then - * by offset, and read requests that refer to contiguous data - * regions are merged into a large read task. Read tasks are - * added to a read_task list. - * - * The read tasks are executed to copy data into a read buffer. - * Data that is copied from shared memory simply uses memcpy(). - * Data that spans shared memory and the spillover file uses - * memcpy with pread. Data that is fully in the spillover file - * are read with async I/O (aio) operations. The aio operations - * are added to a pending_reads queue that is later waited on. - * - * After all data has been read, results are packed into send - * buffers. This is done by matching read tasks with corresponding - * read requests. When send buffers are filled with read replies - * (acks), they are sent back to requesting delegators with - * MPI_Isend calls. The MPI_Request objects for those isends are - * added to a pending_sends list, which is later waited on. - * - * After replying to all sm->service_msgs, the service manager - * thread again starts listening for more incoming requests */ -/* records info needed to build read reply to send back to - * requesting delegator, these are appened in an ack list - * which records a set of read replies before being sent */ -typedef struct { - recv_msg_t msg; /* header information for read reply */ - char* addr; /* address of data in read buffer */ -} ack_meta_t; - -/* this records info about outstanding sends to delegators, - * including the MPI request that must be waited on to - * determine when send has completed, these are added to - * an pending sends list when sent and then iterated over - * while waiting for all pending sends to complete */ -typedef struct { - MPI_Request req; /* MPI request for outstanding isend */ - MPI_Status stat; /* status for test call */ - int src_rank; /* target delegator rank */ - int src_thrd; /* target delegator thread */ -} ack_stat_t; - -/* records info about read tasks, which are generated by - * unpacking incoming read requests from remote delegators, - * it encodes information about a range of read requests - * that generated the read task, read requests that refer - * to contiguous data may be merged into a single read task, - * and the app_id and client_id are used to determine - * the memory/files holding the source data */ -typedef struct { - size_t size; /* size of read operation */ - int start_idx; /* service_msgs_t.msg[] index of starting request */ - int end_idx; /* service_msgs_t.msg[] index of ending request */ - int app_id; /* app id holding requested data */ - int cli_id; /* client id holding requested data */ - int arrival_time; /* time stamp when read request arrived */ -} read_task_t; - -/* defines an array of read tasks */ -typedef struct { - read_task_t* read_tasks; /* list of read tasks */ - int num; /* number of active read tasks */ - int cap; /* total capacity of read task list */ -} task_set_t; - -/* defines an array of read requests, generated by unpacking - * read requests from requesting delegators */ -typedef struct { - send_msg_t* msg; /* buffer of read requests */ - int num; /* number of active read requests in buffer */ - int cap; /* total capacity of read request list */ -} service_msgs_t; - -/* tracks an outstanding read operation, these are generated - * from read tasks and added to a list of pending read operations, - * which are later waited on before sending data back to requesting - * delegators */ -typedef struct { - int err_submit; /* indicates whether read was submitted */ - struct aiocb read_cb; /* structure for outstanding aio read */ - int index; /* index in read task list for this read */ - int start_pos; /* starting byte offset into read request */ - int end_pos; /* ending byte offset into read request */ - char* mem_pos; /* buffer holding read data */ -} pended_read_t; - -/* defines a list of read replies to be sent to a delegator */ -typedef struct { - arraylist_t* ack_list; /* list of read replies for delegator */ - int rank_id; /* rank of remote delegator to send to */ - int thrd_id; /* thread id of remote delegator */ - int src_cli_rank; /* rank of client that initiated read */ - size_t src_sz; /* total data size in read replies */ - int start_cursor; /* offset within ack_list */ -} rank_ack_meta_t; - -/* defines an list of reply data for different delegators, - * list is ordered by (rank,thread) of delegator for fast lookup */ -typedef struct { - rank_ack_meta_t* ack_metas; /* read reply data for a delegator */ - int num; /* number of items in list */ -} rank_ack_task_t; - -/* Service Manager state */ +/* Service Manager (SM) state */ typedef struct { /* the SM thread */ pthread_t thrd; @@ -167,1073 +56,25 @@ typedef struct { /* list of chunk read requests from remote delegators */ arraylist_t* chunk_reads; - /* records list of read requests from requesting delegators */ - service_msgs_t service_msgs; - - /* list of read tasks that must be executed, - * generated from read requests */ - task_set_t read_task_set; - - /* list of read reply data for each delegator */ - rank_ack_task_t rank_ack_task; - - /* list of outstanding read operations */ - arraylist_t* pended_reads; - - /* list of outstanding send operations */ - arraylist_t* pended_sends; - - /* list of buffers to be used in send operations */ - arraylist_t* send_buf_list; - /* tracks running total of bytes in current read burst */ - long burst_data_sz; - - /* buffer to hold read data while gathering it from source - * memory/files and before being packed into send buffers - * for read replies */ - char* read_buf; + size_t burst_data_sz; } svcmgr_state_t; svcmgr_state_t* sm; // = NULL +/* lock macro for debugging SM locking */ #define SM_LOCK() \ do { \ LOGDBG("locking service manager state"); \ pthread_mutex_lock(&(sm->sync)); \ } while (0) +/* unlock macro for debugging SM locking */ #define SM_UNLOCK() \ do { \ LOGDBG("unlocking service manager state"); \ pthread_mutex_unlock(&(sm->sync)); \ } while (0) - -/* sort read requests into log files by - * app id, then client id, then log offset */ -static int compare_send_msg(const void* a, const void* b) -{ - const send_msg_t* ptr_a = a; - const send_msg_t* ptr_b = b; - - if (ptr_a->dest_app_id > ptr_b->dest_app_id) { - return 1; - } - - if (ptr_a->dest_app_id < ptr_b->dest_app_id) { - return -1; - } - - if (ptr_a->dest_client_id > ptr_b->dest_client_id) { - return 1; - } - - if (ptr_a->dest_client_id < ptr_b->dest_client_id) { - return -1; - } - - if (ptr_a->dest_offset > ptr_b->dest_offset) { - return 1; - } - - if (ptr_a->dest_offset < ptr_b->dest_offset) { - return -1; - } - - return 0; -} - -/* starts a new read task based on read request in sm->service_msgs - * at given index */ -static void create_msg_read_task(int index) -{ - /* get pointer to service message at given index */ - send_msg_t* msg = &sm->service_msgs.msg[index]; - - /* get pointer to current read task */ - int idx = sm->read_task_set.num; - read_task_t* read_task = &sm->read_task_set.read_tasks[idx]; - - /* copy fields from message to read task */ - read_task->start_idx = index; - read_task->end_idx = index; - read_task->size = msg->length; - read_task->app_id = msg->dest_app_id; - read_task->cli_id = msg->dest_client_id; - read_task->arrival_time = msg->arrival_time; -} - -/* - * Cluster read requests based on file offset and age. - * Each log file is uniquely identified by client-side app_id - * and client_id, so we can locate the target log file - * (generated by the client-side program). - * - * @return success/error - */ -static int sm_cluster_reads(void) -{ - /* sort service messages by log file (app_id, client_id) - * and then by offset within each log file */ - qsort(sm->service_msgs.msg, sm->service_msgs.num, - sizeof(send_msg_t), compare_send_msg); - - /* initialize count of read tasks */ - sm->read_task_set.num = 0; - - /* create read task given first service message */ - create_msg_read_task(0); - sm->read_task_set.num++; - - /* iterate over each service message and create read tasks, - * will merge multiple read requests into read tasks - * when two requests refer to contiguous data in a log file */ - int i; - for (i = 1; i < sm->service_msgs.num; i++) { - /* get pointer to current service message */ - send_msg_t* msg = &sm->service_msgs.msg[i]; - - /* get pointer to preivous read task */ - read_task_t* last_read = - &sm->read_task_set.read_tasks[sm->read_task_set.num - 1]; - - /* check whether current message reads from the same log file, - * as our last read task */ - if ((last_read->app_id != msg->dest_app_id) || - (last_read->cli_id != msg->dest_client_id)) { - /* reading on a different local log file, - * so create a new read task for this message */ - create_msg_read_task(i); - sm->read_task_set.num++; - } else { - /* this message reads from the same log file as our last - * read request */ - - /* get pointer to previous read request */ - send_msg_t* last_msg = &sm->service_msgs.msg[i - 1]; - - /* see if we can tack current message on to - * previous read request */ - size_t last_offset = last_msg->dest_offset + last_msg->length; - if (last_offset == msg->dest_offset) { - /* current message starts where last read request - * ends, so append it to last read request if no larger - * than read_block_size */ - - /* the size of individual read should be smaller - * than read_block_size, if read size is larger it - * needs to be split into the unit of READ_BLOCK_SIZE */ - if ((last_read->size + msg->length) <= READ_BLOCK_SIZE) { - /* tack current message on previous read request */ - last_read->end_idx = i; - last_read->size += msg->length; - - /* update minimum arrival time */ - if (msg->arrival_time < last_read->arrival_time) { - last_read->arrival_time = msg->arrival_time; - } - } else { - /* if larger than read block size, start a new - * read task, here the data size requested by - * individual read request should be smaller - * than read_block_size. The larger one - * has already been split by the initiator */ - create_msg_read_task(i); - sm->read_task_set.num++; - } - } else { - /* not contiguous from the last offset, - * start a new read request */ - create_msg_read_task(i); - sm->read_task_set.num++; - } - } - } - - return UNIFYFS_SUCCESS; -} - -/* compare by rank and then thread in increasing order */ -static int compare_rank_thrd(int src_rank, int src_thrd, - int cmp_rank, int cmp_thrd) -{ - if (src_rank > cmp_rank) { - return 1; - } - if (src_rank < cmp_rank) { - return -1; - } - if (src_thrd > cmp_thrd) { - return 1; - } - if (src_thrd < cmp_thrd) { - return -1; - } - return 0; -} - -/* Returns index where delegator (rank, thread), - * should be in list, caller must check whether - * delegator at that position matches */ -static int find_ack_meta(int src_rank, int src_thrd, - int* found) -{ - /* assume we won't find item */ - *found = 0; - - /* if nothing in list, place this as first item */ - if (sm->rank_ack_task.num == 0) { - return 0; - } - - rank_ack_meta_t* metas = sm->rank_ack_task.ack_metas; - - /* if list has one item, compare to that item */ - if (sm->rank_ack_task.num == 1) { - /* compare to first item */ - int cmp = compare_rank_thrd(src_rank, src_thrd, - metas[0].rank_id, metas[0].thrd_id); - if (cmp < 0) { - /* item is smaller than first element */ - return 0; - } else if (cmp > 0) { - /* item is smaller than first element */ - return 1; - } else { - /* item matches first element */ - *found = 1; - return 0; - } - } - - /* execute binary search for item location in list */ - int left = 0; - int right = sm->rank_ack_task.num - 1; - int mid = (left + right) / 2; - while (right > left + 1) { - /* compare item to middle element */ - int cmp = compare_rank_thrd(src_rank, src_thrd, - metas[mid].rank_id, metas[mid].thrd_id); - if (cmp > 0) { - /* item is larger than middle item, - * so bump left range up */ - left = mid; - } else if (cmp < 0) { - /* item is smaller than middle item, - * so move right down to middle */ - right = mid; - } else { - /* found an exact match with middle element */ - *found = 1; - return mid; - } - - /* update middle */ - mid = (left + right) / 2; - } - - /* two elements left in the list, compare to left element */ - int cmp_left = compare_rank_thrd(src_rank, src_thrd, - metas[left].rank_id, - metas[left].thrd_id); - if (cmp_left < 0) { - /* item should come before left element */ - return left; - } else if (cmp_left == 0) { - /* item matches left item */ - *found = 1; - return left; - } - - /* item is larger than left element, so compare to right */ - int cmp_right = compare_rank_thrd(src_rank, src_thrd, - metas[right].rank_id, - metas[right].thrd_id); - if (cmp_right < 0) { - /* item should come before right element */ - return right; - } else if (cmp_right == 0) { - /* item matches right element */ - *found = 1; - return right; - } - - /* otherwise, item must be larger than right element */ - return (right + 1); -} - -/* Insert read reply list for specified (rank_id, thrd_id) delegator - * into ack list, keep ordered by rank,thread for fast lookup */ -static int insert_ack_meta(ack_meta_t* ack, - int pos, - int rank_id, - int thrd_id, - int src_cli_rank) -{ - /* get pointer to array of read reply data for delegators */ - rank_ack_meta_t* metas = sm->rank_ack_task.ack_metas; - - /* check whether insert location is in middle of the list */ - if (pos < sm->rank_ack_task.num) { - /* need to insert in the middle, bump all entries - * past pos up a slot */ - int i; - for (i = sm->rank_ack_task.num - 1; i >= pos; i--) { - metas[i + 1] = metas[i]; - } - } - - /* get pointer to ack meta data structure */ - rank_ack_meta_t* ack_meta = &(metas[pos]); - - /* initialize with values */ - ack_meta->ack_list = arraylist_create(); - ack_meta->rank_id = rank_id; - ack_meta->thrd_id = thrd_id; - ack_meta->src_cli_rank = src_cli_rank; - ack_meta->src_sz = ack->msg.length; - ack_meta->start_cursor = 0; - - /* check that we were able to create a new ack_list */ - if (ack_meta->ack_list == NULL) { - return (int)UNIFYFS_ERROR_NOMEM; - } - - /* insert ack_meta into our list */ - arraylist_add(ack_meta->ack_list, ack); - - /* increment the number of entries in our list */ - sm->rank_ack_task.num++; - - return UNIFYFS_SUCCESS; -} - -/* Send back ack to the remote delegator. - * packs read replies into a send buffer, sends data with isend, - * adds record of send to pending sends list */ -static int sm_ack_remote_delegator(rank_ack_meta_t* ack_meta) -{ - int i; - - /* allocate more send buffers if we're at capacity */ - if (sm->send_buf_list->size == sm->send_buf_list->cap) { - /* at capacity in our list, allocate more space, - * double capacity in array */ - size_t new_cap = 2 * sm->send_buf_list->cap; - sm->send_buf_list->elems = (void**)realloc(sm->send_buf_list->elems, - (new_cap * sizeof(void*))); - - /* initialize pointers in new portion of array */ - for (i = sm->send_buf_list->cap; i < new_cap; i++) { - sm->send_buf_list->elems[i] = NULL; - } - - /* record new capacity */ - sm->send_buf_list->cap = new_cap; - } - - /* attempt to reuse allocated buffer if we can */ - if (sm->send_buf_list->elems[sm->send_buf_list->size] == NULL) { - /* need to allocate a new buffer */ - sm->send_buf_list->elems[sm->send_buf_list->size] = - malloc(SENDRECV_BUF_LEN); - } - - /* get pointer to send buffer */ - char* send_msg_buf = sm->send_buf_list->elems[sm->send_buf_list->size]; - sm->send_buf_list->size++; - - /* running total number of bytes we'll send */ - int send_sz = 0; - - /* compute number of read replies we'll send */ - size_t ack_count = arraylist_size(ack_meta->ack_list); - size_t ack_start = ack_meta->start_cursor; - int len = ack_count - ack_start; - - /* copy in number of read replies to message */ - memcpy(send_msg_buf + send_sz, &len, sizeof(int)); - send_sz += sizeof(int); - - /* pack read replies into send buffer */ - for (i = ack_start; i < ack_count; i++) { - /* get pointer to read reply header */ - ack_meta_t* meta = - (ack_meta_t*)arraylist_get(ack_meta->ack_list, i); - - /* copy read reply header to send buffer */ - memcpy(send_msg_buf + send_sz, &(meta->msg), sizeof(recv_msg_t)); - send_sz += sizeof(recv_msg_t); - - /* copy file data to send buffer */ - size_t length = (size_t) meta->msg.length; - memcpy(send_msg_buf + send_sz, meta->addr, length); - send_sz += (int) length; - } - - /* get rank and thread id of remote delegator */ - int del_rank = ack_meta->rank_id; - int del_thread = ack_meta->thrd_id; - - /* allocate a new ack stat structure to track details - * of pending send */ - ack_stat_t* ack_stat = (ack_stat_t*)malloc(sizeof(ack_stat_t)); - ack_stat->src_rank = del_rank; - ack_stat->src_thrd = del_thread; - - /* send read replies to delegator (rank and thread), - * record MPI request in ack stat structure to wait later */ - MPI_Isend(send_msg_buf, send_sz, MPI_BYTE, - del_rank, (int)READ_RESPONSE_TAG + del_thread, - MPI_COMM_WORLD, &(ack_stat->req)); - - /* add item to our list of pending sends */ - arraylist_add(sm->pended_sends, ack_stat); - - return UNIFYFS_SUCCESS; -} - -/* - * Insert a message to an entry of ack (read reply) list corresponding - * to its destination delegator. - * - * @param mem_addr : address of data to be acked in mem_pool - * @param index : identifies msg to be inserted to ack_lst - * @param src_offset: offset of the requested segment on the logical - * file (not the physical log file on SSD). - * e.g., for N-1 pattern, logical offset - * is the offset in the shared file - * @param len : length of the message - */ -static int insert_to_ack_list(char* mem_addr, - int index, - size_t src_offset, - size_t len, - int errcode) -{ - int rc = 0; - - /* get pointer to read request we are replying to */ - send_msg_t* msg = &sm->service_msgs.msg[index]; - - /* allocate a new structure to record ack meta data */ - ack_meta_t* ack = (ack_meta_t*)malloc(sizeof(ack_meta_t)); - - /* the src_offset might start from any position in the message, so - * make it a separate parameter */ - ack->msg.src_fid = msg->src_fid; /* global file id */ - ack->msg.src_offset = src_offset; /* offset in file */ - ack->msg.length = len; /* length of data */ - ack->msg.errcode = errcode; /* error code for read (pass/fail) */ - ack->addr = mem_addr; /* pointer to data in read buffer */ - - /* after setting the ack for this message, link it - * to a ack list based on its destination. */ - int src_rank = msg->src_delegator_rank; - int src_thrd = msg->src_thrd; - int src_cli_rank = msg->src_dbg_rank; - - /* find the position in the list for target delegator */ - int found = 0; - int pos = find_ack_meta(src_rank, src_thrd, &found); - - /* check whether delegator at this position is the target */ - if (!found) { - /* it's not, so insert new entry for the target into the - * list at this position */ - rc = insert_ack_meta(ack, pos, src_rank, src_thrd, src_cli_rank); - } else { - /* found the corresponding entry for target delegator */ - rank_ack_meta_t* ack_meta = &(sm->rank_ack_task.ack_metas[pos]); - - /* compute number of read replies waiting to be sent */ - int num_entries = arraylist_size(ack_meta->ack_list); - int num_to_ack = num_entries - ack_meta->start_cursor; - - /* number of bytes needed to pack existing read replies */ - size_t curr_bytes = (num_to_ack * sizeof(ack_meta_t)) + - ack_meta->src_sz; - - /* number of bytes to pack this read reply */ - size_t bytes = sizeof(ack_meta_t) + ack->msg.length; - - /* check whether we can fit this data into the - * existing send block */ - if (curr_bytes + bytes > SENDRECV_BUF_LEN) { - /* not enough room, send the current list of read replies */ - LOGDBG("early ack due to full send buffer"); - rc = sm_ack_remote_delegator(ack_meta); - - /* start a new list */ - ack_meta->src_sz = ack->msg.length; - arraylist_add(ack_meta->ack_list, ack); - - /* start_cursor records the starting ack - * for the next send */ - ack_meta->start_cursor += num_to_ack; - - /* check that our reads and sends completed ok */ - if (rc < 0) { - return (int)UNIFYFS_ERROR_SEND; - } - } else { - /* current read reply fits in send buffer, - * add it to the list */ - ack_meta->src_sz += ack->msg.length; - arraylist_add(ack_meta->ack_list, ack); - } - } - - return UNIFYFS_SUCCESS; -} - -/* Insert the data read for each element in read task list to read - * reply list, data will be sent later - * @param read_task : data returned from read task - * @param mem_addr : memory loc to copy read data - * @param start_offset: first offset in read task - * @param end_offset : last offset in read task - * @param errcode : error code on read (pass/fail) - */ -static int batch_insert_to_ack_list(read_task_t* read_task, - char* mem_addr, - int start_offset, - int end_offset, - int errcode) -{ - /* search for the starting read request in service msgs for - * a given region of read_task_t identified by start_offset - * and size of read task */ - - /* read_task - * start_offset end_offset - * read_task_t: -----------******************------------ - * service_msgs:[ ],[*********],[***], [******],[ ] - * - */ - - /* find index in read requests such that it contains starting - * offset for read task */ - int idx = read_task->start_idx; - int cur_offset = 0; - while (1) { - /* get pointer to current read request */ - send_msg_t* msg = &sm->service_msgs.msg[idx]; - - /* check whether end offset of this read request comes - * at or after starting offset of read task */ - if (cur_offset + msg->length >= start_offset) { - /* starting offset of read task falls between start - * and end offset of this read request */ - break; - } - - /* move on to next read request */ - cur_offset += msg->length; - idx++; - } - - /* compute leading bytes in read request that this read task - * does not overlap with */ - int skip_offset = start_offset - cur_offset; - - /* insert read replies for leading read request and all middle - * read requests */ - int first = 1; - int mem_pos = 0; - while (1) { - /* get pointer to read request */ - send_msg_t* msg = &sm->service_msgs.msg[idx]; - - /* stop if we reached the read request that contains the last - * byte of our read task */ - if (cur_offset + msg->length >= end_offset + 1) { - /* ending byte in read task comes before ending byte - * in read request */ - break; - } - - /* compute length and offset of read request that we cover with - * this read task, assume it's the whole read request */ - long length = msg->length; - long offset = msg->src_offset; - if (first == 1) { - /* in the first read request, the read task may start - * partway through, so skip any leading bytes */ - length = msg->length - skip_offset; - offset = msg->src_offset + skip_offset; - first = 0; - } - - /* add entry to read reply list */ - insert_to_ack_list(mem_addr + mem_pos, idx, offset, length, errcode); - - /* update our offset into read reply buffer */ - mem_pos += length; - - /* update offset into read task data */ - cur_offset += length; - - /* move on to next read request */ - idx++; - } - - /* the read task may not fully fill the ending read request */ - if (mem_pos < end_offset - start_offset + 1) { - /* compute remaining length of read task */ - long length = (end_offset - start_offset + 1) - mem_pos; - - /* starting offset for read request */ - long offset = sm->service_msgs.msg[idx].src_offset; - - /* add entry to read reply list */ - insert_to_ack_list(mem_addr + mem_pos, idx, offset, length, errcode); - } - - return UNIFYFS_SUCCESS; -} - -/* - * Wait until all data are read and sent. - * @return success/error - */ -static int sm_wait_until_digested(void) -{ - int i, rc, counter; - read_task_t* read_task; - - /* pointer to array of boolean flags for whether we need to test - * outstanding operations */ - int* flags = NULL; - - /* get number of pending read operations */ - int num_pended_reads = arraylist_size(sm->pended_reads); - if (num_pended_reads > 0) { - /* allocate space for pending flags, setting all to 0 */ - flags = (int*) calloc(num_pended_reads, sizeof(int)); - - /* wait for all pending read operations to complete */ - counter = 0; - while (1) { - /* check whether we have processed all reads */ - if (counter == num_pended_reads) { - LOGDBG("all pending reads completed"); - break; - } - LOGDBG("waiting for %d pending reads", - (num_pended_reads - counter)); - - /* iterate over pending reads */ - for (i = 0; i < num_pended_reads; i++) { - if (flags[i] != 1) { - int errcode = UNIFYFS_SUCCESS; - - /* get meta data for this pending read */ - pended_read_t* pr = - (pended_read_t*)arraylist_get(sm->pended_reads, i); - - read_task = NULL; - if (pr->index != -1) { - read_task = &(sm->read_task_set.read_tasks[pr->index]); - } - - if (pr->err_submit) { /* failed to submit */ - /* mark that this read operation failed */ - errcode = (int)UNIFYFS_ERROR_READ; - flags[i] = 1; - counter++; - - if (NULL != read_task) { - /* add read reply to ack_list as failed */ - batch_insert_to_ack_list(read_task, - pr->mem_pos, - pr->start_pos, - pr->end_pos, - errcode); - } - } else if (aio_error(&pr->read_cb) != EINPROGRESS) { - /* mark that this read operation has completed */ - flags[i] = 1; - counter++; - - /* check that read completed without error */ - ssize_t readrc = aio_return(&pr->read_cb); - if (readrc == -1) { - errcode = errno; - } else if (readrc != (ssize_t)pr->read_cb.aio_nbytes) { - /* short read considered as error */ - errcode = (int)UNIFYFS_ERROR_READ; - } - - if (NULL != read_task) { - /* add read reply to ack_list */ - batch_insert_to_ack_list(read_task, - pr->mem_pos, - pr->start_pos, - pr->end_pos, - errcode); - } - } - } - } - } - - free(flags); - flags = NULL; - - /* reset our list of pending read operations */ - arraylist_reset(sm->pended_reads); - } - - /* read operations have completed, - * send data to delegators */ - - /* iterate over list of delegators and send remaining acks */ - for (i = 0; i < sm->rank_ack_task.num; i++) { - /* get data structure for this delegator */ - rank_ack_meta_t* ack_meta = &(sm->rank_ack_task.ack_metas[i]); - - /* get total number of read replies in this list */ - int tmp_sz = arraylist_size(ack_meta->ack_list); - - /* if we have read replies we have yet to send, - * send them now */ - if (ack_meta->start_cursor < tmp_sz) { - /* got some read replies, send them */ - rc = sm_ack_remote_delegator(ack_meta); - if (rc != UNIFYFS_SUCCESS) { - LOGERR("failed to ack delegator"); - } - } - } - - /* sends issued, now wait for them to complete */ - - /* get number of outstanding sends - * initiated in sm_ack_remote_delegator */ - int num_pended_sends = arraylist_size(sm->pended_sends); - if (num_pended_sends > 0) { - /* allocate space for flags, setting all to 0 */ - flags = (int*)calloc(num_pended_sends, sizeof(int)); - - /* wait until all acks are sent */ - counter = 0; - while (1) { - /* check whether we have waited on all sends */ - if (counter == num_pended_sends) { - LOGDBG("all pending sends completed"); - break; - } - LOGDBG("waiting for %d pending sends", - (num_pended_sends - counter)); - - /* iterate over each send we issued */ - for (i = 0; i < num_pended_sends; i++) { - /* if send is still outstanding, check whether it's done */ - if (flags[i] == 0) { - /* still outstanding, get data for this send */ - ack_stat_t* ack_stat = arraylist_get(sm->pended_sends, i); - - /* test whether send is complete */ - rc = MPI_Test(&ack_stat->req, &flags[i], - &ack_stat->stat); - if (rc != MPI_SUCCESS) { - LOGERR("MPI_Test() for pending send failed"); - } - - /* if send completed, bump our counter */ - if (flags[i] != 0) { - counter++; - } - } - } - } - - free(flags); - flags = NULL; - - /* clear our list of sends */ - arraylist_reset(sm->pended_sends); - } - - /* reset our list of read replies */ - for (i = 0; i < sm->rank_ack_task.num; i++) { - rank_ack_meta_t* ack_meta = &(sm->rank_ack_task.ack_metas[i]); - arraylist_reset(ack_meta->ack_list); - ack_meta->rank_id = -1; - ack_meta->thrd_id = -1; - ack_meta->src_sz = 0; - ack_meta->start_cursor = 0; - } - sm->rank_ack_task.num = 0; - - /* TODO: might be nice to free some send buffers here */ - - /* set active send buffers back to 0, - * we do not free send buffers so that we do not have to - * allocate them again */ - sm->send_buf_list->size = 0; - - return UNIFYFS_SUCCESS; -} - -static int compare_read_task(const void* a, const void* b) -{ - const read_task_t* ptr_a = a; - const read_task_t* ptr_b = b; - - if (ptr_a->size > ptr_b->size) { - return 1; - } - - if (ptr_a->size < ptr_b->size) { - return -1; - } - - if (ptr_a->arrival_time > ptr_b->arrival_time) { - return 1; - } - - if (ptr_a->arrival_time < ptr_b->arrival_time) { - return -1; - } - - return 0; -} - -/* - * Read and send the data via pipelined read, copy and send - * @return success/error - */ -static int sm_read_send_pipe(void) -{ - LOGDBG("processing %d reads", sm->read_task_set.num); - - /* sort read tasks by size and then by arrival time */ - qsort(sm->read_task_set.read_tasks, sm->read_task_set.num, - sizeof(read_task_t), compare_read_task); - - /* points to offset in read reply buffer */ - size_t buf_cursor = 0; - - /* iterate over read tasks and pack read replies into send buffer */ - int i; - for (i = 0; i < sm->read_task_set.num; i++) { - /* get pointer to current read task */ - read_task_t* read_task = &(sm->read_task_set.read_tasks[i]); - - /* get size of data we are to read */ - size_t size = read_task->size; - - /* check whether we have room in the read buffer to hold data */ - if ((buf_cursor + size) > READ_BUF_SZ) { - /* no room, wait until reads complete and send - * out replies */ - LOGDBG("read buf exhausted"); - sm_wait_until_digested(); - - /* reset to start of read buffer */ - buf_cursor = 0; - } - - /* get app id and client id for this read task, - * defines log files holding data */ - int app_id = read_task->app_id; - int cli_id = read_task->cli_id; - - /* look up app config for given app id */ - app_config_t* app_config = (app_config_t*) - arraylist_get(app_config_list, app_id); - assert(app_config); - - /* get index of starting service message */ - int start_idx = read_task->start_idx; - - /* get offset in log file */ - send_msg_t* msg = &sm->service_msgs.msg[start_idx]; - size_t offset = msg->dest_offset; - - /* prepare read opertions based on data location */ - if ((offset + read_task->size) <= app_config->data_size) { - /* requested data in read_task is totally in shared memory, - * get pointer to position in shared memory */ - char* log_ptr = app_config->shm_superblocks[cli_id] + - app_config->data_offset + offset; - - /* copy data into read reply buffer */ - char* buf_ptr = sm->read_buf + buf_cursor; - memcpy(buf_ptr, log_ptr, size); - buf_cursor += size; - - /* we assume memcpy worked */ - int errcode = UNIFYFS_SUCCESS; - - /* prepare read reply meta data */ - batch_insert_to_ack_list(read_task, buf_ptr, - 0, size - 1, errcode); - } else if (offset < app_config->data_size) { - /* part of the requested data is in shared memory, - * compute size in shared memory */ - size_t sz_from_mem = app_config->data_size - offset; - - /* get pointer to position in shared memory */ - char* log_ptr = app_config->shm_superblocks[cli_id] + - app_config->data_offset + offset; - - /* copy data into read reply buffer */ - char* buf_ptr = sm->read_buf + buf_cursor; - memcpy(buf_ptr, log_ptr, sz_from_mem); - buf_cursor += sz_from_mem; - - /* we assume memcpy worked */ - int errcode = UNIFYFS_SUCCESS; - - /* compute size in spillover file */ - long sz_from_ssd = size - sz_from_mem; - - /* read data from spillover file */ - int fd = app_config->spill_log_fds[cli_id]; - ssize_t nread = pread(fd, sm->read_buf + buf_cursor, - sz_from_ssd, 0); - if (nread != (ssize_t)sz_from_ssd) { - /* read error or short read, - * consider either case to be an error */ - errcode = (int)UNIFYFS_ERROR_READ; - } - buf_cursor += sz_from_ssd; - - /* prepare read reply meta data */ - batch_insert_to_ack_list(read_task, buf_ptr, - 0, size - 1, errcode); - } else { - /* all requested data in the current read task - * are in spillover files */ - int fd = app_config->spill_log_fds[cli_id]; - - /* allocate empty pending read structure */ - pended_read_t* ptr = - (pended_read_t*)malloc(sizeof(pended_read_t)); - - /* fill in aio fields */ - memset(&ptr->read_cb, 0, sizeof(struct aiocb)); - ptr->read_cb.aio_fildes = fd; - ptr->read_cb.aio_buf = sm->read_buf + buf_cursor; - ptr->read_cb.aio_offset = offset - app_config->data_size; - ptr->read_cb.aio_nbytes = size; - - /* index of read task for this pending read */ - ptr->index = i; - - /* offset locations in generating read request */ - ptr->start_pos = 0; - ptr->end_pos = size - 1; - - /* send buffer location to copy data when complete */ - ptr->mem_pos = sm->read_buf + buf_cursor; - - /* submit read as aio operation */ - ptr->err_submit = 0; - int rc = aio_read(&ptr->read_cb); - if (rc < 0) { - /* remember that we failed to submit this read */ - ptr->err_submit = 1; - } - buf_cursor += size; - - /* enqueue entry in our list of pending reads */ - arraylist_add(sm->pended_reads, ptr); - } - - /* update accounting for burst size */ - sm->burst_data_sz += size; - } - - /* have initiated all read tasks, wait for them to finish - * and send results to delegators */ - sm_wait_until_digested(); - - return UNIFYFS_SUCCESS; -} - -/* Decode the read-request message received from request_manager - * - * @param msg_buf: message buffer containing request(s) - * @return success/error code - */ -int sm_decode_msg(char* msg_buf) -{ - /* get pointer to start of receive buffer */ - char* ptr = msg_buf; - - /* advance past command */ - ptr += sizeof(int); - - /* extract number of read requests in message */ - int num = *((int*)ptr); - ptr += sizeof(int); - - /* get pointer to read request */ - send_msg_t* msg = (send_msg_t*)ptr; - - assert(NULL != sm); - SM_LOCK(); - - /* get current timestamp as integer */ - int now = (int)time(NULL); - - LOGDBG("decoding %d requests", num); - - /* allocate a larger array of service requests if needed */ - if (num + sm->service_msgs.num >= sm->service_msgs.cap) { - /* get a larger buffer (2x what is currently needed) */ - size_t count = 2 * (num + sm->service_msgs.num); - - /* allocate larger buffer (2x what is currently needed) */ - size_t bytes = count * sizeof(send_msg_t); - sm->service_msgs.msg = - (send_msg_t*)realloc(sm->service_msgs.msg, bytes); - if (sm->service_msgs.msg == NULL) { - /* failed to allocate memory */ - SM_UNLOCK(); - return (int)UNIFYFS_ERROR_NOMEM; - } - - /* got the memory, increase the capacity */ - sm->service_msgs.cap = count; - - /* allocate corresponding space for read tasks */ - bytes = count * sizeof(read_task_t); - sm->read_task_set.read_tasks = - (read_task_t*)realloc(sm->read_task_set.read_tasks, bytes); - if (sm->read_task_set.read_tasks == NULL) { - /* failed to allocate memory */ - SM_UNLOCK(); - return (int)UNIFYFS_ERROR_NOMEM; - } - - /* got the memory, increase the capacity */ - sm->read_task_set.cap = count; - } - - /* unpack read requests to fill in service messages */ - int iter; - for (iter = 0; iter < num; iter++) { - /* copy values from read request */ - sm->service_msgs.msg[sm->service_msgs.num] = msg[iter]; - - /* set time stamp on when we received this request */ - sm->service_msgs.msg[sm->service_msgs.num].arrival_time = now; - - /* increment the number of service requests - * and go to next read request */ - sm->service_msgs.num++; - } - - SM_UNLOCK(); - - return UNIFYFS_SUCCESS; -} - /* Decode and issue chunk-reads received from request manager * * @param src_rank : source delegator rank @@ -1403,15 +244,6 @@ int svcmgr_init(void) return (int)UNIFYFS_ERROR_NOMEM; } - /* allocate a buffer to hold read data before packing - * into send buffers */ - sm->read_buf = malloc(READ_BUF_SZ); - if (sm->read_buf == NULL) { - LOGERR("failed to allocate service manager read buffer!"); - svcmgr_fini(); - return (int)UNIFYFS_ERROR_NOMEM; - } - /* tracks how much data we process in each burst */ sm->burst_data_sz = 0; @@ -1423,63 +255,6 @@ int svcmgr_init(void) return (int)UNIFYFS_ERROR_NOMEM; } - /* initialize our data structure for holding read requests */ - size_t bytes = MAX_META_PER_SEND * sizeof(send_msg_t); - sm->service_msgs.msg = (send_msg_t*)malloc(bytes); - sm->service_msgs.num = 0; - sm->service_msgs.cap = MAX_META_PER_SEND; - - /* initialize our data structure for holding read tasks */ - bytes = MAX_META_PER_SEND * sizeof(read_task_t); - sm->read_task_set.read_tasks = (read_task_t*)malloc(bytes); - sm->read_task_set.num = 0; - sm->read_task_set.cap = MAX_META_PER_SEND; - - /* allocate a list to track pending data read operations */ - sm->pended_reads = arraylist_create(); - if (sm->pended_reads == NULL) { - LOGERR("failed to allocate service manager pended_reads!"); - svcmgr_fini(); - return (int)UNIFYFS_ERROR_NOMEM; - } - - /* allocate a list to track outstanding sends back to - * requesting delegators */ - sm->pended_sends = arraylist_create(); - if (sm->pended_sends == NULL) { - LOGERR("failed to allocate service manager pended_sends!"); - svcmgr_fini(); - return (int)UNIFYFS_ERROR_NOMEM; - } - - /* allocate a list to track pending send operations - * that need to be initiated back to requesting delegators */ - sm->send_buf_list = arraylist_create(); - if (sm->send_buf_list == NULL) { - LOGERR("failed to allocate service manager send_buf_list!"); - svcmgr_fini(); - return (int)UNIFYFS_ERROR_NOMEM; - } - - /* allocate memory to hold meta data for read replies */ - bytes = glb_mpi_size * MAX_NUM_CLIENTS * sizeof(rank_ack_meta_t); - sm->rank_ack_task.num = 0; - sm->rank_ack_task.ack_metas = (rank_ack_meta_t*)malloc(bytes); - - /* initialize each ack_meta structure, keep one for each potential - * application client - * (num delegaotrs * max num clients per delegator) */ - int i; - for (i = 0; i < glb_mpi_size * MAX_NUM_CLIENTS; i++) { - /* get pointer to current structure */ - rank_ack_meta_t* ack_meta = &(sm->rank_ack_task.ack_metas[i]); - ack_meta->ack_list = NULL; - ack_meta->rank_id = -1; - ack_meta->thrd_id = -1; - ack_meta->src_sz = 0; - ack_meta->start_cursor = 0; - } - int rc = pthread_mutex_init(&(sm->sync), NULL); if (0 != rc) { LOGERR("failed to initialize service manager mutex!"); @@ -1505,9 +280,6 @@ int svcmgr_fini(void) { if (NULL != sm) { if (sm->thrd) { - // int exit_cmd = (int)SVC_CMD_EXIT; - // MPI_Send(&exit_cmd, sizeof(int), MPI_CHAR, glb_mpi_rank, - // (int)READ_REQUEST_TAG, MPI_COMM_WORLD); sm->time_to_exit = 1; pthread_join(sm->thrd, NULL); } @@ -1516,21 +288,6 @@ int svcmgr_fini(void) } arraylist_free(sm->chunk_reads); - arraylist_free(sm->pended_reads); - arraylist_free(sm->pended_sends); - arraylist_free(sm->send_buf_list); - - if (NULL != sm->service_msgs.msg) { - free(sm->service_msgs.msg); - } - if (NULL != sm->read_task_set.read_tasks) { - free(sm->read_task_set.read_tasks); - } - - int i; - for (i = 0; i < sm->rank_ack_task.num; i++) { - arraylist_free(sm->rank_ack_task.ack_metas[i].ack_list); - } if (sm->initialized) { SM_UNLOCK(); @@ -1556,9 +313,6 @@ static int send_chunk_read_responses(void) remote_chunk_reads_t* rcr = (remote_chunk_reads_t*) arraylist_get(sm->chunk_reads, i); rc = invoke_chunk_read_response_rpc(rcr); - if (rc != UNIFYFS_SUCCESS) { - LOGERR("failed to send chunk read responses"); - } } arraylist_reset(sm->chunk_reads); } @@ -1566,165 +320,61 @@ static int send_chunk_read_responses(void) return rc; } -/* entry point for the service thread, service the read requests - * received from the requesting delegators, executes a loop constantly - * waiting on incoming message, for each message that comes in, - * appends read requests to sm->service_msgs list, if no message has - * arrived, and after some wait time (to catch bursty reads), - * then convert read requests into read tasks, execute read tasks - * to build read replies, and send read replies back to delegators */ -void* sm_service_reads(void* ctx) +/* Entry point for service manager thread. The SM thread + * runs in a loop processing read request replies until + * the main server thread asks it to exit. The read requests + * themselves are handled by Margo RPC threads. + * + * @param arg: pointer to SM thread control structure + * @return NULL */ +void* sm_service_reads(void* arg) { int rc; LOGDBG("I am service manager thread!"); - assert(sm == (svcmgr_state_t*)ctx); - - /* received message format: - * cmd, req_num, a list of read requests */ - - /* buffer to hold received msg, - * since this is large, malloc it instead of declare it on stack - * (mitigate problems thread stack size limits) */ - char* req_msg_buf = (char*)malloc(REQ_BUF_LEN); - - /* counter for timed wait before creating read tasks, - * used to buffer multiple read requests before responding - * with the idea that reads come in bursts */ - int wait_time = 0; - - /* initialize value on how long to wait before processing - * incoming read requests */ - long bursty_interval = MAX_BURSTY_INTERVAL / 10; - - /* listen and server incoming requests until signaled to exit */ - MPI_Status status; - while (!sm->time_to_exit) { - /* post a receive for incoming request */ - MPI_Request request; - MPI_Irecv(req_msg_buf, REQ_BUF_LEN, MPI_BYTE, - MPI_ANY_SOURCE, (int)READ_REQUEST_TAG, - MPI_COMM_WORLD, &request); - - /* test whether we received anything */ - int irecv_flag = 0; - int mpi_rc = MPI_Test(&request, &irecv_flag, &status); - if (mpi_rc != MPI_SUCCESS) { - sm->sm_exit_rc = (int)UNIFYFS_ERROR_RECV; - return NULL; - } - - send_chunk_read_responses(); - - /* as long as we keep receiving requests, we'll keep skipping - * the while loop below (and its sleep) and keep appending - * items to our read request queue */ + assert(sm == (svcmgr_state_t*)arg); - /* - * keep receiving the read request - * until the end of a anticipated - * bursty behavior - * */ - while (!irecv_flag) { - - if (sm->time_to_exit) { - break; - } - - send_chunk_read_responses(); - - /* if we have not received anything, sleep */ - if (bursty_interval > MIN_SLEEP_INTERVAL) { - usleep(SLEEP_INTERVAL); /* wait an interval */ - wait_time += SLEEP_INTERVAL; - } + /* handle chunk reads until signaled to exit */ + while (1) { + rc = send_chunk_read_responses(); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to send chunk read responses"); + } - /* a bursty behavior is considered end when - * wait time is larger than BURSTY_INTERVAL - * */ - if ((wait_time >= bursty_interval) || - (bursty_interval <= MIN_SLEEP_INTERVAL)) { - /* if time to wait has expired, and if we have some - * queued read requests, do some work */ - - pthread_mutex_lock(&(sm->sync)); - if (sm->service_msgs.num > 0) { - /* run through list of read requests and generate - * read tasks, merge requests for contiguous data - * into a single read task */ - rc = sm_cluster_reads(); - if (rc != 0) { - sm->sm_exit_rc = rc; - pthread_mutex_unlock(&(sm->sync)); - return NULL; - } - - /* execute read tasks, wait for them to complete, - * then pack read replies, send to delegators, and - * finally wait for sends to complete */ - rc = sm_read_send_pipe(); - if (rc != 0) { - sm->sm_exit_rc = rc; - pthread_mutex_unlock(&(sm->sync)); - return NULL; - } - - /* have processed all read requests and read tasks, - * prep them for next message */ - sm->service_msgs.num = 0; - sm->read_task_set.num = 0; - - /* determine how long to wait next time based on - * how much data we just processed in this burst */ - if (sm->burst_data_sz >= LARGE_BURSTY_DATA) { - /* for large bursts above a threshold, - * wait for a fixed amount of time */ - bursty_interval = MAX_BURSTY_INTERVAL; - } else { - /* for smaller bursts, set delay proportionally - * to burst size we just processed */ - bursty_interval = - (SLEEP_SLICE_PER_UNIT * sm->burst_data_sz) / MIB; - } - - /* reset our burst size counter */ - sm->burst_data_sz = 0; - } - pthread_mutex_unlock(&(sm->sync)); - - /* reset our timer */ - wait_time = 0; - } + pthread_mutex_lock(&(sm->sync)); - /* test for receive again, will break while loop - * if we find something */ - mpi_rc = MPI_Test(&request, &irecv_flag, &status); - if (mpi_rc != MPI_SUCCESS) { - sm->sm_exit_rc = (int)UNIFYFS_ERROR_RECV; - return NULL; - } + if (sm->time_to_exit) { + pthread_mutex_unlock(&(sm->sync)); + break; } - if (irecv_flag) { - /* got a message, reset wait time */ - wait_time = 0; - - /* first value of receive buffer is integer holding command */ - int reqcmd = *((int*)req_msg_buf); - if (reqcmd == (int)SVC_CMD_RDREQ_MSG) { - /* got a request for data, append read requests in message - * to our sm->service_msgs list */ - sm_decode_msg(req_msg_buf); - } else if (reqcmd == (int)SVC_CMD_EXIT) { - /* time to exit */ - sm->time_to_exit = 1; - } +#ifdef BURSTY_WAIT // REVISIT WHETHER BURSTY WAIT STILL DESIRABLE + /* determine how long to wait next time based on + * how much data we just processed in this burst */ + size_t bursty_interval; + if (sm->burst_data_sz >= LARGE_BURSTY_DATA) { + /* for large bursts above a threshold, + * wait for a fixed amount of time */ + bursty_interval = MAX_BURSTY_INTERVAL; + } else { + /* for smaller bursts, set delay proportionally + * to burst size we just processed */ + bursty_interval = + (SLEEP_SLICE_PER_UNIT * sm->burst_data_sz) / MIB; } - } + if (bursty_interval > MIN_SLEEP_INTERVAL) { + usleep(SLEEP_INTERVAL); /* wait an interval */ + } +#else + /* wait an interval */ + usleep(MIN_SLEEP_INTERVAL); +#endif // REVISIT WHETHER BURSTY WAIT STILL DESIRABLE + + /* reset our burst size counter */ + sm->burst_data_sz = 0; - /* free receive buffer */ - free(req_msg_buf); - req_msg_buf = NULL; + pthread_mutex_unlock(&(sm->sync)); + } LOGDBG("service manager thread exiting"); @@ -1884,21 +534,6 @@ static void server_request_rpc(hg_handle_t handle) } switch (tag) { - case (int)READ_REQUEST_TAG: { - /* verify this is a request for data */ - if (reqcmd == (int)SVC_CMD_RDREQ_MSG) { - LOGDBG("request command: SVC_CMD_RDREQ_MSG"); - /* got a request for data, append read requests in message - * to our sm->service_msgs list */ - sm_decode_msg((char*)reqbuf); - ret = (int32_t)UNIFYFS_SUCCESS; - } else { - LOGERR("invalid request command %d from server %d", - reqcmd, src_rank); - ret = (int32_t)UNIFYFS_ERROR_INVAL; - } - break; - } default: { LOGERR("invalid request tag %d", tag); ret = (int32_t)UNIFYFS_ERROR_INVAL; diff --git a/server/src/unifyfs_service_manager.h b/server/src/unifyfs_service_manager.h index 2e055c37c..9f6dd15af 100644 --- a/server/src/unifyfs_service_manager.h +++ b/server/src/unifyfs_service_manager.h @@ -41,9 +41,6 @@ int svcmgr_init(void); /* join service manager thread and cleanup its state */ int svcmgr_fini(void); -/* process service request message */ -int sm_decode_msg(char* msg_buf); - /* decode and issue chunk reads contained in message buffer */ int sm_issue_chunk_reads(int src_rank, int src_app_id, diff --git a/server/src/unifyfs_sock.c b/server/src/unifyfs_sock.c index 27937fc93..37dca20dd 100644 --- a/server/src/unifyfs_sock.c +++ b/server/src/unifyfs_sock.c @@ -52,10 +52,6 @@ struct sockaddr_un server_address; int detached_sock_idx = -1; int cur_sock_idx = -1; -char cmd_buf[MAX_NUM_CLIENTS][CMD_BUF_SIZE]; -char ack_buf[MAX_NUM_CLIENTS][CMD_BUF_SIZE]; -int ack_msg[3] = {0}; - /* initialize the listening socket on this delegator * @return success/error code */ int sock_init_server(int srvr_id) @@ -201,14 +197,7 @@ int sock_wait_cmd(int poll_timeout) } else { // (i != 0) client sockets rc = 0; if (poll_set[i].revents & POLLIN) { - ssize_t bytes_read = read(poll_set[i].fd, - cmd_buf[i], CMD_BUF_SIZE); - if (bytes_read == 0) { - rc = (int)UNIFYFS_ERROR_SOCK_DISCONNECT; - } else { // read a client command - cur_sock_idx = i; - return UNIFYFS_SUCCESS; - } + assert(!"This is dead code"); } else if (poll_set[i].revents & POLLHUP) { rc = (int)UNIFYFS_ERROR_SOCK_DISCONNECT; } else if (poll_set[i].revents & POLLERR) { @@ -227,63 +216,5 @@ int sock_wait_cmd(int poll_timeout) } return UNIFYFS_SUCCESS; - -} - -#if 0 // DEPRECATED DUE TO MARGO -/* - * send command to the client to let the client digest the - * data in the shared receive buffer - * @param: sock_id: socket index in poll_set - * @param: cmd: command type - * - * */ -int sock_notify_client(int client_idx, int cmd) -{ - LOGDBG("sock notifying fd: %d", client_sockfd); - - memset(ack_buf[client_idx], 0, sizeof(ack_buf[client_idx])); - memcpy(ack_buf[client_idx], &cmd, sizeof(int)); - - ssize_t rc = write(client_sockfd, ack_buf[client_idx], - sizeof(ack_buf[client_idx])); - if (rc < 0) { - return (int)UNIFYFS_ERROR_SOCK_OTHER; - } - return UNIFYFS_SUCCESS; } -int sock_ack_client(int client_idx, int ret_sz) -{ - ssize_t rc = write(poll_set[client_idx].fd, ack_buf[client_idx], ret_sz); - if (rc < 0) { - return (int)UNIFYFS_ERROR_SOCK_OTHER; - } - return UNIFYFS_SUCCESS; -} - -int sock_handle_error(int sock_error_no) -{ - return UNIFYFS_SUCCESS; -} - -char* sock_get_cmd_buf(int client_idx) -{ - return (char*) cmd_buf[client_idx]; -} - -char* sock_get_ack_buf(int client_idx) -{ - return (char*) ack_buf[client_idx]; -} - -int sock_get_id(void) -{ - return cur_sock_idx; -} - -int sock_get_error_id(void) -{ - return detached_sock_idx; -} -#endif // DEPRECATED DUE TO MARGO From 7d085cb73492cd634cdcfdf39aa8f1bb0c845b9e Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Tue, 8 Oct 2019 12:04:40 -0700 Subject: [PATCH 002/168] Implement fread(), add more read/write test cases - Implement fread() for the FILE_STORAGE_LOGIO case - Add a test case for fread/fwrite. It also tests some other stream IO calls like fseek()/rewind()/fgets(). - Add a test case for read/write/lseek. - Add a test case for fflush() for #374. It's currently set to "skip" while we decide what the fflush() behaviour should be. These test cases will clear the way for fixing the various filesize bugs. --- client/src/unifyfs-stdio.c | 29 ++++++---- client/src/unifyfs-sysio.c | 97 +++++++++++++++----------------- client/src/unifyfs-sysio.h | 12 ++-- client/src/unifyfs.c | 32 +---------- t/Makefile.am | 18 ++++-- t/std/fflush.c | 71 ++++++++++++++++++++++++ t/std/fwrite-fread.c | 111 +++++++++++++++++++++++++++++++++++++ t/std/stdio_suite.c | 2 + t/std/stdio_suite.h | 2 + t/sys/sysio_suite.c | 2 + t/sys/sysio_suite.h | 2 + t/sys/write-read.c | 76 +++++++++++++++++++++++++ 12 files changed, 352 insertions(+), 102 deletions(-) create mode 100644 t/std/fflush.c create mode 100644 t/std/fwrite-fread.c create mode 100644 t/sys/write-read.c diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index 0c3787023..5c9910e8d 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -513,6 +513,7 @@ static int unifyfs_stream_read( /* ERROR: invalid file descriptor */ s->err = 1; errno = EBADF; + LOGDBG("Invalid file descriptor"); return UNIFYFS_ERROR_BADF; } @@ -520,6 +521,8 @@ static int unifyfs_stream_read( if (!filedesc->read) { s->err = 1; errno = EBADF; + LOGDBG("Stream not open for reading"); + return UNIFYFS_ERROR_BADF; } @@ -531,12 +534,14 @@ static int unifyfs_stream_read( /* ERROR: failed to associate buffer */ s->err = 1; errno = unifyfs_err_map_to_errno(setvbuf_rc); + LOGDBG("Couldn't setvbuf"); return setvbuf_rc; } } /* don't attempt read if end-of-file indicator is set */ if (s->eof) { + LOGDBG("Stop read, at EOF"); return UNIFYFS_FAILURE; } @@ -594,17 +599,20 @@ static int unifyfs_stream_read( /* read data from file into buffer */ size_t bufcount; - int read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize, &bufcount); - if (read_rc != UNIFYFS_SUCCESS) { - /* ERROR: read error, set error indicator and errno */ + size_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, + s->bufsize); + if (read_rc == -1) { + /* + * ERROR: read error, set error indicator. errno is already set + * by unifyfs_fd_read() + */ s->err = 1; - errno = unifyfs_err_map_to_errno(read_rc); return read_rc; } /* record new buffer range within file */ s->bufpos = current; - s->buflen = bufcount; + s->buflen = read_rc; /* set end-of-file flag if our read was short */ if (bufcount < s->bufsize) { @@ -669,6 +677,7 @@ static int unifyfs_stream_write( unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(s->fd); if (filedesc == NULL) { /* ERROR: invalid file descriptor */ + LOGDBG("Bad file descriptor"); s->err = 1; errno = EBADF; return UNIFYFS_ERROR_BADF; @@ -676,6 +685,7 @@ static int unifyfs_stream_write( /* bail with error if stream not open for writing */ if (!filedesc->write) { + LOGDBG("Stream not open for writing"); s->err = 1; errno = EBADF; return UNIFYFS_ERROR_BADF; @@ -2259,17 +2269,16 @@ static int __srefill(unifyfs_stream_t* stream) /* read data from file into buffer */ size_t bufcount; - int read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize, &bufcount); - if (read_rc != UNIFYFS_SUCCESS) { - /* ERROR: read error, set error indicator and errno */ + size_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize); + if (read_rc < 0) { + /* ERROR: read error, set error indicator */ s->err = 1; - errno = unifyfs_err_map_to_errno(read_rc); return 1; } /* record new buffer range within file */ s->bufpos = current; - s->buflen = bufcount; + s->buflen = read_rc; } /* determine number of bytes to copy from stream buffer */ diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 0dd46c36a..32644f4fb 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -519,35 +519,40 @@ int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) * POSIX wrappers: file descriptors * --------------------------------------- */ -/* read count bytes info buf from file starting at offset pos, - * returns number of bytes actually read in retcount, - * retcount will be less than count only if an error occurs - * or end of file is reached */ -int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, - size_t* retcount) +/* + * Read 'count' bytes info 'buf' from file starting at offset 'pos'. + * + * Returns number of bytes actually read, or -1 on error, in which + * case errno will be set. + */ +size_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) { /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { - return UNIFYFS_ERROR_BADF; + errno = EBADF; + return -1; } /* it's an error to read from a directory */ if (unifyfs_fid_is_dir(fid)) { /* TODO: note that read/pread can return this, but not fread */ - return UNIFYFS_ERROR_ISDIR; + errno = EISDIR; + return -1; } /* check that file descriptor is open for read */ unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); if (!filedesc->read) { - return UNIFYFS_ERROR_BADF; + errno = EBADF; + return -1; } /* TODO: is it safe to assume that off_t is bigger than size_t? */ /* check that we don't overflow the file length */ if (unifyfs_would_overflow_offt(pos, (off_t) count)) { - return UNIFYFS_ERROR_OVERFLOW; + errno = EOVERFLOW; + return -1; } /* TODO: check that file is open for reading */ @@ -566,17 +571,37 @@ int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, } } - /* record number of bytes that we'll actually read */ - *retcount = count; - /* if we don't read any bytes, return success */ if (count == 0) { - return UNIFYFS_SUCCESS; + return 0; } - /* read data from file */ - int read_rc = unifyfs_fid_read(fid, pos, buf, count); - return read_rc; + read_req_t tmp_req; + tmp_req.fid = fid; + tmp_req.offset = (size_t) pos; + tmp_req.length = count; + tmp_req.errcode = UNIFYFS_SUCCESS; + tmp_req.buf = buf; + + int ret = unifyfs_fd_logreadlist(&tmp_req, 1); + + /* + * FIXME: when we can get the global file size correctly, the following + * should be rewritten. currently, we cannot detect EOF reliably. + */ + if (ret != UNIFYFS_SUCCESS) { + if (tmp_req.errcode != UNIFYFS_SUCCESS) { + /* error reading data */ + errno = EIO; + count = -1; + } else { + count = 0; /* possible EOF */ + } + } else { + /* success, update position */ + filedesc->pos += (off_t) count; + } + return count; } /* write count bytes from buf into file starting at offset pos, @@ -1008,40 +1033,7 @@ ssize_t UNIFYFS_WRAP(read)(int fd, void* buf, size_t count) } #endif - /* assume we'll succeed in read */ - size_t retcount = count; - - read_req_t tmp_req; - tmp_req.fid = fid; - tmp_req.offset = (size_t) filedesc->pos; - tmp_req.length = count; - tmp_req.errcode = UNIFYFS_SUCCESS; - tmp_req.buf = buf; - - /* - * this returns error code, which is zero for successful cases. - */ - int ret = unifyfs_fd_logreadlist(&tmp_req, 1); - - /* - * FIXME: when we can get the global file size correctly, the following - * should be rewritten. currently, we cannot detect EOF reliably. - */ - if (ret != UNIFYFS_SUCCESS) { - if (tmp_req.errcode != UNIFYFS_SUCCESS) { - /* error reading data */ - errno = EIO; - retcount = -1; - } else { - retcount = 0; /* possible EOF */ - } - } else { - /* success, update position */ - filedesc->pos += (off_t) retcount; - } - - /* return number of bytes read */ - return (ssize_t) retcount; + return unifyfs_fd_read(fd, filedesc->pos, buf, count); } else { MAP_OR_FAIL(read); ssize_t ret = UNIFYFS_REAL(read)(fd, buf, count); @@ -1052,10 +1044,9 @@ ssize_t UNIFYFS_WRAP(read)(int fd, void* buf, size_t count) /* TODO: find right place to msync spillover mapping */ ssize_t UNIFYFS_WRAP(write)(int fd, const void* buf, size_t count) { - ssize_t ret; - LOGDBG("write was called for fd %d", fd); + size_t ret; /* check whether we should intercept this file descriptor */ if (unifyfs_intercept_fd(&fd)) { /* get pointer to file descriptor structure */ diff --git a/client/src/unifyfs-sysio.h b/client/src/unifyfs-sysio.h index d829874f8..e608d6ff4 100644 --- a/client/src/unifyfs-sysio.h +++ b/client/src/unifyfs-sysio.h @@ -103,12 +103,12 @@ UNIFYFS_DECL(close, int, (int fd)); UNIFYFS_DECL(lio_listio, int, (int mode, struct aiocb* const aiocb_list[], int nitems, struct sigevent* sevp)); -/* read count bytes info buf from file starting at offset pos, - * returns number of bytes actually read in retcount, - * retcount will be less than count only if an error occurs - * or end of file is reached */ -int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, - size_t* retcount); +/* + * Read 'count' bytes info 'buf' from file starting at offset 'pos'. + * Returns number of bytes actually read, or -1 on error, in which + * case errno will be set. + */ +size_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count); /* write count bytes from buf into file starting at offset pos, * allocates new bytes and updates file size as necessary, diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 0776fcc2d..76e1b8317 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -1033,33 +1033,6 @@ int unifyfs_fid_create_directory(const char* path) return UNIFYFS_SUCCESS; } -/* read count bytes from file starting from pos and store into buf, - * all bytes are assumed to exist, so checks on file size should be - * done before calling this routine */ -int unifyfs_fid_read(int fid, off_t pos, void* buf, size_t count) -{ - int rc; - - /* short-circuit a 0-byte read */ - if (count == 0) { - return UNIFYFS_SUCCESS; - } - - /* get meta for this file id */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - - /* determine storage type to read file data */ - if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - /* file stored in fixed-size chunks */ - rc = unifyfs_fid_store_fixed_read(fid, meta, pos, buf, count); - } else { - /* unknown storage type */ - rc = (int)UNIFYFS_ERROR_IO; - } - - return rc; -} - /* write count bytes from buf into file starting at offset pos, * all bytes are assumed to be allocated to file, so file should * be extended before calling this routine */ @@ -1288,8 +1261,9 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, */ if (gfattr.is_laminated && ((flags & (O_CREAT | O_TRUNC | O_APPEND | O_WRONLY)) || - (mode & 0222))) { - LOGDBG("Can't open with a writable flag on laminated file."); + ((mode & 0222) && (flags != O_RDONLY)))) { + LOGDBG("Can't open %s with a writable flag on laminated file.", + path); return EROFS; } diff --git a/t/Makefile.am b/t/Makefile.am index 8394d9e74..c9360b1d5 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -97,7 +97,9 @@ sys_sysio_gotcha_t_SOURCES = sys/sysio_suite.h \ sys/creat64.c \ sys/mkdir-rmdir.c \ sys/open.c \ - sys/open64.c + sys/open64.c \ + sys/write-read.c + sys_sysio_gotcha_t_CPPFLAGS = $(test_cppflags) sys_sysio_gotcha_t_LDADD = $(test_ldadd) sys_sysio_gotcha_t_LDFLAGS = $(AM_LDFLAGS) @@ -108,21 +110,29 @@ sys_sysio_static_t_SOURCES = sys/sysio_suite.h \ sys/creat64.c \ sys/mkdir-rmdir.c \ sys/open.c \ - sys/open64.c + sys/open64.c \ + sys/write-read.c + sys_sysio_static_t_CPPFLAGS = $(test_cppflags) sys_sysio_static_t_LDADD = $(test_static_ldadd) sys_sysio_static_t_LDFLAGS = $(test_static_ldflags) std_stdio_gotcha_t_SOURCES = std/stdio_suite.h \ std/stdio_suite.c \ - std/fopen-fclose.c + std/fopen-fclose.c \ + std/fwrite-fread.c \ + std/fflush.c + std_stdio_gotcha_t_CPPFLAGS = $(test_cppflags) std_stdio_gotcha_t_LDADD = $(test_ldadd) std_stdio_gotcha_t_LDFLAGS = $(AM_LDFLAGS) std_stdio_static_t_SOURCES = std/stdio_suite.h \ std/stdio_suite.c \ - std/fopen-fclose.c + std/fopen-fclose.c \ + std/fwrite-fread.c \ + std/fflush.c + std_stdio_static_t_CPPFLAGS = $(test_cppflags) std_stdio_static_t_LDADD = $(test_static_ldadd) std_stdio_static_t_LDFLAGS = $(test_static_ldflags) diff --git a/t/std/fflush.c b/t/std/fflush.c new file mode 100644 index 000000000..27d1ba6cd --- /dev/null +++ b/t/std/fflush.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + + /* + * Test fflush(). Currently this test is skipped until #374 is addressed. + */ +#include +#include +#include +#include +#include +#include +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +int fflush_test(char* unifyfs_root) +{ + char path[64]; + char buf[64] = {0}; + FILE* fp = NULL; + int rc; + + /* Generate a random file name in the mountpoint path to test on */ + testutil_rand_path(path, sizeof(path), unifyfs_root); + skip(1, 9, "remove when fflush() sync extents (issue #374)"); + + /* Write "hello world" to a file */ + fp = fopen(path, "w"); + ok(fp != NULL, "%s: fopen(%s): %s", __FILE__, path, strerror(errno)); + + rc = fwrite("hello world", 12, 1, fp); + ok(rc == 1, "%s: fwrite(\"hello world\"): %s", __FILE__, strerror(errno)); + + /* Flush the extents */ + rc = fflush(fp); + ok(rc == 0, "%s: fflush() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + rc = fclose(fp); + ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + /* Laminate */ + rc = chmod(path, 0444); + ok(rc == 0, "%s: chmod(0444) (rc=%d): %s", __FILE__, strerror(errno)); + + /* Read it back */ + fp = fopen(path, "r"); + ok(fp != NULL, "%s: fopen(%s): %s", __FILE__, path, strerror(errno)); + + rc = fread(buf, 12, 1, fp); + ok(rc == 1, "%s: fread() buf[]=\"%s\", (rc %d): %s", __FILE__, buf, rc, + strerror(errno)); + is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + + rc = fclose(fp); + ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + end_skip; + + return 0; +} diff --git a/t/std/fwrite-fread.c b/t/std/fwrite-fread.c new file mode 100644 index 000000000..218e0971a --- /dev/null +++ b/t/std/fwrite-fread.c @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + + /* + * Test fwrite/fread/fseek/fgets/rewind/ftell/feof/chmod + */ +#include +#include +#include +#include +#include +#include +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +int fwrite_fread_test(char* unifyfs_root) +{ + char path[64]; + char buf[64] = {0}; + FILE* fp = NULL; + int rc; + char* tmp; + + errno = 0; + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Write "hello world" to a file */ + fp = fopen(path, "w"); + ok(fp != NULL, "%s: fopen(%s): %s", __FILE__, path, strerror(errno)); + + rc = fwrite("hello world", 12, 1, fp); + ok(rc == 1, "%s: fwrite(\"hello world\"): %s", __FILE__, strerror(errno)); + + rc = fclose(fp); + ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + /* Sync extents */ + int fd; + fd = open(path, O_RDWR); + rc = fsync(fd); + ok(rc == 0, "%s: fsync() (rc=%d): %s", __FILE__, rc, strerror(errno)); + close(fd); + + /* Laminate */ + rc = chmod(path, 0444); + ok(rc == 0, "%s: chmod(0444) (rc=%d): %s", __FILE__, strerror(errno)); + + /* Read it back */ + fp = fopen(path, "r"); + ok(fp != NULL, "%s: fopen(%s): %s", __FILE__, path, strerror(errno)); + + rc = fread(buf, 12, 1, fp); + ok(rc == 1, "%s: fread() buf[]=\"%s\", (rc %d): %s", __FILE__, buf, rc, + strerror(errno)); + is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + + fseek(fp, 6, SEEK_SET); + rc = ftell(fp); + ok(rc == 6, "%s: fseek() (rc %d): %s", __FILE__, rc, strerror(errno)); + + rc = fread(buf, 6, 1, fp); + ok(rc == 1, "%s: fread() at offset 6 buf[]=\"%s\", (rc %d): %s", __FILE__, + buf, rc, strerror(errno)); + is(buf, "world", "%s: saw \"world\"", __FILE__); + + rewind(fp); + rc = fread(buf, 12, 1, fp); + ok(rc == 1, "%s: fread() after rewind() buf[]=\"%s\", (rc %d): %s", + __FILE__, buf, rc, strerror(errno)); + is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + + rewind(fp); + memset(buf, 0, sizeof(buf)); + tmp = fgets(buf, 12, fp); + ok(tmp == buf, "%s: fgets() after rewind() buf[]=\"%s\": %s", __FILE__, buf, + strerror(errno)); + is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + + rewind(fp); + memset(buf, 0, sizeof(buf)); + tmp = fgets(buf, 6, fp); + ok(tmp == buf, "%s: fgets() with size = 6 after rewind() buf[]=\"%s\": %s", + __FILE__, buf, strerror(errno)); + is(buf, "hello", "%s: saw \"hello\"", __FILE__); + + rewind(fp); + rc = fread(buf, sizeof(buf), 1, fp); + ok(rc != 1, "%s: fread() past end of file (rc %d): %s", __FILE__, rc, + strerror(errno)); + + rc = feof(fp); + ok(rc != 0, "%s: feof() past end of file (rc %d): %s", __FILE__, rc, + strerror(errno)); + + rc = fclose(fp); + ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + return 0; +} diff --git a/t/std/stdio_suite.c b/t/std/stdio_suite.c index 2a1dd4e8c..cd691f95e 100644 --- a/t/std/stdio_suite.c +++ b/t/std/stdio_suite.c @@ -70,6 +70,8 @@ int main(int argc, char* argv[]) * failures to start passing. */ fopen_fclose_test(unifyfs_root); + fwrite_fread_test(unifyfs_root); + fflush_test(unifyfs_root); MPI_Finalize(); diff --git a/t/std/stdio_suite.h b/t/std/stdio_suite.h index df5033a7b..c3b7d6b91 100644 --- a/t/std/stdio_suite.h +++ b/t/std/stdio_suite.h @@ -32,5 +32,7 @@ /* Tests for UNIFYFS_WRAP(fopen) and UNIFYFS_WRAP(fclose) */ int fopen_fclose_test(char* unifyfs_root); +int fwrite_fread_test(char* unifyfs_root); +int fflush_test(char* unifyfs_root); #endif /* STDIO_SUITE_H */ diff --git a/t/sys/sysio_suite.c b/t/sys/sysio_suite.c index e3db87859..2c656144e 100644 --- a/t/sys/sysio_suite.c +++ b/t/sys/sysio_suite.c @@ -81,6 +81,8 @@ int main(int argc, char* argv[]) open64_test(unifyfs_root); + write_read_test(unifyfs_root); + MPI_Finalize(); done_testing(); diff --git a/t/sys/sysio_suite.h b/t/sys/sysio_suite.h index 01f4e87f6..3ca8a327c 100644 --- a/t/sys/sysio_suite.h +++ b/t/sys/sysio_suite.h @@ -45,4 +45,6 @@ int open_test(char* unifyfs_root); /* Tests for UNIFYFS_WRAP(open64) */ int open64_test(char* unifyfs_root); +int write_read_test(char* unifyfs_root); + #endif /* SYSIO_SUITE_H */ diff --git a/t/sys/write-read.c b/t/sys/write-read.c new file mode 100644 index 000000000..b0a2770ad --- /dev/null +++ b/t/sys/write-read.c @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + + /* + * Test write/read/lseek/fsync/stat/chmod + */ +#include +#include +#include +#include +#include +#include +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +int write_read_test(char* unifyfs_root) +{ + char path[64]; + int rc; + struct stat sb; + int fd; + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Write to the file */ + fd = open(path, O_WRONLY | O_CREAT, 0222); + ok(fd != -1, "%s: open(%s) (fd=%d): %s", __FILE__, path, fd, + strerror(errno)); + + rc = write(fd, "hello world", 12); + ok(rc == 12, "%s: write() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + /* Test writing to a different offset */ + rc = lseek(fd, 6, SEEK_SET); + ok(rc == 6, "%s: lseek() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + rc = write(fd, "universe", 9); + ok(rc == 9, "%s: write() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + /* Filesize should be zero, since we're not-laminated */ + rc = stat(path, &sb); + ok(rc == 0, "%s: stat() on non-synced & non-laminated file (rc %d): %s", + __FILE__, rc, strerror(errno)); + ok(sb.st_size == 0, "%s: file size %ld == 0", __FILE__, sb.st_size); + + rc = fsync(fd); + ok(rc == 0, "%s: fsync() (rc=%d): %s", __FILE__, rc, strerror(errno)); + close(fd); + + /* Size should still be zero, despite syncing, since it's not-laminated */ + ok(rc == 0, "%s: stat() on synced & non-laminated file (rc %d): %s", + __FILE__, rc, strerror(errno)); + ok(sb.st_size == 0, "%s: file size %ld == 0", __FILE__, sb.st_size); + + /* Laminate */ + rc = chmod(path, 0444); + ok(rc == 0, "%s: chmod(0444) (rc=%d): %s", __FILE__, rc, strerror(errno)); + + /* Verify we're getting the correct file size */ + rc = stat(path, &sb); + ok(rc == 0, "%s: stat() (rc %d): %s", __FILE__, rc, strerror(errno)); + ok(sb.st_size == 15, "%s: file size %ld == 15", __FILE__, sb.st_size); + + return 0; +} From 3e2c859142082ec014f64c8409a8334eec254dcd Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Mon, 7 Oct 2019 16:49:16 -0400 Subject: [PATCH 003/168] isolate MPI code with UNIFYFSD_USE_MPI * move MPI include to unifyfs_global.h * rename glb_mpi_rank to glb_pmi_rank * add new keyval fence in place of MPI barrier * fix hostfile vs PMI rank mismatch This commit also includes three fixes: 1. avoid a potential deadlock on access to service manager state that involved an interprocess communication dependency. 2. restore the code in the server to set the log verbosity based on config settings. 3. create a new subdirectory of the path given for UNIFYFS_META_DB_PATH for storing the MDHIM data, which avoids accidental deletion of unrelated user files when someone uses a generic path like /tmp. Finally, there's a minor cleanup of the unit tests to use simplified environment settings and a single temporary directory that gets properly removed. TEST_CHECKPATCH_SKIP_FILES=meta/src/mdhim.c TEST_CHECKPATCH_SKIP_FILES+=,common/src/unifyfs_configurator.h --- common/src/unifyfs_configurator.h | 2 +- common/src/unifyfs_keyval.c | 69 ++++++++++++++-- common/src/unifyfs_keyval.h | 5 +- meta/src/mdhim.c | 4 +- server/src/margo_server.c | 63 ++++++++------- server/src/unifyfs_cmd_handler.c | 7 +- server/src/unifyfs_global.h | 8 +- server/src/unifyfs_init.c | 113 +++++++++++++++++---------- server/src/unifyfs_metadata.c | 23 ++++-- server/src/unifyfs_request_manager.c | 15 ++-- server/src/unifyfs_service_manager.c | 26 +++--- t/0001-setup.t | 25 ++++-- t/9020-mountpoint-empty.t | 4 +- t/9100-metadata-api.t | 6 ++ t/9999-cleanup.t | 11 +++ t/Makefile.am | 6 +- t/lib/testutil.c | 4 +- t/server/metadata_suite.c | 34 +------- t/sharness.d/00-test-env.sh | 3 +- t/sharness.d/01-unifyfs-settings.sh | 35 +++------ t/sharness.d/02-functions.sh | 106 +++++++++++++++++++------ 21 files changed, 364 insertions(+), 205 deletions(-) create mode 100755 t/9999-cleanup.t mode change 100755 => 100644 t/sharness.d/02-functions.sh diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index e2cb36f08..e480ce570 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -77,7 +77,7 @@ UNIFYFS_CFG(logfs, attr_buf_size, INT, UNIFYFS_FATTR_BUF_SIZE, "log file system file attributes buffer size", NULL) \ UNIFYFS_CFG(margo, tcp, BOOL, on, "use TCP for server-server margo RPCs", NULL) \ UNIFYFS_CFG(meta, db_name, STRING, META_DEFAULT_DB_NAME, "metadata database name", NULL) \ - UNIFYFS_CFG(meta, db_path, STRING, /tmp, "metadata database path", NULL) \ + UNIFYFS_CFG(meta, db_path, STRING, RUNDIR, "metadata database path", configurator_directory_check) \ UNIFYFS_CFG(meta, server_ratio, INT, META_DEFAULT_SERVER_RATIO, "metadata server ratio", NULL) \ UNIFYFS_CFG(meta, range_size, INT, META_DEFAULT_RANGE_SZ, "metadata range size", NULL) \ UNIFYFS_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server runstate file") \ diff --git a/common/src/unifyfs_keyval.c b/common/src/unifyfs_keyval.c index e5f528f88..f5147821c 100644 --- a/common/src/unifyfs_keyval.c +++ b/common/src/unifyfs_keyval.c @@ -37,7 +37,7 @@ const char* key_runstate = "unifyfs.runstate"; const char* key_unifyfsd_socket = "unifyfsd.socket"; const char* key_unifyfsd_margo_shm = "unifyfsd.margo-shm"; const char* key_unifyfsd_margo_svr = "unifyfsd.margo-svr"; -const char* key_unifyfsd_mpi_rank = "unifyfsd.mpi-rank"; +const char* key_unifyfsd_pmi_rank = "unifyfsd.pmi-rank"; // key-value store state static int kv_initialized; // = 0 @@ -263,7 +263,14 @@ static int unifyfs_pmi2_publish(const char* key, LOGERR("PMI2_KVS_Put(%s) failed: %s", key, pmi2_errstr); return (int)UNIFYFS_FAILURE; } - rc = PMI2_KVS_Fence(); + return (int)UNIFYFS_SUCCESS; +} + +static int unifyfs_pmi2_fence(void) +{ + /* PMI2_KVS_Fence() is a collective barrier that ensures + * all previous KVS_Put()s are visible */ + int rc = PMI2_KVS_Fence(); if (rc != PMI2_SUCCESS) { unifyfs_pmi2_errstr(rc); LOGERR("PMI2_KVS_Fence() failed: %s", pmi2_errstr); @@ -454,6 +461,20 @@ static int unifyfs_pmix_publish(const char* key, return rc; } +static int unifyfs_pmix_fence(void) +{ + // PMIx_Fence is a collective barrier across all processes in my namespace + int rc = PMIx_Fence(NULL, 0, NULL, 0); + if (rc != PMIX_SUCCESS) { + LOGERR("PMIx rank %d: PMIx_Fence failed: %s", + pmix_myproc.rank, PMIx_Error_string(rc)); + rc = (int)UNIFYFS_FAILURE; + } else { + rc = (int)UNIFYFS_SUCCESS; + } + return rc; +} + #endif // USE_PMIX @@ -753,6 +774,22 @@ static int unifyfs_fskv_publish_remote(const char* key, return (int)UNIFYFS_SUCCESS; } +static int unifyfs_fskv_fence(void) +{ + if (!have_sharedfs_kvstore) { + return (int)UNIFYFS_FAILURE; + } + + if (1 == kv_nranks) { + return (int)UNIFYFS_SUCCESS; + } + + // TODO - use a file as a counting semaphore?? + sleep(10); + + return (int)UNIFYFS_SUCCESS; +} + //--------------------- K-V Store API --------------------- // Initialize key-value store @@ -836,7 +873,7 @@ int unifyfs_keyval_fini(void) int unifyfs_keyval_lookup_local(const char* key, char** oval) { - int rc = UNIFYFS_FAILURE; + int rc; if ((NULL == key) || (NULL == oval)) { LOGERR("NULL parameter"); @@ -870,7 +907,7 @@ int unifyfs_keyval_lookup_remote(int rank, const char* key, char** oval) { - int rc = UNIFYFS_FAILURE; + int rc; if ((NULL == key) || (NULL == oval)) { LOGERR("NULL parameter"); @@ -902,6 +939,8 @@ int unifyfs_keyval_lookup_remote(int rank, rc = unifyfs_pmix_lookup(rank_key, oval); #elif defined(USE_PMI2) rc = unifyfs_pmi2_lookup(rank_key, oval); +#else + rc = (int)UNIFYFS_FAILURE; #endif if (rc != (int)UNIFYFS_SUCCESS) { rc = unifyfs_fskv_lookup_remote(rank, key, oval); @@ -916,7 +955,7 @@ int unifyfs_keyval_lookup_remote(int rank, int unifyfs_keyval_publish_local(const char* key, const char* val) { - int rc = UNIFYFS_FAILURE; + int rc; if (!kv_initialized) { return (int)UNIFYFS_FAILURE; @@ -955,7 +994,7 @@ int unifyfs_keyval_publish_local(const char* key, int unifyfs_keyval_publish_remote(const char* key, const char* val) { - int rc = UNIFYFS_FAILURE; + int rc; if (!kv_initialized) { return (int)UNIFYFS_FAILURE; @@ -980,10 +1019,12 @@ int unifyfs_keyval_publish_remote(const char* key, snprintf(rank_key, sizeof(rank_key), "%d.%s", kv_myrank, key); // publish it - #if defined(USE_PMIX) +#if defined(USE_PMIX) rc = unifyfs_pmix_publish(rank_key, val); #elif defined(USE_PMI2) rc = unifyfs_pmi2_publish(rank_key, val); +#else + rc = (int)UNIFYFS_FAILURE; #endif if (rc != (int)UNIFYFS_SUCCESS) { rc = unifyfs_fskv_publish_remote(key, val); @@ -996,3 +1037,17 @@ int unifyfs_keyval_publish_remote(const char* key, } return rc; } + +// block until a particular key-value pair published by all servers +int unifyfs_keyval_fence_remote(void) +{ + int rc; +#if defined(USE_PMIX) + rc = unifyfs_pmix_fence(); +#elif defined(USE_PMI2) + rc = unifyfs_pmi2_fence(); +#else + rc = unifyfs_fskv_fence(); +#endif + return rc; +} diff --git a/common/src/unifyfs_keyval.h b/common/src/unifyfs_keyval.h index c98cfedae..fa5dc3c30 100644 --- a/common/src/unifyfs_keyval.h +++ b/common/src/unifyfs_keyval.h @@ -26,7 +26,7 @@ const char* key_runstate; // path to runstate file const char* key_unifyfsd_socket; // server domain socket path const char* key_unifyfsd_margo_shm; // client-server margo address const char* key_unifyfsd_margo_svr; // server-server margo address -const char* key_unifyfsd_mpi_rank; // server-server MPI rank +const char* key_unifyfsd_pmi_rank; // server-server pmi rank // initialize key-value store int unifyfs_keyval_init(unifyfs_cfg_t* cfg, @@ -53,6 +53,9 @@ int unifyfs_keyval_lookup_remote(int rank, const char* key, char** oval); +// block until a particular key-value pair published by all servers +int unifyfs_keyval_fence_remote(void); + #ifdef __cplusplus } // extern "C" #endif diff --git a/meta/src/mdhim.c b/meta/src/mdhim.c index d89cd9544..a48e0ee07 100644 --- a/meta/src/mdhim.c +++ b/meta/src/mdhim.c @@ -108,8 +108,8 @@ struct mdhim_t *mdhimInit(void *appComm, struct mdhim_options_t *opts) { } //Quit if MPI didn't initialize with multiple threads if (provided != MPI_THREAD_MULTIPLE) { - mlog(MDHIM_CLIENT_CRIT, "MDHIM - Error while initializing MPI with threads"); - exit(1); + mlog(MDHIM_CLIENT_WARN, "MDHIM - Error while initializing MPI with threads"); + //exit(1); } } diff --git a/server/src/margo_server.c b/server/src/margo_server.c index 96d9ebdcd..dd4b208c9 100644 --- a/server/src/margo_server.c +++ b/server/src/margo_server.c @@ -257,42 +257,51 @@ int margo_connect_servers(void) size_t i; hg_return_t hret; + // block until a margo_svr key pair published by all servers + rc = unifyfs_keyval_fence_remote(); + if ((int)UNIFYFS_SUCCESS != rc) { + LOGERR("keyval fence on margo_svr key failed"); + ret = (int)UNIFYFS_FAILURE; + return ret; + } + for (i = 0; i < glb_num_servers; i++) { - int remote_mpi_rank = -1; - char* mpi_rank_str = NULL; + int remote_pmi_rank = -1; + char* pmi_rank_str = NULL; char* margo_addr_str = NULL; - // NOTE: this really doesn't belong here, and will eventually go away - rc = unifyfs_keyval_lookup_remote(i, key_unifyfsd_mpi_rank, - &mpi_rank_str); - if ((int)UNIFYFS_SUCCESS == rc) { - remote_mpi_rank = atoi(mpi_rank_str); - free(mpi_rank_str); - } else { - LOGERR("server index=%zu - MPI rank lookup failed", i); + rc = unifyfs_keyval_lookup_remote(i, key_unifyfsd_pmi_rank, + &pmi_rank_str); + if ((int)UNIFYFS_SUCCESS != rc) { + LOGERR("server index=%zu - pmi rank lookup failed", i); ret = (int)UNIFYFS_FAILURE; + return ret; + } + if (NULL != pmi_rank_str) { + remote_pmi_rank = atoi(pmi_rank_str); + free(pmi_rank_str); } - glb_servers[i].mpi_rank = remote_mpi_rank; + glb_servers[i].pmi_rank = remote_pmi_rank; margo_addr_str = rpc_lookup_remote_server_addr(i); + if (NULL == margo_addr_str) { + LOGERR("server index=%zu - margo server lookup failed", i); + ret = (int)UNIFYFS_FAILURE; + return ret; + } + glb_servers[i].margo_svr_addr = HG_ADDR_NULL; glb_servers[i].margo_svr_addr_str = margo_addr_str; - if (NULL != margo_addr_str) { - LOGDBG("server index=%zu, mpi_rank=%d, margo_addr=%s", - i, remote_mpi_rank, margo_addr_str); - if (!margo_lazy_connect) { - glb_servers[i].margo_svr_addr = HG_ADDR_NULL; - hret = margo_addr_lookup(unifyfsd_rpc_context->svr_mid, - glb_servers[i].margo_svr_addr_str, - &(glb_servers[i].margo_svr_addr)); - if (hret != HG_SUCCESS) { - LOGERR("server index=%zu - margo_addr_lookup(%s) failed", - i, margo_addr_str); - ret = (int)UNIFYFS_FAILURE; - } + LOGDBG("server index=%zu, pmi_rank=%d, margo_addr=%s", + i, remote_pmi_rank, margo_addr_str); + if (!margo_lazy_connect) { + hret = margo_addr_lookup(unifyfsd_rpc_context->svr_mid, + glb_servers[i].margo_svr_addr_str, + &(glb_servers[i].margo_svr_addr)); + if (hret != HG_SUCCESS) { + LOGERR("server index=%zu - margo_addr_lookup(%s) failed", + i, margo_addr_str); + ret = (int)UNIFYFS_FAILURE; } - } else { - LOGERR("server index=%zu - margo addr string lookup failed", i); - ret = (int)UNIFYFS_FAILURE; } } diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index 5df7f5a78..c0568859c 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -30,7 +30,6 @@ // system headers #include #include -#include // server components #include "unifyfs_global.h" @@ -142,8 +141,7 @@ static int open_log_file(app_config_t* app_config, /* open spill over file for reading */ app_config->spill_log_fds[client_side_id] = open(path, O_RDONLY, 0666); if (app_config->spill_log_fds[client_side_id] < 0) { - printf("rank:%d, opening file %s failure\n", glb_mpi_rank, path); - fflush(stdout); + LOGERR("failed to open spill file %s", path); return (int)UNIFYFS_ERROR_FILE; } @@ -160,8 +158,7 @@ static int open_log_file(app_config_t* app_config, app_config->spill_index_log_fds[client_side_id] = open(path, O_RDONLY, 0666); if (app_config->spill_index_log_fds[client_side_id] < 0) { - printf("rank:%d, opening index file %s failure\n", glb_mpi_rank, path); - fflush(stdout); + LOGERR("failed to open spill index file %s", path); return (int)UNIFYFS_ERROR_FILE; } diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index db8f10527..00b9be855 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -52,11 +52,15 @@ #include #include +#if defined(UNIFYFSD_USE_MPI) +# include +#endif + extern arraylist_t* app_config_list; extern arraylist_t* rm_thrd_list; extern char glb_host[UNIFYFS_MAX_HOSTNAME]; -extern int glb_mpi_rank, glb_mpi_size; +extern int glb_pmi_rank, glb_pmi_size; extern size_t max_recs_per_slice; @@ -178,7 +182,7 @@ typedef struct { //char* hostname; char* margo_svr_addr_str; hg_addr_t margo_svr_addr; - int mpi_rank; + int pmi_rank; } server_info_t; extern char glb_host[UNIFYFS_MAX_HOSTNAME]; diff --git a/server/src/unifyfs_init.c b/server/src/unifyfs_init.c index 4781d341f..a5f49873d 100644 --- a/server/src/unifyfs_init.c +++ b/server/src/unifyfs_init.c @@ -30,7 +30,6 @@ // system headers #include #include -#include // common headers #include "unifyfs_configurator.h" @@ -46,8 +45,11 @@ // margo rpcs #include "margo_server.h" -int glb_mpi_rank, glb_mpi_size; +int glb_pmi_rank; /* = 0 */ +int glb_pmi_size = 1; // for standalone server tests + char glb_host[UNIFYFS_MAX_HOSTNAME]; +size_t glb_host_ndx; // index of localhost in glb_servers size_t glb_num_servers; // size of glb_servers array server_info_t* glb_servers; // array of server_info_t @@ -158,6 +160,32 @@ void exit_request(int sig) } } +#if defined(UNIFYFSD_USE_MPI) +static void init_MPI(void) +{ + int rc, provided; + rc = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + if (rc != MPI_SUCCESS) { + exit(1); + } + + rc = MPI_Comm_rank(MPI_COMM_WORLD, &glb_pmi_rank); + if (rc != MPI_SUCCESS) { + exit(1); + } + + rc = MPI_Comm_size(MPI_COMM_WORLD, &glb_pmi_size); + if (rc != MPI_SUCCESS) { + exit(1); + } +} + +static void fini_MPI(void) +{ + MPI_Finalize(); +} +#endif // UNIFYFSD_USE_MPI + static int allocate_servers(size_t n_servers) { glb_num_servers = n_servers; @@ -207,22 +235,27 @@ static int process_servers_hostfile(const char* hostfile) fclose(fp); return (int)UNIFYFS_FAILURE; } - //glb_servers[i].hostname = strdup(hostbuf); + // NOTE: following assumes one server per host if (0 == strcmp(glb_host, hostbuf)) { - //glb_svr_rank = (int)i; - LOGDBG("found myself at hostfile index=%zu, mpi_rank=%d", - i, glb_mpi_rank); + glb_host_ndx = (int)i; + LOGDBG("found myself at hostfile index=%zu, pmi_rank=%d", + glb_host_ndx, glb_pmi_rank); } } fclose(fp); + if (glb_pmi_size < cnt) { + glb_pmi_rank = (int)glb_host_ndx; + glb_pmi_size = (int)cnt; + LOGDBG("set pmi rank to host index %d", glb_pmi_rank); + } + return (int)UNIFYFS_SUCCESS; } int main(int argc, char* argv[]) { - int provided; int rc; int kv_rank, kv_nranks; bool daemon = true; @@ -244,6 +277,15 @@ int main(int argc, char* argv[]) daemonize(); } + /* unifyfs default log level is LOG_ERR */ + if (server_cfg.log_verbosity != NULL) { + long l; + rc = configurator_int_val(server_cfg.log_verbosity, &l); + if (0 == rc) { + unifyfs_log_level = (int)l; + } + } + // setup clean termination by signal memset(&sa, 0, sizeof(struct sigaction)); sa.sa_handler = exit_request; @@ -264,25 +306,14 @@ int main(int argc, char* argv[]) exit(1); } - rc = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - if (rc != MPI_SUCCESS) { - exit(1); - } - - rc = MPI_Comm_rank(MPI_COMM_WORLD, &glb_mpi_rank); - if (rc != MPI_SUCCESS) { - exit(1); - } - - rc = MPI_Comm_size(MPI_COMM_WORLD, &glb_mpi_size); - if (rc != MPI_SUCCESS) { - exit(1); - } +#if defined(UNIFYFSD_USE_MPI) + init_MPI(); +#endif // start logging gethostname(glb_host, sizeof(glb_host)); - snprintf(dbg_fname, sizeof(dbg_fname), "%s/%s.%s.%d", - server_cfg.log_dir, server_cfg.log_file, glb_host, glb_mpi_rank); + snprintf(dbg_fname, sizeof(dbg_fname), "%s/%s.%s", + server_cfg.log_dir, server_cfg.log_file, glb_host); rc = unifyfs_log_open(dbg_fname); if (rc != UNIFYFS_SUCCESS) { LOGERR("%s", unifyfs_error_enum_description((unifyfs_error_e)rc)); @@ -296,24 +327,25 @@ int main(int argc, char* argv[]) } } - kv_rank = glb_mpi_rank; - kv_nranks = glb_mpi_size; + kv_rank = glb_pmi_rank; + kv_nranks = glb_pmi_size; rc = unifyfs_keyval_init(&server_cfg, &kv_rank, &kv_nranks); if (rc != (int)UNIFYFS_SUCCESS) { exit(1); } - if (glb_mpi_rank != kv_rank) { - LOGDBG("mismatch on MPI (%d) vs kvstore (%d) rank", - glb_mpi_rank, kv_rank); + if (glb_pmi_rank != kv_rank) { + LOGDBG("mismatch on pmi (%d) vs kvstore (%d) rank", + glb_pmi_rank, kv_rank); + glb_pmi_rank = kv_rank; } - if (glb_mpi_size != kv_nranks) { - LOGDBG("mismatch on MPI (%d) vs kvstore (%d) num ranks", - glb_mpi_size, kv_nranks); + if (glb_pmi_size != kv_nranks) { + LOGDBG("mismatch on pmi (%d) vs kvstore (%d) num ranks", + glb_pmi_size, kv_nranks); + glb_pmi_size = kv_nranks; } - // TEMPORARY: remove once we fully eliminate use of MPI in sever - snprintf(rank_str, sizeof(rank_str), "%d", glb_mpi_rank); - rc = unifyfs_keyval_publish_remote(key_unifyfsd_mpi_rank, rank_str); + snprintf(rank_str, sizeof(rank_str), "%d", glb_pmi_rank); + rc = unifyfs_keyval_publish_remote(key_unifyfsd_pmi_rank, rank_str); if (rc != (int)UNIFYFS_SUCCESS) { exit(1); } @@ -336,8 +368,6 @@ int main(int argc, char* argv[]) exit(1); } - MPI_Barrier(MPI_COMM_WORLD); - LOGDBG("connecting rpc servers"); rc = margo_connect_servers(); if (rc != UNIFYFS_SUCCESS) { @@ -348,11 +378,11 @@ int main(int argc, char* argv[]) #if defined(UNIFYFS_USE_DOMAIN_SOCKET) int srvr_rank_idx = 0; #if defined(UNIFYFS_MULTIPLE_DELEGATORS) - rc = CountTasksPerNode(glb_mpi_rank, glb_mpi_size); + rc = CountTasksPerNode(glb_pmi_rank, glb_pmi_size); if (rc < 0) { exit(1); } - srvr_rank_idx = find_rank_idx(glb_mpi_rank); + srvr_rank_idx = find_rank_idx(glb_pmi_rank); #endif // UNIFYFS_MULTIPLE_DELEGATORS LOGDBG("creating server domain socket"); rc = sock_init_server(srvr_rank_idx); @@ -377,7 +407,6 @@ int main(int argc, char* argv[]) exit(1); } - MPI_Barrier(MPI_COMM_WORLD); LOGDBG("finished service initialization"); while (1) { @@ -410,6 +439,7 @@ int main(int argc, char* argv[]) return unifyfs_exit(); } +#if defined(UNIFYFSD_USE_MPI) #if defined(UNIFYFS_MULTIPLE_DELEGATORS) /* count the number of delegators per node, and * the rank of each delegator, the results are stored @@ -580,6 +610,7 @@ static int compare_int(const void* a, const void* b) } #endif // UNIFYFS_MULTIPLE_DELEGATORS +#endif // UNIFYFSD_USE_MPI static int unifyfs_exit(void) @@ -666,8 +697,10 @@ static int unifyfs_exit(void) LOGDBG("stopping metadata service"); meta_sanitize(); +#if defined(UNIFYFSD_USE_MPI) LOGDBG("finalizing MPI"); - MPI_Finalize(); + fini_MPI(); +#endif LOGDBG("all done!"); unifyfs_log_close(); diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index 059e235cc..ad9f5431d 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -99,9 +99,11 @@ int unifyfs_key_compare(unifyfs_key_t* a, unifyfs_key_t* b) int meta_init_store(unifyfs_cfg_t* cfg) { int rc, ratio; + MPI_Comm comm = MPI_COMM_WORLD; size_t path_len; long svr_ratio, range_sz; - MPI_Comm comm = MPI_COMM_WORLD; + struct stat ss; + char db_path[UNIFYFS_MAX_FILENAME] = {0}; if (cfg == NULL) { return -1; @@ -117,7 +119,16 @@ int meta_init_store(unifyfs_cfg_t* cfg) mdhim_options_set_debug_level(db_opts, MLOG_CRIT); /* UNIFYFS_META_DB_PATH: root directory for metadata */ - mdhim_options_set_db_path(db_opts, cfg->meta_db_path); + snprintf(db_path, sizeof(db_path), "%s/mdhim", cfg->meta_db_path); + rc = stat(db_path, &ss); + if (rc != 0) { + rc = mkdir(db_path, 0770); + if (rc != 0) { + LOGERR("failed to create MDHIM metadata directory %s", db_path); + return -1; + } + } + mdhim_options_set_db_path(db_opts, strdup(db_path)); /* number of metadata servers = * number of unifyfs servers / UNIFYFS_META_SERVER_RATIO */ @@ -280,14 +291,16 @@ static int remove_mdhim_db_filetree(char* db_root_path) int meta_sanitize(void) { int rc; - char dbpath[UNIFYFS_MAX_FILENAME] = {0}; + char db_path[UNIFYFS_MAX_FILENAME] = {0}; - snprintf(dbpath, sizeof(dbpath), "%s", md->db_opts->db_path); + // capture db_path before closing MDHIM + snprintf(db_path, sizeof(db_path), "%s", md->db_opts->db_path); mdhimClose(md); md = NULL; - rc = remove_mdhim_db_filetree(dbpath); + // remove the metadata filetree + rc = remove_mdhim_db_filetree(db_path); if (rc) { LOGERR("failure during MDHIM file tree removal"); } diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 301ac832b..807ea59dc 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -33,7 +33,6 @@ #include #include #include -#include // general support #include "unifyfs_global.h" @@ -860,7 +859,7 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) unifyfs_vals[i]->addr = meta_payload[i].mem_pos; unifyfs_vals[i]->len = meta_payload[i].length; - unifyfs_vals[i]->delegator_rank = glb_mpi_rank; + unifyfs_vals[i]->delegator_rank = glb_pmi_rank; unifyfs_vals[i]->app_id = app_id; unifyfs_vals[i]->rank = client_side_id; @@ -1456,7 +1455,7 @@ int invoke_server_hello_rpc(int dst_srvr_rank) /* fill in input struct */ snprintf(hello_msg, sizeof(hello_msg), "hello from %s", glb_host); - in.src_rank = (int32_t)glb_mpi_rank; + in.src_rank = (int32_t)glb_pmi_rank; in.message_str = strdup(hello_msg); LOGDBG("invoking the server-hello rpc function"); @@ -1494,7 +1493,7 @@ int invoke_server_request_rpc(int dst_srvr_rank, int req_id, int tag, hg_addr_t dst_srvr_addr; hg_size_t bulk_sz = buf_sz; - if (dst_srvr_rank == glb_mpi_rank) { + if (dst_srvr_rank == glb_pmi_rank) { // short-circuit for local requests return rc; } @@ -1507,7 +1506,7 @@ int invoke_server_request_rpc(int dst_srvr_rank, int req_id, int tag, assert(hret == HG_SUCCESS); /* fill in input struct */ - in.src_rank = (int32_t)glb_mpi_rank; + in.src_rank = (int32_t)glb_pmi_rank; in.req_id = (int32_t)req_id; in.req_tag = (int32_t)tag; in.bulk_size = bulk_sz; @@ -1556,9 +1555,9 @@ int invoke_chunk_read_request_rpc(int dst_srvr_rank, hg_addr_t dst_srvr_addr; hg_size_t bulk_sz = buf_sz; - if (dst_srvr_rank == glb_mpi_rank) { + if (dst_srvr_rank == glb_pmi_rank) { // short-circuit for local requests - return sm_issue_chunk_reads(glb_mpi_rank, + return sm_issue_chunk_reads(glb_pmi_rank, rdreq->app_id, rdreq->client_id, rdreq->req_ndx, @@ -1575,7 +1574,7 @@ int invoke_chunk_read_request_rpc(int dst_srvr_rank, assert(hret == HG_SUCCESS); /* fill in input struct */ - in.src_rank = (int32_t)glb_mpi_rank; + in.src_rank = (int32_t)glb_pmi_rank; in.app_id = (int32_t)rdreq->app_id; in.client_id = (int32_t)rdreq->client_id; in.req_id = (int32_t)rdreq->req_ndx; diff --git a/server/src/unifyfs_service_manager.c b/server/src/unifyfs_service_manager.c index 83ae8b24d..4adafe5ee 100644 --- a/server/src/unifyfs_service_manager.c +++ b/server/src/unifyfs_service_manager.c @@ -29,7 +29,6 @@ #include #include -#include #include "unifyfs_global.h" #include "unifyfs_request_manager.h" @@ -212,7 +211,7 @@ int sm_issue_chunk_reads(int src_rank, sm->burst_data_sz += size; } - if (src_rank != glb_mpi_rank) { + if (src_rank != glb_pmi_rank) { /* add chunk_reads to svcmgr response list */ LOGDBG("adding to svcmgr chunk_reads"); assert(NULL != sm); @@ -304,19 +303,24 @@ int svcmgr_fini(void) static int send_chunk_read_responses(void) { int rc = (int)UNIFYFS_SUCCESS; + arraylist_t* chunk_reads = NULL; pthread_mutex_lock(&(sm->sync)); int num_chunk_reads = arraylist_size(sm->chunk_reads); if (num_chunk_reads) { LOGDBG("processing %d chunk read responses", num_chunk_reads); - for (int i = 0; i < num_chunk_reads; i++) { - /* get data structure */ - remote_chunk_reads_t* rcr = (remote_chunk_reads_t*) - arraylist_get(sm->chunk_reads, i); - rc = invoke_chunk_read_response_rpc(rcr); - } - arraylist_reset(sm->chunk_reads); + chunk_reads = sm->chunk_reads; + sm->chunk_reads = arraylist_create(); } pthread_mutex_unlock(&(sm->sync)); + for (int i = 0; i < num_chunk_reads; i++) { + /* get data structure */ + remote_chunk_reads_t* rcr = (remote_chunk_reads_t*) + arraylist_get(chunk_reads, i); + rc = invoke_chunk_read_response_rpc(rcr); + } + if (NULL != chunk_reads) { + arraylist_free(chunk_reads); + } return rc; } @@ -406,7 +410,7 @@ int invoke_chunk_read_response_rpc(remote_chunk_reads_t* rcr) assert(hret == HG_SUCCESS); /* fill in input struct */ - in.src_rank = (int32_t)glb_mpi_rank; + in.src_rank = (int32_t)glb_pmi_rank; in.app_id = (int32_t)rcr->app_id; in.client_id = (int32_t)rcr->client_id; in.req_id = (int32_t)rcr->rdreq_id; @@ -470,7 +474,7 @@ static void server_hello_rpc(hg_handle_t handle) } /* fill output structure to return to caller */ - out.ret = (int32_t)glb_mpi_rank; + out.ret = (int32_t)glb_pmi_rank; /* send output back to caller */ hret = margo_respond(handle, &out); diff --git a/t/0001-setup.t b/t/0001-setup.t index e9fd500a0..b3ad338f2 100755 --- a/t/0001-setup.t +++ b/t/0001-setup.t @@ -14,8 +14,13 @@ echo 1..1 # common metadata directory across multiple tests. Save the value to a # script in known location that later test scripts can source. # -export UNIFYFS_MOUNT_POINT=$(mktemp -d) -export UNIFYFS_META_DB_PATH=$(mktemp -d) +export UNIFYFS_TEST_TMPDIR=$(mktemp -d) +mkdir -p $UNIFYFS_TEST_TMPDIR/{meta,mount,share,spill,state} +export UNIFYFS_TEST_META=$UNIFYFS_TEST_TMPDIR/meta +export UNIFYFS_TEST_MOUNT=$UNIFYFS_TEST_TMPDIR/mount +export UNIFYFS_TEST_SHARE=$UNIFYFS_TEST_TMPDIR/share +export UNIFYFS_TEST_SPILL=$UNIFYFS_TEST_TMPDIR/spill +export UNIFYFS_TEST_STATE=$UNIFYFS_TEST_TMPDIR/state # # Source test environment first to pick up UNIFYFS_TEST_RUN_SCRIPT @@ -23,25 +28,29 @@ export UNIFYFS_META_DB_PATH=$(mktemp -d) . $(dirname $0)/sharness.d/00-test-env.sh cat >"$UNIFYFS_TEST_RUN_SCRIPT" <<-EOF -export UNIFYFS_MOUNT_POINT=$UNIFYFS_MOUNT_POINT -export UNIFYFS_META_DB_PATH=$UNIFYFS_META_DB_PATH +export UNIFYFS_TEST_TMPDIR=$UNIFYFS_TEST_TMPDIR +export UNIFYFS_TEST_META=$UNIFYFS_TEST_META +export UNIFYFS_TEST_MOUNT=$UNIFYFS_TEST_MOUNT +export UNIFYFS_TEST_SHARE=$UNIFYFS_TEST_SHARE +export UNIFYFS_TEST_SPILL=$UNIFYFS_TEST_SPILL +export UNIFYFS_TEST_STATE=$UNIFYFS_TEST_STATE EOF . $(dirname $0)/sharness.d/01-unifyfs-settings.sh . $(dirname $0)/sharness.d/02-functions.sh # -# Start the UnifyFS daemon after killing and cleanup up after any previously +# Start the UnifyFS daemon after killing any previously # running instance. # unifyfsd_stop_daemon -unifyfsd_cleanup unifyfsd_start_daemon # # Make sure the unifyfsd process starts. # if ! process_is_running unifyfsd 5 ; then + cat $UNIFYFS_LOG_DIR/${UNIFYFS_LOG_FILE}* >&3 echo not ok 1 - unifyfsd started exit 1 fi @@ -51,6 +60,7 @@ fi # it dies during initialization. # if process_is_not_running unifyfsd 5; then + cat $UNIFYFS_LOG_DIR/${UNIFYFS_LOG_FILE}* >&3 echo not ok 1 - unifyfsd running exit 1 fi @@ -59,7 +69,8 @@ fi # Make sure unifyfsd successfully generated client runstate file # uid=$(id -u) -if ! test -f $UNIFYFS_META_DB_PATH/unifyfs-runstate.conf.$uid ; then +if ! test -f $UNIFYFS_RUNSTATE_DIR/unifyfs-runstate.conf.$uid ; then + cat $UNIFYFS_LOG_DIR/${UNIFYFS_LOG_FILE}* >&3 echo not ok 1 - unifyfsd runstate exit 1 fi diff --git a/t/9020-mountpoint-empty.t b/t/9020-mountpoint-empty.t index 54c20b59e..82141dd9d 100755 --- a/t/9020-mountpoint-empty.t +++ b/t/9020-mountpoint-empty.t @@ -11,8 +11,8 @@ test_description="Verify UnifyFS intercepted mount point is empty" . $(dirname $0)/sharness.sh -test_expect_success "Intercepted mount point $UNIFYFS_MOUNT_POINT is empty" ' - test_dir_is_empty $UNIFYFS_MOUNT_POINT +test_expect_success "Intercepted mount point $UNIFYFS_MOUNTPOINT is empty" ' + test_dir_is_empty $UNIFYFS_MOUNTPOINT ' test_done diff --git a/t/9100-metadata-api.t b/t/9100-metadata-api.t index 7ae09a671..f02ea7f51 100755 --- a/t/9100-metadata-api.t +++ b/t/9100-metadata-api.t @@ -7,4 +7,10 @@ test_description="Test Metadata API" # and UnifyFS runtime settings. # . $(dirname $0)/sharness.d/00-test-env.sh +. $(dirname $0)/sharness.d/01-unifyfs-settings.sh + +# create a new directory for MDHIM data +export UNIFYFS_META_DB_PATH=$UNIFYFS_TEST_TMPDIR/meta2 +mkdir -p $UNIFYFS_META_DB_PATH + $UNIFYFS_BUILD_DIR/t/server/metadata.t diff --git a/t/9999-cleanup.t b/t/9999-cleanup.t new file mode 100755 index 000000000..a72e1dd24 --- /dev/null +++ b/t/9999-cleanup.t @@ -0,0 +1,11 @@ +#!/bin/bash + +test_description="Cleanup test environment" + +. $(dirname $0)/sharness.sh + +test_expect_success "Cleanup" ' + unifyfsd_cleanup +' + +test_done diff --git a/t/Makefile.am b/t/Makefile.am index c9360b1d5..630319caf 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -12,7 +12,8 @@ TESTS = \ 9005-unifyfs-unmount.t \ 9010-stop-unifyfsd.t \ 9020-mountpoint-empty.t \ - 9100-metadata-api.t + 9100-metadata-api.t \ + 9999-cleanup.t check_SCRIPTS = \ 0001-setup.t \ @@ -23,7 +24,8 @@ check_SCRIPTS = \ 9005-unifyfs-unmount.t \ 9010-stop-unifyfsd.t \ 9020-mountpoint-empty.t \ - 9100-metadata-api.t + 9100-metadata-api.t \ + 9999-cleanup.t EXTRA_DIST = \ $(check_SCRIPTS) \ diff --git a/t/lib/testutil.c b/t/lib/testutil.c index 75ec78a2a..8e66d738f 100644 --- a/t/lib/testutil.c +++ b/t/lib/testutil.c @@ -80,14 +80,14 @@ void testutil_rand_path(char* buf, size_t len, const char* pfx) /* * Return a pointer to the path name of the UnifyFS mount point. Use the - * value of the environment variable UNIFYFS_MOUNT_POINT if it exists, + * value of the environment variable UNIFYFS_MOUNTPOINT if it exists, * otherwise use P_tmpdir which is defined in stdio.h and is typically * /tmp. */ char* testutil_get_mount_point(void) { char* path; - char* env = getenv("UNIFYFS_MOUNT_POINT"); + char* env = getenv("UNIFYFS_MOUNTPOINT"); if (env != NULL) { path = env; diff --git a/t/server/metadata_suite.c b/t/server/metadata_suite.c index 88d7c31c3..db91f57fc 100644 --- a/t/server/metadata_suite.c +++ b/t/server/metadata_suite.c @@ -1,17 +1,12 @@ -#include - #include "unifyfs_configurator.h" #include "unifyfs_metadata.h" -#include "unifyfs_log.h" -#include "unifyfs_runstate.h" #include "t/lib/tap.h" - #include "metadata_suite.h" int main(int argc, char* argv[]) { - /* need to initialize enougth of the server to use the metadata API */ + /* need to initialize enough of the server to use the metadata API */ unifyfs_cfg_t server_cfg; int rc, provided, glb_rank, glb_size; @@ -21,27 +16,6 @@ int main(int argc, char* argv[]) exit(1); } - rc = unifyfs_write_runstate(&server_cfg); - if (rc != (int)UNIFYFS_SUCCESS) { - exit(1); - } - - rc = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - if (rc != MPI_SUCCESS) { - exit(1); - } - - rc = MPI_Comm_rank(MPI_COMM_WORLD, &glb_rank); - if (rc != MPI_SUCCESS) { - exit(1); - } - - rc = MPI_Comm_size(MPI_COMM_WORLD, &glb_size); - if (rc != MPI_SUCCESS) { - exit(1); - } - rc = meta_init_store(&server_cfg); if (rc != 0) { LOG(LOG_ERR, "%s", @@ -56,8 +30,7 @@ int main(int argc, char* argv[]) plan(NO_PLAN); - // keep the order - + // keep the following two calls in order unifyfs_set_file_attribute_test(); unifyfs_get_file_attribute_test(); @@ -69,9 +42,6 @@ int main(int argc, char* argv[]) // shutdown the metadata service meta_sanitize(); - // finalize mpi - MPI_Finalize(); - // finish the testing // needs to be last call done_testing(); diff --git a/t/sharness.d/00-test-env.sh b/t/sharness.d/00-test-env.sh index 5ab26fa28..566473019 100644 --- a/t/sharness.d/00-test-env.sh +++ b/t/sharness.d/00-test-env.sh @@ -29,13 +29,12 @@ elif test -n "$(which srun 2>/dev/null)"; then elif test -n "$(which mpirun 2>/dev/null)"; then JOB_RUN_COMMAND="mpirun -wd $UNIFYFS_BUILD_DIR -np 1" fi - if test -z "$JOB_RUN_COMMAND"; then echo >&2 "Failed to find a suitable parallel job launcher" echo >&2 "Do you need to install OpenMPI or SLURM?" return 1 fi - +#echo >&2 "Using JOB_RUN_COMMAND: $JOB_RUN_COMMAND" export JOB_RUN_COMMAND # diff --git a/t/sharness.d/01-unifyfs-settings.sh b/t/sharness.d/01-unifyfs-settings.sh index a3155f3cc..154631dac 100644 --- a/t/sharness.d/01-unifyfs-settings.sh +++ b/t/sharness.d/01-unifyfs-settings.sh @@ -5,33 +5,20 @@ # # Source a script that is dynamically generated by 0001-setup.t. # -. $UNIFYFS_TEST_RUN_SCRIPT - +source $UNIFYFS_TEST_RUN_SCRIPT # Common settings -UNIFYFS_MOUNTPOINT=${UNIFYFS_MOUNT_POINT:-$(mktemp -d)} -export UNIFYFS_MOUNTPOINT +export UNIFYFS_LOG_VERBOSITY=${UNIFYFS_LOG_VERBOSITY:-5} +export UNIFYFS_MOUNTPOINT=${UNIFYFS_TEST_MOUNT:-"/unifyfs"} # Server settings -UNIFYFS_META_DB_PATH=${UNIFYFS_META_DB_PATH:-$(mktemp -d)} -UNIFYFS_META_DB_NAME=${UNIFYFS_META_DB_NAME:-unifyfs_db} -UNIFYFS_META_SERVER_RATIO=${UNIFYFS_META_SERVER_RATIO:-1} -UNIFYFS_LOG_DIR=${UNIFYFS_LOG_DIRECTORY:-$UNIFYFS_META_DB_PATH} -UNIFYFS_LOG_FILE=${UNIFYFS_LOG_FILE:-unifyfsd_debuglog} -UNIFYFS_RUNSTATE_DIR=${UNIFYFS_RUNSTATE_DIR:-$UNIFYFS_META_DB_PATH} -UNIFYFS_SHAREDFS_DIR=${UNIFYFS_SHAREDFS_DIR:-$UNIFYFS_META_DB_PATH} -export UNIFYFS_LOG_DIR -export UNIFYFS_LOG_FILE -export UNIFYFS_META_DB_NAME -export UNIFYFS_META_DB_PATH -export UNIFYFS_META_SERVER_RATIO -export UNIFYFS_RUNSTATE_DIR -export UNIFYFS_SHAREDFS_DIR +export UNIFYFS_LOG_DIR=${UNIFYFS_LOG_DIR:-$UNIFYFS_TEST_STATE} +export UNIFYFS_LOG_FILE="unifyfsd.log" +export UNIFYFS_META_DB_PATH=${UNIFYFS_TEST_META} +export UNIFYFS_RUNSTATE_DIR=${UNIFYFS_TEST_STATE} +export UNIFYFS_SHAREDFS_DIR=${UNIFYFS_TEST_SHARE} # Client settings -UNIFYFS_SPILLOVER_ENABLED=${UNIFYFS_SPILLOVER_ENABLED:-"Y"} -UNIFYFS_SPILLOVER_DATA_DIR=${UNIFYFS_SPILLOVER_DATA_DIR:-$UNIFYFS_META_DB_PATH} -UNIFYFS_SPILLOVER_META_DIR=${UNIFYFS_SPILLOVER_META_DIR:-$UNIFYFS_META_DB_PATH} -export UNIFYFS_SPILLOVER_DATA_DIR -export UNIFYFS_SPILLOVER_META_DIR -export UNIFYFS_SPILLOVER_ENABLED +export UNIFYFS_SPILLOVER_ENABLED=${UNIFYFS_SPILLOVER_ENABLED:-"Y"} +export UNIFYFS_SPILLOVER_META_DIR=${UNIFYFS_TEST_SPILL} +export UNIFYFS_SPILLOVER_DATA_DIR=${UNIFYFS_TEST_SPILL} diff --git a/t/sharness.d/02-functions.sh b/t/sharness.d/02-functions.sh old mode 100755 new mode 100644 index e4bb0b256..4aa91bc20 --- a/t/sharness.d/02-functions.sh +++ b/t/sharness.d/02-functions.sh @@ -61,31 +61,85 @@ process_is_not_running() return 1 } +# Dump test state (for debugging) +unifyfsd_dump_state() +{ + if ! test -d "$UNIFYFS_TEST_TMPDIR"; then + return 1 + fi + + dumpfile=$UNIFYFS_TEST_TMPDIR/unifyfsd.dump.$$ + [ -f $dumpfile ] || touch $dumpfile + + metadir=$UNIFYFS_TEST_META + if [ -d $metadir ]; then + echo "Listing meta directory $metadir :" >> $dumpfile + ls -lR $metadir >> $dumpfile + echo >> $dumpfile + fi + + sharedir=$UNIFYFS_TEST_SHARE + if [ -d $sharedir ]; then + echo "Listing share directory $sharedir :" >> $dumpfile + ls -lR $sharedir >> $dumpfile + echo >> $dumpfile + fi + + spilldir=$UNIFYFS_TEST_SPILL + if [ -d $spilldir ]; then + echo "Listing spill directory $spilldir :" >> $dumpfile + ls -lR $spilldir >> $dumpfile + echo >> $dumpfile + fi + + statedir=$UNIFYFS_TEST_STATE + if [ -d $statedir ]; then + echo "Listing state directory $statedir :" >> $dumpfile + ls -lR $statedir >> $dumpfile + echo >> $dumpfile + echo "Dumping state directory $statedir file contents :" >> $dumpfile + for f in $statedir/* ; do + if [ -f $f ]; then + echo "========= $f ==========" >> $dumpfile + cat $f >> $dumpfile + echo "+++++++++++++++++++++++" >> $dumpfile + echo >> $dumpfile + fi + done + fi + # print out dumpfile contents to current test log + cat $dumpfile >&3 + return 0 +} + +# Remove the test directory. +unifyfsd_cleanup() +{ + unifyfsd_dump_state + # remove test directory if it exists + test -d "$UNIFYFS_TEST_TMPDIR" && /bin/rm -r $UNIFYFS_TEST_TMPDIR + return 0 +} + # Create metadata directory if needed and start daemon. unifyfsd_start_daemon() { - # Make sure metadata directory exists - if test -z "$UNIFYFS_META_DB_PATH"; then - return 1 - elif ! test -d "$UNIFYFS_META_DB_PATH" && - ! mkdir $UNIFYFS_META_DB_PATH; then + # Make sure test directory exists + if ! test -d "$UNIFYFS_TEST_TMPDIR"; then return 1 fi # Generate servers hostfile - # if test -z "$UNIFYFS_SHAREDFS_DIR"; then - # return 1 - # elif ! test -d "$UNIFYFS_SHAREDFS_DIR" && - # ! mkdir $UNIFYFS_SHAREDFS_DIR; then - # return 1 - # fi - # srvr_hosts=$UNIFYFS_SHAREDFS_DIR/unifyfsd.hosts - # if [ ! -f $srvr_hosts ]; then - # touch $srvr_hosts - # echo "1" >> $srvr_hosts - # hostname >> $srvr_hosts - # fi - # export UNIFYFS_SERVER_HOSTFILE=$srvr_hosts + if test -z "$UNIFYFS_SHAREDFS_DIR"; then + return 1 + fi + srvr_hosts=$UNIFYFS_SHAREDFS_DIR/unifyfsd.hosts + if [ ! -f $srvr_hosts ]; then + touch $srvr_hosts + echo "1" >> $srvr_hosts + hostname >> $srvr_hosts + fi + export UNIFYFS_SERVER_HOSTFILE=$srvr_hosts # run server daemon $UNIFYFSD @@ -94,12 +148,14 @@ unifyfsd_start_daemon() # Kill UnifyFS daemon. unifyfsd_stop_daemon() { - while killall -q -s TERM unifyfsd 2>/dev/null; do :; done + killsig="TERM" + srvrpids="$(pgrep unifyfsd)" + while [ -n "$srvrpids" ]; do + killall -q -s $killsig unifyfsd 2>/dev/null + sleep 5 + srvrpids="$(pgrep unifyfsd)" + killsig="KILL" + done } -# Remove the metadata directory. -unifyfsd_cleanup() -{ - test -d "$UNIFYFS_META_DB_PATH" && rm -rf $UNIFYFS_META_DB_PATH - # test -d "$UNIFYFS_SHAREDFS_DIR" && rm -rf $UNIFYFS_SHAREDFS_DIR -} + From 49790fb7fd53ba5cdc293c08944e945f166957d5 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Wed, 9 Oct 2019 18:06:55 -0700 Subject: [PATCH 004/168] Fix log_size bugs, cleanup *stat(), add local append - Separate meta->size into meta->local_size and meta->global_size - Hijack stat.st_rdev to store our local_size for debugging - Cleanup the stat() functions. Add fstat() back in. - Make fseek(SEEK_END) and file append work on local files - Add append mode for both fopen() and open(). - Add/update tests to include global and local file size checking, and test append mode. --- client/check_fns/unifyfs_list.txt | 1 + client/src/gotcha_map_unifyfs_list.h | 2 + client/src/unifyfs-dirops.c | 14 +- client/src/unifyfs-fixed.c | 4 +- client/src/unifyfs-internal.h | 32 +++- client/src/unifyfs-stdio.c | 7 +- client/src/unifyfs-sysio.c | 267 +++++++++++---------------- client/src/unifyfs-sysio.h | 4 +- client/src/unifyfs.c | 108 +++++++---- configure.ac | 1 + t/0001-setup.t | 1 + t/Makefile.am | 6 +- t/std/size.c | 216 ++++++++++++++++++++++ t/std/stdio_suite.c | 1 + t/std/stdio_suite.h | 1 + t/sys/write-read.c | 92 +++++++-- 16 files changed, 528 insertions(+), 229 deletions(-) create mode 100644 t/std/size.c diff --git a/client/check_fns/unifyfs_list.txt b/client/check_fns/unifyfs_list.txt index 343f8bfc8..83693605a 100644 --- a/client/check_fns/unifyfs_list.txt +++ b/client/check_fns/unifyfs_list.txt @@ -8,6 +8,7 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) int UNIFYFS_WRAP(unlink)(const char *path) int UNIFYFS_WRAP(remove)(const char *path) int UNIFYFS_WRAP(stat)(const char *path, struct stat *buf) +int UNIFYFS_WRAP(fstat)(int fd, struct stat *buf) int UNIFYFS_WRAP(__xstat)(int vers, const char *path, struct stat *buf) int UNIFYFS_WRAP(__lxstat)(int vers, const char *path, struct stat *buf) int UNIFYFS_WRAP(creat)(const char* path, mode_t mode) diff --git a/client/src/gotcha_map_unifyfs_list.h b/client/src/gotcha_map_unifyfs_list.h index 412b560f9..9a9ac3412 100644 --- a/client/src/gotcha_map_unifyfs_list.h +++ b/client/src/gotcha_map_unifyfs_list.h @@ -13,6 +13,7 @@ UNIFYFS_DEF(truncate, int, (const char* path, off_t length)); UNIFYFS_DEF(unlink, int, (const char* path)); UNIFYFS_DEF(remove, int, (const char* path)); UNIFYFS_DEF(stat, int, (const char* path, struct stat* buf)); +UNIFYFS_DEF(fstat, int, (int fd, struct stat* buf)); UNIFYFS_DEF(__xstat, int, (int vers, const char* path, struct stat* buf)); UNIFYFS_DEF(__lxstat, int, (int vers, const char* path, struct stat* buf)); UNIFYFS_DEF(creat, int, (const char* path, mode_t mode)); @@ -117,6 +118,7 @@ struct gotcha_binding_t wrap_unifyfs_list[] = { { "unlink", UNIFYFS_WRAP(unlink), &UNIFYFS_REAL(unlink) }, { "remove", UNIFYFS_WRAP(remove), &UNIFYFS_REAL(remove) }, { "stat", UNIFYFS_WRAP(stat), &UNIFYFS_REAL(stat) }, + { "fstat", UNIFYFS_WRAP(fstat), &UNIFYFS_REAL(fstat) }, { "__xstat", UNIFYFS_WRAP(__xstat), &UNIFYFS_REAL(__xstat) }, { "__lxstat", UNIFYFS_WRAP(__lxstat), &UNIFYFS_REAL(__lxstat) }, { "creat", UNIFYFS_WRAP(creat), &UNIFYFS_REAL(creat) }, diff --git a/client/src/unifyfs-dirops.c b/client/src/unifyfs-dirops.c index 8325152f5..49dda8e23 100644 --- a/client/src/unifyfs-dirops.c +++ b/client/src/unifyfs-dirops.c @@ -131,13 +131,6 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) errno = EIO; return NULL; } - - /* - * FIXME: also, is it safe to oeverride this local data? - */ - meta->size = sb.st_size; - meta->chunks = sb.st_blocks; - meta->log_size = 0; /* no need of local storage for dir operations */ } else { fid = unifyfs_fid_create_file(name); if (fid < 0) { @@ -147,11 +140,12 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) meta = unifyfs_get_meta_from_fid(fid); meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; /* set as directory */ - meta->size = sb.st_size; - meta->chunks = sb.st_blocks; - meta->log_size = 0; } + meta->global_size = sb.st_size; + meta->chunks = sb.st_blocks; + meta->local_size = 0; /* no need of local storage for dir operations */ + unifyfs_dirstream_t* dirp = unifyfs_dirstream_alloc(fid); return (DIR*) dirp; diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 88b4c72a3..f2befb2b1 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -629,8 +629,8 @@ int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, chunk_id = pos >> unifyfs_chunk_bits; chunk_offset = pos & unifyfs_chunk_mask; } else if (meta->storage == FILE_STORAGE_LOGIO) { - chunk_id = meta->size >> unifyfs_chunk_bits; - chunk_offset = meta->size & unifyfs_chunk_mask; + chunk_id = meta->log_size >> unifyfs_chunk_bits; + chunk_offset = meta->log_size & unifyfs_chunk_mask; } else { return UNIFYFS_ERROR_IO; } diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index f67361cfe..f08edbfeb 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -204,6 +204,7 @@ typedef struct { off_t pos; /* current file pointer */ int read; /* whether file is opened for read */ int write; /* whether file is opened for write */ + int append; /* whether file is opened for append */ } unifyfs_fd_t; enum unifyfs_stream_orientation { @@ -267,8 +268,10 @@ typedef struct { } unifyfs_chunkmeta_t; typedef struct { - off_t size; /* current file size */ - off_t log_size; /* real size of the file for logio*/ + off_t global_size; /* Global size of the file */ + off_t local_size; /* Local size of the file */ + off_t log_size; /* Log size. This is the sum of all the + * write counts. */ pthread_spinlock_t fspinlock; /* file lock variable */ enum flock_enum flock_status; /* file lock status */ @@ -469,9 +472,18 @@ unifyfs_fd_t* unifyfs_get_filedesc_from_fd(int fd); * otherwise return NULL */ unifyfs_filemeta_t* unifyfs_get_meta_from_fid(int fid); +/* Return 1 if fid is laminated, 0 if not */ +int unifyfs_fid_is_laminated(int fid); + +/* Return 1 if fd is laminated, 0 if not */ +int unifyfs_fd_is_laminated(int fd); + /* Given a fid, return the path. */ const char* unifyfs_path_from_fid(int fid); +/* Given a fid, return a gfid */ +int unifyfs_gfid_from_fid(const int fid); + /* given an UNIFYFS error code, return corresponding errno code */ int unifyfs_err_map_to_errno(int rc); @@ -491,8 +503,20 @@ int unifyfs_fid_is_dir(int fid); * returns 0 for no */ int unifyfs_fid_is_dir_empty(const char* path); -/* return current size of given file id */ -off_t unifyfs_fid_size(int fid); +/* Return current global size of given file id */ +off_t unifyfs_fid_global_size(int fid); + +/* Return current local size of given file id */ +off_t unifyfs_fid_local_size(int fid); + +/* Return current local size of given file id */ +off_t unifyfs_fid_log_size(int fid); + +/* + * Return current size of given file id. If the file is laminated, return the + * global size. Otherwise, return the local size. + */ +off_t unifyfs_fid_logical_size(int fid); /* fill in limited amount of stat information for global file id */ int unifyfs_gfid_stat(int gfid, struct stat* buf); diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index 5c9910e8d..9cfa6600f 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -366,7 +366,7 @@ static int unifyfs_fopen( filedesc->fid = fid; filedesc->pos = pos; filedesc->read = read || plus; - filedesc->write = write || plus; + filedesc->write = write || plus || append; /* record our stream id value */ s->sid = sid; @@ -704,7 +704,7 @@ static int unifyfs_stream_write( errno = EBADF; return UNIFYFS_ERROR_BADF; } - current = unifyfs_fid_size(fid); + current = unifyfs_fid_logical_size(fid); /* like a seek, we discard push back bytes */ s->ubuflen = 0; @@ -902,10 +902,11 @@ static int unifyfs_fseek(FILE* stream, off_t offset, int whence) return -1; } current_pos += offset; + break; case SEEK_END: /* seek to EOF + offset */ - filesize = unifyfs_fid_size(fid); + filesize = unifyfs_fid_logical_size(fid); if (unifyfs_would_overflow_offt(filesize, offset)) { s->err = 1; errno = EOVERFLOW; diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 32644f4fb..4e41c2486 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -52,6 +52,8 @@ extern int unifyfs_spilloverblock; extern int unifyfs_use_spillover; +#define MAX(a, b) (a > b ? a : b) + /* --------------------------------------- * POSIX wrappers: paths * --------------------------------------- */ @@ -333,71 +335,86 @@ int UNIFYFS_WRAP(remove)(const char* path) } } -int UNIFYFS_WRAP(stat)(const char* path, struct stat* buf) +/* The main stat call for all the *stat() functions */ +static int __stat(const char* path, struct stat* buf) { - LOGDBG("stat was called for %s", path); + int gfid, fid; + unifyfs_file_attr_t fattr; + int ret; - if (unifyfs_intercept_path(path)) { - /* check that caller gave us a buffer to write to */ - if (!buf) { - errno = EFAULT; - return -1; - } + gfid = unifyfs_generate_gfid(path); + fid = unifyfs_get_fid_from_path(path); - /* get global file id for path */ - int gfid = unifyfs_generate_gfid(path); + /* check that caller gave us a buffer to write to */ + if (!buf) { + errno = EFAULT; + return -1; + } - /* lookup stat info for global file id */ - int ret = unifyfs_gfid_stat(gfid, buf); - if (ret != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(ret); - return -1; - } + /* lookup stat data for global file id */ + ret = invoke_client_metaget_rpc(gfid, &fattr); + if (ret != UNIFYFS_SUCCESS) { + LOGDBG("metaget failed"); + return ret; + } - /* success */ - return 0; + memset(buf, 0, sizeof(*buf)); + + /* copy attributes to stat struct */ + unifyfs_file_attr_to_stat(&fattr, buf); + + /* + * For debugging purposes, we hijack st_rdev to store our local, + * non-laminated file size. Note: st_rdev is only a 32-bit number, + * so don't depend on it if the file is really big (we use the + * lower 32-bits of the size). + * + * We also hijack st_dev to store our log_size for debugging. + */ + buf->st_rdev = fid; + if (fid >= 0) { /* If we have a local file */ + buf->st_rdev = unifyfs_fid_local_size(fid) & 0xFFFFFFFF; + buf->st_dev = unifyfs_fid_log_size(fid) & 0xFFFFFFFF; + } + + if (!fattr.is_laminated) { + /* + * It was decided that all non-laminated files would report a global + * filesize of zero. + */ + buf->st_size = 0; + } + + return 0; +} + +int UNIFYFS_WRAP(stat)(const char* path, struct stat* buf) +{ + LOGDBG("stat was called for %s", path); + if (unifyfs_intercept_path(path)) { + return __stat(path, buf); } else { MAP_OR_FAIL(stat); - int ret = UNIFYFS_REAL(stat)(path, buf); - return ret; + return UNIFYFS_REAL(stat)(path, buf); } } -#if 0 int UNIFYFS_WRAP(fstat)(int fd, struct stat* buf) { + int fid; + const char* path; LOGDBG("fstat was called for fd: %d", fd); /* check whether we should intercept this file descriptor */ if (unifyfs_intercept_fd(&fd)) { - if (!buf) { - errno = EFAULT; - return -1; - } - - /* get the file id for this file descriptor */ - int fid = unifyfs_get_fid_from_fd(fd); - if (fid < 0) { - errno = EBADF; - return -1; - } - - /* lookup stat info for this file id */ - int ret = unifyfs_fid_stat(fid, buf); - if (ret < 0) { - errno = unifyfs_err_map_to_errno(ret); - return -1; - } - - /* success */ - return 0; + fid = unifyfs_get_fid_from_fd(fd); + path = unifyfs_path_from_fid(fid); + return __stat(path, buf); } else { MAP_OR_FAIL(fstat); - int ret = UNIFYFS_REAL(fstat)(fd, buf); - return ret; + return UNIFYFS_REAL(fstat)(fd, buf); } } -#endif /* * NOTE on __xstat(2), __lxstat(2), and __fxstat(2) @@ -417,24 +434,7 @@ int UNIFYFS_WRAP(__xstat)(int vers, const char* path, struct stat* buf) errno = EINVAL; return -1; } - - if (!buf) { - errno = EFAULT; - return -1; - } - - /* get global file id for path */ - int gfid = unifyfs_generate_gfid(path); - - /* lookup stat info for global file id */ - int ret = unifyfs_gfid_stat(gfid, buf); - if (ret != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(ret); - return -1; - } - - /* success */ - return 0; + return __stat(path, buf); } else { MAP_OR_FAIL(__xstat); int ret = UNIFYFS_REAL(__xstat)(vers, path, buf); @@ -451,34 +451,18 @@ int UNIFYFS_WRAP(__lxstat)(int vers, const char* path, struct stat* buf) errno = EINVAL; return -1; } - - if (!buf) { - errno = EFAULT; - return -1; - } - - /* get global file id for path */ - int gfid = unifyfs_generate_gfid(path); - - /* lookup stat info for global file id */ - int ret = unifyfs_gfid_stat(gfid, buf); - if (ret != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(ret); - return -1; - } - - /* success */ - return 0; + return __stat(path, buf); } else { MAP_OR_FAIL(__lxstat); - int ret = UNIFYFS_REAL(__lxstat)(vers, path, buf); - return ret; + return UNIFYFS_REAL(__lxstat)(vers, path, buf); } } int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) { LOGDBG("fxstat was called for fd %d", fd); + int fid; + const char* path; /* check whether we should intercept this file descriptor */ if (unifyfs_intercept_fd(&fd)) { @@ -487,31 +471,12 @@ int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) return -1; } - if (!buf) { - errno = EINVAL; - return -1; - } - - /* get the file id for this file descriptor */ - int fid = unifyfs_get_fid_from_fd(fd); - if (fid < 0) { - errno = EBADF; - return -1; - } - - /* lookup stat info for this file id */ - int ret = unifyfs_fid_stat(fid, buf); - if (ret != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(ret); - return -1; - } - - /* success */ - return 0; + fid = unifyfs_get_fid_from_fd(fd); + path = unifyfs_path_from_fid(fid); + return __stat(path, buf); } else { MAP_OR_FAIL(__fxstat); - int ret = UNIFYFS_REAL(__fxstat)(vers, fd, buf); - return ret; + return UNIFYFS_REAL(__fxstat)(vers, fd, buf); } } @@ -525,7 +490,7 @@ int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) * Returns number of bytes actually read, or -1 on error, in which * case errno will be set. */ -size_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) +ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) { /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); @@ -559,7 +524,7 @@ size_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) /* check that we don't try to read past the end of the file */ off_t lastread = pos + (off_t) count; - off_t filesize = unifyfs_fid_size(fid); + off_t filesize = unifyfs_fid_logical_size(fid); if (filesize < lastread) { /* adjust count so we don't read past end of file */ if (filesize > pos) { @@ -604,9 +569,12 @@ size_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) return count; } -/* write count bytes from buf into file starting at offset pos, - * allocates new bytes and updates file size as necessary, - * fills any gaps with zeros */ +/* + * Write 'count' bytes from 'buf' into file starting at offset' pos'. + * Allocates new bytes and updates file size as necessary. It is assumed + * that 'pos' is actually where you want to write, and so O_APPEND behavior + * is ignored. Fills any gaps with zeros + */ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) { /* get the file id for this file descriptor */ @@ -633,48 +601,28 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) return UNIFYFS_ERROR_OVERFLOW; } + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + /* TODO: check that file is open for writing */ /* get current file size before extending the file */ - off_t filesize = unifyfs_fid_size(fid); + off_t logsize = unifyfs_fid_log_size(fid); /* compute new position based on storage type */ - off_t newpos; - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - /* extend file size and allocate chunks if needed */ - newpos = pos + (off_t) count; - int extend_rc = unifyfs_fid_extend(fid, newpos); - if (extend_rc != UNIFYFS_SUCCESS) { - return extend_rc; - } - - /* fill any new bytes between old size and pos with zero values */ - if (filesize < pos) { - off_t gap_size = pos - filesize; - int zero_rc = unifyfs_fid_write_zero(fid, filesize, gap_size); - if (zero_rc != UNIFYFS_SUCCESS) { - return zero_rc; - } - } - } else if (meta->storage == FILE_STORAGE_LOGIO) { - newpos = filesize + (off_t)count; - int extend_rc = unifyfs_fid_extend(fid, newpos); - if (extend_rc != UNIFYFS_SUCCESS) { - return extend_rc; - } - } else { - return UNIFYFS_ERROR_IO; + off_t newlogsize; + + newlogsize = logsize + count; + int extend_rc = unifyfs_fid_extend(fid, newlogsize); + if (extend_rc != UNIFYFS_SUCCESS) { + return extend_rc; } /* finally write specified data to file */ int write_rc = unifyfs_fid_write(fid, pos, buf, count); if (write_rc == 0) { meta->needs_sync = 1; - if (meta->storage == FILE_STORAGE_LOGIO) { - meta->size = newpos; - meta->log_size = pos + count; - } + meta->local_size = MAX(meta->local_size, pos + count); + meta->log_size = newlogsize; } return write_rc; } @@ -779,6 +727,7 @@ int UNIFYFS_WRAP(open)(const char* path, int flags, ...) || ((flags & O_RDWR) == O_RDWR); filedesc->write = ((flags & O_WRONLY) == O_WRONLY) || ((flags & O_RDWR) == O_RDWR); + filedesc->append = ((flags & O_APPEND)); LOGDBG("UNIFYFS_open generated fd %d for file %s", fd, path); /* don't conflict with active system fds that range from 0 - (fd_limit) */ @@ -895,7 +844,6 @@ off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) /* TODO: support SEEK_DATA and SEEK_HOLE? */ /* compute final file position */ - LOGDBG("seeking from %ld", current_pos); switch (whence) { case SEEK_SET: /* seek to offset */ @@ -907,13 +855,12 @@ off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) break; case SEEK_END: /* seek to EOF + offset */ - current_pos = meta->size + offset; + current_pos = unifyfs_fid_logical_size(fid); break; default: errno = EINVAL; return (off_t)(-1); } - LOGDBG("seeking to %ld", current_pos); /* set and return final file position */ filedesc->pos = current_pos; @@ -1045,8 +992,9 @@ ssize_t UNIFYFS_WRAP(read)(int fd, void* buf, size_t count) ssize_t UNIFYFS_WRAP(write)(int fd, const void* buf, size_t count) { LOGDBG("write was called for fd %d", fd); - size_t ret; + off_t pos; + /* check whether we should intercept this file descriptor */ if (unifyfs_intercept_fd(&fd)) { /* get pointer to file descriptor structure */ @@ -1057,8 +1005,19 @@ ssize_t UNIFYFS_WRAP(write)(int fd, const void* buf, size_t count) return (ssize_t)(-1); } + if (filedesc->append) { + /* + * With O_APPEND we always write to the end, despite the current + * file position. + */ + int fid = unifyfs_get_fid_from_fd(fd); + pos = unifyfs_fid_local_size(fid); + } else { + pos = filedesc->pos; + } + /* write data to file */ - int write_rc = unifyfs_fd_write(fd, filedesc->pos, buf, count); + int write_rc = unifyfs_fd_write(fd, pos, buf, count); if (write_rc != UNIFYFS_SUCCESS) { errno = unifyfs_err_map_to_errno(write_rc); return (ssize_t)(-1); @@ -1066,7 +1025,7 @@ ssize_t UNIFYFS_WRAP(write)(int fd, const void* buf, size_t count) ret = count; /* update file position */ - filedesc->pos += ret; + filedesc->pos = pos + count; } else { MAP_OR_FAIL(write); ret = UNIFYFS_REAL(write)(fd, buf, count); @@ -2112,6 +2071,7 @@ int UNIFYFS_WRAP(fsync)(int fd) /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { + LOGERR("Couldn't get fid from fd %d", fd); errno = EBADF; return -1; } @@ -2132,12 +2092,9 @@ int UNIFYFS_WRAP(fsync)(int fd) } } - /* if using LOGIO, call fsync rpc */ - if (meta->storage == FILE_STORAGE_LOGIO) { - /* invoke fsync rpc to register index metadata with server */ - int gfid = get_gfid(fid); - invoke_client_fsync_rpc(gfid); - } + /* invoke fsync rpc to register index metadata with server */ + int gfid = get_gfid(fid); + invoke_client_fsync_rpc(gfid); meta->needs_sync = 0; return 0; @@ -2409,7 +2366,7 @@ static int __chmod(int fid, mode_t mode) * We're laminating. Calculate the file size so we can cache it * (both locally and on the server). */ - ret = invoke_client_filesize_rpc(gfid, &meta->size); + ret = invoke_client_filesize_rpc(gfid, &meta->global_size); if (ret) { LOGERR("chmod: couldn't get the global file size on laminate"); errno = EIO; diff --git a/client/src/unifyfs-sysio.h b/client/src/unifyfs-sysio.h index e608d6ff4..fb65d7a68 100644 --- a/client/src/unifyfs-sysio.h +++ b/client/src/unifyfs-sysio.h @@ -60,7 +60,7 @@ UNIFYFS_DECL(remove, int, (const char* path)); UNIFYFS_DECL(rename, int, (const char* oldpath, const char* newpath)); UNIFYFS_DECL(truncate, int, (const char* path, off_t length)); UNIFYFS_DECL(stat, int, (const char* path, struct stat* buf)); -//UNIFYFS_DECL(fstat, int, (int fd, struct stat* buf)); +UNIFYFS_DECL(fstat, int, (int fd, struct stat* buf)); UNIFYFS_DECL(__xstat, int, (int vers, const char* path, struct stat* buf)); UNIFYFS_DECL(__lxstat, int, (int vers, const char* path, struct stat* buf)); UNIFYFS_DECL(__fxstat, int, (int vers, int fd, struct stat* buf)); @@ -108,7 +108,7 @@ UNIFYFS_DECL(lio_listio, int, (int mode, struct aiocb* const aiocb_list[], * Returns number of bytes actually read, or -1 on error, in which * case errno will be set. */ -size_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count); +ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count); /* write count bytes from buf into file starting at offset pos, * allocates new bytes and updates file size as necessary, diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 76e1b8317..0e7463455 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -609,6 +609,18 @@ unifyfs_filemeta_t* unifyfs_get_meta_from_fid(int fid) return NULL; } +int unifyfs_fid_is_laminated(int fid) +{ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + return meta->is_laminated; +} + +int unifyfs_fd_is_laminated(int fd) +{ + int fid = unifyfs_get_fid_from_fd(fd); + return unifyfs_fid_is_laminated(fid); +} + /* --------------------------------------- * Operations on file storage * --------------------------------------- */ @@ -671,7 +683,7 @@ int unifyfs_generate_gfid(const char* path) return abs(ival[0]); } -static int unifyfs_gfid_from_fid(const int fid) +int unifyfs_gfid_from_fid(const int fid) { /* check that local file id is in range */ if (fid < 0 || fid >= unifyfs_max_files) { @@ -732,12 +744,43 @@ int unifyfs_fid_is_dir_empty(const char* path) return 1; } -/* return current size of given file id */ -off_t unifyfs_fid_size(int fid) +/* Return the global (laminated) size of the file */ +off_t unifyfs_fid_global_size(int fid) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - return meta->size; + return meta->global_size; +} + +/* Return the log size of the file */ +off_t unifyfs_fid_local_size(int fid) +{ + /* get meta data for this file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + return meta->local_size; +} + +/* + * Return the size of the file. If the file is laminated, return the + * laminated size. If the file is not laminated, return the local + * size. + */ +off_t unifyfs_fid_logical_size(int fid) +{ + /* get meta data for this file */ + if (unifyfs_fid_is_laminated(fid)) { + return unifyfs_fid_global_size(fid); + } else { + return unifyfs_fid_local_size(fid); + } +} + +/* Return the local (un-laminated) size of the file */ +off_t unifyfs_fid_log_size(int fid) +{ + /* get meta data for this file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + return meta->log_size; } /* @@ -808,11 +851,12 @@ int unifyfs_set_global_file_meta(int fid, int gfid) if (meta->is_laminated) { /* * If is_laminated is set, we're either laminating for the first time, - * in which case meta->size will have already been calculated and + * in which case meta->global_size will have already been calculated and * filled in with the global file size, or, the file was already - * laminated. In either case, we write meta->size to the server. + * laminated. In either case, we write meta->global_size to the server. */ - new_fmeta.size = meta->size; + new_fmeta.size = meta->global_size; + LOGDBG("setting new_fmeta to %d", new_fmeta.size); } else { new_fmeta.size = 0; } @@ -875,27 +919,6 @@ int unifyfs_gfid_stat(int gfid, struct stat* buf) return UNIFYFS_SUCCESS; } -/* fill in limited amount of stat information */ -int unifyfs_fid_stat(int fid, struct stat* buf) -{ - /* check that fid is defined */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (meta == NULL) { - return UNIFYFS_ERROR_IO; - } - - /* get global file id corresponding to local file id */ - int gfid = unifyfs_gfid_from_fid(fid); - - /* lookup stat info for global file id */ - int ret = unifyfs_gfid_stat(gfid, buf); - if (ret != UNIFYFS_SUCCESS) { - return UNIFYFS_ERROR_IO; - } - - return UNIFYFS_SUCCESS; -} - /* allocate a file id slot for a new file * return the fid or -1 on error */ int unifyfs_fid_alloc() @@ -946,9 +969,10 @@ int unifyfs_fid_create_file(const char* path) /* initialize meta data */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - meta->size = 0; - meta->chunks = 0; + meta->global_size = 0; + meta->local_size = 0; meta->log_size = 0; + meta->chunks = 0; meta->storage = FILE_STORAGE_NULL; meta->needs_sync = 0; meta->flock_status = UNLOCKED; @@ -1129,8 +1153,8 @@ int unifyfs_fid_extend(int fid, off_t length) /* TODO: move this statement elsewhere */ /* increase file size up to length */ if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - if (length > meta->size) { - meta->size = length; + if (length > meta->local_size) { + meta->local_size = length; } } @@ -1164,9 +1188,12 @@ int unifyfs_fid_truncate(int fid, off_t length) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if (meta->is_laminated) { + return EINVAL; /* Can't truncate a laminated file */ + } /* get current size of file */ - off_t size = meta->size; + off_t size = meta->local_size; /* drop data if length is less than current size, * allocate new space and zero fill it if bigger */ @@ -1192,7 +1219,7 @@ int unifyfs_fid_truncate(int fid, off_t length) } /* set the new size */ - meta->size = length; + meta->local_size = length; return UNIFYFS_SUCCESS; } @@ -1233,6 +1260,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, int found_local = 0; off_t pos = 0; /* set the pointer to the start of the file */ unifyfs_file_attr_t gfattr = { 0, }; + unifyfs_filemeta_t* meta = NULL; if (pathlen > UNIFYFS_MAX_FILENAME) { return (int) UNIFYFS_ERROR_NAMETOOLONG; @@ -1290,7 +1318,6 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, * create a local meta cache and also initialize the local storage * space. */ - unifyfs_filemeta_t* meta = NULL; fid = unifyfs_fid_create_file(path); if (fid < 0) { @@ -1309,7 +1336,8 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, meta = unifyfs_get_meta_from_fid(fid); - meta->size = gfattr.size; + meta->global_size = gfattr.size; + meta->local_size = 0; gfattr.fid = fid; gfattr.gfid = gfid; @@ -1333,8 +1361,12 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } if (flags & O_APPEND) { - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - pos = meta->size; + meta = unifyfs_get_meta_from_fid(fid); + /* + * We only support O_APPEND on non-laminated (local) files, so + * use local_size here. + */ + pos = meta->local_size; } } else { /* !found_local && !found_global diff --git a/configure.ac b/configure.ac index 8d7b802c3..c1a14ed8e 100755 --- a/configure.ac +++ b/configure.ac @@ -204,6 +204,7 @@ CP_WRAPPERS+=",-wrap,remove" CP_WRAPPERS+=",-wrap,rename" CP_WRAPPERS+=",-wrap,truncate" CP_WRAPPERS+=",-wrap,stat" +CP_WRAPPERS+=",-wrap,fstat" CP_WRAPPERS+=",-wrap,__lxstat" CP_WRAPPERS+=",-wrap,__xstat" diff --git a/t/0001-setup.t b/t/0001-setup.t index b3ad338f2..18cfbc5fe 100755 --- a/t/0001-setup.t +++ b/t/0001-setup.t @@ -34,6 +34,7 @@ export UNIFYFS_TEST_MOUNT=$UNIFYFS_TEST_MOUNT export UNIFYFS_TEST_SHARE=$UNIFYFS_TEST_SHARE export UNIFYFS_TEST_SPILL=$UNIFYFS_TEST_SPILL export UNIFYFS_TEST_STATE=$UNIFYFS_TEST_STATE +export UNIFYFS_LOG_VERBOSITY=5 EOF . $(dirname $0)/sharness.d/01-unifyfs-settings.sh diff --git a/t/Makefile.am b/t/Makefile.am index 630319caf..840333ddb 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -123,7 +123,8 @@ std_stdio_gotcha_t_SOURCES = std/stdio_suite.h \ std/stdio_suite.c \ std/fopen-fclose.c \ std/fwrite-fread.c \ - std/fflush.c + std/fflush.c \ + std/size.c std_stdio_gotcha_t_CPPFLAGS = $(test_cppflags) std_stdio_gotcha_t_LDADD = $(test_ldadd) @@ -133,7 +134,8 @@ std_stdio_static_t_SOURCES = std/stdio_suite.h \ std/stdio_suite.c \ std/fopen-fclose.c \ std/fwrite-fread.c \ - std/fflush.c + std/fflush.c \ + std/size.c std_stdio_static_t_CPPFLAGS = $(test_cppflags) std_stdio_static_t_LDADD = $(test_static_ldadd) diff --git a/t/std/size.c b/t/std/size.c new file mode 100644 index 000000000..7140121c4 --- /dev/null +++ b/t/std/size.c @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + + /* + * Test fwrite/fread/fseek/fgets/rewind/ftell/feof/chmod + */ +#include +#include +#include +#include +#include +#include +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +/* + * Test correctness of local and global file size. Also, test opening a file + * for append, and test file positioning (fseek, ftell, etc). + */ + +/* Get global, local, or log sizes (or all) */ +static +void get_size(char* path, size_t* global, size_t* local, size_t* log) +{ + struct stat sb = {0}; + int rc; + + rc = stat(path, &sb); + if (rc != 0) { + printf("Error: %s\n", strerror(errno)); + exit(1); /* die on failure */ + } + if (global) { + *global = sb.st_size; + } + + if (local) { + *local = sb.st_rdev; + } + + if (log) { + *log = sb.st_dev; + } +} + +int size_test(char* unifyfs_root) +{ + char path[64]; + char buf[64] = {0}; + FILE* fp = NULL; + int rc; + char* tmp; + size_t global, local, log; + + errno = 0; + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Write "hello world" to a file */ + fp = fopen(path, "w"); + ok(fp != NULL, "%s: fopen(%s): %s", __FILE__, path, strerror(errno)); + + rc = fwrite("hello world", 12, 1, fp); + ok(rc == 1, "%s: fwrite(\"hello world\"): %s", __FILE__, strerror(errno)); + + rc = fclose(fp); + ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + get_size(path, &global, &local, &log); + ok(global == 0, "%s: global size is %d: %s", __FILE__, global, + strerror(errno)); + ok(local == 12, "%s: local size is %d: %s", __FILE__, local, + strerror(errno)); + ok(log == 12, "%s: log size is %d: %s", __FILE__, local, + strerror(errno)); + + /* Open the file again with append, write to it. */ + fp = fopen(path, "a"); + ok(fp != NULL, "%s: fopen(%s) in append mode: %s", __FILE__, path, + strerror(errno)); + + rc = fwrite("HELLO WORLD", 12, 1, fp); + ok(rc == 1, "%s: fwrite(\"HELLO WORLD\"): %s", __FILE__, strerror(errno)); + + rc = ftell(fp); + ok(rc == 24, "%s: ftell() (rc=%d) %s", __FILE__, rc, strerror(errno)); + + + /* + * Set our position to somewhere in the middle of the file. Since the file + * is in append mode, this new position should be ignored, and writes + * should still go to the end of the file. + */ + rc = fseek(fp, 11, SEEK_SET); + ok(rc == 0, "%s: fseek(11) (rc=%d) %s", __FILE__, rc, strerror(errno)); + + rc = fwrite("", 6, 1, fp); + ok(rc == 1, "%s: fwrite(\" \") (rc=%d): %s", __FILE__, rc, strerror(errno)); + + /* Test seeking to SEEK_END */ + rc = fseek(fp, 0, SEEK_END); + ok(rc == 0, "%s: fseek(SEEK_END) (rc=%d) %s", __FILE__, rc, + strerror(errno)); + + rc = ftell(fp); + ok(rc == 30, "%s: ftell() (rc=%d) %s", __FILE__, rc, strerror(errno)); + + rc = fclose(fp); + ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + get_size(path, &global, &local, &log); + ok(global == 0, "%s: global size is %d: %s", __FILE__, global, + strerror(errno)); + ok(local == 30, "%s: local size is %d: %s", __FILE__, local, + strerror(errno)); + ok(log == 30, "%s: log size is %d: %s", __FILE__, local, + strerror(errno)); + + + /* Sync extents */ + int fd; + fd = open(path, O_RDWR); + ok(fd >= 0, "%s: open() (fd=%d): %s", __FILE__, fd, strerror(errno)); + + rc = fsync(fd); + ok(rc == 0, "%s: fsync() (rc=%d): %s", __FILE__, rc, strerror(errno)); + close(fd); + + /* Laminate */ + rc = chmod(path, 0444); + ok(rc == 0, "%s: chmod(0444) (rc=%d): %s", __FILE__, rc, strerror(errno)); + + /* Both local and global size should be correct */ + get_size(path, &global, &local, &log); + ok(global == 30, "%s: global size is %d: %s", __FILE__, global, + strerror(errno)); + ok(local == 30, "%s: local size is %d: %s", __FILE__, local, + strerror(errno)); + ok(local == 30, "%s: log size is %d: %s", __FILE__, local, + strerror(errno)); + + /* Read it back */ + fp = fopen(path, "r"); + ok(fp != NULL, "%s: fopen(%s): %s", __FILE__, path, strerror(errno)); + + memset(buf, 0, sizeof(buf)); + rc = fread(buf, 30, 1, fp); + ok(rc == 1, "%s: fread() buf[]=\"%s\", (rc %d): %s", __FILE__, buf, rc, + strerror(errno)); + + /* + * We wrote three strings to the file: "hello world" "HELLO WORLD" and + * "". Replace the '\0' after the first two strings with spaces + * so we can compare the file contents as one big string. + */ + buf[11] = ' '; /* after "hello world" */ + buf[23] = ' '; /* after "HELLO WORLD" */ + + is(buf, "hello world HELLO WORLD ", + "%s: saw \"hello world HELLO WORLD \"", __FILE__); + + /* Try seeking and reading at various positions */ + fseek(fp, 6, SEEK_SET); + rc = ftell(fp); + ok(rc == 6, "%s: fseek() (rc %d): %s", __FILE__, rc, strerror(errno)); + + rc = fread(buf, 6, 1, fp); + ok(rc == 1, "%s: fread() at offset 6 buf[]=\"%s\", (rc %d): %s", __FILE__, + buf, rc, strerror(errno)); + is(buf, "world", "%s: saw \"world\"", __FILE__); + + rewind(fp); + rc = fread(buf, 12, 1, fp); + ok(rc == 1, "%s: fread() after rewind() buf[]=\"%s\", (rc %d): %s", + __FILE__, buf, rc, strerror(errno)); + is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + + rewind(fp); + memset(buf, 0, sizeof(buf)); + tmp = fgets(buf, 12, fp); + ok(tmp == buf, "%s: fgets() after rewind() buf[]=\"%s\": %s", __FILE__, buf, + strerror(errno)); + is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + + rewind(fp); + memset(buf, 0, sizeof(buf)); + tmp = fgets(buf, 6, fp); + ok(tmp == buf, "%s: fgets() with size = 6 after rewind() buf[]=\"%s\": %s", + __FILE__, buf, strerror(errno)); + is(buf, "hello", "%s: saw \"hello\"", __FILE__); + + rewind(fp); + rc = fread(buf, sizeof(buf), 1, fp); + ok(rc != 1, "%s: fread() past end of file (rc %d): %s", __FILE__, rc, + strerror(errno)); + + rc = feof(fp); + ok(rc != 0, "%s: feof() past end of file (rc %d): %s", __FILE__, rc, + strerror(errno)); + + rc = fclose(fp); + ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + return 0; +} diff --git a/t/std/stdio_suite.c b/t/std/stdio_suite.c index cd691f95e..6c07c6c94 100644 --- a/t/std/stdio_suite.c +++ b/t/std/stdio_suite.c @@ -72,6 +72,7 @@ int main(int argc, char* argv[]) fopen_fclose_test(unifyfs_root); fwrite_fread_test(unifyfs_root); fflush_test(unifyfs_root); + size_test(unifyfs_root); MPI_Finalize(); diff --git a/t/std/stdio_suite.h b/t/std/stdio_suite.h index c3b7d6b91..eceaefd3a 100644 --- a/t/std/stdio_suite.h +++ b/t/std/stdio_suite.h @@ -34,5 +34,6 @@ int fopen_fclose_test(char* unifyfs_root); int fwrite_fread_test(char* unifyfs_root); int fflush_test(char* unifyfs_root); +int size_test(char* unifyfs_root); #endif /* STDIO_SUITE_H */ diff --git a/t/sys/write-read.c b/t/sys/write-read.c index b0a2770ad..38e77d30f 100644 --- a/t/sys/write-read.c +++ b/t/sys/write-read.c @@ -24,12 +24,37 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" +/* Get global, local, or log sizes (or all) */ +static +void get_size(char* path, size_t* global, size_t* local, size_t* log) +{ + struct stat sb = {0}; + int rc; + + rc = stat(path, &sb); + if (rc != 0) { + printf("Error: %s\n", strerror(errno)); + exit(1); /* die on failure */ + } + if (global) { + *global = sb.st_size; + } + + if (local) { + *local = sb.st_rdev; + } + + if (log) { + *log = sb.st_dev; + } +} + int write_read_test(char* unifyfs_root) { char path[64]; int rc; - struct stat sb; int fd; + size_t global, local, log; testutil_rand_path(path, sizeof(path), unifyfs_root); @@ -48,29 +73,70 @@ int write_read_test(char* unifyfs_root) rc = write(fd, "universe", 9); ok(rc == 9, "%s: write() (rc=%d): %s", __FILE__, rc, strerror(errno)); - /* Filesize should be zero, since we're not-laminated */ - rc = stat(path, &sb); - ok(rc == 0, "%s: stat() on non-synced & non-laminated file (rc %d): %s", - __FILE__, rc, strerror(errno)); - ok(sb.st_size == 0, "%s: file size %ld == 0", __FILE__, sb.st_size); + /* Check global and local size on our un-laminated file */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s: global size is %d: %s", __FILE__, global, + strerror(errno)); + ok(local == 15, "%s: local size is %d: %s", __FILE__, local, + strerror(errno)); + ok(log == 21, "%s: log size is %d: %s", __FILE__, log, + strerror(errno)); + rc = fsync(fd); ok(rc == 0, "%s: fsync() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + /* Check global and local size on our un-laminated file */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s: global size is %d: %s", __FILE__, global, + strerror(errno)); + ok(local == 15, "%s: local size is %d: %s", __FILE__, local, + strerror(errno)); + ok(log == 21, "%s: log size is %d: %s", __FILE__, log, + strerror(errno)); + + + close(fd); + + /* Test O_APPEND */ + fd = open(path, O_WRONLY | O_APPEND, 0222); + ok(fd != -1, "%s: open(%s, O_APPEND) (fd=%d): %s", __FILE__, path, fd, + strerror(errno)); + + /* + * Seek to an offset in the file and write. Since it's O_APPEND, the + * offset we seeked to doesn't matter - all writes go to the end. + */ + rc = lseek(fd, 3, SEEK_SET); + ok(rc == 3, "%s: lseek() (rc=%d): %s", __FILE__, rc, strerror(errno)); + + rc = write(fd, "", 6); + ok(rc == 6, "%s: write() (rc=%d): %s", __FILE__, rc, strerror(errno)); close(fd); - /* Size should still be zero, despite syncing, since it's not-laminated */ - ok(rc == 0, "%s: stat() on synced & non-laminated file (rc %d): %s", - __FILE__, rc, strerror(errno)); - ok(sb.st_size == 0, "%s: file size %ld == 0", __FILE__, sb.st_size); + /* Check global and local size on our un-laminated file */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s: global size is %d: %s", __FILE__, global, + strerror(errno)); + ok(local == 21, "%s: local size is %d: %s", __FILE__, local, + strerror(errno)); + ok(log == 27, "%s: log size is %d: %s", __FILE__, log, + strerror(errno)); + /* Laminate */ rc = chmod(path, 0444); ok(rc == 0, "%s: chmod(0444) (rc=%d): %s", __FILE__, rc, strerror(errno)); /* Verify we're getting the correct file size */ - rc = stat(path, &sb); - ok(rc == 0, "%s: stat() (rc %d): %s", __FILE__, rc, strerror(errno)); - ok(sb.st_size == 15, "%s: file size %ld == 15", __FILE__, sb.st_size); + get_size(path, &global, &local, &log); + ok(global == 21, "%s: global size is %d: %s", __FILE__, global, + strerror(errno)); + ok(local == 21, "%s: local size is %d: %s", __FILE__, local, + strerror(errno)); + ok(log == 27, "%s: log size is %d: %s", __FILE__, log, + strerror(errno)); + return 0; } From d6a11a2ba3017948505bbb4ced7041ca1f6ed778 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 18 Oct 2019 12:28:38 -0700 Subject: [PATCH 005/168] docs: dropping dotkit --- docs/build-intercept.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/build-intercept.rst b/docs/build-intercept.rst index 2989c4cb0..1ce63b710 100644 --- a/docs/build-intercept.rst +++ b/docs/build-intercept.rst @@ -29,11 +29,10 @@ Building with Spack ******************** These instructions assume that you do not already have a module system installed -such as LMod, Dotkit, or Environment Modules. If your system already has Dotkit -or LMod installed then installing the environment-modules package with Spack +such as LMod or Environment Modules. If your system already has +LMod installed then installing the environment-modules package with Spack is unnecessary (so you can safely skip that step). -If you use Dotkit then replace ``spack load`` with ``spack use``. First, install Spack if you don't already have it: .. code-block:: Bash @@ -108,8 +107,6 @@ Build the Dependencies with Spack Once Spack is installed on your system (see :ref:`above `), you can install just the dependencies for an easier manual installation of UnifyFS. -If you use Dotkit then replace ``spack load`` with ``spack use``. - .. code-block:: Bash $ spack install leveldb From fe262ba2e46d44401f17fe258e8a33456446911c Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 18 Oct 2019 15:37:24 -0700 Subject: [PATCH 006/168] server: allocate buffer to return key/value pairs in get_file_extents --- server/src/unifyfs_cmd_handler.c | 2 +- server/src/unifyfs_metadata.c | 88 ++++++++++++++++++---------- server/src/unifyfs_request_manager.c | 66 +++++++++++---------- 3 files changed, 95 insertions(+), 61 deletions(-) diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index c0568859c..f4981c787 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -480,7 +480,7 @@ static void unifyfs_filesize_rpc(hg_handle_t handle) /* read data for a single read request from client, * returns data to client through shared memory */ - size_t filesize; + size_t filesize = 0; int ret = rm_cmd_filesize(in.app_id, in.local_rank_idx, in.gfid, &filesize); diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index ad9f5431d..465d505aa 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -401,47 +401,75 @@ int unifyfs_get_file_extents(int num_keys, unifyfs_key_t** keys, * key-value pairs within the range of the key tuple. * We need to re-evaluate this function to use different key-value stores. */ - - int i; int rc = UNIFYFS_SUCCESS; - int tot_num = 0; - unifyfs_key_t* tmp_key; - unifyfs_val_t* tmp_val; - unifyfs_keyval_t* kviter = *keyvals; + /* initialize output values */ + *num_values = 0; + *keyvals = NULL; md->primary_index = unifyfs_indexes[0]; - bgrm = mdhimBGet(md, md->primary_index, (void**)keys, - key_lens, num_keys, MDHIM_RANGE_BGET); - while (bgrm) { - bgrmp = bgrm; - if (bgrmp->error < 0) { - // TODO: need better error handling - rc = (int)UNIFYFS_ERROR_MDHIM; - return rc; + /* execute range query */ + struct mdhim_bgetrm_t* bkvlist = mdhimBGet(md, md->primary_index, + (void**)keys, key_lens, num_keys, MDHIM_RANGE_BGET); + + /* iterate over each item in list, check for errors + * and sum up total number of key/value pairs we got back */ + size_t tot_num = 0; + struct mdhim_bgetrm_t* ptr = bkvlist; + while (ptr) { + /* check that we don't have an error condition */ + if (ptr->error < 0) { + /* hit an error */ + LOGERR("MDHIM Range Query error"); + return (int)UNIFYFS_ERROR_MDHIM; } - if (tot_num < MAX_META_PER_SEND) { - for (i = 0; i < bgrmp->num_keys; i++) { - tmp_key = (unifyfs_key_t*)bgrmp->keys[i]; - tmp_val = (unifyfs_val_t*)bgrmp->values[i]; - memcpy(&(kviter->key), tmp_key, sizeof(unifyfs_key_t)); - memcpy(&(kviter->val), tmp_val, sizeof(unifyfs_val_t)); - kviter++; - tot_num++; - if (MAX_META_PER_SEND == tot_num) { - LOGERR("Error: maximum number of values!"); - rc = UNIFYFS_FAILURE; - break; - } - } + /* total up number of key/values returned */ + tot_num += (size_t) ptr->num_keys; + + /* get pointer to next item in the list */ + ptr = ptr->next; + } + + /* allocate memory to copy key/value data */ + unifyfs_keyval_t* kvs = (unifyfs_keyval_t*) calloc( + tot_num, sizeof(unifyfs_keyval_t)); + if (NULL == kvs) { + LOGERR("failed to allocate keyvals"); + return (int)UNIFYFS_ERROR_MDHIM; + } + + /* iterate over list and copy each key/value into output array */ + ptr = bkvlist; + unifyfs_keyval_t* kviter = kvs; + while (ptr) { + /* iterate over key/value in list element */ + int i; + for (i = 0; i < ptr->num_keys; i++) { + /* get pointer to current key and value */ + unifyfs_key_t* tmp_key = (unifyfs_key_t*)ptr->keys[i]; + unifyfs_val_t* tmp_val = (unifyfs_val_t*)ptr->values[i]; + + /* copy contents over to output array */ + memcpy(&(kviter->key), tmp_key, sizeof(unifyfs_key_t)); + memcpy(&(kviter->val), tmp_val, sizeof(unifyfs_val_t)); + + /* bump up to next element in output array */ + kviter++; } - bgrm = bgrmp->next; - mdhim_full_release_msg(bgrmp); + + /* get pointer to next item in the list */ + struct mdhim_bgetrm_t* next = ptr->next; + + /* release resources for the curren item */ + mdhim_full_release_msg(ptr); + ptr = next; } + /* set output values */ *num_values = tot_num; + *keyvals = kvs; return rc; } diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 807ea59dc..78e19521d 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -477,14 +477,16 @@ int rm_cmd_filesize( int gfid, /* global file id of read request */ size_t* outsize) /* output file size */ { + /* initialize output file size to something deterministic, + * in case we drop out with an error */ + *outsize = 0; + /* set offset and length to request *all* key/value pairs * for this file */ size_t offset = 0; /* want to pick the highest integer offset value a file * could have here */ - // TODO: would like to unsed max for unsigned long, but - // that fails to return any keys for some reason size_t length = (SIZE_MAX >> 1) - 1; /* get the locations of all the read requests from the @@ -499,27 +501,17 @@ int rm_cmd_filesize( key2.fid = gfid; key2.offset = offset + length - 1; - unifyfs_keyval_t* keyvals; + /* set up input params to specify range lookup */ unifyfs_key_t* unifyfs_keys[2] = {&key1, &key2}; int key_lens[2] = {sizeof(unifyfs_key_t), sizeof(unifyfs_key_t)}; /* look up all entries in this range */ int num_vals = 0; - keyvals = (unifyfs_keyval_t*) calloc(UNIFYFS_MAX_SPLIT_CNT, - sizeof(unifyfs_keyval_t)); - if (NULL == keyvals) { - LOGERR("failed to allocate keyvals"); - return UNIFYFS_ERROR_NOMEM; - } - + unifyfs_keyval_t* keyvals = NULL; int rc = unifyfs_get_file_extents(2, unifyfs_keys, key_lens, &num_vals, &keyvals); - /* TODO: if there are file extents not accounted for we should - * either return 0 for that date (holes) or EOF if reading past - * the end of the file */ if (UNIFYFS_SUCCESS != rc) { - // we need to let the client know that there was an error - free(keyvals); + /* failed to look up extents, bail with error */ return UNIFYFS_FAILURE; } @@ -540,8 +532,11 @@ int rm_cmd_filesize( } } - // cleanup - free(keyvals); + /* free off key/value buffer returned from get_file_extents */ + if (NULL != keyvals) { + free(keyvals); + keyvals = NULL; + } *outsize = filesize; return rc; @@ -551,17 +546,24 @@ int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, int gfid, int app_id, int client_id, int num_keys, unifyfs_key_t** keys, int* keylens) { - // TODO: might want to get this from a memory pool - unifyfs_keyval_t* keyvals = calloc(UNIFYFS_MAX_SPLIT_CNT, - sizeof(unifyfs_keyval_t)); - if (NULL == keyvals) { - LOGERR("failed to allocate keyvals"); - return UNIFYFS_ERROR_NOMEM; - } - + /* lookup all key/value pairs for given range */ int num_vals = 0; + unifyfs_keyval_t* keyvals = NULL; int rc = unifyfs_get_file_extents(num_keys, keys, keylens, &num_vals, &keyvals); + + /* this is to maintain limits imposed in previous code + * that would throw fatal errors */ + if (num_vals >= UNIFYFS_MAX_SPLIT_CNT || + num_vals >= MAX_META_PER_SEND) { + LOGERR("too many key/values returned in range lookup"); + if (NULL != keyvals) { + free(keyvals); + keyvals = NULL; + } + return UNIFYFS_ERROR_NOMEM; + } + /* TODO: if there are file extents not accounted for we should * either return 0 for that data (holes) or EOF if reading past * the end of the file */ @@ -574,13 +576,14 @@ int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, qsort(keyvals, (size_t)num_vals, sizeof(unifyfs_keyval_t), compare_kv_gfid_rank); } + server_read_req_t* rdreq = reserve_read_req(thrd_ctrl); if (NULL == rdreq) { rc = UNIFYFS_FAILURE; } else { - rdreq->app_id = app_id; - rdreq->client_id = client_id; - rdreq->extent.gfid = gfid; + rdreq->app_id = app_id; + rdreq->client_id = client_id; + rdreq->extent.gfid = gfid; rdreq->extent.errcode = EINPROGRESS; rc = create_chunk_requests(thrd_ctrl, rdreq, num_vals, keyvals); @@ -590,8 +593,11 @@ int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, } } - // cleanup - free(keyvals); + /* free off key/value buffer returned from get_file_extents */ + if (NULL != keyvals) { + free(keyvals); + keyvals = NULL; + } return rc; } From 6cc407bd32b0d155e7032cd008681294e0bed7ac Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Mon, 21 Oct 2019 10:10:07 -0700 Subject: [PATCH 007/168] Remove old FILE_STORAGE_FIXED_CHUNK code Remove all references to the old FILE_STORAGE_FIXED_CHUNK code. We use the FILE_STORAGE_LOGIO codepath exclusively now. --- client/src/unifyfs-fixed.c | 18 ++++-------------- client/src/unifyfs-internal.h | 5 +---- client/src/unifyfs.c | 30 ++++-------------------------- 3 files changed, 9 insertions(+), 44 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index f2befb2b1..743a0ac0e 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -625,10 +625,7 @@ int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, int chunk_id; off_t chunk_offset; - if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - chunk_id = pos >> unifyfs_chunk_bits; - chunk_offset = pos & unifyfs_chunk_mask; - } else if (meta->storage == FILE_STORAGE_LOGIO) { + if (meta->storage == FILE_STORAGE_LOGIO) { chunk_id = meta->log_size >> unifyfs_chunk_bits; chunk_offset = meta->log_size & unifyfs_chunk_mask; } else { @@ -639,9 +636,7 @@ int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, size_t remaining = unifyfs_chunk_size - chunk_offset; if (count <= remaining) { /* all bytes for this write fit within the current chunk */ - if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - rc = unifyfs_chunk_write(meta, chunk_id, chunk_offset, buf, count); - } else if (meta->storage == FILE_STORAGE_LOGIO) { + if (meta->storage == FILE_STORAGE_LOGIO) { rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, chunk_offset, buf, count); } else { @@ -650,10 +645,7 @@ int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, } else { /* otherwise, fill up the remainder of the current chunk */ char* ptr = (char*) buf; - if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - rc = unifyfs_chunk_write(meta, chunk_id, - chunk_offset, (void*)ptr, remaining); - } else if (meta->storage == FILE_STORAGE_LOGIO) { + if (meta->storage == FILE_STORAGE_LOGIO) { rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, chunk_offset, (void*)ptr, remaining); } else { @@ -677,9 +669,7 @@ int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, } /* write data */ - if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - rc = unifyfs_chunk_write(meta, chunk_id, 0, (void*)ptr, num); - } else if (meta->storage == FILE_STORAGE_LOGIO) { + if (meta->storage == FILE_STORAGE_LOGIO) { rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, 0, (void*)ptr, num); } else { diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index f08edbfeb..387113f91 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -252,10 +252,7 @@ enum flock_enum { SH_LOCKED }; -/* TODO: make this an enum */ -#define FILE_STORAGE_NULL 0 -#define FILE_STORAGE_FIXED_CHUNK 1 -#define FILE_STORAGE_LOGIO 2 +enum {FILE_STORAGE_NULL = 0, FILE_STORAGE_LOGIO}; /* TODO: make this an enum */ #define CHUNK_LOCATION_NULL 0 diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 0e7463455..468d262ec 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -1073,8 +1073,7 @@ int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count) unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); /* determine storage type to write file data */ - if (meta->storage == FILE_STORAGE_FIXED_CHUNK || - meta->storage == FILE_STORAGE_LOGIO) { + if (meta->storage == FILE_STORAGE_LOGIO) { /* file stored in fixed-size chunks */ rc = unifyfs_fid_store_fixed_write(fid, meta, pos, buf, count); } else { @@ -1141,8 +1140,7 @@ int unifyfs_fid_extend(int fid, off_t length) unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); /* determine file storage type */ - if (meta->storage == FILE_STORAGE_FIXED_CHUNK || - meta->storage == FILE_STORAGE_LOGIO) { + if (meta->storage == FILE_STORAGE_LOGIO) { /* file stored in fixed-size chunks */ rc = unifyfs_fid_store_fixed_extend(fid, meta, length); } else { @@ -1150,35 +1148,15 @@ int unifyfs_fid_extend(int fid, off_t length) rc = (int)UNIFYFS_ERROR_IO; } - /* TODO: move this statement elsewhere */ - /* increase file size up to length */ - if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - if (length > meta->local_size) { - meta->local_size = length; - } - } - return rc; } /* if length is less than reserved space, give back space down to length */ int unifyfs_fid_shrink(int fid, off_t length) { - int rc; + /* TODO: implement this function */ - /* get meta data for this file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - - /* determine file storage type */ - if (meta->storage == FILE_STORAGE_FIXED_CHUNK) { - /* file stored in fixed-size chunks */ - rc = unifyfs_fid_store_fixed_shrink(fid, meta, length); - } else { - /* unknown storage type */ - rc = (int)UNIFYFS_ERROR_IO; - } - - return rc; + return UNIFYFS_ERROR_IO; } /* truncate file id to given length, frees resources if length is From bc2579685d2d91ec4730fbadd011e4a1612349d8 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 23 Oct 2019 17:11:23 -0700 Subject: [PATCH 008/168] client: clarify some comments --- client/src/unifyfs-fixed.c | 17 ++++++++++------- client/src/unifyfs-sysio.c | 13 +++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 743a0ac0e..d5ae00505 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -349,7 +349,7 @@ static int unifyfs_logio_chunk_write( unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); if (chunk_meta->location != CHUNK_LOCATION_MEMFS && - chunk_meta->location != CHUNK_LOCATION_SPILLOVER) { + chunk_meta->location != CHUNK_LOCATION_SPILLOVER) { /* unknown chunk type */ LOGERR("unknown chunk type"); return UNIFYFS_ERROR_IO; @@ -363,6 +363,9 @@ static int unifyfs_logio_chunk_write( meta, chunk_id, chunk_offset); memcpy(chunk_buf, buf, count); + /* record byte offset position within log, which is the number + * of bytes between the memory location we write to and the + * starting memory location of the shared memory data chunk region */ log_offset = chunk_buf - unifyfs_chunks; } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { /* spill over to a file, so write to file descriptor */ @@ -373,9 +376,13 @@ static int unifyfs_logio_chunk_write( LOGERR("pwrite failed: errno=%d (%s)", errno, strerror(errno)); } + /* record byte offset position within log, for spill over + * locations we take the offset within the spill file + * added to the total size of all shared memory data chunks */ log_offset = spill_offset + unifyfs_max_chunks * (1 << unifyfs_chunk_bits); } + /* TODO: pass in gfid for this file or call function to look it up? */ /* find the corresponding file attr entry and update attr*/ unifyfs_file_attr_t tmp_meta_entry; tmp_meta_entry.fid = fid; @@ -385,9 +392,6 @@ static int unifyfs_logio_chunk_write( *unifyfs_fattrs.ptr_num_entries, sizeof(unifyfs_file_attr_t), compare_fattr); - if (ptr_meta_entry != NULL) { - ptr_meta_entry->size = pos + count; - } /* define an new index entry for this write operation */ unifyfs_index_t cur_idx; @@ -397,8 +401,7 @@ static int unifyfs_logio_chunk_write( cur_idx.length = count; /* split the write requests larger than unifyfs_key_slice_range into - * the ones smaller than unifyfs_key_slice_range - * */ + * the ones smaller than unifyfs_key_slice_range */ index_set_t tmp_index_set; memset(&tmp_index_set, 0, sizeof(tmp_index_set)); unifyfs_split_index(&cur_idx, &tmp_index_set, @@ -468,7 +471,7 @@ static int unifyfs_logio_chunk_write( } else { /* TODO: no room to write additional index metadata entries, * swap out existing metadata buffer to disk*/ - printf("exhausted metadata"); + LOGERR("exhausted metadata"); } /* assume read was successful if we get to here */ diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 4e41c2486..7c66d0f3b 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -601,17 +601,13 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) return UNIFYFS_ERROR_OVERFLOW; } - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - - /* TODO: check that file is open for writing */ - - /* get current file size before extending the file */ + /* get current log size before extending the log */ off_t logsize = unifyfs_fid_log_size(fid); - /* compute new position based on storage type */ - off_t newlogsize; + /* compute size log will be after we append data */ + off_t newlogsize = logsize + count; - newlogsize = logsize + count; + /* allocate storage space to hold data for this write */ int extend_rc = unifyfs_fid_extend(fid, newlogsize); if (extend_rc != UNIFYFS_SUCCESS) { return extend_rc; @@ -620,6 +616,7 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) /* finally write specified data to file */ int write_rc = unifyfs_fid_write(fid, pos, buf, count); if (write_rc == 0) { + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); meta->needs_sync = 1; meta->local_size = MAX(meta->local_size, pos + count); meta->log_size = newlogsize; From 5f724a7bc05acdc6efe0b7bef8a3de574943126b Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 22 Oct 2019 12:10:29 -0700 Subject: [PATCH 009/168] examples: add timers for open, close, and laminate add timers for open, close, and laminate, time global cost of write and stat in write.c, define functions to include barriers when starting/stopping timers --- examples/src/read.c | 34 ++++++++++++++--- examples/src/testutil.h | 35 +++++++++++++++++ examples/src/testutil_rdwr.h | 18 +++++++++ examples/src/write.c | 73 +++++++++++++++++++++++++++++------- examples/src/writeread.c | 51 +++++++++++++++++++------ t/ci/100-writeread-tests.sh | 4 +- t/ci/110-write-tests.sh | 4 +- t/ci/120-read-tests.sh | 4 +- 8 files changed, 186 insertions(+), 37 deletions(-) diff --git a/examples/src/read.c b/examples/src/read.c index c69d2b749..5616d74cb 100644 --- a/examples/src/read.c +++ b/examples/src/read.c @@ -112,11 +112,17 @@ int main(int argc, char* argv[]) test_cfg test_config; test_cfg* cfg = &test_config; + test_timer time_stat; + test_timer time_open; test_timer time_rd; test_timer time_check; + test_timer time_close; + timer_init(&time_stat, "stat"); + timer_init(&time_open, "open"); timer_init(&time_rd, "read"); timer_init(&time_check, "check"); + timer_init(&time_close, "close"); rc = test_init(argc, argv, cfg); if (rc) { @@ -138,6 +144,7 @@ int main(int argc, char* argv[]) target_file); // file size check + timer_start_barrier(cfg, &time_stat); size_t rank_bytes = test_config.n_blocks * test_config.block_sz; size_t total_bytes = rank_bytes * test_config.n_ranks; size_t expected = total_bytes; @@ -158,12 +165,17 @@ int main(int argc, char* argv[]) } } } + timer_stop_barrier(cfg, &time_stat); + test_print_verbose_once(cfg, "DEBUG: finished stat"); // open file + timer_start_barrier(cfg, &time_open); rc = test_open_file(cfg, target_file, O_RDONLY); if (rc) { test_abort(cfg, rc); } + timer_stop_barrier(cfg, &time_open); + test_print_verbose_once(cfg, "DEBUG: finished open"); // generate read requests test_print_verbose_once(cfg, "DEBUG: generating read requests"); @@ -178,8 +190,7 @@ int main(int argc, char* argv[]) // do reads test_print_verbose_once(cfg, "DEBUG: starting read requests"); - test_barrier(cfg); - timer_start(&time_rd); + timer_start_barrier(cfg, &time_rd); rc = issue_read_req_batch(cfg, num_reqs, reqs); if (rc) { test_abort(cfg, rc); @@ -188,18 +199,17 @@ int main(int argc, char* argv[]) if (rc) { test_abort(cfg, rc); } - timer_stop(&time_rd); + timer_stop_barrier(cfg, &time_rd); test_print_verbose_once(cfg, "DEBUG: finished read requests"); // check file data - test_barrier(cfg); test_print_verbose_once(cfg, "DEBUG: starting data check"); - timer_start(&time_check); + timer_start_barrier(cfg, &time_check); rc = check_read_req_batch(cfg, num_reqs, reqs); if (rc) { test_abort(cfg, rc); } - timer_stop(&time_check); + timer_stop_barrier(cfg, &time_check); test_print_verbose_once(cfg, "DEBUG: finished data check"); // post-read cleanup @@ -208,10 +218,13 @@ int main(int argc, char* argv[]) reqs = NULL; // close file + timer_start_barrier(cfg, &time_close); rc = test_close_file(cfg); if (rc) { test_abort(cfg, rc); } + timer_stop_barrier(cfg, &time_close); + test_print_verbose_once(cfg, "DEBUG: finished close"); // calculate achieved bandwidth rates double max_read_time, max_check_time; @@ -232,8 +245,11 @@ int main(int argc, char* argv[]) "Number of processes: %d\n" "Each process wrote: %.2lf MiB\n" "Total data written: %.2lf MiB\n" + "File stat time: %.6lf sec\n" + "File open time: %.6lf sec\n" "Maximum read time: %.6lf sec\n" "Maximum check time: %.6lf sec\n" + "File close time: %.6lf sec\n" "Aggregate read bandwidth: %.3lf MiB/s\n" "Effective read bandwidth: %.3lf MiB/s\n", io_pattern_str(test_config.io_pattern), @@ -242,16 +258,22 @@ int main(int argc, char* argv[]) test_config.n_ranks, bytes_to_mib(rank_bytes), bytes_to_mib(total_bytes), + time_stat.elapsed_sec_all, + time_open.elapsed_sec_all, max_read_time, max_check_time, + time_close.elapsed_sec_all, aggr_read_bw, eff_read_bw); // cleanup free(target_file); + timer_fini(&time_stat); + timer_fini(&time_open); timer_fini(&time_rd); timer_fini(&time_check); + timer_fini(&time_close); test_fini(cfg); diff --git a/examples/src/testutil.h b/examples/src/testutil.h index 7ee02ad89..6bc82506b 100644 --- a/examples/src/testutil.h +++ b/examples/src/testutil.h @@ -329,8 +329,10 @@ void test_print_verbose_once(test_cfg* cfg, const char* fmt, ...) typedef struct { struct timeval start; struct timeval stop; + struct timeval stop_all; char* name; double elapsed_sec; + double elapsed_sec_all; } test_timer; static inline @@ -384,6 +386,39 @@ void timer_stop(test_timer* timer) &(timer->stop)); } +static inline +void timer_start_barrier(test_cfg* cfg, test_timer* timer) +{ + /* execute a barrier to ensure procs don't start + * next phase before all have reached this point */ + if (cfg->use_mpi) { + MPI_Barrier(MPI_COMM_WORLD); + } + + /* everyone has reached, start the timer, + * the start field is used in both local and global timers */ + timer_start(timer); +} + +static inline +void timer_stop_barrier(test_cfg* cfg, test_timer* timer) +{ + /* stop the local timer and compute elapsed_secs */ + timer_stop(timer); + + /* execute a barrier to ensure procs have reached this point + * before stopping the global timer */ + if (cfg->use_mpi) { + MPI_Barrier(MPI_COMM_WORLD); + } + + /* everyone has reached, stop the global timer and + * compute elapsed global time */ + gettimeofday(&(timer->stop_all), NULL); + timer->elapsed_sec_all = timediff_sec(&(timer->start), + &(timer->stop_all)); +} + /* ---------- Option Parsing Utilities ---------- */ static const char* unifyfs_mntpt = "/unifyfs"; diff --git a/examples/src/testutil_rdwr.h b/examples/src/testutil_rdwr.h index 1ed21813f..bc17667a0 100644 --- a/examples/src/testutil_rdwr.h +++ b/examples/src/testutil_rdwr.h @@ -206,6 +206,24 @@ int write_sync(test_cfg* cfg) return 0; } +static inline +int write_laminate(test_cfg* cfg, const char* filepath) +{ + /* need one process to laminate each file, + * we use the same process that created the file */ + int rc = 0; + if (cfg->rank == 0 || cfg->io_pattern == IO_PATTERN_NN) { + /* laminate by setting permissions to read-only */ + int chmod_rc = chmod(filepath, 0444); + if (-1 == chmod_rc) { + /* lamination failed */ + test_print(cfg, "chmod() lamination failed"); + rc = -1; + } + } + return rc; +} + /* -------- Read Helper Methods -------- */ static inline diff --git a/examples/src/write.c b/examples/src/write.c index 5e8e794f2..4405f10f2 100644 --- a/examples/src/write.c +++ b/examples/src/write.c @@ -125,11 +125,21 @@ int main(int argc, char* argv[]) test_cfg test_config; test_cfg* cfg = &test_config; + test_timer time_create2laminate; + test_timer time_create; test_timer time_wr; test_timer time_sync; + test_timer time_close; + test_timer time_laminate; + test_timer time_stat; + timer_init(&time_create2laminate, "create2laminate"); + timer_init(&time_create, "create"); timer_init(&time_wr, "write"); timer_init(&time_sync, "sync"); + timer_init(&time_close, "close"); + timer_init(&time_laminate, "laminate"); + timer_init(&time_stat, "stat"); rc = test_init(argc, argv, cfg); if (rc) { @@ -146,13 +156,20 @@ int main(int argc, char* argv[]) return -1; } + // timer to wrap all parts of write operation + timer_start_barrier(cfg, &time_create2laminate); + + // create file target_file = test_target_filename(cfg); test_print_verbose_once(cfg, "DEBUG: creating target file %s", target_file); + timer_start_barrier(cfg, &time_create); rc = test_create_file(cfg, target_file, O_RDWR); if (rc) { test_abort(cfg, rc); } + timer_stop_barrier(cfg, &time_create); + test_print_verbose_once(cfg, "DEBUG: finished create"); // generate write requests test_print_verbose_once(cfg, "DEBUG: generating write requests"); @@ -167,8 +184,7 @@ int main(int argc, char* argv[]) // do writes test_print_verbose_once(cfg, "DEBUG: starting write requests"); - test_barrier(cfg); - timer_start(&time_wr); + timer_start_barrier(cfg, &time_wr); rc = issue_write_req_batch(cfg, num_reqs, reqs); if (rc) { test_abort(cfg, rc); @@ -177,38 +193,46 @@ int main(int argc, char* argv[]) if (rc) { test_abort(cfg, rc); } - timer_stop(&time_wr); + timer_stop_barrier(cfg, &time_wr); test_print_verbose_once(cfg, "DEBUG: finished write requests"); - // sync/laminate - timer_start(&time_sync); + // sync + timer_start_barrier(cfg, &time_sync); rc = write_sync(cfg); if (rc) { test_abort(cfg, rc); } - timer_stop(&time_sync); - test_barrier(cfg); + timer_stop_barrier(cfg, &time_sync); test_print_verbose_once(cfg, "DEBUG: finished sync"); - if ((test_config.rank == 0) || - (IO_PATTERN_NN == test_config.io_pattern)) { - /* laminate by removing write bits */ - chmod(target_file, 0400); - } - test_print_verbose_once(cfg, "DEBUG: finished lamination"); - // post-write cleanup free(wr_buf); free(reqs); reqs = NULL; // close file + timer_start_barrier(cfg, &time_close); rc = test_close_file(cfg); if (rc) { test_abort(cfg, rc); } + timer_stop_barrier(cfg, &time_close); + test_print_verbose_once(cfg, "DEBUG: finished close"); + + // laminate + timer_start_barrier(cfg, &time_laminate); + rc = write_laminate(cfg, target_file); + if (rc) { + test_abort(cfg, rc); + } + timer_stop_barrier(cfg, &time_laminate); + test_print_verbose_once(cfg, "DEBUG: finished laminate"); + + // timer to wrap all parts of write operation + timer_stop_barrier(cfg, &time_create2laminate); // file size check + timer_start_barrier(cfg, &time_stat); size_t rank_bytes = test_config.n_blocks * test_config.block_sz; size_t total_bytes = rank_bytes * test_config.n_ranks; size_t expected = total_bytes; @@ -229,6 +253,8 @@ int main(int argc, char* argv[]) } } } + timer_stop_barrier(cfg, &time_stat); + test_print_verbose_once(cfg, "DEBUG: finished stat"); // calculate achieved bandwidth rates double max_write_time, max_sync_time; @@ -249,8 +275,15 @@ int main(int argc, char* argv[]) "Number of processes: %d\n" "Each process wrote: %.2lf MiB\n" "Total data written: %.2lf MiB\n" + "File create time: %.6lf sec\n" "Maximum write time: %.6lf sec\n" + "Global write time: %.6lf sec\n" "Maximum sync time: %.6lf sec\n" + "Global sync time: %.6lf sec\n" + "File close time: %.6lf sec\n" + "File laminate time: %.6lf sec\n" + "Full write time: %.6lf sec\n" + "File stat time: %.6lf sec\n" "Aggregate write bandwidth: %.3lf MiB/s\n" "Effective write bandwidth: %.3lf MiB/s\n", io_pattern_str(test_config.io_pattern), @@ -259,16 +292,28 @@ int main(int argc, char* argv[]) test_config.n_ranks, bytes_to_mib(rank_bytes), bytes_to_mib(total_bytes), + time_create.elapsed_sec_all, max_write_time, + time_wr.elapsed_sec_all, max_sync_time, + time_sync.elapsed_sec_all, + time_close.elapsed_sec_all, + time_laminate.elapsed_sec_all, + time_create2laminate.elapsed_sec_all, + time_stat.elapsed_sec_all, aggr_write_bw, eff_write_bw); // cleanup free(target_file); + timer_fini(&time_create2laminate); + timer_fini(&time_create); timer_fini(&time_wr); timer_fini(&time_sync); + timer_fini(&time_close); + timer_fini(&time_laminate); + timer_fini(&time_stat); test_fini(cfg); diff --git a/examples/src/writeread.c b/examples/src/writeread.c index c104a5ee9..b9967947c 100644 --- a/examples/src/writeread.c +++ b/examples/src/writeread.c @@ -177,13 +177,19 @@ int main(int argc, char* argv[]) test_cfg test_config; test_cfg* cfg = &test_config; + test_timer time_create2laminate; + test_timer time_create; test_timer time_wr; test_timer time_rd; test_timer time_sync; + test_timer time_laminate; + timer_init(&time_create2laminate, "create2laminate"); + timer_init(&time_create, "create"); timer_init(&time_wr, "write"); timer_init(&time_rd, "read"); timer_init(&time_sync, "sync"); + timer_init(&time_laminate, "laminate"); rc = test_init(argc, argv, cfg); if (rc) { @@ -200,13 +206,20 @@ int main(int argc, char* argv[]) return -1; } + // timer to wrap all parts of write operation + timer_start_barrier(cfg, &time_create2laminate); + + // create file target_file = test_target_filename(cfg); test_print_verbose_once(cfg, "DEBUG: creating target file %s", target_file); + timer_start_barrier(cfg, &time_create); rc = test_create_file(cfg, target_file, O_RDWR); if (rc) { test_abort(cfg, rc); } + timer_stop_barrier(cfg, &time_create); + test_print_verbose_once(cfg, "DEBUG: finished create"); // generate write requests test_print_verbose_once(cfg, "DEBUG: generating write requests"); @@ -221,8 +234,7 @@ int main(int argc, char* argv[]) // do writes test_print_verbose_once(cfg, "DEBUG: starting write requests"); - test_barrier(cfg); - timer_start(&time_wr); + timer_start_barrier(cfg, &time_wr); rc = issue_write_req_batch(cfg, num_reqs, reqs); if (rc) { test_abort(cfg, rc); @@ -231,24 +243,36 @@ int main(int argc, char* argv[]) if (rc) { test_abort(cfg, rc); } - timer_stop(&time_wr); + timer_stop_barrier(cfg, &time_wr); test_print_verbose_once(cfg, "DEBUG: finished write requests"); - // sync/laminate - timer_start(&time_sync); + // sync + timer_start_barrier(cfg, &time_sync); rc = write_sync(cfg); if (rc) { test_abort(cfg, rc); } - timer_stop(&time_sync); - test_barrier(cfg); + timer_stop_barrier(cfg, &time_sync); test_print_verbose_once(cfg, "DEBUG: finished sync"); + // close file + + // laminate + timer_start_barrier(cfg, &time_laminate); + rc = write_laminate(cfg, target_file); + if (rc) { + test_abort(cfg, rc); + } + timer_stop_barrier(cfg, &time_laminate); + test_print_verbose_once(cfg, "DEBUG: finished laminate"); + // post-write cleanup free(wr_buf); free(reqs); reqs = NULL; + // open file + // generate read requests test_print_verbose_once(cfg, "DEBUG: generating read requests"); rd_buf = calloc(test_config.n_blocks, test_config.block_sz); @@ -262,8 +286,7 @@ int main(int argc, char* argv[]) // do reads test_print_verbose_once(cfg, "DEBUG: starting read requests"); - test_barrier(cfg); - timer_start(&time_rd); + timer_start_barrier(cfg, &time_rd); rc = issue_read_req_batch(cfg, num_reqs, reqs); if (rc) { test_abort(cfg, rc); @@ -272,8 +295,7 @@ int main(int argc, char* argv[]) if (rc) { test_abort(cfg, rc); } - timer_stop(&time_rd); - test_barrier(cfg); + timer_stop_barrier(cfg, &time_rd); test_print_verbose_once(cfg, "DEBUG: finished read requests"); if (test_config.io_check) { @@ -310,6 +332,8 @@ int main(int argc, char* argv[]) eff_write_bw = bandwidth_mib(total_bytes, max_write_time); eff_read_bw = bandwidth_mib(total_bytes, max_read_time); + printf("File create time is %.3lf s\n", + time_create.elapsed_sec_all); printf("Aggregate Write BW is %.3lf MiB/s\n" "Effective Write BW is %.3lf MiB/s\n\n", aggr_write_bw, eff_write_bw); @@ -319,15 +343,20 @@ int main(int argc, char* argv[]) printf("Aggregate Read BW is %.3lf MiB/s\n" "Effective Read BW is %.3lf MiB/s\n\n", aggr_read_bw, eff_read_bw); + printf("File laminate time is %.3lf s\n", + time_laminate.elapsed_sec_all); fflush(stdout); } // cleanup free(target_file); + timer_fini(&time_create2laminate); + timer_fini(&time_create); timer_fini(&time_wr); timer_fini(&time_rd); timer_fini(&time_sync); + timer_fini(&time_laminate); test_fini(cfg); diff --git a/t/ci/100-writeread-tests.sh b/t/ci/100-writeread-tests.sh index 6969dd607..cb50c2a32 100755 --- a/t/ci/100-writeread-tests.sh +++ b/t/ci/100-writeread-tests.sh @@ -68,7 +68,7 @@ unify_test_writeread() { # Evaluate output test_expect_success "$app_name $app_args: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 8 + test $lcount = 10 ' } @@ -86,7 +86,7 @@ unify_test_writeread_posix() { # Evaluate output test_expect_success POSIX "$app_name $1: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 8 && + test $lcount = 10 && if [[ $io_pattern =~ (n1)$ ]]; then test_path_is_file ${CI_POSIX_MP}/$filename else diff --git a/t/ci/110-write-tests.sh b/t/ci/110-write-tests.sh index bba5d1ea9..27474c857 100755 --- a/t/ci/110-write-tests.sh +++ b/t/ci/110-write-tests.sh @@ -68,7 +68,7 @@ unify_test_write() { # Evaluate output test_expect_success "$app_name $app_args: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 11 + test $lcount = 18 ' } @@ -86,7 +86,7 @@ unify_test_write_posix() { # Evaluate output test_expect_success POSIX "$app_name $1: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 11 && + test $lcount = 18 && if [[ $io_pattern =~ (n1)$ ]]; then test_path_is_file ${CI_POSIX_MP}/$filename else diff --git a/t/ci/120-read-tests.sh b/t/ci/120-read-tests.sh index 686c4e94f..c61549321 100755 --- a/t/ci/120-read-tests.sh +++ b/t/ci/120-read-tests.sh @@ -68,7 +68,7 @@ unify_test_read() { # Evaluate output test_expect_success "$app_name $app_args: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 11 + test $lcount = 14 ' } @@ -85,7 +85,7 @@ unify_test_read_posix() { # Evaluate output test_expect_success POSIX "$app_name $1: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 11 + test $lcount = 14 ' } From 0dd8a5d78ef8d83eec251d9a975d644c21322d3d Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 24 Oct 2019 16:38:17 -0700 Subject: [PATCH 010/168] server: cleanup metadata functions --- server/src/unifyfs_metadata.c | 163 +++++++++++++++++++++------------- 1 file changed, 99 insertions(+), 64 deletions(-) diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index 465d505aa..a922903dc 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -43,22 +43,21 @@ #include "indexes.h" #include "mdhim.h" - unifyfs_key_t** unifyfs_keys; unifyfs_val_t** unifyfs_vals; fattr_key_t** fattr_keys; fattr_val_t** fattr_vals; -char* manifest_path; - -struct mdhim_brm_t* brm, *brmp; -struct mdhim_bgetrm_t* bgrm, *bgrmp; - struct mdhim_t* md; -int md_size; +/* we use two MDHIM indexes: + * 0) for file extents + * 1) for file attributes */ +#define IDX_FILE_EXTENTS (0) +#define IDX_FILE_ATTR (1) struct index_t* unifyfs_indexes[2]; + size_t max_recs_per_slice; void debug_log_key_val(const char* ctx, @@ -152,15 +151,12 @@ int meta_init_store(unifyfs_cfg_t* cfg) md = mdhimInit(&comm, db_opts); - /*this index is created for storing index metadata*/ - unifyfs_indexes[0] = md->primary_index; + /* index for storing file extent metadata */ + unifyfs_indexes[IDX_FILE_EXTENTS] = md->primary_index; - /*this index is created for storing file attribute metadata*/ - unifyfs_indexes[1] = create_global_index(md, ratio, 1, - LEVELDB, MDHIM_INT_KEY, - "file_attr"); - - MPI_Comm_size(md->mdhim_comm, &md_size); + /* index for storing file attribute metadata */ + unifyfs_indexes[IDX_FILE_ATTR] = create_global_index(md, + ratio, 1, LEVELDB, MDHIM_INT_KEY, "file_attr"); rc = meta_init_indices(); if (rc != 0) { @@ -257,6 +253,7 @@ void meta_free_indices(void) free(unifyfs_keys); free(unifyfs_vals); } + if (NULL != fattr_keys) { for (i = 0; i < MAX_FILE_CNT_PER_NODE; i++) { if (NULL != fattr_keys[i]) { @@ -318,18 +315,24 @@ int unifyfs_set_file_attribute(unifyfs_file_attr_t* fattr_ptr) { int rc = UNIFYFS_SUCCESS; + /* select index for file attributes */ + md->primary_index = unifyfs_indexes[IDX_FILE_ATTR]; + + /* insert file attribute for given global file id */ int gfid = fattr_ptr->gfid; + struct mdhim_brm_t* brm = mdhimPut(md, + &gfid, sizeof(int), + fattr_ptr, sizeof(unifyfs_file_attr_t), + NULL, NULL); - md->primary_index = unifyfs_indexes[1]; - brm = mdhimPut(md, &gfid, sizeof(int), - fattr_ptr, sizeof(unifyfs_file_attr_t), - NULL, NULL); if (!brm || brm->error) { - // return UNIFYFS_ERROR_MDHIM on error + LOGERR("Error inserting file attribute into MDHIM"); rc = (int)UNIFYFS_ERROR_MDHIM; } - mdhim_full_release_msg(brm); + if (brm) { + mdhim_full_release_msg(brm); + } return rc; } @@ -343,46 +346,66 @@ int unifyfs_set_file_attributes(int num_entries, { int rc = UNIFYFS_SUCCESS; - md->primary_index = unifyfs_indexes[1]; - brm = mdhimBPut(md, (void**)keys, key_lens, (void**)fattr_ptr, - val_lens, num_entries, NULL, NULL); - brmp = brm; - if (!brmp || brmp->error) { + /* select index for file attributes */ + md->primary_index = unifyfs_indexes[IDX_FILE_ATTR]; + + /* put list of key/value pairs */ + struct mdhim_brm_t* brm = mdhimBPut(md, + (void**)keys, key_lens, + (void**)fattr_ptr, val_lens, + num_entries, NULL, NULL); + + /* check for errors and free resources */ + if (!brm) { + LOGERR("Error inserting file attributes into MDHIM"); rc = (int)UNIFYFS_ERROR_MDHIM; - LOGERR("Error inserting keys/values into MDHIM"); - } + } else { + /* step through linked list of messages, + * scan for any error and free messages */ + struct mdhim_brm_t* brmp = brm; + while (brmp) { + /* check current item for error */ + if (brmp->error < 0) { + LOGERR("Error inserting file attributes into MDHIM"); + rc = (int)UNIFYFS_ERROR_MDHIM; + } - while (brmp) { - if (brmp->error < 0) { - rc = (int)UNIFYFS_ERROR_MDHIM; - break; + /* record pointer to current item, + * advance loop pointer to next item in list, + * free resources for current item */ + brm = brmp; + brmp = brmp->next; + mdhim_full_release_msg(brm); } - - brm = brmp; - brmp = brmp->next; - mdhim_full_release_msg(brm); } return rc; } -/* - * - */ -int unifyfs_get_file_attribute(int gfid, - unifyfs_file_attr_t* attr_val_ptr) +/* given a global file id, lookup and return file attributes */ +int unifyfs_get_file_attribute( + int gfid, + unifyfs_file_attr_t* attr) { int rc = UNIFYFS_SUCCESS; - unifyfs_file_attr_t* tmp_ptr_attr; - md->primary_index = unifyfs_indexes[1]; - bgrm = mdhimGet(md, md->primary_index, &gfid, - sizeof(int), MDHIM_GET_EQ); + /* select index holding file attributes, + * execute lookup for given file id */ + md->primary_index = unifyfs_indexes[IDX_FILE_ATTR]; + struct mdhim_bgetrm_t* bgrm = mdhimGet(md, md->primary_index, + &gfid, sizeof(int), MDHIM_GET_EQ); + if (!bgrm || bgrm->error) { + /* failed to find info for this file id */ rc = (int)UNIFYFS_ERROR_MDHIM; } else { - tmp_ptr_attr = (unifyfs_file_attr_t*)bgrm->values[0]; - memcpy(attr_val_ptr, tmp_ptr_attr, sizeof(unifyfs_file_attr_t)); + /* copy file attribute from value into output parameter */ + unifyfs_file_attr_t* ptr = (unifyfs_file_attr_t*)bgrm->values[0]; + memcpy(attr, ptr, sizeof(unifyfs_file_attr_t)); + } + + /* free resources returned from lookup */ + if (bgrm) { mdhim_full_release_msg(bgrm); } @@ -407,7 +430,8 @@ int unifyfs_get_file_extents(int num_keys, unifyfs_key_t** keys, *num_values = 0; *keyvals = NULL; - md->primary_index = unifyfs_indexes[0]; + /* select index for file extents */ + md->primary_index = unifyfs_indexes[IDX_FILE_EXTENTS]; /* execute range query */ struct mdhim_bgetrm_t* bkvlist = mdhimBGet(md, md->primary_index, @@ -483,26 +507,37 @@ int unifyfs_set_file_extents(int num_entries, { int rc = UNIFYFS_SUCCESS; - md->primary_index = unifyfs_indexes[0]; + /* select index for file extents */ + md->primary_index = unifyfs_indexes[IDX_FILE_EXTENTS]; + + /* put list of key/value pairs */ + struct mdhim_brm_t* brm = mdhimBPut(md, + (void**)(keys), key_lens, + (void**)(vals), val_lens, + num_entries, NULL, NULL); - brm = mdhimBPut(md, (void**)(keys), key_lens, - (void**)(vals), val_lens, num_entries, - NULL, NULL); - brmp = brm; - if (!brmp || brmp->error) { + /* check for errors and free resources */ + if (!brm) { + LOGERR("Error inserting file extents into MDHIM"); rc = (int)UNIFYFS_ERROR_MDHIM; - LOGERR("Error inserting keys/values into MDHIM"); - } + } else { + /* step through linked list of messages, + * scan for any error and free messages */ + struct mdhim_brm_t* brmp = brm; + while (brmp) { + /* check current item for error */ + if (brmp->error < 0) { + LOGERR("Error inserting file extents into MDHIM"); + rc = (int)UNIFYFS_ERROR_MDHIM; + } - while (brmp) { - if (brmp->error < 0) { - rc = (int)UNIFYFS_ERROR_MDHIM; - break; + /* record pointer to current item, + * advance loop pointer to next item in list, + * free resources for current item */ + brm = brmp; + brmp = brmp->next; + mdhim_full_release_msg(brm); } - - brm = brmp; - brmp = brmp->next; - mdhim_full_release_msg(brm); } return rc; From e2de49073ebd70aa5251c26f5bda32d1e24a8369 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 24 Oct 2019 11:36:19 -0700 Subject: [PATCH 011/168] client: cleanup index slice code --- client/src/unifyfs-fixed.c | 145 +++++++++++++++++++++++-------------- 1 file changed, 89 insertions(+), 56 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index d5ae00505..7ea79f6a7 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -265,72 +265,105 @@ static int unifyfs_chunk_read( } /* given an index, split it into multiple indices whose range is equal or - * smaller than slice_range size @param cur_idx: the index to split + * smaller than slice_range size + * @param cur_idx: the index to split * @param slice_range: the slice size of the key-value store * @return index_set: the set of split indices */ int unifyfs_split_index(unifyfs_index_t* cur_idx, index_set_t* index_set, long slice_range) { - - long cur_idx_start = cur_idx->file_pos; - long cur_idx_end = cur_idx->file_pos + cur_idx->length - 1; - - long cur_slice_start = cur_idx->file_pos / slice_range * slice_range; - long cur_slice_end = cur_slice_start + slice_range - 1; - - - index_set->count = 0; - - long cur_mem_pos = cur_idx->mem_pos; - if (cur_idx_end <= cur_slice_end) { - /* - cur_slice_start cur_slice_end - cur_idx_start cur_idx_end - - */ - index_set->idxes[index_set->count] = *cur_idx; - index_set->count++; - + /* first byte offset this write will write to */ + long idx_start = cur_idx->file_pos; + + /* last byte offset this write will write to */ + long idx_end = cur_idx->file_pos + cur_idx->length - 1; + + /* starting byte offset of slice that first write offset falls in */ + long slice_start = (idx_start / slice_range) * slice_range; + + /* last byte offset in slice that first write offset falls in */ + long slice_end = slice_start + slice_range - 1; + + /* get pointer to first output index structure */ + unifyfs_index_t* set = index_set->idxes; + + /* initialize count of output index entries */ + int count = 0; + + /* define new index entries in index_set by splitting write index + * at slice boundaries */ + if (idx_end <= slice_end) { + /* index falls fully within one slice + * + * slice_start slice_end + * idx_start idx_end + */ + set[count] = *cur_idx; + count++; } else { - /* - cur_slice_start cur_slice_endnext_slice_start next_slice_end - cur_idx_start cur_idx_end - - */ - index_set->idxes[index_set->count] = *cur_idx; - index_set->idxes[index_set->count].length = - cur_slice_end - cur_idx_start + 1; - - cur_mem_pos += index_set->idxes[index_set->count].length; - - cur_slice_start = cur_slice_end + 1; - cur_slice_end = cur_slice_start + slice_range - 1; - index_set->count++; - - while (1) { - if (cur_idx_end <= cur_slice_end) { - break; - } - - index_set->idxes[index_set->count].fid = cur_idx->fid; - index_set->idxes[index_set->count].file_pos = cur_slice_start; - index_set->idxes[index_set->count].length = slice_range; - index_set->idxes[index_set->count].mem_pos = cur_mem_pos; - cur_mem_pos += index_set->idxes[index_set->count].length; - - cur_slice_start = cur_slice_end + 1; - cur_slice_end = cur_slice_start + slice_range - 1; - index_set->count++; - + /* ending offset of index is beyond last offset in first slice, + * so this index spans across multiple slices + * + * slice_start slice_end next_slice_start next_slice_end + * idx_start idx_end + */ + + /* get starting position in log */ + long log_pos = cur_idx->mem_pos; + + /* copy over all fields in current index */ + set[count] = *cur_idx; + + /* update length field to adjust for boundary of first slice */ + long length = slice_end - idx_start + 1; + set[count].length = length; + + /* advance offset into log */ + log_pos += length; + + /* increment our output index count */ + count++; + + /* advance slice boundary offsets to next slice */ + slice_start = slice_end + 1; + slice_end = slice_start + slice_range - 1; + + /* loop until we find the slice that contains + * ending offset of write */ + while (idx_end > slice_end) { + /* ending offset of write is beyond end of this slice, + * so write spans the full length of this slice */ + length = slice_range; + + /* define index for this slice */ + set[count].fid = cur_idx->fid; + set[count].file_pos = slice_start; + set[count].length = length; + set[count].mem_pos = log_pos; + + /* advance offset into log */ + log_pos += length; + + /* increment our output index count */ + count++; + + /* advance slice boundary offsets to next slice */ + slice_start = slice_end + 1; + slice_end = slice_start + slice_range - 1; } - index_set->idxes[index_set->count].fid = cur_idx->fid; - index_set->idxes[index_set->count].file_pos = cur_slice_start; - index_set->idxes[index_set->count].length = cur_idx_end - cur_slice_start + 1; - index_set->idxes[index_set->count].mem_pos = cur_mem_pos; - index_set->count++; + /* this slice contains the remainder of write */ + length = idx_end - slice_start + 1; + set[count].fid = cur_idx->fid; + set[count].file_pos = slice_start; + set[count].length = length; + set[count].mem_pos = log_pos; + count++; } + /* record number of entires in output index set */ + index_set->count = count; + return 0; } From 21b2c527f57a33397997fce8bbf03acfc44390de Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 24 Oct 2019 12:02:45 -0700 Subject: [PATCH 012/168] client: check for overflow when splitting write index --- client/src/unifyfs-fixed.c | 55 ++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 7ea79f6a7..1d18f1b1d 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -266,11 +266,15 @@ static int unifyfs_chunk_read( /* given an index, split it into multiple indices whose range is equal or * smaller than slice_range size - * @param cur_idx: the index to split - * @param slice_range: the slice size of the key-value store - * @return index_set: the set of split indices */ -int unifyfs_split_index(unifyfs_index_t* cur_idx, index_set_t* index_set, - long slice_range) + * @param cur_idx: the index to split + * @param slice_range: the slice size of the key-value store + * @return index_set: the set of split indices + * @param maxcount: number of entries in output array */ +int unifyfs_split_index( + unifyfs_index_t* cur_idx, /* write index to split (offset and length) */ + long slice_range, /* number of bytes in each slice */ + index_set_t* index_set, /* output array to store new indexes in */ + int maxcount) /* max number of items in output array */ { /* first byte offset this write will write to */ long idx_start = cur_idx->file_pos; @@ -281,7 +285,7 @@ int unifyfs_split_index(unifyfs_index_t* cur_idx, index_set_t* index_set, /* starting byte offset of slice that first write offset falls in */ long slice_start = (idx_start / slice_range) * slice_range; - /* last byte offset in slice that first write offset falls in */ + /* last byte offset of slice that first write offset falls in */ long slice_end = slice_start + slice_range - 1; /* get pointer to first output index structure */ @@ -290,6 +294,15 @@ int unifyfs_split_index(unifyfs_index_t* cur_idx, index_set_t* index_set, /* initialize count of output index entries */ int count = 0; + /* no room to write index values */ + if (count >= maxcount) { + /* no room to write more index values, + * and we have at least one more, + * record number we wrote and return with error */ + index_set->count = 0; + return UNIFYFS_FAILURE; + } + /* define new index entries in index_set by splitting write index * at slice boundaries */ if (idx_end <= slice_end) { @@ -324,6 +337,15 @@ int unifyfs_split_index(unifyfs_index_t* cur_idx, index_set_t* index_set, /* increment our output index count */ count++; + /* check that we have room to write more index values */ + if (count >= maxcount) { + /* no room to write more index values, + * and we have at least one more, + * record number we wrote and return with error */ + index_set->count = count; + return UNIFYFS_FAILURE; + } + /* advance slice boundary offsets to next slice */ slice_start = slice_end + 1; slice_end = slice_start + slice_range - 1; @@ -347,6 +369,15 @@ int unifyfs_split_index(unifyfs_index_t* cur_idx, index_set_t* index_set, /* increment our output index count */ count++; + /* check that we have room to write more index values */ + if (count >= maxcount) { + /* no room to write more index values, + * and we have at least one more, + * record number we wrote and return with error */ + index_set->count = count; + return UNIFYFS_FAILURE; + } + /* advance slice boundary offsets to next slice */ slice_start = slice_end + 1; slice_end = slice_start + slice_range - 1; @@ -437,8 +468,16 @@ static int unifyfs_logio_chunk_write( * the ones smaller than unifyfs_key_slice_range */ index_set_t tmp_index_set; memset(&tmp_index_set, 0, sizeof(tmp_index_set)); - unifyfs_split_index(&cur_idx, &tmp_index_set, - unifyfs_key_slice_range); + int split_rc = unifyfs_split_index(&cur_idx, unifyfs_key_slice_range, + &tmp_index_set, UNIFYFS_MAX_SPLIT_CNT); + if (split_rc != UNIFYFS_SUCCESS) { + /* in this case, we have copied data to the log, + * but we failed to generate index entries, + * we're returning with an error and leaving the data + * in the log */ + LOGERR("exhausted space when splitting write index"); + return UNIFYFS_ERROR_IO; + } /* lookup number of existing index entries */ off_t num_entries = *(unifyfs_indices.ptr_num_entries); From f7dcb9624679aae8439e3e752b8023b0d2e64142 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 24 Oct 2019 13:25:07 -0700 Subject: [PATCH 013/168] client: split write index directly into shared memory buffer --- client/src/unifyfs-fixed.c | 224 ++++++++++++++++++++-------------- client/src/unifyfs-internal.h | 5 - 2 files changed, 130 insertions(+), 99 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 1d18f1b1d..099dd5401 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -264,17 +264,88 @@ static int unifyfs_chunk_read( return UNIFYFS_SUCCESS; } +/* merges next_idx into prev_idx if possible, + * updates both prev_idx and next_idx leaving any + * portion that could not be merged in next_idx */ +static int unifyfs_coalesce_index( + unifyfs_index_t* prev_idx, /* existing index entry to coalesce into */ + unifyfs_index_t* next_idx, /* new index entry we'd like to add */ + long slice_size) /* byte size of slice of key-value store */ +{ + /* check whether last index and next index refer to same file */ + if (prev_idx->fid != next_idx->fid) { + /* two index values are for different files, can't combine */ + return UNIFYFS_SUCCESS; + } + + /* got same file, + * check whether last index and next index refer to + * contiguous bytes */ + off_t prev_offset = prev_idx->file_pos + prev_idx->length; + off_t next_offset = next_idx->file_pos; + if (prev_offset != next_offset) { + /* index values are not contiguous, can't combine */ + return UNIFYFS_SUCCESS; + } + + /* got contiguous bytes in the same file, + * check whether both index values fall in the same slice */ + off_t prev_slice = prev_idx->file_pos / slice_size; + off_t next_slice = next_idx->file_pos / slice_size; + if (prev_slice != next_slice) { + /* index values refer to different slices, can't combine */ + return UNIFYFS_SUCCESS; + } + + /* check whether last index and next index refer to + * contiguous bytes in the log */ + off_t prev_log = prev_idx->log_pos + prev_idx->length; + off_t next_log = next_idx->log_pos; + if (prev_log != next_log) { + /* index values are not contiguous in log, can't combine */ + return UNIFYFS_SUCCESS; + } + + /* if we get here, we can coalesce next index value into previous */ + + /* get ending offset of write in next index, + * and ending offset of current slice */ + off_t next_end = next_idx->file_pos + next_idx->length; + off_t slice_end = (next_slice * slice_size) + slice_size; + + /* determine number of bytes in next index that lie in current slice, + * assume all bytes in the index will fit */ + long length = next_idx->length; + if (next_end > slice_end) { + /* current index writes beyond end of slice, + * so we can only coalesce bytes that fall within slice */ + length = slice_end - next_offset; + } + + /* extend length of last index to include beginning portion + * of current index */ + prev_idx->length += length; + + /* adjust current index to subtract off those bytes */ + next_idx->file_pos += length; + next_idx->mem_pos += length; + next_idx->length -= length; + + return UNIFYFS_SUCCESS; +} + /* given an index, split it into multiple indices whose range is equal or * smaller than slice_range size * @param cur_idx: the index to split * @param slice_range: the slice size of the key-value store * @return index_set: the set of split indices * @param maxcount: number of entries in output array */ -int unifyfs_split_index( - unifyfs_index_t* cur_idx, /* write index to split (offset and length) */ - long slice_range, /* number of bytes in each slice */ - index_set_t* index_set, /* output array to store new indexes in */ - int maxcount) /* max number of items in output array */ +static int unifyfs_split_index( + unifyfs_index_t* cur_idx, /* write index to split (offset and length) */ + long slice_range, /* number of bytes in each slice */ + unifyfs_index_t* index_set, /* output array to store new indexes in */ + off_t maxcount, /* max number of items in output array */ + off_t* used_count) /* number of entries we added in split */ { /* first byte offset this write will write to */ long idx_start = cur_idx->file_pos; @@ -289,17 +360,17 @@ int unifyfs_split_index( long slice_end = slice_start + slice_range - 1; /* get pointer to first output index structure */ - unifyfs_index_t* set = index_set->idxes; + unifyfs_index_t* set = index_set; /* initialize count of output index entries */ - int count = 0; + off_t count = 0; /* no room to write index values */ if (count >= maxcount) { /* no room to write more index values, * and we have at least one more, * record number we wrote and return with error */ - index_set->count = 0; + *used_count = 0; return UNIFYFS_FAILURE; } @@ -342,7 +413,7 @@ int unifyfs_split_index( /* no room to write more index values, * and we have at least one more, * record number we wrote and return with error */ - index_set->count = count; + *used_count = count; return UNIFYFS_FAILURE; } @@ -374,7 +445,7 @@ int unifyfs_split_index( /* no room to write more index values, * and we have at least one more, * record number we wrote and return with error */ - index_set->count = count; + *used_count = count; return UNIFYFS_FAILURE; } @@ -392,10 +463,10 @@ int unifyfs_split_index( count++; } - /* record number of entires in output index set */ - index_set->count = count; + /* record number of entires we added */ + *used_count = count; - return 0; + return UNIFYFS_SUCCESS; } /* read data from specified chunk id, chunk offset, and count into user buffer, @@ -412,6 +483,7 @@ static int unifyfs_logio_chunk_write( /* get chunk meta data */ unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); + /* sanity check on the chunk type */ if (chunk_meta->location != CHUNK_LOCATION_MEMFS && chunk_meta->location != CHUNK_LOCATION_SPILLOVER) { /* unknown chunk type */ @@ -419,12 +491,15 @@ static int unifyfs_logio_chunk_write( return UNIFYFS_ERROR_IO; } - /* determine location of chunk */ + /* copy data into chunk and record its starting offset within the log */ off_t log_offset = 0; if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { - /* just need a memcpy to write data */ + /* store data in shared memory chunk, + * compute address to copy data to */ char* chunk_buf = unifyfs_compute_chunk_buf( meta, chunk_id, chunk_offset); + + /* just need a memcpy to record data */ memcpy(chunk_buf, buf, count); /* record byte offset position within log, which is the number @@ -432,9 +507,12 @@ static int unifyfs_logio_chunk_write( * starting memory location of the shared memory data chunk region */ log_offset = chunk_buf - unifyfs_chunks; } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { - /* spill over to a file, so write to file descriptor */ - //MAP_OR_FAIL(pwrite); + /* spill over to a file, so write to file descriptor, + * compute offset within spill over file */ off_t spill_offset = unifyfs_compute_spill_offset(meta, chunk_id, chunk_offset); + + /* write data into file at appropriate offset */ + //MAP_OR_FAIL(pwrite); ssize_t rc = __real_pwrite(unifyfs_spilloverblock, buf, count, spill_offset); if (rc < 0) { LOGERR("pwrite failed: errno=%d (%s)", errno, strerror(errno)); @@ -464,88 +542,50 @@ static int unifyfs_logio_chunk_write( cur_idx.mem_pos = log_offset; cur_idx.length = count; - /* split the write requests larger than unifyfs_key_slice_range into - * the ones smaller than unifyfs_key_slice_range */ - index_set_t tmp_index_set; - memset(&tmp_index_set, 0, sizeof(tmp_index_set)); - int split_rc = unifyfs_split_index(&cur_idx, unifyfs_key_slice_range, - &tmp_index_set, UNIFYFS_MAX_SPLIT_CNT); - if (split_rc != UNIFYFS_SUCCESS) { - /* in this case, we have copied data to the log, - * but we failed to generate index entries, - * we're returning with an error and leaving the data - * in the log */ - LOGERR("exhausted space when splitting write index"); - return UNIFYFS_ERROR_IO; - } - /* lookup number of existing index entries */ off_t num_entries = *(unifyfs_indices.ptr_num_entries); - /* number of new entries we may add */ - off_t tmp_entries = (off_t) tmp_index_set.count; - - /* check whether there is room to add new entries */ - if (num_entries + tmp_entries < unifyfs_max_index_entries) { - /* get pointer to index array */ - unifyfs_index_t* idxs = unifyfs_indices.index_entry; - - /* coalesce contiguous indices */ - int i = 0; - if (num_entries > 0) { - /* pointer to last element in index array */ - unifyfs_index_t* prev_idx = &idxs[num_entries - 1]; - - /* pointer to first element in temp list */ - unifyfs_index_t* next_idx = &tmp_index_set.idxes[0]; - - /* offset of last byte for last index in list */ - off_t prev_offset = prev_idx->file_pos + prev_idx->length; - - /* check whether last index and temp index refer to - * contiguous bytes in the same file */ - if (prev_idx->fid == next_idx->fid && - prev_offset == next_idx->file_pos) { - /* got contiguous bytes in the same file, - * check if both index values fall in the same slice */ - off_t prev_slice = prev_idx->file_pos / unifyfs_key_slice_range; - off_t next_slice = next_idx->file_pos / unifyfs_key_slice_range; - if (prev_slice == next_slice) { - /* index values also are in same slice, - * so append first index in temp list to - * last index in list */ - prev_idx->length += next_idx->length; - - /* advance to next index in temp list */ - i++; - } - } - } + /* remaining entries we can fit in the shared memory region */ + off_t remaining_entries = unifyfs_max_index_entries - num_entries; + + /* get pointer to index array */ + unifyfs_index_t* idxs = unifyfs_indices.index_entry; - /* pointer to temp index list */ - unifyfs_index_t* newidxs = tmp_index_set.idxes; + /* attempt to coalesce contiguous index entries if we + * have an existing index in the buffer */ + if (num_entries > 0) { + /* get pointer to last element in index array */ + unifyfs_index_t* prev_idx = &idxs[num_entries - 1]; - /* copy remaining items in temp index list to index list */ - while (i < tmp_index_set.count) { - /* copy index fields */ - idxs[num_entries].fid = newidxs[i].fid; - idxs[num_entries].file_pos = newidxs[i].file_pos; - idxs[num_entries].mem_pos = newidxs[i].mem_pos; - idxs[num_entries].length = newidxs[i].length; + /* attempt to coalesce current index with last index, + * updates fields in last index and current index + * accordingly */ + unifyfs_coalesce_index(prev_idx, &cur_idx, + unifyfs_key_slice_range); + } - /* advance to next element in each list */ - num_entries++; - i++; + /* split any remaining write index at boundaries of + * unifyfs_key_slice_range */ + if (cur_idx.length > 0) { + off_t used_entries = 0; + int split_rc = unifyfs_split_index(&cur_idx, unifyfs_key_slice_range, + &idxs[num_entries], remaining_entries, &used_entries); + if (split_rc != UNIFYFS_SUCCESS) { + /* in this case, we have copied data to the log, + * but we failed to generate index entries, + * we're returning with an error and leaving the data + * in the log */ + LOGERR("exhausted space when splitting write index"); + return UNIFYFS_ERROR_IO; } - /* update number of entries in index array */ - (*unifyfs_indices.ptr_num_entries) = num_entries; - } else { - /* TODO: no room to write additional index metadata entries, - * swap out existing metadata buffer to disk*/ - LOGERR("exhausted metadata"); + /* account for entries we just added */ + num_entries += used_entries; } + /* update number of entries in index array */ + (*unifyfs_indices.ptr_num_entries) = num_entries; + /* assume read was successful if we get to here */ return UNIFYFS_SUCCESS; } @@ -604,14 +644,10 @@ int unifyfs_fid_store_fixed_extend(int fid, unifyfs_filemeta_t* meta, /* compute number of additional bytes we need */ off_t additional = length - maxsize; while (additional > 0) { - /* check that we don't overrun max number of chunks for file */ - if (meta->chunks == unifyfs_max_chunks + unifyfs_spillover_max_chunks) { - return UNIFYFS_ERROR_NOSPC; - } - /* allocate a new chunk */ int rc = unifyfs_chunk_alloc(fid, meta, meta->chunks); if (rc != UNIFYFS_SUCCESS) { + /* ran out of space to store data */ LOGERR("failed to allocate chunk"); return UNIFYFS_ERROR_NOSPC; } diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 387113f91..49143875f 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -314,11 +314,6 @@ typedef struct { unifyfs_file_attr_t* meta_entry; } unifyfs_fattr_buf_t; -typedef struct { - unifyfs_index_t idxes[UNIFYFS_MAX_SPLIT_CNT]; - int count; -} index_set_t; - typedef struct { read_req_t read_reqs[UNIFYFS_MAX_READ_CNT]; int count; From ff4bd3beae1ccbbe8558175d15e40a73dcebc625 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 24 Oct 2019 15:06:37 -0700 Subject: [PATCH 014/168] rename mem_pos field to log_pos in index structure --- client/src/unifyfs-fixed.c | 10 +++++----- common/src/unifyfs_meta.h | 8 ++++---- server/src/unifyfs_request_manager.c | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 099dd5401..a6e0d1118 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -328,7 +328,7 @@ static int unifyfs_coalesce_index( /* adjust current index to subtract off those bytes */ next_idx->file_pos += length; - next_idx->mem_pos += length; + next_idx->log_pos += length; next_idx->length -= length; return UNIFYFS_SUCCESS; @@ -393,7 +393,7 @@ static int unifyfs_split_index( */ /* get starting position in log */ - long log_pos = cur_idx->mem_pos; + long log_pos = cur_idx->log_pos; /* copy over all fields in current index */ set[count] = *cur_idx; @@ -432,7 +432,7 @@ static int unifyfs_split_index( set[count].fid = cur_idx->fid; set[count].file_pos = slice_start; set[count].length = length; - set[count].mem_pos = log_pos; + set[count].log_pos = log_pos; /* advance offset into log */ log_pos += length; @@ -459,7 +459,7 @@ static int unifyfs_split_index( set[count].fid = cur_idx->fid; set[count].file_pos = slice_start; set[count].length = length; - set[count].mem_pos = log_pos; + set[count].log_pos = log_pos; count++; } @@ -539,7 +539,7 @@ static int unifyfs_logio_chunk_write( unifyfs_index_t cur_idx; cur_idx.fid = ptr_meta_entry->gfid; cur_idx.file_pos = pos; - cur_idx.mem_pos = log_offset; + cur_idx.log_pos = log_offset; cur_idx.length = count; /* lookup number of existing index entries */ diff --git a/common/src/unifyfs_meta.h b/common/src/unifyfs_meta.h index ea5a159b6..5e3be3391 100644 --- a/common/src/unifyfs_meta.h +++ b/common/src/unifyfs_meta.h @@ -97,10 +97,10 @@ void unifyfs_file_attr_to_stat(unifyfs_file_attr_t* fattr, struct stat* sb) } typedef struct { - off_t file_pos; - off_t mem_pos; - size_t length; - int fid; + off_t file_pos; /* starting logical offset of data in file */ + off_t log_pos; /* starting physical offset of data in log */ + size_t length; /* length of data */ + int fid; /* global file id */ } unifyfs_index_t; /* Header for read request reply in client shared memory region. diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 78e19521d..aa4a52781 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -863,7 +863,7 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) unifyfs_keys[i]->fid = meta_payload[i].fid; unifyfs_keys[i]->offset = meta_payload[i].file_pos; - unifyfs_vals[i]->addr = meta_payload[i].mem_pos; + unifyfs_vals[i]->addr = meta_payload[i].log_pos; unifyfs_vals[i]->len = meta_payload[i].length; unifyfs_vals[i]->delegator_rank = glb_pmi_rank; unifyfs_vals[i]->app_id = app_id; From 509f2d5f925bb8da4a618b17673008f4600c8a5f Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 24 Oct 2019 17:27:07 -0700 Subject: [PATCH 015/168] client: check for short writes and errors in spillover pwrite --- client/src/unifyfs-fixed.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index a6e0d1118..f1425fd93 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -511,11 +511,25 @@ static int unifyfs_logio_chunk_write( * compute offset within spill over file */ off_t spill_offset = unifyfs_compute_spill_offset(meta, chunk_id, chunk_offset); - /* write data into file at appropriate offset */ + /* write data into file at appropriate offset, + * loop to keep trying short writes */ //MAP_OR_FAIL(pwrite); - ssize_t rc = __real_pwrite(unifyfs_spilloverblock, buf, count, spill_offset); - if (rc < 0) { - LOGERR("pwrite failed: errno=%d (%s)", errno, strerror(errno)); + size_t nwritten = 0; + while (nwritten < count) { + /* attempt to write */ + void* bufpos = (void*)((char*)buf + nwritten); + size_t remaining = count - nwritten; + off_t filepos = spill_offset + (off_t)nwritten; + errno = 0; + ssize_t rc = __real_pwrite(unifyfs_spilloverblock, + bufpos, remaining, filepos); + if (rc < 0) { + LOGERR("pwrite failed: errno=%d (%s)", errno, strerror(errno)); + return UNIFYFS_ERROR_IO; + } + + /* wrote without an error, total up bytes written so far */ + nwritten += (size_t)rc; } /* record byte offset position within log, for spill over From 31f5447d1e46f79ac162c717b05c3557ce0def91 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 25 Oct 2019 11:09:38 -0700 Subject: [PATCH 016/168] client: avoid calling split_index if no room to add them --- client/src/unifyfs-fixed.c | 43 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index f1425fd93..b31675cd2 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -365,15 +365,6 @@ static int unifyfs_split_index( /* initialize count of output index entries */ off_t count = 0; - /* no room to write index values */ - if (count >= maxcount) { - /* no room to write more index values, - * and we have at least one more, - * record number we wrote and return with error */ - *used_count = 0; - return UNIFYFS_FAILURE; - } - /* define new index entries in index_set by splitting write index * at slice boundaries */ if (idx_end <= slice_end) { @@ -559,9 +550,6 @@ static int unifyfs_logio_chunk_write( /* lookup number of existing index entries */ off_t num_entries = *(unifyfs_indices.ptr_num_entries); - /* remaining entries we can fit in the shared memory region */ - off_t remaining_entries = unifyfs_max_index_entries - num_entries; - /* get pointer to index array */ unifyfs_index_t* idxs = unifyfs_indices.index_entry; @@ -578,13 +566,29 @@ static int unifyfs_logio_chunk_write( unifyfs_key_slice_range); } - /* split any remaining write index at boundaries of - * unifyfs_key_slice_range */ + /* add new index entries if needed */ if (cur_idx.length > 0) { - off_t used_entries = 0; - int split_rc = unifyfs_split_index(&cur_idx, unifyfs_key_slice_range, - &idxs[num_entries], remaining_entries, &used_entries); - if (split_rc != UNIFYFS_SUCCESS) { + /* remaining entries we can fit in the shared memory region */ + off_t remaining_entries = unifyfs_max_index_entries - num_entries; + if (remaining_entries > 0) { + /* split any remaining write index at boundaries of + * unifyfs_key_slice_range */ + off_t used_entries = 0; + int split_rc = unifyfs_split_index(&cur_idx, + unifyfs_key_slice_range, &idxs[num_entries], + remaining_entries, &used_entries); + if (split_rc != UNIFYFS_SUCCESS) { + /* in this case, we have copied data to the log, + * but we failed to generate index entries, + * we're returning with an error and leaving the data + * in the log */ + LOGERR("exhausted space when splitting write index"); + return UNIFYFS_ERROR_IO; + } + + /* account for entries we just added */ + num_entries += used_entries; + } else { /* in this case, we have copied data to the log, * but we failed to generate index entries, * we're returning with an error and leaving the data @@ -592,9 +596,6 @@ static int unifyfs_logio_chunk_write( LOGERR("exhausted space when splitting write index"); return UNIFYFS_ERROR_IO; } - - /* account for entries we just added */ - num_entries += used_entries; } /* update number of entries in index array */ From 47b176b0eddbfa48254df2274d0494359bff9be1 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 25 Oct 2019 11:28:14 -0700 Subject: [PATCH 017/168] client: fix typo: replace read with write in some comments --- client/src/unifyfs-fixed.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index b31675cd2..e4ecc312c 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -460,7 +460,7 @@ static int unifyfs_split_index( return UNIFYFS_SUCCESS; } -/* read data from specified chunk id, chunk offset, and count into user buffer, +/* write count bytes from user buffer into specified chunk id at chunk offset, * count should fit within chunk starting from specified offset */ static int unifyfs_logio_chunk_write( int fid, /* local file id */ @@ -601,11 +601,11 @@ static int unifyfs_logio_chunk_write( /* update number of entries in index array */ (*unifyfs_indices.ptr_num_entries) = num_entries; - /* assume read was successful if we get to here */ + /* assume write was successful if we get to here */ return UNIFYFS_SUCCESS; } -/* read data from specified chunk id, chunk offset, and count into user buffer, +/* write count bytes from user buffer into specified chunk id at chunk offset, * count should fit within chunk starting from specified offset */ static int unifyfs_chunk_write( unifyfs_filemeta_t* meta, /* pointer to file meta data */ @@ -641,7 +641,7 @@ static int unifyfs_chunk_write( return UNIFYFS_ERROR_IO; } - /* assume read was successful if we get to here */ + /* assume write was successful if we get to here */ return UNIFYFS_SUCCESS; } From 6af9a71389f24b5e7074f1c0614afa9e8720c9ec Mon Sep 17 00:00:00 2001 From: Swen Boehm Date: Fri, 25 Oct 2019 14:54:23 -0400 Subject: [PATCH 018/168] Fix duplicate symbols --- client/src/unifyfs-internal.h | 2 +- common/src/unifyfs_keyval.c | 10 +++++----- common/src/unifyfs_keyval.h | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 49143875f..3fec8b47b 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -399,7 +399,7 @@ extern void* free_chunk_stack; extern void* free_spillchunk_stack; extern char* unifyfs_chunks; extern unifyfs_chunkmeta_t* unifyfs_chunkmetas; -int unifyfs_spilloverblock; +extern int unifyfs_spilloverblock; /* ------------------------------- * Common functions diff --git a/common/src/unifyfs_keyval.c b/common/src/unifyfs_keyval.c index f5147821c..8bdcc02d6 100644 --- a/common/src/unifyfs_keyval.c +++ b/common/src/unifyfs_keyval.c @@ -33,11 +33,11 @@ #include // UnifyFS keys -const char* key_runstate = "unifyfs.runstate"; -const char* key_unifyfsd_socket = "unifyfsd.socket"; -const char* key_unifyfsd_margo_shm = "unifyfsd.margo-shm"; -const char* key_unifyfsd_margo_svr = "unifyfsd.margo-svr"; -const char* key_unifyfsd_pmi_rank = "unifyfsd.pmi-rank"; +const char* const key_runstate = "unifyfs.runstate"; +const char* const key_unifyfsd_socket = "unifyfsd.socket"; +const char* const key_unifyfsd_margo_shm = "unifyfsd.margo-shm"; +const char* const key_unifyfsd_margo_svr = "unifyfsd.margo-svr"; +const char* const key_unifyfsd_pmi_rank = "unifyfsd.pmi-rank"; // key-value store state static int kv_initialized; // = 0 diff --git a/common/src/unifyfs_keyval.h b/common/src/unifyfs_keyval.h index fa5dc3c30..b3ff18883 100644 --- a/common/src/unifyfs_keyval.h +++ b/common/src/unifyfs_keyval.h @@ -22,11 +22,11 @@ extern "C" { #endif // keys we use -const char* key_runstate; // path to runstate file -const char* key_unifyfsd_socket; // server domain socket path -const char* key_unifyfsd_margo_shm; // client-server margo address -const char* key_unifyfsd_margo_svr; // server-server margo address -const char* key_unifyfsd_pmi_rank; // server-server pmi rank +extern const char* const key_runstate; // path to runstate file +extern const char* const key_unifyfsd_socket; // server domain socket path +extern const char* const key_unifyfsd_margo_shm; // client-server margo address +extern const char* const key_unifyfsd_margo_svr; // server-server margo address +extern const char* const key_unifyfsd_pmi_rank; // server-server pmi rank // initialize key-value store int unifyfs_keyval_init(unifyfs_cfg_t* cfg, From 40276cc3a023b8a4307a8bab41d6096b42772542 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 25 Oct 2019 13:42:00 -0700 Subject: [PATCH 019/168] server: clean up fsync rpc handler --- server/src/unifyfs_request_manager.c | 142 ++++++++++++++++----------- 1 file changed, 85 insertions(+), 57 deletions(-) diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index aa4a52781..48f47a476 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -816,121 +816,149 @@ int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl) */ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) { - int ret = 0; + size_t i; - unifyfs_key_t** unifyfs_keys; - unifyfs_val_t** unifyfs_vals; - int* unifyfs_key_lens = NULL; - int* unifyfs_val_lens = NULL; + /* assume we'll succeed */ + int ret = (int)UNIFYFS_SUCCESS; + + /* pointers to memory we'll dynamically allocate for file extents */ + unifyfs_key_t** unifyfs_keys = NULL; + unifyfs_val_t** unifyfs_vals = NULL; + int* unifyfs_key_lens = NULL; + int* unifyfs_val_lens = NULL; - fattr_key_t** fattr_keys = NULL; + /* pointers to memory we'll dynamically allocate for file attributes */ + fattr_key_t** fattr_keys = NULL; unifyfs_file_attr_t** fattr_vals = NULL; - int* fattr_key_lens = NULL; - int* fattr_val_lens = NULL; - size_t i, attr_num_entries, extent_num_entries; + int* fattr_key_lens = NULL; + int* fattr_val_lens = NULL; + + /* get memory page size on this machine */ + int page_sz = getpagesize(); + /* get app config struct for this client */ app_config_t* app_config = (app_config_t*) arraylist_get(app_config_list, app_id); - extent_num_entries = *(size_t*) - (app_config->shm_superblocks[client_side_id] - + app_config->meta_offset); + /* get pointer to superblock for this client and app */ + char* superblk = app_config->shm_superblocks[client_side_id]; - /* - * indices are stored in the superblock shared memory - * created by the client - */ - int page_sz = getpagesize(); - unifyfs_index_t* meta_payload = (unifyfs_index_t*) - (app_config->shm_superblocks[client_side_id] - + app_config->meta_offset + page_sz); - - // allocate storage for values - // TODO: possibly get this from memory pool - unifyfs_keys = alloc_key_array(extent_num_entries); - unifyfs_vals = alloc_value_array(extent_num_entries); + /* get pointer to start of key/value region in superblock */ + char* meta = superblk + app_config->meta_offset; + + /* get pointer to start of file attribute region in superblock */ + char* fmeta = superblk + app_config->fmeta_offset; + + /* get number of file extent index values client has for us, + * stored as a size_t value in meta region of shared memory */ + size_t extent_num_entries = *(size_t*)(meta); + + /* indices are stored in the superblock shared memory + * created by the client, these are stored as index_t + * structs starting one page size offset into meta region */ + char* ptr_extents = meta + page_sz; + unifyfs_index_t* meta_payload = (unifyfs_index_t*)(ptr_extents); + + /* allocate storage for file extent key/values */ + /* TODO: possibly get this from memory pool */ + unifyfs_keys = alloc_key_array(extent_num_entries); + unifyfs_vals = alloc_value_array(extent_num_entries); unifyfs_key_lens = calloc(extent_num_entries, sizeof(int)); unifyfs_val_lens = calloc(extent_num_entries, sizeof(int)); if ((NULL == unifyfs_keys) || (NULL == unifyfs_vals) || (NULL == unifyfs_key_lens) || (NULL == unifyfs_val_lens)) { - return (int)UNIFYFS_ERROR_NOMEM; + LOGERR("failed to allocate memory for file extents"); + ret = (int)UNIFYFS_ERROR_NOMEM; + goto rm_cmd_fsync_exit; } - // file extents + /* create file extent key/values for insertion into MDHIM */ for (i = 0; i < extent_num_entries; i++) { - unifyfs_keys[i]->fid = meta_payload[i].fid; - unifyfs_keys[i]->offset = meta_payload[i].file_pos; - - unifyfs_vals[i]->addr = meta_payload[i].log_pos; - unifyfs_vals[i]->len = meta_payload[i].length; - unifyfs_vals[i]->delegator_rank = glb_pmi_rank; - unifyfs_vals[i]->app_id = app_id; - unifyfs_vals[i]->rank = client_side_id; + /* for a key, we store the global file id and logical file offset */ + unifyfs_key_t* key = unifyfs_keys[i]; + key->fid = meta_payload[i].fid; + key->offset = meta_payload[i].file_pos; + + /* for the value, we store the log position, the length, + * the host server (delegator rank), the mount point id (app id), + * and the client id (rank) */ + unifyfs_val_t* val = unifyfs_vals[i]; + val->addr = meta_payload[i].log_pos; + val->len = meta_payload[i].length; + val->delegator_rank = glb_pmi_rank; + val->app_id = app_id; + val->rank = client_side_id; LOGDBG("extent - fid:%d, offset:%zu, length:%zu, app:%d, clid:%d", - unifyfs_keys[i]->fid, unifyfs_keys[i]->offset, - unifyfs_vals[i]->len, unifyfs_vals[i]->app_id, - unifyfs_vals[i]->rank); + key->fid, key->offset, + val->len, val->app_id, val->rank); + /* MDHIM needs to know the byte size of each key and value */ unifyfs_key_lens[i] = sizeof(unifyfs_key_t); unifyfs_val_lens[i] = sizeof(unifyfs_val_t); } + /* batch insert file extent key/values into MDHIM */ ret = unifyfs_set_file_extents((int)extent_num_entries, unifyfs_keys, unifyfs_key_lens, unifyfs_vals, unifyfs_val_lens); if (ret != UNIFYFS_SUCCESS) { - // TODO: need proper error handling + /* TODO: need proper error handling */ LOGERR("unifyfs_set_file_extents() failed"); goto rm_cmd_fsync_exit; } - // file attributes - attr_num_entries = *(size_t*) - (app_config->shm_superblocks[client_side_id] - + app_config->fmeta_offset); + /* get number of file attribute values client has for us, + * stored as a size_t value in fmeta region of shared memory */ + size_t attr_num_entries = *(size_t*)(fmeta); - /* - * file attributes are stored in the superblock shared memory - * created by the client - */ - unifyfs_file_attr_t* attr_payload = (unifyfs_file_attr_t*) - (app_config->shm_superblocks[client_side_id] - + app_config->fmeta_offset + page_sz); - - // allocate storage for values - // TODO: possibly get this from memory pool - fattr_keys = alloc_attr_key_array(attr_num_entries); - fattr_vals = calloc(attr_num_entries, sizeof(unifyfs_file_attr_t*)); + /* file attributes are stored in the superblock shared memory + * created by the client */ + char* ptr_fattr = fmeta + page_sz; + unifyfs_file_attr_t* attr_payload = (unifyfs_file_attr_t*)(ptr_fattr); + + /* allocate storage for file attribute key/values */ + /* TODO: possibly get this from memory pool */ + fattr_keys = alloc_attr_key_array(attr_num_entries); + fattr_vals = calloc(attr_num_entries, sizeof(unifyfs_file_attr_t*)); fattr_key_lens = calloc(attr_num_entries, sizeof(int)); fattr_val_lens = calloc(attr_num_entries, sizeof(int)); if ((NULL == fattr_keys) || (NULL == fattr_vals) || (NULL == fattr_key_lens) || (NULL == fattr_val_lens)) { + LOGERR("failed to allocate memory for file attributes"); ret = (int)UNIFYFS_ERROR_NOMEM; goto rm_cmd_fsync_exit; } + /* create file attribute key/values for insertion into MDHIM */ for (i = 0; i < attr_num_entries; i++) { + /* for a key, we use the global file id */ *fattr_keys[i] = attr_payload[i].gfid; + + /* for the value, we'll store a file_attr structure */ fattr_vals[i] = &(attr_payload[i]); + + /* MDHIM needs to know the byte size of each key and value */ fattr_key_lens[i] = sizeof(fattr_key_t); fattr_val_lens[i] = sizeof(fattr_val_t); } + /* batch insert file attribute key/values into MDHIM */ ret = unifyfs_set_file_attributes((int)attr_num_entries, fattr_keys, fattr_key_lens, fattr_vals, fattr_val_lens); if (ret != UNIFYFS_SUCCESS) { - // TODO: need proper error handling + /* TODO: need proper error handling */ + LOGERR("unifyfs_set_file_attributes() failed"); goto rm_cmd_fsync_exit; } rm_cmd_fsync_exit: - // clean up memory + /* clean up memory */ if (NULL != unifyfs_keys) { free_key_array(unifyfs_keys); From e3c628ec3afdb16acd6a882239b9b6ad8d148511 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 25 Oct 2019 14:35:26 -0700 Subject: [PATCH 020/168] server: fix: use proper type when inserting file metadata in MDHIM --- server/src/unifyfs_request_manager.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 48f47a476..3fb4fc6e6 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -944,7 +944,7 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) /* MDHIM needs to know the byte size of each key and value */ fattr_key_lens[i] = sizeof(fattr_key_t); - fattr_val_lens[i] = sizeof(fattr_val_t); + fattr_val_lens[i] = sizeof(unifyfs_file_attr_t); } /* batch insert file attribute key/values into MDHIM */ From feaca4e9c086c6b9ffaebc0583344c3b3fe7e89e Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 25 Oct 2019 14:42:51 -0700 Subject: [PATCH 021/168] server: drop fattr_val_t and defunct init/free_indices functions --- server/src/unifyfs_global.h | 5 -- server/src/unifyfs_metadata.c | 102 ---------------------------------- server/src/unifyfs_metadata.h | 2 - 3 files changed, 109 deletions(-) diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index 00b9be855..d0d9b86c2 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -171,11 +171,6 @@ typedef struct { typedef int fattr_key_t; -typedef struct { - char fname[UNIFYFS_MAX_FILENAME]; - struct stat file_attr; -} fattr_val_t; - int invert_sock_ids[MAX_NUM_CLIENTS]; typedef struct { diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index a922903dc..fd3ac103d 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -43,12 +43,6 @@ #include "indexes.h" #include "mdhim.h" -unifyfs_key_t** unifyfs_keys; -unifyfs_val_t** unifyfs_vals; - -fattr_key_t** fattr_keys; -fattr_val_t** fattr_vals; - struct mdhim_t* md; /* we use two MDHIM indexes: @@ -158,70 +152,6 @@ int meta_init_store(unifyfs_cfg_t* cfg) unifyfs_indexes[IDX_FILE_ATTR] = create_global_index(md, ratio, 1, LEVELDB, MDHIM_INT_KEY, "file_attr"); - rc = meta_init_indices(); - if (rc != 0) { - return -1; - } - - return 0; -} - -/* initialize the key and value list used to put/get key-value pairs - * TODO: split once the number of metadata exceeds MAX_META_PER_SEND */ -int meta_init_indices(void) -{ - int i; - - /* init index metadata */ - unifyfs_keys = (unifyfs_key_t**) - calloc(MAX_META_PER_SEND, sizeof(unifyfs_key_t*)); - if (unifyfs_keys == NULL) { - return (int)UNIFYFS_ERROR_NOMEM; - } - - unifyfs_vals = (unifyfs_val_t**) - calloc(MAX_META_PER_SEND, sizeof(unifyfs_val_t*)); - if (unifyfs_vals == NULL) { - return (int)UNIFYFS_ERROR_NOMEM; - } - - for (i = 0; i < MAX_META_PER_SEND; i++) { - unifyfs_keys[i] = (unifyfs_key_t*) calloc(1, sizeof(unifyfs_key_t)); - if (unifyfs_keys[i] == NULL) { - return (int)UNIFYFS_ERROR_NOMEM; - } - - unifyfs_vals[i] = (unifyfs_val_t*) calloc(1, sizeof(unifyfs_val_t)); - if (unifyfs_vals[i] == NULL) { - return (int)UNIFYFS_ERROR_NOMEM; - } - } - - /* init attribute metadata */ - fattr_keys = (fattr_key_t**) - calloc(MAX_FILE_CNT_PER_NODE, sizeof(fattr_key_t*)); - if (fattr_keys == NULL) { - return (int)UNIFYFS_ERROR_NOMEM; - } - - fattr_vals = (fattr_val_t**) - calloc(MAX_FILE_CNT_PER_NODE, sizeof(fattr_val_t*)); - if (fattr_vals == NULL) { - return (int)UNIFYFS_ERROR_NOMEM; - } - - for (i = 0; i < MAX_FILE_CNT_PER_NODE; i++) { - fattr_keys[i] = (fattr_key_t*) calloc(1, sizeof(fattr_key_t)); - if (fattr_keys[i] == NULL) { - return (int)UNIFYFS_ERROR_NOMEM; - } - - fattr_vals[i] = (fattr_val_t*) calloc(1, sizeof(fattr_val_t)); - if (fattr_vals[i] == NULL) { - return (int)UNIFYFS_ERROR_NOMEM; - } - } - return 0; } @@ -238,36 +168,6 @@ void print_fsync_indices(unifyfs_key_t** keys, } } -void meta_free_indices(void) -{ - int i; - if (NULL != unifyfs_keys) { - for (i = 0; i < MAX_META_PER_SEND; i++) { - if (NULL != unifyfs_keys[i]) { - free(unifyfs_keys[i]); - } - if (NULL != unifyfs_vals[i]) { - free(unifyfs_vals[i]); - } - } - free(unifyfs_keys); - free(unifyfs_vals); - } - - if (NULL != fattr_keys) { - for (i = 0; i < MAX_FILE_CNT_PER_NODE; i++) { - if (NULL != fattr_keys[i]) { - free(fattr_keys[i]); - } - if (NULL != fattr_vals[i]) { - free(fattr_vals[i]); - } - } - free(fattr_keys); - free(fattr_vals); - } -} - static int remove_cb(const char* fpath, const struct stat* sb, int typeflag, struct FTW* ftwbuf) { @@ -302,8 +202,6 @@ int meta_sanitize(void) LOGERR("failure during MDHIM file tree removal"); } - meta_free_indices(); - return UNIFYFS_SUCCESS; } diff --git a/server/src/unifyfs_metadata.h b/server/src/unifyfs_metadata.h index 8e76102ca..46fe81ffb 100644 --- a/server/src/unifyfs_metadata.h +++ b/server/src/unifyfs_metadata.h @@ -80,8 +80,6 @@ void debug_log_key_val(const char* ctx, int meta_sanitize(void); int meta_init_store(unifyfs_cfg_t* cfg); -int meta_init_indices(void); -void meta_free_indices(void); void print_fsync_indices(unifyfs_key_t** unifyfs_keys, unifyfs_val_t** unifyfs_vals, size_t num_entries); From 280e25a1ded99583bf92dc1171db678567fe1e33 Mon Sep 17 00:00:00 2001 From: Swen Boehm Date: Fri, 25 Oct 2019 14:33:50 -0400 Subject: [PATCH 022/168] Clean up configure.ac - Add check for -wrap in linker - Add checks for wrapped functions - make gotcha optional - remove leveldb and flatcc from the optional dependencies --- client/src/Makefile.am | 15 +- client/src/gotcha_map_unifyfs_list.h | 15 ++ client/src/unifyfs-sysio.c | 12 ++ configure.ac | 223 +++++++++++++++------------ examples/src/Makefile.am | 64 ++++++-- m4/flatcc.m4 | 14 +- m4/gotcha.m4 | 10 +- m4/leveldb.m4 | 14 +- 8 files changed, 231 insertions(+), 136 deletions(-) diff --git a/client/src/Makefile.am b/client/src/Makefile.am index 399eb80fb..d7679af31 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -1,12 +1,15 @@ -lib_LTLIBRARIES = libunifyfs.la libunifyfs_gotcha.la +lib_LTLIBRARIES = libunifyfs.la +libunifyfsdir = $(includedir) + +if HAVE_GOTCHA +lib_LTLIBRARIES += libunifyfs_gotcha.la +libunifyfs_gotchadir = $(includedir) +endif if HAVE_FORTRAN lib_LTLIBRARIES += libunifyfsf.la endif -libunifyfsdir = $(includedir) -libunifyfs_gotchadir = $(includedir) - AM_CFLAGS = -Wall -Wno-strict-aliasing include_HEADERS = unifyfs.h @@ -69,12 +72,16 @@ libunifyfs_la_CFLAGS = $(CLIENT_COMMON_CFLAGS) libunifyfs_la_LDFLAGS = $(CLIENT_COMMON_LDFLAGS) libunifyfs_la_LIBADD = $(CLIENT_COMMON_LIBADD) +if HAVE_GOTCHA + libunifyfs_gotcha_la_SOURCES = $(CLIENT_COMMON_SOURCES) gotcha_map_unifyfs_list.h libunifyfs_gotcha_la_CPPFLAGS = $(CLIENT_COMMON_CPPFLAGS) -DUNIFYFS_GOTCHA libunifyfs_gotcha_la_CFLAGS = $(CLIENT_COMMON_CFLAGS) $(GOTCHA_CFLAGS) libunifyfs_gotcha_la_LDFLAGS = $(CLIENT_COMMON_LDFLAGS) $(GOTCHA_LDFLAGS) libunifyfs_gotcha_la_LIBADD = $(CLIENT_COMMON_LIBADD) -lgotcha +endif + if HAVE_FORTRAN libunifyfsf_la_SOURCES = unifyfsf.c diff --git a/client/src/gotcha_map_unifyfs_list.h b/client/src/gotcha_map_unifyfs_list.h index 9a9ac3412..97a9c2317 100644 --- a/client/src/gotcha_map_unifyfs_list.h +++ b/client/src/gotcha_map_unifyfs_list.h @@ -1,5 +1,6 @@ //Generated by translate.py +#include "config.h" #include #include @@ -14,18 +15,30 @@ UNIFYFS_DEF(unlink, int, (const char* path)); UNIFYFS_DEF(remove, int, (const char* path)); UNIFYFS_DEF(stat, int, (const char* path, struct stat* buf)); UNIFYFS_DEF(fstat, int, (int fd, struct stat* buf)); +#ifdef HAVE___XSTAT UNIFYFS_DEF(__xstat, int, (int vers, const char* path, struct stat* buf)); +#endif +#ifdef HAVE___LXSTAT UNIFYFS_DEF(__lxstat, int, (int vers, const char* path, struct stat* buf)); +#endif UNIFYFS_DEF(creat, int, (const char* path, mode_t mode)); UNIFYFS_DEF(creat64, int, (const char* path, mode_t mode)); UNIFYFS_DEF(open, int, (const char* path, int flags, ...)); +#ifdef HAVE_OPEN64 UNIFYFS_DEF(open64, int, (const char* path, int flags, ...)); +#endif UNIFYFS_DEF(__open_2, int, (const char* path, int flags, ...)); + +#ifdef HAVE_LIO_LISTIO UNIFYFS_DEF(lio_listio, int, (int mode, struct aiocb* const aiocb_list[], int nitems, struct sigevent* sevp)); +#endif UNIFYFS_DEF(lseek, off_t, (int fd, off_t offset, int whence)); UNIFYFS_DEF(lseek64, off64_t, (int fd, off64_t offset, int whence)); + +#ifdef HAVE_POSIX_FADVISE UNIFYFS_DEF(posix_fadvise, int, (int fd, off_t offset, off_t len, int advice)); +#endif UNIFYFS_DEF(read, ssize_t, (int fd, void* buf, size_t count)); UNIFYFS_DEF(write, ssize_t, (int fd, const void* buf, size_t count)); UNIFYFS_DEF(readv, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); @@ -47,7 +60,9 @@ UNIFYFS_DEF(msync, int, (void* addr, size_t length, int flags)); UNIFYFS_DEF(munmap, int, (void* addr, size_t length)); UNIFYFS_DEF(mmap64, void*, (void* addr, size_t length, int prot, int flags, int fd, off_t offset)); +#ifdef HAVE___FXSTAT UNIFYFS_DEF(__fxstat, int, (int vers, int fd, struct stat* buf)); +#endif UNIFYFS_DEF(close, int, (int fd)); UNIFYFS_DEF(opendir, DIR*, (const char* name)); UNIFYFS_DEF(fdopendir, DIR*, (int fd)); diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 7c66d0f3b..0084e5bc4 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -425,6 +425,7 @@ int UNIFYFS_WRAP(fstat)(int fd, struct stat* buf) * instead of using the absolute value 3. */ +#ifdef HAVE___XSTAT int UNIFYFS_WRAP(__xstat)(int vers, const char* path, struct stat* buf) { LOGDBG("xstat was called for %s", path); @@ -441,7 +442,9 @@ int UNIFYFS_WRAP(__xstat)(int vers, const char* path, struct stat* buf) return ret; } } +#endif +#ifdef HAVE___LXSTAT int UNIFYFS_WRAP(__lxstat)(int vers, const char* path, struct stat* buf) { LOGDBG("lxstat was called for %s", path); @@ -457,7 +460,9 @@ int UNIFYFS_WRAP(__lxstat)(int vers, const char* path, struct stat* buf) return UNIFYFS_REAL(__lxstat)(vers, path, buf); } } +#endif +#ifdef HAVE___FXSTAT int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) { LOGDBG("fxstat was called for fd %d", fd); @@ -479,6 +484,7 @@ int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) return UNIFYFS_REAL(__fxstat)(vers, fd, buf); } } +#endif /* --------------------------------------- * POSIX wrappers: file descriptors @@ -741,6 +747,7 @@ int UNIFYFS_WRAP(open)(const char* path, int flags, ...) } } +#ifdef HAVE_OPEN64 int UNIFYFS_WRAP(open64)(const char* path, int flags, ...) { /* if O_CREAT is set, we should also have some mode flags */ @@ -772,6 +779,7 @@ int UNIFYFS_WRAP(open64)(const char* path, int flags, ...) return ret; } +#endif int UNIFYFS_WRAP(__open_2)(const char* path, int flags, ...) { @@ -894,6 +902,7 @@ off64_t UNIFYFS_WRAP(lseek64)(int fd, off64_t offset, int whence) } } +#ifdef HAVE_POSIX_FADVISE int UNIFYFS_WRAP(posix_fadvise)(int fd, off_t offset, off_t len, int advice) { /* check whether we should intercept this file descriptor */ @@ -942,6 +951,7 @@ int UNIFYFS_WRAP(posix_fadvise)(int fd, off_t offset, off_t len, int advice) return ret; } } +#endif ssize_t UNIFYFS_WRAP(read)(int fd, void* buf, size_t count) { @@ -1088,6 +1098,7 @@ ssize_t UNIFYFS_WRAP(writev)(int fd, const struct iovec* iov, int iovcnt) } } +#ifdef HAVE_LIO_LISTIO int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], int nitems, struct sigevent* sevp) { @@ -1182,6 +1193,7 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], } return ret; } +#endif /* order by file id then by file position */ static int compare_index_entry(const void* a, const void* b) diff --git a/configure.ac b/configure.ac index c1a14ed8e..492932075 100755 --- a/configure.ac +++ b/configure.ac @@ -1,12 +1,15 @@ -dnl +dnl dnl This file is a part of UnifyFS. Please see LICENSE for the license dnl information. dnl Process this file with autoconf to produce a configure script. +AC_LANG([C]) + AC_INIT([unifyfs], m4_esyscmd([git describe --always | awk '/.*/{sub(/^v/,""); printf "%s",$1; exit}']), [unifycr@llnl.gov]) +AC_PREREQ(2.60) AC_CONFIG_SRCDIR([configure.ac]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) @@ -16,32 +19,27 @@ AM_SILENT_RULES([yes]) AM_MAINTAINER_MODE([disable]) -LT_INIT - AC_PROG_CC_STDC AC_PROG_AWK AC_PROG_CPP AC_PROG_INSTALL AC_PROG_LN_S AC_PROG_MAKE_SET -AC_PROG_RANLIB - # fortran support -AC_ARG_ENABLE([fortran], - AC_HELP_STRING([--enable-fortran], - [Enable fortran compatibility and features])) +AC_ARG_ENABLE([fortran],[AS_HELP_STRING([--enable-fortran],[Enable fortran compatibility and features])]) AC_MSG_CHECKING(if fortran is wanted ) -if test "x$enable_fortran" = "xyes"; then -AC_MSG_RESULT(yes) -AC_PROG_FC -AC_FC_LIBRARY_LDFLAGS -AC_FC_DUMMY_MAIN -AM_CONDITIONAL([HAVE_FORTRAN], [true]) -else -AC_MSG_RESULT(no) -AM_CONDITIONAL([HAVE_FORTRAN], [false]) -fi +AS_IF([test "x$enable_fortran" = "xyes"],[ + AC_MSG_RESULT(yes) + AC_PROG_FC + AM_CONDITIONAL([HAVE_FORTRAN], [true]) +],[ + AC_MSG_RESULT(no) + AM_CONDITIONAL([HAVE_FORTRAN], [false]) +]) + +dnl Need to do Fortran checks before initializing LIBTOOL +LT_INIT # Checks for typedefs, structures, and compiler characteristics. AC_TYPE_MODE_T @@ -66,8 +64,8 @@ AC_CHECK_HEADERS([fcntl.h limits.h stdlib.h string.h sys/socket.h sys/time.h]) AC_CHECK_HEADERS([unistd.h arpa/inet.h inttypes.h netdb.h netinet/in.h]) AC_CHECK_HEADERS([stddef.h stdint.h libgen.h strings.h syslog.h]) AC_CHECK_HEADERS([inttypes.h wchar.h wctype.h]) -AC_CHECK_HEADER([openssl/md5.h], [], [AC_MSG_FAILURE([ - *** openssl/md5.h missing, openssl-devel package required])]) +AC_CHECK_HEADER([openssl/md5.h], [], + [AC_MSG_FAILURE([*** openssl/md5.h missing, openssl-devel package required])]) # Checks for library functions. AC_FUNC_MALLOC @@ -79,69 +77,83 @@ AC_CHECK_FUNCS([gethostbyname strcasecmp strdup strerror strncasecmp strrchr]) AC_CHECK_FUNCS([gethostname strstr strtoumax strtol uname posix_fallocate]) # PMPI Init/Fini mount/unmount option -AC_ARG_ENABLE([mpi-mount], - AC_HELP_STRING([--enable-mpi-mount], - [Enable transparent mount/unmount at MPI_Init/Finalize.])) -if test "x$enable_mpi_mount" = "xyes"; then -AM_CONDITIONAL([USE_PMPI_WRAPPERS], [true]) -else -AM_CONDITIONAL([USE_PMPI_WRAPPERS], [false]) -fi +AC_ARG_ENABLE([mpi-mount],[AS_HELP_STRING([--enable-mpi-mount],[Enable transparent mount/unmount at MPI_Init/Finalize.])]) +AS_IF([test "x$enable_mpi_mount" = "xyes"],[ + AM_CONDITIONAL([USE_PMPI_WRAPPERS], [true]) +],[ + AM_CONDITIONAL([USE_PMPI_WRAPPERS], [false]) +]) # PMIx support build option -AC_ARG_ENABLE([pmix], - AC_HELP_STRING([--enable-pmix], - [Enable PMIx build options.])) -if test "x$enable_pmix" = "xyes"; then -AC_CHECK_HEADERS([pmix.h], - [AM_CONDITIONAL([USE_PMIX], [true])], - [AM_CONDITIONAL([USE_PMIX], [false])]) -else -AM_CONDITIONAL([USE_PMIX], [false]) -fi +AC_ARG_ENABLE([pmix],[AS_HELP_STRING([--enable-pmix],[Enable PMIx build options.])]) +AS_IF([test "x$enable_pmix" = "xyes"],[ + AC_CHECK_HEADERS([pmix.h], + [AM_CONDITIONAL([USE_PMIX], [true])], + [AM_CONDITIONAL([USE_PMIX], [false])]) +],[ + AM_CONDITIONAL([USE_PMIX], [false]) +]) # PMI2 support build option -AC_ARG_ENABLE([pmi], - AC_HELP_STRING([--enable-pmi], - [Enable PMI2 build options.])) -if test "x$enable_pmi" = "xyes"; then -AC_CHECK_HEADERS([pmi2.h], - [AM_CONDITIONAL([USE_PMI2], [true])], - [AM_CONDITIONAL([USE_PMI2], [false])]) -else -AM_CONDITIONAL([USE_PMI2], [false]) -fi +AC_ARG_ENABLE([pmi],[AS_HELP_STRING([--enable-pmi],[Enable PMI2 build options.])]) +AS_IF([test "x$enable_pmi" = "xyes"],[ + AC_CHECK_HEADERS([pmi2.h], + [AM_CONDITIONAL([USE_PMI2], [true])], + [AM_CONDITIONAL([USE_PMI2], [false])]) +],[ + AM_CONDITIONAL([USE_PMI2], [false]) +]) CHECK_NUMA AC_ARG_WITH(pkgconfigdir, - [ --with-pkgconfigdir=DIR pkgconfig file in DIR @<:@LIBDIR/pkgconfig@:>@], + [AS_HELP_STRING([--with-pkgconfigdir=DIR],[pkgconfig file in DIR @<:@LIBDIR/pkgconfig@:>@])], [pkgconfigdir=$withval], [pkgconfigdir='${libdir}/pkgconfig']) AC_SUBST(pkgconfigdir) ## unifyfs options -AC_ARG_ENABLE(cuserid, -[ --disable-cuserid Disables attempted use of cuserid() at run time], -[if test "x$enableval" = "xno" ; then - AC_DEFINE(CRUISE_DISABLE_CUSERID, 1, Define if cuserid() should be disabled), -fi] +AC_ARG_ENABLE([cuserid],[AS_HELP_STRING([--disable-cuserid],[Disables attempted use of cuserid() at run time])],[ + AS_IF([test "x$enableval" = "xno"],[ + AC_DEFINE(CRUISE_DISABLE_CUSERID, 1, Define if cuserid() should be disabled),],[])] ,) -AC_ARG_ENABLE(ld-preload, -[ --disable-ld-preload Disables support for LD_PRELOAD library], -[if test "x$enableval" = "xno" ; then - DISABLE_LDPRELOAD="1" -fi] -,) +AC_ARG_ENABLE(ld-preload,[AS_HELP_STRING([--disable-ld-preload],[Disables support for LD_PRELOAD library])],[ + AS_IF([test "x$enableval" = "xno"],[ + DISABLE_LDPRELOAD="1" + ],[]) + ] +,[]) + +AC_ARG_ENABLE(st-dev-workaround, + [AS_HELP_STRING([--enable-st-dev-workaround],[Gather device id from parent directory instead of file])],[ + AS_IF([test "x$enableval" = "xyes"],[ + AC_DEFINE(__CP_ST_DEV_WORKAROUND, 1, Define if device id should be taken from parent directory rather than file) + DISABLE_LDPRELOAD="1" + ],[]) + ] +,[]) -AC_ARG_ENABLE(st-dev-workaround, -[ --enable-st-dev-workaround Gather device id from parent directory instead of file], -[if test "x$enableval" = "xyes" ; then - AC_DEFINE(__CP_ST_DEV_WORKAROUND, 1, Define if device id should be taken from parent directory rather than file) - DISABLE_LDPRELOAD="1" -fi] -,) +# look for MPI and set flags +LX_FIND_MPI +AS_IF([test "x$enable_fortran" = "xyes"],[ + AC_LANG_PUSH([Fortran]) + LX_FIND_MPI + AC_LANG_POP +],[]) + +AS_IF([test "$have_C_mpi" != "yes"],[ + AC_MSG_ERROR(["Couldn't find MPI"]) +],[]) + +# look for leveldb library, sets LEVELDB_CFLAGS/LDFLAGS/LIBS +UNIFYFS_AC_LEVELDB + +# look for gotcha library, sets GOTCHA_INCLUDE, GOTCHA_LIB +UNIFYFS_AC_GOTCHA + +UNIFYFS_AC_MARGO +UNIFYFS_AC_FLATCC # checks to see how we can print 64 bit values on this architecture gt_INTTYPES_PRI @@ -164,28 +176,21 @@ AC_TRY_COMPILE( AC_MSG_RESULT(no) ) -AC_CHECK_HEADERS(mntent.h sys/mount.h) - -# look for MPI and set flags -LX_FIND_MPI -if test "x$enable_fortran" = "xyes"; then -AC_LANG_PUSH([Fortran]) -LX_FIND_MPI -AC_LANG_POP -fi - -if test "$have_C_mpi" != "yes" ; then - AC_MSG_ERROR(["Couldn't find MPI"]) -fi - -# look for leveldb library, sets LEVELDB_CFLAGS/LDFLAGS/LIBS -UNIFYFS_AC_LEVELDB - -# look for gotcha library, sets GOTCHA_INCLUDE, GOTCHA_LIB -UNIFYFS_AC_GOTCHA +AC_MSG_CHECKING(if linker suppots -wrap) +OLD_LDFLAGS=$LDFLAGS +LDFLAGS=$LDFLAGS +LDFLAGS+="-Wl,-wrap,malloc" +AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]],[[int *test = malloc(sizeof(int));]])], +[ + AC_MSG_RESULT([yes]) + AM_CONDITIONAL([HAVE_LD_WRAP],[true]) +],[ + AC_MSG_RESULT([no]) + AM_CONDITIONAL([HAVE_LD_WRAP],[false]) +]) +LDFLAGS=$OLD_LDFLAGS -UNIFYFS_AC_MARGO -UNIFYFS_AC_FLATCC +AC_CHECK_HEADERS(mntent.h sys/mount.h) # HDF found? AX_LIB_HDF5 @@ -196,7 +201,13 @@ AM_CONDITIONAL([HAVE_HDF5], [test x$with_hdf5 = xyes]) CP_WRAPPERS+="-Wl,-wrap,access" CP_WRAPPERS+=",-wrap,chmod" CP_WRAPPERS+=",-wrap,fchmod" -CP_WRAPPERS+=",-wrap,lio_listio" + +OLD_LIBS=$LIBS +LIBS+="-lrt" +AC_CHECK_FUNCS(lio_listio,[ + CP_WRAPPERS+=",-wrap,lio_listio" +], []) +LIBS=$OLD_LIBS CP_WRAPPERS+=",-wrap,mkdir" CP_WRAPPERS+=",-wrap,rmdir" CP_WRAPPERS+=",-wrap,unlink" @@ -205,13 +216,24 @@ CP_WRAPPERS+=",-wrap,rename" CP_WRAPPERS+=",-wrap,truncate" CP_WRAPPERS+=",-wrap,stat" CP_WRAPPERS+=",-wrap,fstat" -CP_WRAPPERS+=",-wrap,__lxstat" -CP_WRAPPERS+=",-wrap,__xstat" + +AC_CHECK_FUNCS(__lxstat,[ + CP_WRAPPERS+=",-wrap,__lxstat" +],[]) +AC_CHECK_FUNCS(__xstat,[ + CP_WRAPPERS+=",-wrap,__xstat" +],[]) + +AC_CHECK_FUNCS(__fxstat,[ + CP_WRAPPERS+=",-wrap,__fxstat" +],[]) CP_WRAPPERS+=",-wrap,creat" CP_WRAPPERS+=",-wrap,creat64" CP_WRAPPERS+=",-wrap,open" -CP_WRAPPERS+=",-wrap,open64" +AC_CHECK_FUNCS(open64, [ + CP_WRAPPERS+=",-wrap,open64" +],[]) CP_WRAPPERS+=",-wrap,__open_2" CP_WRAPPERS+=",-wrap,read" CP_WRAPPERS+=",-wrap,write" @@ -221,7 +243,9 @@ CP_WRAPPERS+=",-wrap,pread" CP_WRAPPERS+=",-wrap,pread64" CP_WRAPPERS+=",-wrap,pwrite" CP_WRAPPERS+=",-wrap,pwrite64" -CP_WRAPPERS+=",-wrap,posix_fadvise" +AC_CHECK_FUNCS(posix_fadvise, [ + CP_WRAPPERS+=",-wrap,posix_fadvise" +],[]) CP_WRAPPERS+=",-wrap,lseek" CP_WRAPPERS+=",-wrap,lseek64" CP_WRAPPERS+=",-wrap,ftruncate" @@ -232,7 +256,6 @@ CP_WRAPPERS+=",-wrap,mmap" CP_WRAPPERS+=",-wrap,mmap64" CP_WRAPPERS+=",-wrap,munmap" CP_WRAPPERS+=",-wrap,msync" -CP_WRAPPERS+=",-wrap,__fxstat" CP_WRAPPERS+=",-wrap,close" # FILE* functions @@ -289,11 +312,11 @@ CP_WRAPPERS+=",-wrap,ungetwc" # ,-u,__wrap___fxstat64,-u,pthread_mutex_lock,-u,pthread_mutex_unlock -# We need to know the value of the $libdir and $bindir variables so that +# We need to know the value of the $libdir and $bindir variables so that # we can reference the correct path in the unifyfs compiler wrappers. -# Unfortunately, those two variables are not normally evaluated by autoconf. -# They are evaluated at build time using Makefile variable substitutions. -# +# Unfortunately, those two variables are not normally evaluated by autoconf. +# They are evaluated at build time using Makefile variable substitutions. +# # The following logic was copied from mpich2 1.3.1 to resolve the $libdir # variable at configure time. # @@ -365,9 +388,9 @@ AC_MSG_RESULT([ compiler ${CC} CFLAGS ${CFLAGS} ========================== - + Supported POSIX wrappers: - + ${CP_WRAPPERS} ]) diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index 6835decac..eec7ff892 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -1,21 +1,51 @@ libexec_PROGRAMS = \ - cr-posix cr-gotcha cr-static \ - read-posix read-gotcha read-static \ - write-posix write-gotcha write-static \ - writeread-posix writeread-gotcha writeread-static \ - sysio-write-gotcha sysio-write-static \ - sysio-read-gotcha sysio-read-static \ - sysio-writeread-gotcha sysio-writeread-static sysio-writeread-posix \ - sysio-writeread2-gotcha sysio-writeread2-static \ - sysio-dir-gotcha sysio-dir-static \ - sysio-stat-gotcha sysio-stat-static \ - sysio-cp-gotcha sysio-cp-static \ - app-mpiio-gotcha app-mpiio-static \ - app-btio-gotcha app-btio-static \ - app-tileio-gotcha app-tileio-static \ - transfer-gotcha transfer-static \ - size-gotcha size-static \ - chmod-gotcha chmod-static + cr-posix \ + read-posix \ + write-posix \ + writeread-posix \ + sysio-writeread-posix + +if HAVE_LD_WRAP + libexec_PROGRAMS += \ + cr-static \ + read-static \ + write-static \ + writeread-static \ + sysio-write-static \ + sysio-read-static \ + sysio-writeread-static \ + sysio-writeread2-static \ + sysio-dir-static \ + sysio-stat-static \ + sysio-cp-static \ + app-mpiio-static \ + app-btio-static \ + app-tileio-static \ + transfer-static \ + size-static \ + chmod-static +endif + +if HAVE_GOTCHA + libexec_PROGRAMS += \ + cr-gotcha \ + read-gotcha \ + write-gotcha \ + writeread-gotcha \ + sysio-write-gotcha \ + sysio-read-gotcha \ + sysio-writeread-gotcha \ + sysio-writeread2-gotcha \ + sysio-dir-gotcha \ + sysio-stat-gotcha \ + sysio-cp-gotcha \ + app-mpiio-gotcha \ + app-btio-gotcha \ + app-tileio-gotcha \ + transfer-gotcha \ + size-gotcha \ + chmod-gotcha +endif if HAVE_FORTRAN libexec_PROGRAMS += \ diff --git a/m4/flatcc.m4 b/m4/flatcc.m4 index f4d7f6658..65596a194 100644 --- a/m4/flatcc.m4 +++ b/m4/flatcc.m4 @@ -3,13 +3,15 @@ AC_DEFUN([UNIFYFS_AC_FLATCC], [ FLATCC_OLD_CFLAGS=$CFLAGS FLATCC_OLD_LDFLAGS=$LDFLAGS - AC_ARG_WITH([flatcc], [AC_HELP_STRING([--with-flatcc=PATH], - [path to installed libflatcc [default=/usr/local]])], [ - FLATCC_CFLAGS="-I${withval}/include" - FLATCC_LDFLAGS="-L${withval}/lib" + AC_ARG_VAR([FLATCC_ROOT],[Set the path to the LevelDB installation directory]) + + AS_IF([test -n "$FLATCC_ROOT"],[ + AC_MSG_NOTICE([FLATCC_ROOT is set, checking for flatcc in $FLATCC_ROOT]) + FLATCC_CFLAGS="-I${FLATCC_ROOT}/include" + FLATCC_LDFLAGS="-L${FLATCC_ROOT}/lib" CFLAGS="$CFLAGS ${FLATCC_CFLAGS}" LDFLAGS="$LDFLAGS ${FLATCC_LDFLAGS}" - ], []) + ],[]) AC_CHECK_LIB([flatcc], [flatcc_create_context], [FLATCC_LIBS="-lflatcc" @@ -17,7 +19,7 @@ AC_DEFUN([UNIFYFS_AC_FLATCC], [ AC_SUBST(FLATCC_LDFLAGS) AC_SUBST(FLATCC_LIBS) ], - [AC_MSG_ERROR([couldn't find a suitable libflatcc, use --with-flatcc=PATH])], + [AC_MSG_ERROR([couldn't find a suitable libflatcc, use FLATCC_ROOT to set the path to the flatcc installation directory.])], [] ) diff --git a/m4/gotcha.m4 b/m4/gotcha.m4 index 6d32ac5f3..5138a7247 100644 --- a/m4/gotcha.m4 +++ b/m4/gotcha.m4 @@ -14,10 +14,14 @@ AC_DEFUN([UNIFYFS_AC_GOTCHA], [ ], []) AC_CHECK_LIB([gotcha], [gotcha_wrap], - [AC_SUBST(GOTCHA_CFLAGS) - AC_SUBST(GOTCHA_LDFLAGS) + [ + AC_SUBST(GOTCHA_CFLAGS) + AC_SUBST(GOTCHA_LDFLAGS) + AM_CONDITIONAL([HAVE_GOTCHA], [true]) + ],[ + AC_MSG_WARN([couldn't find a suitable libgotcha, use --with-gotcha=PATH]) + AM_CONDITIONAL([HAVE_GOTCHA], [false]) ], - [AC_MSG_ERROR([couldn't find a suitable libgotcha, use --with-gotcha=PATH])], [] ) diff --git a/m4/leveldb.m4 b/m4/leveldb.m4 index 2eed55189..1b08ce5dd 100644 --- a/m4/leveldb.m4 +++ b/m4/leveldb.m4 @@ -3,13 +3,15 @@ AC_DEFUN([UNIFYFS_AC_LEVELDB], [ LEVELDB_OLD_CFLAGS=$CFLAGS LEVELDB_OLD_LDFLAGS=$LDFLAGS - AC_ARG_WITH([leveldb], [AC_HELP_STRING([--with-leveldb=PATH], - [path to installed libleveldb [default=/usr/local]])], [ - LEVELDB_CFLAGS="-I${withval}/include" - LEVELDB_LDFLAGS="-L${withval}/lib -L${withval}/lib64" + AC_ARG_VAR([LEVELDB_ROOT], [Set the path to the LevelDB installation Directory]) + + AS_IF([test -n "$LEVELDB_ROOT"],[ + AC_MSG_NOTICE([LEVELDB_ROOT is set, checking for LevelDB in $LEVELDB_ROOT]) + LEVELDB_CFLAGS="-I${LEVELDB_ROOT}/include" + LEVELDB_LDFLAGS="-L${LEVELDB_ROOT}/lib -L${LEVELDB_ROOT}/lib64" CFLAGS="$CFLAGS ${LEVELDB_CFLAGS}" LDFLAGS="$LDFLAGS ${LEVELDB_LDFLAGS}" - ], []) + ],[]) AC_CHECK_LIB([leveldb], [leveldb_open], [LEVELDB_LIBS="-lleveldb" @@ -17,7 +19,7 @@ AC_DEFUN([UNIFYFS_AC_LEVELDB], [ AC_SUBST(LEVELDB_LDFLAGS) AC_SUBST(LEVELDB_LIBS) ], - [AC_MSG_ERROR([couldn't find a suitable libleveldb, use --with-leveldb=PATH])], + [AC_MSG_ERROR([couldn't find a suitable libleveldb, use LEVELDB_ROOT to set the path to the flatcc installation directory.])], [] ) From f2f67f3da7b85f9033799c1ddddac568482ebf06 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 25 Oct 2019 15:41:17 -0700 Subject: [PATCH 023/168] client: flush extent buffer to server when full --- client/src/unifyfs-fixed.c | 119 +++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 51 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index e4ecc312c..a12fc9010 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -42,6 +42,7 @@ #include "unifyfs-fixed.h" #include "unifyfs_log.h" +#include "margo_client.h" static inline unifyfs_chunkmeta_t* filemeta_get_chunkmeta(const unifyfs_filemeta_t* meta, @@ -375,6 +376,12 @@ static int unifyfs_split_index( */ set[count] = *cur_idx; count++; + + /* update write index to account for index we just added */ + long length = cur_idx->length; + cur_idx->file_pos += length; + cur_idx->log_pos += length; + cur_idx->length -= length; } else { /* ending offset of index is beyond last offset in first slice, * so this index spans across multiple slices @@ -383,34 +390,33 @@ static int unifyfs_split_index( * idx_start idx_end */ - /* get starting position in log */ - long log_pos = cur_idx->log_pos; - - /* copy over all fields in current index */ - set[count] = *cur_idx; - - /* update length field to adjust for boundary of first slice */ + /* compute number of bytes until end of first slice */ long length = slice_end - idx_start + 1; - set[count].length = length; - - /* advance offset into log */ - log_pos += length; - /* increment our output index count */ + /* copy over all fields in current index, + * update length field to adjust for boundary of first slice */ + set[count].fid = cur_idx->fid; + set[count].file_pos = cur_idx->file_pos; + set[count].length = length; + set[count].log_pos = cur_idx->log_pos; count++; + /* update write index to account for index we just added */ + cur_idx->file_pos += length; + cur_idx->log_pos += length; + cur_idx->length -= length; + /* check that we have room to write more index values */ if (count >= maxcount) { /* no room to write more index values, * and we have at least one more, - * record number we wrote and return with error */ + * record number we wrote and return with success */ *used_count = count; - return UNIFYFS_FAILURE; + return UNIFYFS_SUCCESS; } /* advance slice boundary offsets to next slice */ - slice_start = slice_end + 1; - slice_end = slice_start + slice_range - 1; + slice_end += slice_range; /* loop until we find the slice that contains * ending offset of write */ @@ -421,37 +427,41 @@ static int unifyfs_split_index( /* define index for this slice */ set[count].fid = cur_idx->fid; - set[count].file_pos = slice_start; + set[count].file_pos = cur_idx->file_pos; set[count].length = length; - set[count].log_pos = log_pos; - - /* advance offset into log */ - log_pos += length; - - /* increment our output index count */ + set[count].log_pos = cur_idx->log_pos; count++; + /* update write index to account for index we just added */ + cur_idx->file_pos += length; + cur_idx->log_pos += length; + cur_idx->length -= length; + /* check that we have room to write more index values */ if (count >= maxcount) { /* no room to write more index values, * and we have at least one more, - * record number we wrote and return with error */ + * record number we wrote and return with success */ *used_count = count; - return UNIFYFS_FAILURE; + return UNIFYFS_SUCCESS; } /* advance slice boundary offsets to next slice */ - slice_start = slice_end + 1; - slice_end = slice_start + slice_range - 1; + slice_end += slice_range; } /* this slice contains the remainder of write */ - length = idx_end - slice_start + 1; + length = cur_idx->length; set[count].fid = cur_idx->fid; - set[count].file_pos = slice_start; + set[count].file_pos = cur_idx->file_pos; set[count].length = length; - set[count].log_pos = log_pos; + set[count].log_pos = cur_idx->log_pos; count++; + + /* update write index to account for index we just added */ + cur_idx->file_pos += length; + cur_idx->log_pos += length; + cur_idx->length -= length; } /* record number of entires we added */ @@ -567,39 +577,46 @@ static int unifyfs_logio_chunk_write( } /* add new index entries if needed */ - if (cur_idx.length > 0) { + while (cur_idx.length > 0) { /* remaining entries we can fit in the shared memory region */ off_t remaining_entries = unifyfs_max_index_entries - num_entries; - if (remaining_entries > 0) { - /* split any remaining write index at boundaries of - * unifyfs_key_slice_range */ - off_t used_entries = 0; - int split_rc = unifyfs_split_index(&cur_idx, - unifyfs_key_slice_range, &idxs[num_entries], - remaining_entries, &used_entries); - if (split_rc != UNIFYFS_SUCCESS) { - /* in this case, we have copied data to the log, - * but we failed to generate index entries, - * we're returning with an error and leaving the data - * in the log */ - LOGERR("exhausted space when splitting write index"); + + /* if we have filled the key/value buffer, flush it to server */ + if (0 == remaining_entries) { + /* index buffer is full, flush it */ + int ret = invoke_client_fsync_rpc(cur_idx.fid); + if (ret != UNIFYFS_SUCCESS) { + /* something went wrong when trying to flush key/values */ + LOGERR("failed to flush key/value index to server"); return UNIFYFS_ERROR_IO; } - /* account for entries we just added */ - num_entries += used_entries; - } else { + /* flushed, refresh number of entries and number remaining */ + num_entries = *(unifyfs_indices.ptr_num_entries); + remaining_entries = unifyfs_max_index_entries - num_entries; + } + + /* split any remaining write index at boundaries of + * unifyfs_key_slice_range */ + off_t used_entries = 0; + int split_rc = unifyfs_split_index(&cur_idx, + unifyfs_key_slice_range, &idxs[num_entries], + remaining_entries, &used_entries); + if (split_rc != UNIFYFS_SUCCESS) { /* in this case, we have copied data to the log, * but we failed to generate index entries, * we're returning with an error and leaving the data * in the log */ - LOGERR("exhausted space when splitting write index"); + LOGERR("failed to split write index"); return UNIFYFS_ERROR_IO; } - } - /* update number of entries in index array */ - (*unifyfs_indices.ptr_num_entries) = num_entries; + /* account for entries we just added */ + num_entries += used_entries; + + /* update number of entries in index array */ + (*unifyfs_indices.ptr_num_entries) = num_entries; + } /* assume write was successful if we get to here */ return UNIFYFS_SUCCESS; From 1a0f89e5cd5b4d4eb824cfcbe4db53901fd17580 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 25 Oct 2019 17:06:26 -0700 Subject: [PATCH 024/168] client: clear index buffer after fsync --- client/src/unifyfs-fixed.c | 6 ++++-- client/src/unifyfs-sysio.c | 10 +++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index a12fc9010..6f69cd7ae 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -591,8 +591,10 @@ static int unifyfs_logio_chunk_write( return UNIFYFS_ERROR_IO; } - /* flushed, refresh number of entries and number remaining */ - num_entries = *(unifyfs_indices.ptr_num_entries); + /* flushed, clear buffer and refresh number of entries + * and number remaining */ + num_entries = 0; + *(unifyfs_indices.ptr_num_entries) = num_entries; remaining_entries = unifyfs_max_index_entries - num_entries; } diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 0084e5bc4..212277c6b 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -2103,7 +2103,15 @@ int UNIFYFS_WRAP(fsync)(int fd) /* invoke fsync rpc to register index metadata with server */ int gfid = get_gfid(fid); - invoke_client_fsync_rpc(gfid); + int ret = invoke_client_fsync_rpc(gfid); + if (ret != UNIFYFS_SUCCESS) { + /* sync failed for some reason, set errno and return error */ + errno = unifyfs_err_map_to_errno(ret); + return -1; + } + + /* server has processed entries in index buffer, reset it */ + *(unifyfs_indices.ptr_num_entries) = 0; meta->needs_sync = 0; return 0; From caa11487f28712e6ace2534cff922b73cac3b6c9 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sat, 26 Oct 2019 15:45:01 -0700 Subject: [PATCH 025/168] server: avoid setting file attributes on fsync --- server/src/unifyfs_request_manager.c | 95 +--------------------------- 1 file changed, 1 insertion(+), 94 deletions(-) diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 3fb4fc6e6..36a2f61d0 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -218,22 +218,6 @@ unifyfs_key_t** alloc_key_array(int elems) return (unifyfs_key_t**)mem_block; } -fattr_key_t** alloc_attr_key_array(int elems) -{ - int size = elems * (sizeof(fattr_key_t*) + sizeof(fattr_key_t)); - - void* mem_block = calloc(size, sizeof(char)); - - fattr_key_t** array_ptr = mem_block; - fattr_key_t* key_ptr = (fattr_key_t*)(array_ptr + elems); - - for (int i = 0; i < elems; i++) { - array_ptr[i] = &key_ptr[i]; - } - - return (fattr_key_t**)mem_block; -} - unifyfs_val_t** alloc_value_array(int elems) { int size = elems * (sizeof(unifyfs_val_t*) + sizeof(unifyfs_val_t)); @@ -260,11 +244,6 @@ void free_value_array(unifyfs_val_t** array) free(array); } -void free_attr_key_array(fattr_key_t** array) -{ - free(array); -} - static void debug_print_read_req(server_read_req_t* req) { if (NULL != req) { @@ -806,7 +785,7 @@ int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl) } /* - * synchronize all the indices and file attributes + * synchronize all the indices * to the key-value store * * @param app_id: the application id @@ -827,12 +806,6 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) int* unifyfs_key_lens = NULL; int* unifyfs_val_lens = NULL; - /* pointers to memory we'll dynamically allocate for file attributes */ - fattr_key_t** fattr_keys = NULL; - unifyfs_file_attr_t** fattr_vals = NULL; - int* fattr_key_lens = NULL; - int* fattr_val_lens = NULL; - /* get memory page size on this machine */ int page_sz = getpagesize(); @@ -846,9 +819,6 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) /* get pointer to start of key/value region in superblock */ char* meta = superblk + app_config->meta_offset; - /* get pointer to start of file attribute region in superblock */ - char* fmeta = superblk + app_config->fmeta_offset; - /* get number of file extent index values client has for us, * stored as a size_t value in meta region of shared memory */ size_t extent_num_entries = *(size_t*)(meta); @@ -910,53 +880,6 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) goto rm_cmd_fsync_exit; } - /* get number of file attribute values client has for us, - * stored as a size_t value in fmeta region of shared memory */ - size_t attr_num_entries = *(size_t*)(fmeta); - - /* file attributes are stored in the superblock shared memory - * created by the client */ - char* ptr_fattr = fmeta + page_sz; - unifyfs_file_attr_t* attr_payload = (unifyfs_file_attr_t*)(ptr_fattr); - - /* allocate storage for file attribute key/values */ - /* TODO: possibly get this from memory pool */ - fattr_keys = alloc_attr_key_array(attr_num_entries); - fattr_vals = calloc(attr_num_entries, sizeof(unifyfs_file_attr_t*)); - fattr_key_lens = calloc(attr_num_entries, sizeof(int)); - fattr_val_lens = calloc(attr_num_entries, sizeof(int)); - if ((NULL == fattr_keys) || - (NULL == fattr_vals) || - (NULL == fattr_key_lens) || - (NULL == fattr_val_lens)) { - LOGERR("failed to allocate memory for file attributes"); - ret = (int)UNIFYFS_ERROR_NOMEM; - goto rm_cmd_fsync_exit; - } - - /* create file attribute key/values for insertion into MDHIM */ - for (i = 0; i < attr_num_entries; i++) { - /* for a key, we use the global file id */ - *fattr_keys[i] = attr_payload[i].gfid; - - /* for the value, we'll store a file_attr structure */ - fattr_vals[i] = &(attr_payload[i]); - - /* MDHIM needs to know the byte size of each key and value */ - fattr_key_lens[i] = sizeof(fattr_key_t); - fattr_val_lens[i] = sizeof(unifyfs_file_attr_t); - } - - /* batch insert file attribute key/values into MDHIM */ - ret = unifyfs_set_file_attributes((int)attr_num_entries, - fattr_keys, fattr_key_lens, - fattr_vals, fattr_val_lens); - if (ret != UNIFYFS_SUCCESS) { - /* TODO: need proper error handling */ - LOGERR("unifyfs_set_file_attributes() failed"); - goto rm_cmd_fsync_exit; - } - rm_cmd_fsync_exit: /* clean up memory */ @@ -976,22 +899,6 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) free(unifyfs_val_lens); } - if (NULL != fattr_keys) { - free_attr_key_array(fattr_keys); - } - - if (NULL != fattr_vals) { - free(fattr_vals); - } - - if (NULL != fattr_key_lens) { - free(fattr_key_lens); - } - - if (NULL != fattr_val_lens) { - free(fattr_val_lens); - } - return ret; } From a8b86dc92a522e1a425fd80199bc64fdd9562469 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sat, 26 Oct 2019 16:04:19 -0700 Subject: [PATCH 026/168] client: lookup gfid with gfid_from_fid --- client/src/unifyfs-fixed.c | 15 ++++-------- client/src/unifyfs-sysio.c | 48 ++++++-------------------------------- client/src/unifyfs.c | 20 ++-------------- client/src/unifyfs.h | 1 - 4 files changed, 13 insertions(+), 71 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 6f69cd7ae..3405d539f 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -40,6 +40,7 @@ * Please also read this file LICENSE.CRUISE */ +#include "unifyfs-internal.h" #include "unifyfs-fixed.h" #include "unifyfs_log.h" #include "margo_client.h" @@ -539,20 +540,12 @@ static int unifyfs_logio_chunk_write( log_offset = spill_offset + unifyfs_max_chunks * (1 << unifyfs_chunk_bits); } - /* TODO: pass in gfid for this file or call function to look it up? */ - /* find the corresponding file attr entry and update attr*/ - unifyfs_file_attr_t tmp_meta_entry; - tmp_meta_entry.fid = fid; - unifyfs_file_attr_t* ptr_meta_entry - = (unifyfs_file_attr_t*)bsearch(&tmp_meta_entry, - unifyfs_fattrs.meta_entry, - *unifyfs_fattrs.ptr_num_entries, - sizeof(unifyfs_file_attr_t), - compare_fattr); + /* get global file id for this file */ + int gfid = unifyfs_gfid_from_fid(fid); /* define an new index entry for this write operation */ unifyfs_index_t cur_idx; - cur_idx.fid = ptr_meta_entry->gfid; + cur_idx.fid = gfid; cur_idx.file_pos = pos; cur_idx.log_pos = log_offset; cur_idx.length = count; diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 212277c6b..3ddcea478 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -1793,21 +1793,12 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) * */ /* convert local fid to global fid */ - unifyfs_file_attr_t tmp_meta_entry; - unifyfs_file_attr_t* ptr_meta_entry; for (i = 0; i < count; i++) { - /* look for global meta data for this local file id */ - tmp_meta_entry.fid = read_reqs[i].fid; - ptr_meta_entry = - (unifyfs_file_attr_t*) bsearch(&tmp_meta_entry, - unifyfs_fattrs.meta_entry, - *unifyfs_fattrs.ptr_num_entries, - sizeof(unifyfs_file_attr_t), - compare_fattr); - - /* replace local file id with global file id in request */ - if (ptr_meta_entry != NULL) { - read_reqs[i].fid = ptr_meta_entry->gfid; + /* get global file id for each request */ + int gfid = unifyfs_gfid_from_fid(read_reqs[i].fid); + if (gfid != -1) { + /* replace local file id with global file id in request */ + read_reqs[i].fid = gfid; } else { /* failed to find gfid for this request */ return UNIFYFS_ERROR_BADF; @@ -1863,7 +1854,7 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) free(buffer); } else { /* got a single read request */ - int gfid = ptr_meta_entry->gfid; + int gfid = read_req_set.read_reqs[0].fid; size_t offset = read_req_set.read_reqs[0].offset; size_t length = read_req_set.read_reqs[0].length; LOGDBG("read: offset:%zu, len:%zu", offset, length); @@ -2048,31 +2039,6 @@ int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) } } -/* get the gfid for use in fsync wrapper - * TODO: maybe move this somewhere else */ -uint32_t get_gfid(int fid) -{ - unifyfs_file_attr_t target; - target.fid = fid; - - const void* entries = unifyfs_fattrs.meta_entry; - size_t num = *unifyfs_fattrs.ptr_num_entries; - size_t size = sizeof(unifyfs_file_attr_t); - - unifyfs_file_attr_t* entry = - (unifyfs_file_attr_t*) bsearch(&target, entries, num, size, - compare_fattr); - - uint32_t gfid; - if (entry != NULL) { - gfid = (uint32_t)entry->gfid; - } else { - return -1; - } - - return gfid; -} - int UNIFYFS_WRAP(fsync)(int fd) { /* check whether we should intercept this file descriptor */ @@ -2102,7 +2068,7 @@ int UNIFYFS_WRAP(fsync)(int fd) } /* invoke fsync rpc to register index metadata with server */ - int gfid = get_gfid(fid); + int gfid = unifyfs_gfid_from_fid(fid); int ret = invoke_client_fsync_rpc(gfid); if (ret != UNIFYFS_SUCCESS) { /* sync failed for some reason, set errno and return error */ diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 468d262ec..019a3dcf0 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -687,7 +687,7 @@ int unifyfs_gfid_from_fid(const int fid) { /* check that local file id is in range */ if (fid < 0 || fid >= unifyfs_max_files) { - return -EINVAL; + return -1; } /* lookup file name structure for this local file id */ @@ -699,7 +699,7 @@ int unifyfs_gfid_from_fid(const int fid) return gfid; } - return -EINVAL; + return -1; } /* Given a fid, return the path. */ @@ -2206,22 +2206,6 @@ static int unifyfs_init_socket(int proc_id, int l_num_procs_per_node, } #endif // UNIFYFS_USE_DOMAIN_SOCKET -int compare_fattr(const void* a, const void* b) -{ - const unifyfs_file_attr_t* ptr_a = a; - const unifyfs_file_attr_t* ptr_b = b; - - if (ptr_a->fid > ptr_b->fid) { - return 1; - } - - if (ptr_a->fid < ptr_b->fid) { - return -1; - } - - return 0; -} - static int compare_int(const void* a, const void* b) { const int* ptr_a = a; diff --git a/client/src/unifyfs.h b/client/src/unifyfs.h index a2a11b8d1..d7413e498 100644 --- a/client/src/unifyfs.h +++ b/client/src/unifyfs.h @@ -73,7 +73,6 @@ typedef struct { int unifyfs_mount(const char prefix[], int rank, size_t size, int l_app_id); int unifyfs_unmount(void); -int compare_fattr(const void* a, const void* b); /** * @brief transfer a single file between unifyfs and other file system. either From c1bd0f347b20e1e040495b15e9eb2be797ddb312 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sat, 26 Oct 2019 16:11:40 -0700 Subject: [PATCH 027/168] client: do not insert file attributes into shared memory --- client/src/unifyfs.c | 46 -------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 019a3dcf0..a3963e681 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -783,48 +783,6 @@ off_t unifyfs_fid_log_size(int fid) return meta->log_size; } -/* - * insert file attribute to attributed shared memory buffer, - * keep entries ordered by file id - */ -static int ins_file_meta(unifyfs_fattr_buf_t* ptr_f_meta_log, - unifyfs_file_attr_t* ins_fattr) -{ - /* get pointer to start of stat structures in shared memory buffer */ - unifyfs_file_attr_t* meta_entry = ptr_f_meta_log->meta_entry; - - /* get number of active entries currently in the buffer */ - int meta_cnt = *(ptr_f_meta_log->ptr_num_entries); - - /* TODO: Improve the search time */ - /* search backwards until we find an entry whose - * file id is less than the current file id */ - int i; - for (i = meta_cnt - 1; i >= 0; i--) { - if (meta_entry[i].fid <= ins_fattr->fid) { - /* sort in acsending order */ - break; - } - } - - /* compute position to store stat info for this file */ - int ins_pos = i + 1; - - /* we need to move some entries up a slot to make room - * for this one */ - for (i = meta_cnt - 1; i >= ins_pos; i--) { - meta_entry[i + 1] = meta_entry[i]; - } - - /* insert stat data for this file into buffer */ - meta_entry[ins_pos] = *ins_fattr; - - /* increment our count of active entries */ - (*ptr_f_meta_log->ptr_num_entries)++; - - return 0; -} - unifyfs_filemeta_t* meta; int unifyfs_set_global_file_meta(int fid, int gfid) { @@ -869,8 +827,6 @@ int unifyfs_set_global_file_meta(int fid, int gfid) return ret; } - ins_file_meta(&unifyfs_fattrs, &new_fmeta); - return 0; } @@ -1318,8 +1274,6 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, meta->local_size = 0; gfattr.fid = fid; gfattr.gfid = gfid; - - ins_file_meta(&unifyfs_fattrs, &gfattr); } else if (found_local && found_global) { /* file exists and is valid. */ if ((flags & O_CREAT) && (flags & O_EXCL)) { From 52f353f03dcd4f0d72dfca5a13160017b68a15ed Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sat, 26 Oct 2019 16:26:03 -0700 Subject: [PATCH 028/168] drop shared memory segment for file attributes --- client/src/unifyfs-internal.h | 6 ------ client/src/unifyfs.c | 29 ----------------------------- common/src/unifyfs_client_rpcs.h | 2 -- server/src/unifyfs_cmd_handler.c | 4 ---- server/src/unifyfs_global.h | 2 -- 5 files changed, 43 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 3fec8b47b..57f96292e 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -309,11 +309,6 @@ typedef struct { unifyfs_index_t* index_entry; } unifyfs_index_buf_t; -typedef struct { - size_t* ptr_num_entries; - unifyfs_file_attr_t* meta_entry; -} unifyfs_fattr_buf_t; - typedef struct { read_req_t read_reqs[UNIFYFS_MAX_READ_CNT]; int count; @@ -330,7 +325,6 @@ extern int client_sockfd; extern struct pollfd cmd_fd; extern void* shm_req_buf; extern void* shm_recv_buf; -extern unifyfs_fattr_buf_t unifyfs_fattrs; extern int app_id; extern size_t unifyfs_key_slice_range; diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index a3963e681..ec00dd62e 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -71,7 +71,6 @@ static int unifyfs_fpos_enabled = 1; /* whether we can use fgetpos/fsetpos */ unifyfs_cfg_t client_cfg; unifyfs_index_buf_t unifyfs_indices; -unifyfs_fattr_buf_t unifyfs_fattrs; static size_t unifyfs_index_buf_size; /* size of metadata log */ static size_t unifyfs_fattr_buf_size; unsigned long unifyfs_max_index_entries; /* max metadata log entries */ @@ -1436,12 +1435,6 @@ int unifyfs_fid_unlink(int fid) * - array of index metadata to track physical offset * of logical file data, of length unifyfs_max_index_entries, * entries added during write operations - * - * - count of number of active file metadata entries - * - array of file metadata to track stat info for each - * file, of length unifyfs_max_fattr_entries, filed - * in by client and read by server to record file meta - * data */ /* compute memory size of superblock in bytes, @@ -1495,10 +1488,6 @@ static size_t unifyfs_superblock_size(void) sb_size += unifyfs_page_size; sb_size += unifyfs_max_index_entries * sizeof(unifyfs_index_t); - /* attribute region size */ - sb_size += unifyfs_page_size; - sb_size += unifyfs_max_fattr_entries * sizeof(unifyfs_file_attr_t); - /* return number of bytes */ return sb_size; } @@ -1576,14 +1565,6 @@ static void* unifyfs_init_pointers(void* superblock) unifyfs_indices.index_entry = (unifyfs_index_t*)ptr; ptr += unifyfs_max_index_entries * sizeof(unifyfs_index_t); - /* pointer to number of file metadata entries */ - unifyfs_fattrs.ptr_num_entries = (size_t*)ptr; - - /* pointer to array of file metadata entries */ - ptr += unifyfs_page_size; - unifyfs_fattrs.meta_entry = (unifyfs_file_attr_t*)ptr; - ptr += unifyfs_max_fattr_entries * sizeof(unifyfs_file_attr_t); - /* compute size of memory we're using and check that * it matches what we allocated */ size_t ptr_size = (size_t)(ptr - (char*)superblock); @@ -1634,9 +1615,6 @@ static int unifyfs_init_structures() /* initialize count of key/value entries */ *(unifyfs_indices.ptr_num_entries) = 0; - /* initialize count of file stat structures */ - *(unifyfs_fattrs.ptr_num_entries) = 0; - LOGDBG("Meta-stacks initialized!"); return UNIFYFS_SUCCESS; @@ -2000,11 +1978,6 @@ void fill_client_mount_info(unifyfs_mount_in_t* in) size_t meta_size = unifyfs_max_index_entries * sizeof(unifyfs_index_t); - size_t fmeta_offset = (char*)unifyfs_fattrs.ptr_num_entries - - (char*)shm_super_buf; - size_t fmeta_size = unifyfs_max_fattr_entries - * sizeof(unifyfs_file_attr_t); - size_t data_offset = (char*)unifyfs_chunks - (char*)shm_super_buf; size_t data_size = (size_t)unifyfs_max_chunks * unifyfs_chunk_size; @@ -2017,8 +1990,6 @@ void fill_client_mount_info(unifyfs_mount_in_t* in) in->superblock_sz = shm_super_size; in->meta_offset = meta_offset; in->meta_size = meta_size; - in->fmeta_offset = fmeta_offset; - in->fmeta_size = fmeta_size; in->data_offset = data_offset; in->data_size = data_size; in->external_spill_dir = strdup(external_data_dir); diff --git a/common/src/unifyfs_client_rpcs.h b/common/src/unifyfs_client_rpcs.h index 546810937..a9420bd64 100644 --- a/common/src/unifyfs_client_rpcs.h +++ b/common/src/unifyfs_client_rpcs.h @@ -30,8 +30,6 @@ MERCURY_GEN_PROC(unifyfs_mount_in_t, ((hg_size_t)(superblock_sz)) ((hg_size_t)(meta_offset)) ((hg_size_t)(meta_size)) - ((hg_size_t)(fmeta_offset)) - ((hg_size_t)(fmeta_size)) ((hg_size_t)(data_offset)) ((hg_size_t)(data_size)) ((hg_const_string_t)(external_spill_dir))) diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index f4981c787..879b05729 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -214,10 +214,6 @@ static void unifyfs_mount_rpc(hg_handle_t handle) tmp_config->meta_offset = in.meta_offset; tmp_config->meta_size = in.meta_size; - /* record offset and size of file meta data entries */ - tmp_config->fmeta_offset = in.fmeta_offset; - tmp_config->fmeta_size = in.fmeta_size; - /* record offset and size of file data */ tmp_config->data_offset = in.data_offset; tmp_config->data_size = in.data_size; diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index d0d9b86c2..2ffc9f746 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -131,8 +131,6 @@ typedef struct { size_t superblock_sz; /* size of memory region used to store data */ size_t meta_offset; /* superblock offset to index metadata */ size_t meta_size; /* size of index metadata region in bytes */ - size_t fmeta_offset; /* superblock offset to file attribute metadata */ - size_t fmeta_size; /* size of file attribute metadata region in bytes */ size_t data_offset; /* superblock offset to data log */ size_t data_size; /* size of data log in bytes */ size_t req_buf_sz; /* buffer size for client to issue read requests */ From 26865bc498c180ad782ea824666be0675cc84f80 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sat, 26 Oct 2019 16:33:32 -0700 Subject: [PATCH 029/168] client: drop config for file attribute shared memory --- client/src/unifyfs.c | 15 --------------- common/src/unifyfs_configurator.h | 1 - common/src/unifyfs_const.h | 1 - 3 files changed, 17 deletions(-) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index ec00dd62e..88cbef59b 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -74,7 +74,6 @@ unifyfs_index_buf_t unifyfs_indices; static size_t unifyfs_index_buf_size; /* size of metadata log */ static size_t unifyfs_fattr_buf_size; unsigned long unifyfs_max_index_entries; /* max metadata log entries */ -unsigned int unifyfs_max_fattr_entries; int unifyfs_spillmetablock; int global_rank_cnt; /* count of world ranks */ @@ -1813,20 +1812,6 @@ static int unifyfs_init(int rank) unifyfs_max_index_entries = unifyfs_index_buf_size / sizeof(unifyfs_index_t); - /* define size of buffer used to cache stat structures - * for files we create before passing this info - * to the server */ - unifyfs_fattr_buf_size = UNIFYFS_FATTR_BUF_SIZE; - cfgval = client_cfg.logfs_attr_buf_size; - if (cfgval != NULL) { - rc = configurator_int_val(cfgval, &l); - if (rc == 0) { - unifyfs_fattr_buf_size = (size_t)l; - } - } - unifyfs_max_fattr_entries = - unifyfs_fattr_buf_size / sizeof(unifyfs_file_attr_t); - /* if we're using NUMA, process some configuration settings */ #ifdef HAVE_LIBNUMA char* env = getenv("UNIFYFS_NUMA_POLICY"); diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index e480ce570..1ea97aeba 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -74,7 +74,6 @@ UNIFYFS_CFG_CLI(log, file, STRING, unifyfsd.log, "log file name", NULL, 'l', "specify log file name") \ UNIFYFS_CFG_CLI(log, dir, STRING, LOGDIR, "log file directory", configurator_directory_check, 'L', "specify full path to directory to contain log file") \ UNIFYFS_CFG(logfs, index_buf_size, INT, UNIFYFS_INDEX_BUF_SIZE, "log file system index buffer size", NULL) \ - UNIFYFS_CFG(logfs, attr_buf_size, INT, UNIFYFS_FATTR_BUF_SIZE, "log file system file attributes buffer size", NULL) \ UNIFYFS_CFG(margo, tcp, BOOL, on, "use TCP for server-server margo RPCs", NULL) \ UNIFYFS_CFG(meta, db_name, STRING, META_DEFAULT_DB_NAME, "metadata database name", NULL) \ UNIFYFS_CFG(meta, db_path, STRING, RUNDIR, "metadata database path", configurator_directory_check) \ diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h index 31a223739..cdca51240 100644 --- a/common/src/unifyfs_const.h +++ b/common/src/unifyfs_const.h @@ -85,7 +85,6 @@ #define UNIFYFS_SHMEM_REQ_SIZE (8 * MIB) #define UNIFYFS_SHMEM_RECV_SIZE (32 * MIB) #define UNIFYFS_INDEX_BUF_SIZE (20 * MIB) -#define UNIFYFS_FATTR_BUF_SIZE MIB #define UNIFYFS_MAX_READ_CNT KIB /* NOTE: max read size = UNIFYFS_MAX_SPLIT_CNT * META_DEFAULT_RANGE_SZ */ From 210a61f76b54d03074b97218fde38549a4ec9727 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sat, 26 Oct 2019 21:49:36 -0700 Subject: [PATCH 030/168] rename fid to gfid in index structure --- client/src/unifyfs-fixed.c | 12 +++---- client/src/unifyfs-sysio.c | 4 +-- common/src/unifyfs_meta.h | 2 +- server/src/unifyfs_metadata.c | 16 +++++----- server/src/unifyfs_metadata.h | 4 +-- server/src/unifyfs_request_manager.c | 48 ++++++++++++++-------------- 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 3405d539f..0f78bc02d 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -275,7 +275,7 @@ static int unifyfs_coalesce_index( long slice_size) /* byte size of slice of key-value store */ { /* check whether last index and next index refer to same file */ - if (prev_idx->fid != next_idx->fid) { + if (prev_idx->gfid != next_idx->gfid) { /* two index values are for different files, can't combine */ return UNIFYFS_SUCCESS; } @@ -396,7 +396,7 @@ static int unifyfs_split_index( /* copy over all fields in current index, * update length field to adjust for boundary of first slice */ - set[count].fid = cur_idx->fid; + set[count].gfid = cur_idx->gfid; set[count].file_pos = cur_idx->file_pos; set[count].length = length; set[count].log_pos = cur_idx->log_pos; @@ -427,7 +427,7 @@ static int unifyfs_split_index( length = slice_range; /* define index for this slice */ - set[count].fid = cur_idx->fid; + set[count].gfid = cur_idx->gfid; set[count].file_pos = cur_idx->file_pos; set[count].length = length; set[count].log_pos = cur_idx->log_pos; @@ -453,7 +453,7 @@ static int unifyfs_split_index( /* this slice contains the remainder of write */ length = cur_idx->length; - set[count].fid = cur_idx->fid; + set[count].gfid = cur_idx->gfid; set[count].file_pos = cur_idx->file_pos; set[count].length = length; set[count].log_pos = cur_idx->log_pos; @@ -545,7 +545,7 @@ static int unifyfs_logio_chunk_write( /* define an new index entry for this write operation */ unifyfs_index_t cur_idx; - cur_idx.fid = gfid; + cur_idx.gfid = gfid; cur_idx.file_pos = pos; cur_idx.log_pos = log_offset; cur_idx.length = count; @@ -577,7 +577,7 @@ static int unifyfs_logio_chunk_write( /* if we have filled the key/value buffer, flush it to server */ if (0 == remaining_entries) { /* index buffer is full, flush it */ - int ret = invoke_client_fsync_rpc(cur_idx.fid); + int ret = invoke_client_fsync_rpc(cur_idx.gfid); if (ret != UNIFYFS_SUCCESS) { /* something went wrong when trying to flush key/values */ LOGERR("failed to flush key/value index to server"); diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 3ddcea478..1ba7fe9ec 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -1201,8 +1201,8 @@ static int compare_index_entry(const void* a, const void* b) const unifyfs_index_t* ptr_a = a; const unifyfs_index_t* ptr_b = b; - if (ptr_a->fid != ptr_b->fid) { - if (ptr_a->fid < ptr_b->fid) { + if (ptr_a->gfid != ptr_b->gfid) { + if (ptr_a->gfid < ptr_b->gfid) { return -1; } else { return 1; diff --git a/common/src/unifyfs_meta.h b/common/src/unifyfs_meta.h index 5e3be3391..461558391 100644 --- a/common/src/unifyfs_meta.h +++ b/common/src/unifyfs_meta.h @@ -100,7 +100,7 @@ typedef struct { off_t file_pos; /* starting logical offset of data in file */ off_t log_pos; /* starting physical offset of data in log */ size_t length; /* length of data */ - int fid; /* global file id */ + int gfid; /* global file id */ } unifyfs_index_t; /* Header for read request reply in client shared memory region. diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index fd3ac103d..6e6f113eb 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -59,21 +59,21 @@ void debug_log_key_val(const char* ctx, unifyfs_val_t* val) { if ((key != NULL) && (val != NULL)) { - LOGDBG("@%s - key(fid=%d, offset=%lu), " + LOGDBG("@%s - key(gfid=%d, offset=%lu), " "val(del=%d, len=%lu, addr=%lu, app=%d, rank=%d)", - ctx, key->fid, key->offset, + ctx, key->gfid, key->offset, val->delegator_rank, val->len, val->addr, val->app_id, val->rank); } else if (key != NULL) { - LOGDBG("@%s - key(fid=%d, offset=%lu)", - ctx, key->fid, key->offset); + LOGDBG("@%s - key(gfid=%d, offset=%lu)", + ctx, key->gfid, key->offset); } } int unifyfs_key_compare(unifyfs_key_t* a, unifyfs_key_t* b) { assert((NULL != a) && (NULL != b)); - if (a->fid == b->fid) { + if (a->gfid == b->gfid) { if (a->offset == b->offset) { return 0; } else if (a->offset < b->offset) { @@ -81,7 +81,7 @@ int unifyfs_key_compare(unifyfs_key_t* a, unifyfs_key_t* b) } else { return 1; } - } else if (a->fid < b->fid) { + } else if (a->gfid < b->gfid) { return -1; } else { return 1; @@ -161,8 +161,8 @@ void print_fsync_indices(unifyfs_key_t** keys, { size_t i; for (i = 0; i < num_entries; i++) { - LOGDBG("fid:%d, offset:%lu, addr:%lu, len:%lu, del_id:%d", - keys[i]->fid, keys[i]->offset, + LOGDBG("gfid:%d, offset:%lu, addr:%lu, len:%lu, del_id:%d", + keys[i]->gfid, keys[i]->offset, vals[i]->addr, vals[i]->len, vals[i]->delegator_rank); } diff --git a/server/src/unifyfs_metadata.h b/server/src/unifyfs_metadata.h index 46fe81ffb..a326ef10e 100644 --- a/server/src/unifyfs_metadata.h +++ b/server/src/unifyfs_metadata.h @@ -40,13 +40,13 @@ */ typedef struct { /** global file id */ - int fid; + int gfid; /** logical file offset */ size_t offset; } unifyfs_key_t; #define UNIFYFS_KEY_SZ (sizeof(unifyfs_key_t)) -#define UNIFYFS_KEY_FID(keyp) (((unifyfs_key_t*)keyp)->fid) +#define UNIFYFS_KEY_FID(keyp) (((unifyfs_key_t*)keyp)->gfid) #define UNIFYFS_KEY_OFF(keyp) (((unifyfs_key_t*)keyp)->offset) typedef struct { diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 36a2f61d0..4f7ffad01 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -183,8 +183,8 @@ static int compare_kv_gfid_rank(const void* a, const void* b) const unifyfs_keyval_t* kv_a = a; const unifyfs_keyval_t* kv_b = b; - int gfid_a = kv_a->key.fid; - int gfid_b = kv_b->key.fid; + int gfid_a = kv_a->key.gfid; + int gfid_b = kv_b->key.gfid; if (gfid_a == gfid_b) { int rank_a = kv_a->val.delegator_rank; int rank_b = kv_b->val.delegator_rank; @@ -473,11 +473,11 @@ int rm_cmd_filesize( unifyfs_key_t key1, key2; /* create key to describe first byte we'll read */ - key1.fid = gfid; + key1.gfid = gfid; key1.offset = offset; /* create key to describe last byte we'll read */ - key2.fid = gfid; + key2.gfid = gfid; key2.offset = offset + length - 1; /* set up input params to specify range lookup */ @@ -617,11 +617,11 @@ int rm_cmd_read( unifyfs_key_t key1, key2; /* create key to describe first byte we'll read */ - key1.fid = gfid; + key1.gfid = gfid; key1.offset = offset; /* create key to describe last byte we'll read */ - key2.fid = gfid; + key2.gfid = gfid; key2.offset = offset + length - 1; unifyfs_key_t* unifyfs_keys[2] = {&key1, &key2}; @@ -678,28 +678,28 @@ int rm_cmd_mread(int app_id, int client_id, /* get chunks corresponding to requested client read extents */ int rc, num_keys; - int fid = -1; - int last_fid = -1; + int gfid = -1; + int last_gfid = -1; int ndx = 0; size_t j, eoff, elen; for (j = 0; j < req_num; j++) { - fid = unifyfs_Extent_fid(unifyfs_Extent_vec_at(extents, j)); - if (j && (fid != last_fid)) { - // create requests for all extents of last_fid + gfid = unifyfs_Extent_fid(unifyfs_Extent_vec_at(extents, j)); + if (j && (gfid != last_gfid)) { + // create requests for all extents of last_gfid num_keys = ndx; - rc = create_gfid_chunk_reads(thrd_ctrl, last_fid, app_id, + rc = create_gfid_chunk_reads(thrd_ctrl, last_gfid, app_id, client_id, num_keys, unifyfs_keys, key_lens); if (rc != UNIFYFS_SUCCESS) { - LOGERR("Error creating chunk reads for gfid=%d", last_fid); + LOGERR("Error creating chunk reads for gfid=%d", last_gfid); } - // reset ndx for current fid + // reset ndx for current gfid ndx = 0; } eoff = unifyfs_Extent_offset(unifyfs_Extent_vec_at(extents, j)); elen = unifyfs_Extent_length(unifyfs_Extent_vec_at(extents, j)); - LOGDBG("gfid:%d, offset:%zu, length:%zu", fid, eoff, elen); + LOGDBG("gfid:%d, offset:%zu, length:%zu", gfid, eoff, elen); /* Generate a pair of keys for each read request, representing * the start and end offsets. MDHIM returns all key-value pairs that @@ -714,24 +714,24 @@ int rm_cmd_mread(int app_id, int client_id, key_lens[ndx + 1] = sizeof(unifyfs_key_t); /* create key to describe first byte we'll read */ - unifyfs_keys[ndx]->fid = fid; + unifyfs_keys[ndx]->gfid = gfid; unifyfs_keys[ndx]->offset = eoff; /* create key to describe last byte we'll read */ - unifyfs_keys[ndx + 1]->fid = fid; + unifyfs_keys[ndx + 1]->gfid = gfid; unifyfs_keys[ndx + 1]->offset = eoff + elen - 1; ndx += 2; - last_fid = fid; + last_gfid = gfid; } - // create requests for all extents of last_fid + // create requests for all extents of last_gfid num_keys = ndx; - rc = create_gfid_chunk_reads(thrd_ctrl, last_fid, app_id, + rc = create_gfid_chunk_reads(thrd_ctrl, last_gfid, app_id, client_id, num_keys, unifyfs_keys, key_lens); if (rc != UNIFYFS_SUCCESS) { - LOGERR("Error creating chunk reads for gfid=%d", last_fid); + LOGERR("Error creating chunk reads for gfid=%d", last_gfid); } // cleanup @@ -848,7 +848,7 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) for (i = 0; i < extent_num_entries; i++) { /* for a key, we store the global file id and logical file offset */ unifyfs_key_t* key = unifyfs_keys[i]; - key->fid = meta_payload[i].fid; + key->gfid = meta_payload[i].gfid; key->offset = meta_payload[i].file_pos; /* for the value, we store the log position, the length, @@ -861,8 +861,8 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) val->app_id = app_id; val->rank = client_side_id; - LOGDBG("extent - fid:%d, offset:%zu, length:%zu, app:%d, clid:%d", - key->fid, key->offset, + LOGDBG("extent - gfid:%d, offset:%zu, length:%zu, app:%d, clid:%d", + key->gfid, key->offset, val->len, val->app_id, val->rank); /* MDHIM needs to know the byte size of each key and value */ From 2eaf909200c29c92b85e33fcd89718587e498cf7 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sun, 27 Oct 2019 17:04:25 -0700 Subject: [PATCH 031/168] client: cache gfid in file metadata struct, add comments in set/get_global_meta --- client/src/unifyfs-internal.h | 1 + client/src/unifyfs.c | 197 ++++++++++++++++++---------------- 2 files changed, 104 insertions(+), 94 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 57f96292e..7afc0f55a 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -274,6 +274,7 @@ typedef struct { int storage; /* FILE_STORAGE type */ + int gfid; /* global file id for this file */ int needs_sync; /* have unsynced writes */ off_t chunks; /* number of chunks allocated to file */ diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 88cbef59b..3d0a4f209 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -688,16 +688,9 @@ int unifyfs_gfid_from_fid(const int fid) return -1; } - /* lookup file name structure for this local file id */ - unifyfs_filename_t* fname = &unifyfs_filelist[fid]; - - /* generate global file id from path if file is valid */ - if (fname->in_use) { - int gfid = unifyfs_generate_gfid(fname->filename); - return gfid; - } - - return -1; + /* return global file id, cached in file meta struct */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + return meta->gfid; } /* Given a fid, return the path. */ @@ -781,29 +774,35 @@ off_t unifyfs_fid_log_size(int fid) return meta->log_size; } -unifyfs_filemeta_t* meta; int unifyfs_set_global_file_meta(int fid, int gfid) { - int ret = 0; + /* lookup local metadata for file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - unifyfs_file_attr_t new_fmeta = {0}; - struct timespec tp = {0}; - const char* path = unifyfs_path_from_fid(fid); + /* initialize an empty file attributes structure */ + unifyfs_file_attr_t fattr = {0}; - sprintf(new_fmeta.filename, "%s", path); + /* copy our file name */ + const char* path = unifyfs_path_from_fid(fid); + sprintf(fattr.filename, "%s", path); - new_fmeta.fid = fid; - new_fmeta.gfid = gfid; + /* get local and global file ids */ + fattr.fid = fid; + fattr.gfid = gfid; + /* use current time for atime/mtime/ctime */ + struct timespec tp = {0}; clock_gettime(CLOCK_REALTIME, &tp); - new_fmeta.atime = tp; - new_fmeta.mtime = tp; - new_fmeta.ctime = tp; + fattr.atime = tp; + fattr.mtime = tp; + fattr.ctime = tp; - new_fmeta.mode = meta->mode; - new_fmeta.is_laminated = meta->is_laminated; + /* copy file mode bits and lamination flag */ + fattr.mode = meta->mode; + fattr.is_laminated = meta->is_laminated; + /* set file size, we use the global size field if laminated + * set size to 0 otherwise */ if (meta->is_laminated) { /* * If is_laminated is set, we're either laminating for the first time, @@ -811,32 +810,37 @@ int unifyfs_set_global_file_meta(int fid, int gfid) * filled in with the global file size, or, the file was already * laminated. In either case, we write meta->global_size to the server. */ - new_fmeta.size = meta->global_size; - LOGDBG("setting new_fmeta to %d", new_fmeta.size); + fattr.size = meta->global_size; + LOGDBG("setting fattr to %d", fattr.size); } else { - new_fmeta.size = 0; + fattr.size = 0; } - new_fmeta.uid = getuid(); - new_fmeta.gid = getgid(); + /* capture current uid and gid */ + fattr.uid = getuid(); + fattr.gid = getgid(); - ret = invoke_client_metaset_rpc(&new_fmeta); + /* submit file attributes to global key/value store */ + int ret = invoke_client_metaset_rpc(&fattr); if (ret < 0) { return ret; } - return 0; + return UNIFYFS_SUCCESS; } int unifyfs_get_global_file_meta(int fid, int gfid, unifyfs_file_attr_t* gfattr) { + /* check that we have an output buffer to write to */ if (!gfattr) { return -EINVAL; } + /* attempt to lookup file attributes in key/value store */ unifyfs_file_attr_t fmeta; int ret = invoke_client_metaget_rpc(gfid, &fmeta); if (ret == UNIFYFS_SUCCESS) { + /* found it, copy attributes to output struct */ *gfattr = fmeta; gfattr->fid = fid; } @@ -902,6 +906,13 @@ int unifyfs_fid_free(int fid) * returns the new fid, or negative value on error */ int unifyfs_fid_create_file(const char* path) { + /* check that pathname is within bounds */ + size_t pathlen = strlen(path) + 1; + if (pathlen > UNIFYFS_MAX_FILENAME) { + return (int) UNIFYFS_ERROR_NAMETOOLONG; + } + + /* allocate an id for this file */ int fid = unifyfs_fid_alloc(); if (fid < 0) { /* was there an error? if so, return it */ @@ -909,13 +920,9 @@ int unifyfs_fid_create_file(const char* path) return fid; } - /* mark this slot as in use and copy the filename */ + /* mark this slot as in use */ unifyfs_filelist[fid].in_use = 1; - /* TODO: check path length to see if it is < 128 bytes - * and return appropriate error if it is greater - */ - /* copy file name into slot */ strcpy((void*)&unifyfs_filelist[fid].filename, path); LOGDBG("Filename %s got unifyfs fd %d", @@ -923,17 +930,18 @@ int unifyfs_fid_create_file(const char* path) /* initialize meta data */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - meta->global_size = 0; - meta->local_size = 0; - meta->log_size = 0; - meta->chunks = 0; - meta->storage = FILE_STORAGE_NULL; - meta->needs_sync = 0; + meta->global_size = 0; + meta->local_size = 0; + meta->log_size = 0; meta->flock_status = UNLOCKED; + meta->storage = FILE_STORAGE_NULL; + meta->gfid = unifyfs_generate_gfid(path); + meta->needs_sync = 0; + meta->chunks = 0; meta->is_laminated = 0; - meta->mode = UNIFYFS_STAT_DEFAULT_FILE_MODE; + meta->mode = UNIFYFS_STAT_DEFAULT_FILE_MODE; - /* PTHREAD_PROCESS_SHARED allows Process-Shared Synchronization*/ + /* PTHREAD_PROCESS_SHARED allows Process-Shared Synchronization */ pthread_spin_init(&meta->fspinlock, PTHREAD_PROCESS_SHARED); return fid; @@ -941,28 +949,30 @@ int unifyfs_fid_create_file(const char* path) int unifyfs_fid_create_directory(const char* path) { - int ret = 0; - int fid = 0; - int gfid = 0; - int found_global = 0; - int found_local = 0; + /* check that pathname is within bounds */ size_t pathlen = strlen(path) + 1; - struct stat sb = { 0, }; - unifyfs_file_attr_t gfattr = { 0, }; - unifyfs_filemeta_t* meta = NULL; - if (pathlen > UNIFYFS_MAX_FILENAME) { return (int) UNIFYFS_ERROR_NAMETOOLONG; } - fid = unifyfs_get_fid_from_path(path); - gfid = unifyfs_generate_gfid(path); + /* get local and global file ids */ + int fid = unifyfs_get_fid_from_path(path); + int gfid = unifyfs_generate_gfid(path); - found_global = - (unifyfs_get_global_file_meta(fid, gfid, &gfattr) == UNIFYFS_SUCCESS); - found_local = (fid >= 0); + /* test whether we have info for file in our local file list */ + int found_local = (fid >= 0); + /* test whether we have metadata for file in global key/value store */ + int found_global = 0; + unifyfs_file_attr_t gfattr = { 0, }; + if (unifyfs_get_global_file_meta(fid, gfid, &gfattr) == UNIFYFS_SUCCESS) { + found_global = 1; + } + + /* check whether we already have info for this item + * locally and globally */ if (found_local && found_global) { + /* this directory already exists */ return (int) UNIFYFS_ERROR_EXIST; } @@ -993,15 +1003,16 @@ int unifyfs_fid_create_directory(const char* path) /* now, we need to create a new directory. */ fid = unifyfs_fid_create_file(path); if (fid < 0) { - return (int) UNIFYFS_ERROR_IO; /* FIXME: ENOSPC or EIO? */ + /* FIXME: ENOSPC or EIO? */ + return (int) UNIFYFS_ERROR_IO; } - meta = unifyfs_get_meta_from_fid(fid); - /* Set as directory */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; - ret = unifyfs_set_global_file_meta(fid, gfid); + /* insert global meta data for directory */ + int ret = unifyfs_set_global_file_meta(fid, gfid); if (ret) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); @@ -1183,17 +1194,13 @@ static int unifyfs_get_global_fid(const char* path, int* gfid) int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, off_t* outpos) { - /* check that path is short enough */ - int ret = 0; - size_t pathlen = strlen(path) + 1; - int fid = 0; - int gfid = -1; - int found_global = 0; - int found_local = 0; - off_t pos = 0; /* set the pointer to the start of the file */ - unifyfs_file_attr_t gfattr = { 0, }; - unifyfs_filemeta_t* meta = NULL; + int ret; + + /* set the pointer to the start of the file */ + off_t pos = 0; + /* check that pathname is within bounds */ + size_t pathlen = strlen(path) + 1; if (pathlen > UNIFYFS_MAX_FILENAME) { return (int) UNIFYFS_ERROR_NAMETOOLONG; } @@ -1206,14 +1213,21 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, * the broadcast for cache invalidation has not been implemented, yet. */ - gfid = unifyfs_generate_gfid(path); - fid = unifyfs_get_fid_from_path(path); + /* get local and global file ids */ + int fid = unifyfs_get_fid_from_path(path); + int gfid = unifyfs_generate_gfid(path); LOGDBG("unifyfs_get_fid_from_path() gave %d (gfid = %d)", fid, gfid); - found_global = - (unifyfs_get_global_file_meta(fid, gfid, &gfattr) == UNIFYFS_SUCCESS); - found_local = (fid >= 0); + /* test whether we have info for file in our local file list */ + int found_local = (fid >= 0); + + /* test whether we have metadata for file in global key/value store */ + int found_global = 0; + unifyfs_file_attr_t gfattr = { 0, }; + if (unifyfs_get_global_file_meta(fid, gfid, &gfattr) == UNIFYFS_SUCCESS) { + found_global = 1; + } /* * Catch any case where we could potentially want to write to a laminated @@ -1251,14 +1265,15 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, * space. */ + /* initialize local metadata for this file */ fid = unifyfs_fid_create_file(path); if (fid < 0) { - LOGERR("failed to create a new file %s", path); - /* FIXME: UNIFYFS_ERROR_NFILE or UNIFYFS_ERROR_IO ? */ + LOGERR("failed to create a new file %s", path); return (int) UNIFYFS_ERROR_IO; } + /* initialize local storage for this file */ ret = unifyfs_fid_store_alloc(fid); if (ret != UNIFYFS_SUCCESS) { LOGERR("failed to allocate storage space for file %s (fid=%d)", @@ -1266,12 +1281,9 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, return (int) UNIFYFS_ERROR_IO; } - meta = unifyfs_get_meta_from_fid(fid); - + /* initialize global size of file from global metadata */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); meta->global_size = gfattr.size; - meta->local_size = 0; - gfattr.fid = fid; - gfattr.gfid = gfid; } else if (found_local && found_global) { /* file exists and is valid. */ if ((flags & O_CREAT) && (flags & O_EXCL)) { @@ -1291,19 +1303,15 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } if (flags & O_APPEND) { - meta = unifyfs_get_meta_from_fid(fid); - /* - * We only support O_APPEND on non-laminated (local) files, so - * use local_size here. - */ + /* We only support O_APPEND on non-laminated (local) files, so + * use local_size here. */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); pos = meta->local_size; } } else { /* !found_local && !found_global * If we reach here, we need to create a brand new file. */ - struct stat sb = { 0, }; - if (!(flags & O_CREAT)) { LOGERR("%s does not exist (O_CREAT not given).", path); return (int) UNIFYFS_ERROR_NOENT; @@ -1329,7 +1337,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, return (int) UNIFYFS_ERROR_IO; } - /*create a file and send its attribute to key-value store*/ + /* insert file attribute for file in key-value store */ ret = unifyfs_set_global_file_meta(fid, gfid); if (ret) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", @@ -1339,8 +1347,9 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } /* TODO: allocate a free file descriptor and associate it with fid set - * in_use flag and file pointer - */ + * in_use flag and file pointer */ + + /* return local file id and starting file position */ *outfid = fid; *outpos = pos; @@ -1404,7 +1413,7 @@ int unifyfs_fid_unlink(int fid) * structures used to track file names, meta data, the list of * storage blocks used for each file, and optional blocks. * It also contains a fixed-size region for keeping log - * index entries and stat info for each file. + * index entries for each file. * * - stack of free local file ids of length max_files, * the local file id is used to index into other data From 8d5089f4b91172e8b0088f90885e760ee47725ca Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sun, 27 Oct 2019 17:19:43 -0700 Subject: [PATCH 032/168] drop fid from global file attributes --- client/src/margo_client.c | 1 - client/src/unifyfs.c | 4 +--- common/src/unifyfs_client_rpcs.h | 2 -- common/src/unifyfs_meta.h | 1 - t/server/unifyfs_meta_get_test.c | 6 ++---- 5 files changed, 3 insertions(+), 11 deletions(-) diff --git a/client/src/margo_client.c b/client/src/margo_client.c index ea3676e8f..c5b7a6c54 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -256,7 +256,6 @@ int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta) assert(hret == HG_SUCCESS); /* fill in input struct */ - in.fid = f_meta->fid; in.gfid = f_meta->gfid; in.filename = f_meta->filename; in.mode = f_meta->mode; diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 3d0a4f209..2860fa7a0 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -786,8 +786,7 @@ int unifyfs_set_global_file_meta(int fid, int gfid) const char* path = unifyfs_path_from_fid(fid); sprintf(fattr.filename, "%s", path); - /* get local and global file ids */ - fattr.fid = fid; + /* set global file id */ fattr.gfid = gfid; /* use current time for atime/mtime/ctime */ @@ -842,7 +841,6 @@ int unifyfs_get_global_file_meta(int fid, int gfid, unifyfs_file_attr_t* gfattr) if (ret == UNIFYFS_SUCCESS) { /* found it, copy attributes to output struct */ *gfattr = fmeta; - gfattr->fid = fid; } return ret; diff --git a/common/src/unifyfs_client_rpcs.h b/common/src/unifyfs_client_rpcs.h index a9420bd64..c4176e4d8 100644 --- a/common/src/unifyfs_client_rpcs.h +++ b/common/src/unifyfs_client_rpcs.h @@ -59,7 +59,6 @@ MERCURY_GEN_STRUCT_PROC(sys_timespec_t, * record key/value entry for this file */ MERCURY_GEN_PROC(unifyfs_metaset_in_t, ((hg_const_string_t)(filename)) - ((int32_t)(fid)) ((int32_t)(gfid)) ((uint32_t)(mode)) ((uint32_t)(uid)) @@ -81,7 +80,6 @@ MERCURY_GEN_PROC(unifyfs_metaget_in_t, MERCURY_GEN_PROC(unifyfs_metaget_out_t, ((int32_t)(ret)) ((hg_const_string_t)(filename)) - ((int32_t)(fid)) ((int32_t)(gfid)) ((uint32_t)(mode)) ((uint32_t)(uid)) diff --git a/common/src/unifyfs_meta.h b/common/src/unifyfs_meta.h index 461558391..ce909ea49 100644 --- a/common/src/unifyfs_meta.h +++ b/common/src/unifyfs_meta.h @@ -42,7 +42,6 @@ typedef enum { } cmd_lst_t; typedef struct { - int fid; int gfid; char filename[UNIFYFS_MAX_FILENAME]; diff --git a/t/server/unifyfs_meta_get_test.c b/t/server/unifyfs_meta_get_test.c index db2c304e6..d1ecc180f 100644 --- a/t/server/unifyfs_meta_get_test.c +++ b/t/server/unifyfs_meta_get_test.c @@ -17,7 +17,6 @@ int unifyfs_set_file_attribute_test(void) unifyfs_file_attr_t fattr = {0}; fattr.gfid = TEST_META_GFID_VALUE; - fattr.fid = TEST_META_FID_VALUE; snprintf(fattr.filename, sizeof(fattr.filename), TEST_META_FILE); fflush(NULL); @@ -35,10 +34,9 @@ int unifyfs_get_file_attribute_test(void) rc = unifyfs_get_file_attribute(TEST_META_GFID_VALUE, &fattr); ok(UNIFYFS_SUCCESS == rc && TEST_META_GFID_VALUE == fattr.gfid && - TEST_META_FID_VALUE == fattr.fid && (0 == strcmp(fattr.filename, TEST_META_FILE)), - "Retrieve file attributes (rc = %d, gfid = 0x%02X, fid = 0x%02X)", - rc, fattr.gfid, fattr.fid + "Retrieve file attributes (rc = %d, gfid = 0x%02X)", + rc, fattr.gfid ); return 0; } From cd14c60c5ee8e4f76b45c20dc98b8436b89d0d99 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sun, 27 Oct 2019 17:24:56 -0700 Subject: [PATCH 033/168] client: drop fid from get_global_file_meta --- client/src/unifyfs-dirops.c | 2 +- client/src/unifyfs-internal.h | 2 +- client/src/unifyfs.c | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client/src/unifyfs-dirops.c b/client/src/unifyfs-dirops.c index 49dda8e23..0ad0f5f68 100644 --- a/client/src/unifyfs-dirops.c +++ b/client/src/unifyfs-dirops.c @@ -103,7 +103,7 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) int gfid = unifyfs_generate_gfid(name); unifyfs_file_attr_t gfattr = { 0, }; - int ret = unifyfs_get_global_file_meta(fid, gfid, &gfattr); + int ret = unifyfs_get_global_file_meta(gfid, &gfattr); if (ret != UNIFYFS_SUCCESS) { errno = ENOENT; return NULL; diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 7afc0f55a..93fb796c0 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -568,7 +568,7 @@ int unifyfs_generate_gfid(const char* path); int unifyfs_set_global_file_meta(int fid, int gfid); -int unifyfs_get_global_file_meta(int fid, int gfid, +int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr); // These require types/structures defined above diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 2860fa7a0..48ba8ec4c 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -828,11 +828,11 @@ int unifyfs_set_global_file_meta(int fid, int gfid) return UNIFYFS_SUCCESS; } -int unifyfs_get_global_file_meta(int fid, int gfid, unifyfs_file_attr_t* gfattr) +int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) { /* check that we have an output buffer to write to */ if (!gfattr) { - return -EINVAL; + return UNIFYFS_FAILURE; } /* attempt to lookup file attributes in key/value store */ @@ -963,7 +963,7 @@ int unifyfs_fid_create_directory(const char* path) /* test whether we have metadata for file in global key/value store */ int found_global = 0; unifyfs_file_attr_t gfattr = { 0, }; - if (unifyfs_get_global_file_meta(fid, gfid, &gfattr) == UNIFYFS_SUCCESS) { + if (unifyfs_get_global_file_meta(gfid, &gfattr) == UNIFYFS_SUCCESS) { found_global = 1; } @@ -1223,7 +1223,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, /* test whether we have metadata for file in global key/value store */ int found_global = 0; unifyfs_file_attr_t gfattr = { 0, }; - if (unifyfs_get_global_file_meta(fid, gfid, &gfattr) == UNIFYFS_SUCCESS) { + if (unifyfs_get_global_file_meta(gfid, &gfattr) == UNIFYFS_SUCCESS) { found_global = 1; } From 25153ee269b904e1a0cdb4bbeb91f432f98c0433 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sun, 27 Oct 2019 17:46:26 -0700 Subject: [PATCH 034/168] client: drop fid from set/get global file meta functions --- client/src/unifyfs-internal.h | 5 ++- client/src/unifyfs-sysio.c | 2 +- client/src/unifyfs.c | 70 +++++++++++++++++++---------------- 3 files changed, 44 insertions(+), 33 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 93fb796c0..ceec8a801 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -566,7 +566,10 @@ int unifyfs_fid_unlink(int fid); int unifyfs_generate_gfid(const char* path); -int unifyfs_set_global_file_meta(int fid, int gfid); +int unifyfs_set_global_file_meta_from_fid(int fid); + +int unifyfs_set_global_file_meta(int gfid, + unifyfs_file_attr_t* gfattr); int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr); diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 1ba7fe9ec..3722ab82b 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -2362,7 +2362,7 @@ static int __chmod(int fid, mode_t mode) meta->mode = meta->mode & ~0777; meta->mode = meta->mode | mode; - ret = unifyfs_set_global_file_meta(fid, gfid); + ret = unifyfs_set_global_file_meta_from_fid(fid); if (ret) { LOGERR("chmod: can't set global meta entry for %s (fid:%d)", path, fid); diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 48ba8ec4c..1e9ba35ba 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -774,20 +774,50 @@ off_t unifyfs_fid_log_size(int fid) return meta->log_size; } -int unifyfs_set_global_file_meta(int fid, int gfid) +int unifyfs_set_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) { - /* lookup local metadata for file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + /* check that we have an input buffer */ + if (!gfattr) { + return UNIFYFS_FAILURE; + } + + /* submit file attributes to global key/value store */ + int ret = invoke_client_metaset_rpc(gfattr); + return ret; +} + +int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) +{ + /* check that we have an output buffer to write to */ + if (!gfattr) { + return UNIFYFS_FAILURE; + } + + /* attempt to lookup file attributes in key/value store */ + unifyfs_file_attr_t fmeta; + int ret = invoke_client_metaget_rpc(gfid, &fmeta); + if (ret == UNIFYFS_SUCCESS) { + /* found it, copy attributes to output struct */ + *gfattr = fmeta; + } + return ret; +} + +int unifyfs_set_global_file_meta_from_fid(int fid) +{ /* initialize an empty file attributes structure */ unifyfs_file_attr_t fattr = {0}; + /* lookup local metadata for file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + /* copy our file name */ const char* path = unifyfs_path_from_fid(fid); sprintf(fattr.filename, "%s", path); /* set global file id */ - fattr.gfid = gfid; + fattr.gfid = meta->gfid; /* use current time for atime/mtime/ctime */ struct timespec tp = {0}; @@ -820,29 +850,7 @@ int unifyfs_set_global_file_meta(int fid, int gfid) fattr.gid = getgid(); /* submit file attributes to global key/value store */ - int ret = invoke_client_metaset_rpc(&fattr); - if (ret < 0) { - return ret; - } - - return UNIFYFS_SUCCESS; -} - -int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) -{ - /* check that we have an output buffer to write to */ - if (!gfattr) { - return UNIFYFS_FAILURE; - } - - /* attempt to lookup file attributes in key/value store */ - unifyfs_file_attr_t fmeta; - int ret = invoke_client_metaget_rpc(gfid, &fmeta); - if (ret == UNIFYFS_SUCCESS) { - /* found it, copy attributes to output struct */ - *gfattr = fmeta; - } - + int ret = unifyfs_set_global_file_meta(meta->gfid, &fattr); return ret; } @@ -1010,8 +1018,8 @@ int unifyfs_fid_create_directory(const char* path) meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; /* insert global meta data for directory */ - int ret = unifyfs_set_global_file_meta(fid, gfid); - if (ret) { + int ret = unifyfs_set_global_file_meta_from_fid(fid); + if (ret != UNIFYFS_SUCCESS) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); return (int) UNIFYFS_ERROR_IO; @@ -1336,8 +1344,8 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } /* insert file attribute for file in key-value store */ - ret = unifyfs_set_global_file_meta(fid, gfid); - if (ret) { + ret = unifyfs_set_global_file_meta_from_fid(fid); + if (ret != UNIFYFS_SUCCESS) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); return (int) UNIFYFS_ERROR_IO; From b1d862dfc432f3a1098a63c332cf3f66224066aa Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sun, 27 Oct 2019 18:14:25 -0700 Subject: [PATCH 035/168] client: style edits around margo calls --- client/src/margo_client.c | 216 ++++++++++++++++++++------------------ 1 file changed, 112 insertions(+), 104 deletions(-) diff --git a/client/src/margo_client.c b/client/src/margo_client.c index c5b7a6c54..bb197c938 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -63,24 +63,17 @@ static void register_client_rpcs(void) /* initialize margo client-server rpc */ int unifyfs_client_rpc_init(void) { - /* initialize margo */ hg_return_t hret; - char addr_self_string[128]; - hg_size_t addr_self_string_sz = sizeof(addr_self_string); - client_rpc_context_t* rpc_ctx; - - rpc_ctx = calloc(1, sizeof(client_rpc_context_t)); - if (NULL == rpc_ctx) { - LOGERR("Failed to allocate client RPC context"); - return UNIFYFS_FAILURE; - } - /* initialize margo */ + /* lookup margo server address string */ char* svr_addr_string = rpc_lookup_local_server_addr(); if (svr_addr_string == NULL) { LOGERR("Failed to find local margo RPC server address"); return UNIFYFS_FAILURE; } + + /* duplicate server address string, + * then parse address to pick out protocol portion */ char* proto = strdup(svr_addr_string); char* colon = strchr(proto, ':'); if (NULL != colon) { @@ -88,9 +81,21 @@ int unifyfs_client_rpc_init(void) } LOGDBG("svr_addr:'%s' proto:'%s'", svr_addr_string, proto); + /* allocate memory for rpc context struct */ + client_rpc_context_t* rpc_ctx = calloc(1, sizeof(client_rpc_context_t)); + if (NULL == rpc_ctx) { + LOGERR("Failed to allocate client RPC context"); + free(proto); + return UNIFYFS_FAILURE; + } + + /* initialize margo */ rpc_ctx->mid = margo_init(proto, MARGO_SERVER_MODE, 1, 1); assert(rpc_ctx->mid); + + /* done with the protocol string, free it */ free(proto); + margo_diag_start(rpc_ctx->mid); /* get server margo address */ @@ -113,6 +118,8 @@ int unifyfs_client_rpc_init(void) return UNIFYFS_FAILURE; } + char addr_self_string[128]; + hg_size_t addr_self_string_sz = sizeof(addr_self_string); hret = margo_addr_to_string(rpc_ctx->mid, addr_self_string, &addr_self_string_sz, rpc_ctx->client_addr); if (hret != HG_SUCCESS) { @@ -121,9 +128,12 @@ int unifyfs_client_rpc_init(void) free(rpc_ctx); return UNIFYFS_FAILURE; } + + /* make a copy of our own margo address string */ rpc_ctx->client_addr_str = strdup(addr_self_string); client_rpc_context = rpc_ctx; + register_client_rpcs(); return UNIFYFS_SUCCESS; @@ -159,35 +169,36 @@ int unifyfs_client_rpc_finalize(void) /* invokes the mount rpc function by calling unifyfs_sync_to_del */ int invoke_client_mount_rpc(void) { - hg_handle_t handle; - unifyfs_mount_in_t in; - unifyfs_mount_out_t out; - hg_return_t hret; - int32_t ret; - if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } - hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.mount_id, &handle); + /* get handle to rpc function */ + hg_handle_t handle; + hg_return_t hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.mount_id, &handle); assert(hret == HG_SUCCESS); /* fill in input struct */ + unifyfs_mount_in_t in; fill_client_mount_info(&in); in.client_addr_str = strdup(client_rpc_context->client_addr_str); + /* call rpc function */ LOGDBG("invoking the mount rpc function in client"); hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); + + /* free memory on input struct */ free((void*)in.external_spill_dir); free((void*)in.client_addr_str); /* decode response */ + unifyfs_mount_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); - ret = out.ret; + int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); unifyfs_key_slice_range = out.max_recs_per_slice; @@ -195,42 +206,42 @@ int invoke_client_mount_rpc(void) margo_free_output(handle, &out); margo_destroy(handle); - return ret; + return (int)ret; } /* function invokes the unmount rpc */ int invoke_client_unmount_rpc(void) { - hg_handle_t handle; - unifyfs_unmount_in_t in; - unifyfs_unmount_out_t out; - hg_return_t hret; - int32_t ret; - if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } - hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.unmount_id, - &handle); + /* get handle to rpc function */ + hg_handle_t handle; + hg_return_t hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.unmount_id, + &handle); assert(hret == HG_SUCCESS); /* fill in input struct */ - in.app_id = app_id; + unifyfs_unmount_in_t in; + in.app_id = app_id; in.local_rank_idx = local_rank_idx; + /* call rpc function */ LOGDBG("invoking the unmount rpc function in client"); hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ + unifyfs_unmount_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); - ret = out.ret; + int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); + /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); return (int)ret; @@ -239,44 +250,44 @@ int invoke_client_unmount_rpc(void) /* invokes the client metaset rpc function */ int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta) { - hg_handle_t handle; - unifyfs_metaset_in_t in; - unifyfs_metaset_out_t out; - hg_return_t hret; - int32_t ret; - if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } - hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.metaset_id, - &handle); + /* get handle to rpc function */ + hg_handle_t handle; + hg_return_t hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.metaset_id, + &handle); assert(hret == HG_SUCCESS); /* fill in input struct */ - in.gfid = f_meta->gfid; - in.filename = f_meta->filename; - in.mode = f_meta->mode; - in.uid = f_meta->uid; - in.gid = f_meta->gid; - in.size = f_meta->size; - in.atime = f_meta->atime; - in.mtime = f_meta->mtime; - in.ctime = f_meta->ctime; + unifyfs_metaset_in_t in; + in.gfid = f_meta->gfid; + in.filename = f_meta->filename; + in.mode = f_meta->mode; + in.uid = f_meta->uid; + in.gid = f_meta->gid; + in.size = f_meta->size; + in.atime = f_meta->atime; + in.mtime = f_meta->mtime; + in.ctime = f_meta->ctime; in.is_laminated = f_meta->is_laminated; + /* call rpc function */ LOGDBG("invoking the metaset rpc function in client"); hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ + unifyfs_metaset_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); - ret = out.ret; + int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); + /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); return (int)ret; @@ -286,32 +297,32 @@ int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta) int invoke_client_metaget_rpc(int gfid, unifyfs_file_attr_t* file_meta) { - hg_handle_t handle; - unifyfs_metaget_in_t in; - unifyfs_metaget_out_t out; - hg_return_t hret; - int32_t ret; - if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } - hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.metaget_id, - &handle); + /* get handle to rpc function */ + hg_handle_t handle; + hg_return_t hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.metaget_id, + &handle); assert(hret == HG_SUCCESS); /* fill in input struct */ + unifyfs_metaget_in_t in; in.gfid = (int32_t)gfid; + + /* call rpc function */ LOGDBG("invoking the metaget rpc function in client"); hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ + unifyfs_metaget_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); - ret = out.ret; + int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); if (ret == (int32_t)UNIFYFS_SUCCESS) { @@ -329,6 +340,7 @@ int invoke_client_metaget_rpc(int gfid, file_meta->is_laminated = out.is_laminated; } + /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); return (int)ret; @@ -337,37 +349,37 @@ int invoke_client_metaget_rpc(int gfid, /* invokes the client fsync rpc function */ int invoke_client_fsync_rpc(int gfid) { - hg_handle_t handle; - unifyfs_fsync_in_t in; - unifyfs_fsync_out_t out; - hg_return_t hret; - int32_t ret; - if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } - hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.fsync_id, - &handle); + /* get handle to rpc function */ + hg_handle_t handle; + hg_return_t hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.fsync_id, + &handle); assert(hret == HG_SUCCESS); /* fill in input struct */ + unifyfs_fsync_in_t in; in.app_id = (int32_t)app_id; in.local_rank_idx = (int32_t)local_rank_idx; in.gfid = (int32_t)gfid; + /* call rpc function */ LOGDBG("invoking the fsync rpc function in client"); hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ + unifyfs_fsync_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); - ret = out.ret; + int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); + /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); return (int)ret; @@ -377,14 +389,12 @@ int invoke_client_fsync_rpc(int gfid) int invoke_client_filesize_rpc(int gfid, size_t* outsize) { - int32_t ret; - hg_handle_t handle; - if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } /* get handle to rpc function */ + hg_handle_t handle; hg_return_t hret = margo_create(client_rpc_context->mid, client_rpc_context->svr_addr, client_rpc_context->rpcs.filesize_id, @@ -406,7 +416,7 @@ int invoke_client_filesize_rpc(int gfid, unifyfs_filesize_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); - ret = out.ret; + int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIu32, ret); /* save output from function */ @@ -423,40 +433,39 @@ int invoke_client_read_rpc(int gfid, size_t offset, size_t length) { - hg_handle_t handle; - unifyfs_read_in_t in; - unifyfs_read_out_t out; - hg_return_t hret; - int32_t ret; - if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } - /* fill in input struct */ - hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.read_id, - &handle); + /* get handle to rpc function */ + hg_handle_t handle; + hg_return_t hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.read_id, + &handle); assert(hret == HG_SUCCESS); /* fill in input struct */ + unifyfs_read_in_t in; in.app_id = (int32_t)app_id; in.local_rank_idx = (int32_t)local_rank_idx; in.gfid = (int32_t)gfid; in.offset = (hg_size_t)offset; in.length = (hg_size_t)length; + /* call rpc function */ LOGDBG("invoking the read rpc function in client"); hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ + unifyfs_read_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); - ret = out.ret; + int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); + /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); return (int)ret; @@ -467,23 +476,19 @@ int invoke_client_mread_rpc(int read_count, size_t size, void* buffer) { - hg_handle_t handle; - unifyfs_mread_in_t in; - unifyfs_mread_out_t out; - hg_return_t hret; - int32_t ret; - if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } - /* fill in input struct */ - hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.mread_id, - &handle); + /* get handle to rpc function */ + hg_handle_t handle; + hg_return_t hret = margo_create(client_rpc_context->mid, + client_rpc_context->svr_addr, + client_rpc_context->rpcs.mread_id, + &handle); assert(hret == HG_SUCCESS); + unifyfs_mread_in_t in; hret = margo_bulk_create(client_rpc_context->mid, 1, &buffer, &size, HG_BULK_READ_ONLY, &in.bulk_handle); assert(hret == HG_SUCCESS); @@ -494,16 +499,19 @@ int invoke_client_mread_rpc(int read_count, in.read_count = (int32_t)read_count; in.bulk_size = (hg_size_t)size; + /* call rpc function */ LOGDBG("invoking the read rpc function in client"); hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ + unifyfs_mread_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); - ret = out.ret; + int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); + /* free resources */ margo_bulk_free(in.bulk_handle); margo_free_output(handle, &out); margo_destroy(handle); From 48ed1c42ac59eda4b03fe134bacbc7332c9dc78a Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 28 Oct 2019 15:38:28 -0700 Subject: [PATCH 036/168] client: style edits to margo calls --- client/src/margo_client.c | 268 ++++++++++++++++++-------------------- client/src/margo_client.h | 24 ++-- 2 files changed, 139 insertions(+), 153 deletions(-) diff --git a/client/src/margo_client.c b/client/src/margo_client.c index bb197c938..a4e6fbf3a 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -7,57 +7,53 @@ #include "margo_client.h" /* global rpc context */ -client_rpc_context_t* client_rpc_context; // = NULL +static client_rpc_context_t* client_rpc_context; // = NULL /* register client RPCs */ -static void register_client_rpcs(void) +static void register_client_rpcs(client_rpc_context_t* ctx) { - margo_instance_id mid = client_rpc_context->mid; + /* shorter name for our margo instance id */ + margo_instance_id mid = ctx->mid; - client_rpc_context->rpcs.read_id = - MARGO_REGISTER(mid, "unifyfs_read_rpc", - unifyfs_read_in_t, - unifyfs_read_out_t, - NULL); - - client_rpc_context->rpcs.mread_id = - MARGO_REGISTER(mid, "unifyfs_mread_rpc", - unifyfs_mread_in_t, - unifyfs_mread_out_t, - NULL); - - client_rpc_context->rpcs.mount_id = - MARGO_REGISTER(mid, "unifyfs_mount_rpc", + ctx->rpcs.mount_id = MARGO_REGISTER(mid, "unifyfs_mount_rpc", unifyfs_mount_in_t, unifyfs_mount_out_t, NULL); - client_rpc_context->rpcs.unmount_id = - MARGO_REGISTER(mid, "unifyfs_unmount_rpc", + ctx->rpcs.unmount_id = MARGO_REGISTER(mid, "unifyfs_unmount_rpc", unifyfs_unmount_in_t, unifyfs_unmount_out_t, NULL); - client_rpc_context->rpcs.metaget_id = - MARGO_REGISTER(mid, "unifyfs_metaget_rpc", - unifyfs_metaget_in_t, unifyfs_metaget_out_t, - NULL); - - client_rpc_context->rpcs.metaset_id = - MARGO_REGISTER(mid, "unifyfs_metaset_rpc", - unifyfs_metaset_in_t, unifyfs_metaset_out_t, + ctx->rpcs.metaset_id = MARGO_REGISTER(mid, "unifyfs_metaset_rpc", + unifyfs_metaset_in_t, + unifyfs_metaset_out_t, NULL); - client_rpc_context->rpcs.fsync_id = - MARGO_REGISTER(mid, "unifyfs_fsync_rpc", - unifyfs_fsync_in_t, unifyfs_fsync_out_t, + ctx->rpcs.metaget_id = MARGO_REGISTER(mid, "unifyfs_metaget_rpc", + unifyfs_metaget_in_t, + unifyfs_metaget_out_t, NULL); - client_rpc_context->rpcs.filesize_id = - MARGO_REGISTER(mid, "unifyfs_filesize_rpc", + ctx->rpcs.filesize_id = MARGO_REGISTER(mid, "unifyfs_filesize_rpc", unifyfs_filesize_in_t, unifyfs_filesize_out_t, NULL); + + ctx->rpcs.fsync_id = MARGO_REGISTER(mid, "unifyfs_fsync_rpc", + unifyfs_fsync_in_t, + unifyfs_fsync_out_t, + NULL); + + ctx->rpcs.read_id = MARGO_REGISTER(mid, "unifyfs_read_rpc", + unifyfs_read_in_t, + unifyfs_read_out_t, + NULL); + + ctx->rpcs.mread_id = MARGO_REGISTER(mid, "unifyfs_mread_rpc", + unifyfs_mread_in_t, + unifyfs_mread_out_t, + NULL); } /* initialize margo client-server rpc */ @@ -65,7 +61,8 @@ int unifyfs_client_rpc_init(void) { hg_return_t hret; - /* lookup margo server address string */ + /* lookup margo server address string, + * should be something like: "na+sm://7170/0" */ char* svr_addr_string = rpc_lookup_local_server_addr(); if (svr_addr_string == NULL) { LOGERR("Failed to find local margo RPC server address"); @@ -73,7 +70,8 @@ int unifyfs_client_rpc_init(void) } /* duplicate server address string, - * then parse address to pick out protocol portion */ + * then parse address to pick out protocol portion + * which is the piece before the colon like: "na+sm" */ char* proto = strdup(svr_addr_string); char* colon = strchr(proto, ':'); if (NULL != colon) { @@ -82,59 +80,70 @@ int unifyfs_client_rpc_init(void) LOGDBG("svr_addr:'%s' proto:'%s'", svr_addr_string, proto); /* allocate memory for rpc context struct */ - client_rpc_context_t* rpc_ctx = calloc(1, sizeof(client_rpc_context_t)); - if (NULL == rpc_ctx) { + client_rpc_context_t* ctx = calloc(1, sizeof(client_rpc_context_t)); + if (NULL == ctx) { LOGERR("Failed to allocate client RPC context"); free(proto); + free(svr_addr_string); return UNIFYFS_FAILURE; } /* initialize margo */ - rpc_ctx->mid = margo_init(proto, MARGO_SERVER_MODE, 1, 1); - assert(rpc_ctx->mid); + ctx->mid = margo_init(proto, MARGO_SERVER_MODE, 1, 1); + assert(ctx->mid); - /* done with the protocol string, free it */ - free(proto); - - margo_diag_start(rpc_ctx->mid); + /* TODO: want to keep this enabled all the time */ + /* what's this do? */ + margo_diag_start(ctx->mid); /* get server margo address */ - rpc_ctx->svr_addr = HG_ADDR_NULL; - margo_addr_lookup(rpc_ctx->mid, svr_addr_string, - &(rpc_ctx->svr_addr)); + ctx->svr_addr = HG_ADDR_NULL; + margo_addr_lookup(ctx->mid, svr_addr_string, &(ctx->svr_addr)); + + /* done with the protocol and address strings, free them */ + free(proto); free(svr_addr_string); - if (rpc_ctx->svr_addr == HG_ADDR_NULL) { + + /* check that we got a valid margo address for the server */ + if (ctx->svr_addr == HG_ADDR_NULL) { LOGERR("Failed to resolve margo server RPC address"); - free(rpc_ctx); + margo_finalize(ctx->mid); + free(ctx); return UNIFYFS_FAILURE; } - /* get client margo address */ - hret = margo_addr_self(rpc_ctx->mid, &(rpc_ctx->client_addr)); + /* get our own margo address */ + hret = margo_addr_self(ctx->mid, &(ctx->client_addr)); if (hret != HG_SUCCESS) { - LOGERR("margo_addr_self()"); - margo_finalize(rpc_ctx->mid); - free(rpc_ctx); + LOGERR("Failed to acquire our margo address"); + margo_addr_free(ctx->mid, ctx->svr_addr); + margo_finalize(ctx->mid); + free(ctx); return UNIFYFS_FAILURE; } + /* convert our margo address to a string */ char addr_self_string[128]; hg_size_t addr_self_string_sz = sizeof(addr_self_string); - hret = margo_addr_to_string(rpc_ctx->mid, addr_self_string, - &addr_self_string_sz, rpc_ctx->client_addr); + hret = margo_addr_to_string(ctx->mid, + addr_self_string, &addr_self_string_sz, ctx->client_addr); if (hret != HG_SUCCESS) { - LOGERR("margo_addr_to_string()"); - margo_finalize(rpc_ctx->mid); - free(rpc_ctx); + LOGERR("Failed to convert our margo address to string"); + margo_addr_free(ctx->mid, ctx->client_addr); + margo_addr_free(ctx->mid, ctx->svr_addr); + margo_finalize(ctx->mid); + free(ctx); return UNIFYFS_FAILURE; } /* make a copy of our own margo address string */ - rpc_ctx->client_addr_str = strdup(addr_self_string); + ctx->client_addr_str = strdup(addr_self_string); - client_rpc_context = rpc_ctx; + /* look up and record id values for each rpc */ + register_client_rpcs(ctx); - register_client_rpcs(); + /* cache context in global variable */ + client_rpc_context = ctx; return UNIFYFS_SUCCESS; } @@ -163,31 +172,45 @@ int unifyfs_client_rpc_finalize(void) free(ctx->client_addr_str); free(ctx); } + return UNIFYFS_SUCCESS; } +/* create and return a margo handle for given rpc id */ +static hg_handle_t create_handle(hg_id_t id) +{ + /* define a temporary to refer to global context */ + client_rpc_context_t* ctx = client_rpc_context; + + /* create handle for specified rpc */ + hg_handle_t handle; + hg_return_t hret = margo_create(ctx->mid, ctx->svr_addr, id, &handle); + assert(hret == HG_SUCCESS); + + return handle; +} + /* invokes the mount rpc function by calling unifyfs_sync_to_del */ int invoke_client_mount_rpc(void) { + /* check that we have initialized margo */ if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } /* get handle to rpc function */ - hg_handle_t handle; - hg_return_t hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.mount_id, &handle); - assert(hret == HG_SUCCESS); + hg_handle_t handle = create_handle(client_rpc_context->rpcs.mount_id); /* fill in input struct */ unifyfs_mount_in_t in; fill_client_mount_info(&in); + + /* pass our margo address to the server */ in.client_addr_str = strdup(client_rpc_context->client_addr_str); /* call rpc function */ LOGDBG("invoking the mount rpc function in client"); - hret = margo_forward(handle, &in); + hg_return_t hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* free memory on input struct */ @@ -201,9 +224,11 @@ int invoke_client_mount_rpc(void) int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); + /* get slice size for write index key/value store */ unifyfs_key_slice_range = out.max_recs_per_slice; LOGDBG("set unifyfs_key_slice_range=%zu", unifyfs_key_slice_range); + /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); return (int)ret; @@ -212,17 +237,13 @@ int invoke_client_mount_rpc(void) /* function invokes the unmount rpc */ int invoke_client_unmount_rpc(void) { + /* check that we have initialized margo */ if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } /* get handle to rpc function */ - hg_handle_t handle; - hg_return_t hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.unmount_id, - &handle); - assert(hret == HG_SUCCESS); + hg_handle_t handle = create_handle(client_rpc_context->rpcs.unmount_id); /* fill in input struct */ unifyfs_unmount_in_t in; @@ -231,7 +252,7 @@ int invoke_client_unmount_rpc(void) /* call rpc function */ LOGDBG("invoking the unmount rpc function in client"); - hret = margo_forward(handle, &in); + hg_return_t hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ @@ -250,17 +271,13 @@ int invoke_client_unmount_rpc(void) /* invokes the client metaset rpc function */ int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta) { + /* check that we have initialized margo */ if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } /* get handle to rpc function */ - hg_handle_t handle; - hg_return_t hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.metaset_id, - &handle); - assert(hret == HG_SUCCESS); + hg_handle_t handle = create_handle(client_rpc_context->rpcs.metaset_id); /* fill in input struct */ unifyfs_metaset_in_t in; @@ -277,7 +294,7 @@ int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta) /* call rpc function */ LOGDBG("invoking the metaset rpc function in client"); - hret = margo_forward(handle, &in); + hg_return_t hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ @@ -294,20 +311,15 @@ int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta) } /* invokes the client metaget rpc function */ -int invoke_client_metaget_rpc(int gfid, - unifyfs_file_attr_t* file_meta) +int invoke_client_metaget_rpc(int gfid, unifyfs_file_attr_t* file_meta) { + /* check that we have initialized margo */ if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } /* get handle to rpc function */ - hg_handle_t handle; - hg_return_t hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.metaget_id, - &handle); - assert(hret == HG_SUCCESS); + hg_handle_t handle = create_handle(client_rpc_context->rpcs.metaget_id); /* fill in input struct */ unifyfs_metaget_in_t in; @@ -315,7 +327,7 @@ int invoke_client_metaget_rpc(int gfid, /* call rpc function */ LOGDBG("invoking the metaget rpc function in client"); - hret = margo_forward(handle, &in); + hg_return_t hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ @@ -346,82 +358,73 @@ int invoke_client_metaget_rpc(int gfid, return (int)ret; } -/* invokes the client fsync rpc function */ -int invoke_client_fsync_rpc(int gfid) +/* invokes the client filesize rpc function */ +int invoke_client_filesize_rpc(int gfid, size_t* outsize) { + /* check that we have initialized margo */ if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } /* get handle to rpc function */ - hg_handle_t handle; - hg_return_t hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.fsync_id, - &handle); - assert(hret == HG_SUCCESS); + hg_handle_t handle = create_handle(client_rpc_context->rpcs.filesize_id); /* fill in input struct */ - unifyfs_fsync_in_t in; + unifyfs_filesize_in_t in; in.app_id = (int32_t)app_id; in.local_rank_idx = (int32_t)local_rank_idx; in.gfid = (int32_t)gfid; /* call rpc function */ - LOGDBG("invoking the fsync rpc function in client"); - hret = margo_forward(handle, &in); + LOGDBG("invoking the filesize rpc function in client"); + hg_return_t hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ - unifyfs_fsync_out_t out; + unifyfs_filesize_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); + /* save output from function */ + *outsize = (size_t) out.filesize; + /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); return (int)ret; } -/* invokes the client filesize rpc function */ -int invoke_client_filesize_rpc(int gfid, - size_t* outsize) +/* invokes the client fsync rpc function */ +int invoke_client_fsync_rpc(int gfid) { + /* check that we have initialized margo */ if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } /* get handle to rpc function */ - hg_handle_t handle; - hg_return_t hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.filesize_id, - &handle); - assert(hret == HG_SUCCESS); + hg_handle_t handle = create_handle(client_rpc_context->rpcs.fsync_id); /* fill in input struct */ - unifyfs_filesize_in_t in; + unifyfs_fsync_in_t in; in.app_id = (int32_t)app_id; in.local_rank_idx = (int32_t)local_rank_idx; in.gfid = (int32_t)gfid; /* call rpc function */ - LOGDBG("invoking the filesize rpc function in client"); - hret = margo_forward(handle, &in); + LOGDBG("invoking the fsync rpc function in client"); + hg_return_t hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ - unifyfs_filesize_out_t out; + unifyfs_fsync_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIu32, ret); - /* save output from function */ - *outsize = (size_t) out.filesize; - /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); @@ -429,21 +432,15 @@ int invoke_client_filesize_rpc(int gfid, } /* invokes the client read rpc function */ -int invoke_client_read_rpc(int gfid, - size_t offset, - size_t length) +int invoke_client_read_rpc(int gfid, size_t offset, size_t length) { + /* check that we have initialized margo */ if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } /* get handle to rpc function */ - hg_handle_t handle; - hg_return_t hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.read_id, - &handle); - assert(hret == HG_SUCCESS); + hg_handle_t handle = create_handle(client_rpc_context->rpcs.read_id); /* fill in input struct */ unifyfs_read_in_t in; @@ -455,7 +452,7 @@ int invoke_client_read_rpc(int gfid, /* call rpc function */ LOGDBG("invoking the read rpc function in client"); - hret = margo_forward(handle, &in); + hg_return_t hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ @@ -472,25 +469,20 @@ int invoke_client_read_rpc(int gfid, } /* invokes the client mread rpc function */ -int invoke_client_mread_rpc(int read_count, - size_t size, - void* buffer) +int invoke_client_mread_rpc(int read_count, size_t size, void* buffer) { + /* check that we have initialized margo */ if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } /* get handle to rpc function */ - hg_handle_t handle; - hg_return_t hret = margo_create(client_rpc_context->mid, - client_rpc_context->svr_addr, - client_rpc_context->rpcs.mread_id, - &handle); - assert(hret == HG_SUCCESS); + hg_handle_t handle = create_handle(client_rpc_context->rpcs.mread_id); unifyfs_mread_in_t in; - hret = margo_bulk_create(client_rpc_context->mid, 1, &buffer, &size, - HG_BULK_READ_ONLY, &in.bulk_handle); + hg_return_t hret = margo_bulk_create( + client_rpc_context->mid, 1, &buffer, &size, + HG_BULK_READ_ONLY, &in.bulk_handle); assert(hret == HG_SUCCESS); /* fill in input struct */ diff --git a/client/src/margo_client.h b/client/src/margo_client.h index d4b0861c7..6fc76fb65 100644 --- a/client/src/margo_client.h +++ b/client/src/margo_client.h @@ -10,14 +10,14 @@ #include "unifyfs_client_rpcs.h" typedef struct ClientRpcIds { - hg_id_t filesize_id; - hg_id_t read_id; - hg_id_t mread_id; hg_id_t mount_id; hg_id_t unmount_id; - hg_id_t metaget_id; hg_id_t metaset_id; + hg_id_t metaget_id; + hg_id_t filesize_id; hg_id_t fsync_id; + hg_id_t read_id; + hg_id_t mread_id; } client_rpcs_t; typedef struct ClientRpcContext { @@ -41,20 +41,14 @@ int invoke_client_unmount_rpc(void); int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta); -int invoke_client_metaget_rpc(int gfid, - unifyfs_file_attr_t* f_meta); +int invoke_client_metaget_rpc(int gfid, unifyfs_file_attr_t* f_meta); -int invoke_client_fsync_rpc(int gfid); +int invoke_client_filesize_rpc(int gfid, size_t* filesize); -int invoke_client_filesize_rpc(int gfid, - size_t* filesize); +int invoke_client_fsync_rpc(int gfid); -int invoke_client_read_rpc(int gfid, - size_t offset, - size_t length); +int invoke_client_read_rpc(int gfid, size_t offset, size_t length); -int invoke_client_mread_rpc(int read_count, - size_t size, - void* buffer); +int invoke_client_mread_rpc(int read_count, size_t size, void* buffer); #endif // MARGO_CLIENT_H From b1e2876e170d78fa377a685942f2d7669acb6cbb Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 30 Oct 2019 20:26:35 -0700 Subject: [PATCH 037/168] client: change fid to gfid in read request structure --- client/src/unifyfs-internal.h | 2 +- client/src/unifyfs-sysio.c | 37 ++++++++++++----------------------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index ceec8a801..14eee542f 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -298,7 +298,7 @@ typedef struct { /*unifyfs structures*/ typedef struct { - int fid; + int gfid; int errcode; size_t offset; size_t length; diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 3722ab82b..0911f48cd 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -548,7 +548,7 @@ ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) } read_req_t tmp_req; - tmp_req.fid = fid; + tmp_req.gfid = unifyfs_gfid_from_fid(fid); tmp_req.offset = (size_t) pos; tmp_req.length = count; tmp_req.errcode = UNIFYFS_SUCCESS; @@ -1138,7 +1138,7 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], if (fid < 0) { AIOCB_ERROR_CODE(cbp) = EINVAL; } else { - reqs[reqcnt].fid = fid; + reqs[reqcnt].gfid = unifyfs_gfid_from_fid(fid); reqs[reqcnt].offset = (size_t)(cbp->aio_offset); reqs[reqcnt].length = cbp->aio_nbytes; reqs[reqcnt].errcode = EINPROGRESS; @@ -1224,8 +1224,8 @@ static int compare_read_req(const void* a, const void* b) const read_req_t* ptr_a = a; const read_req_t* ptr_b = b; - if (ptr_a->fid != ptr_b->fid) { - if (ptr_a->fid < ptr_b->fid) { + if (ptr_a->gfid != ptr_b->gfid) { + if (ptr_a->gfid < ptr_b->gfid) { return -1; } else { return 1; @@ -1370,7 +1370,7 @@ static int unifyfs_split_read_requests(read_req_t* req, } /* full slice is contained in read request */ - out_set->read_reqs[count].fid = req->fid; + out_set->read_reqs[count].gfid = req->gfid; out_set->read_reqs[count].offset = slice_start; out_set->read_reqs[count].length = slice_range; out_set->read_reqs[count].errcode = UNIFYFS_SUCCESS; @@ -1378,7 +1378,7 @@ static int unifyfs_split_read_requests(read_req_t* req, } while (1); /* account for bytes in final slice */ - out_set->read_reqs[count].fid = req->fid; + out_set->read_reqs[count].gfid = req->gfid; out_set->read_reqs[count].offset = slice_start; out_set->read_reqs[count].length = req_end - slice_start + 1; out_set->read_reqs[count].errcode = UNIFYFS_SUCCESS; @@ -1439,7 +1439,7 @@ static int unifyfs_coalesce_read_reqs(read_req_t* read_req, int count, read_req_t* tmp_req = &(tmp_set.read_reqs[0]); /* look to merge these items if they are contiguous */ - if (out_req->fid == tmp_req->fid && + if (out_req->gfid == tmp_req->gfid && out_req->offset + out_req->length == tmp_req->offset) { /* refers to contiguous range in the same file, * coalesce if also in the same slice */ @@ -1720,13 +1720,13 @@ static int process_read_data(read_req_t* read_reqs, int count, int* done) /* define request object */ read_req_t req; - req.fid = msg->gfid; + req.gfid = msg->gfid; req.offset = msg->offset; req.length = msg->length; req.errcode = msg->errcode; LOGDBG("read reply: gfid=%d offset=%zu size=%zu", - req.fid, req.offset, req.length); + req.gfid, req.offset, req.length); /* get pointer to data */ req.buf = shmptr; @@ -1792,19 +1792,6 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) * sends and transfer in bulks * */ - /* convert local fid to global fid */ - for (i = 0; i < count; i++) { - /* get global file id for each request */ - int gfid = unifyfs_gfid_from_fid(read_reqs[i].fid); - if (gfid != -1) { - /* replace local file id with global file id in request */ - read_reqs[i].fid = gfid; - } else { - /* failed to find gfid for this request */ - return UNIFYFS_ERROR_BADF; - } - } - /* order read request by increasing file id, then increasing offset */ qsort(read_reqs, count, sizeof(read_req_t), compare_read_req); @@ -1830,7 +1817,7 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) /* fill in values for each request entry */ for (i = 0; i < read_req_set.count; i++) { unifyfs_Extent_vec_push_create(&builder, - read_req_set.read_reqs[i].fid, + read_req_set.read_reqs[i].gfid, read_req_set.read_reqs[i].offset, read_req_set.read_reqs[i].length); } @@ -1854,7 +1841,7 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) free(buffer); } else { /* got a single read request */ - int gfid = read_req_set.read_reqs[0].fid; + int gfid = read_req_set.read_reqs[0].gfid; size_t offset = read_req_set.read_reqs[0].offset; size_t length = read_req_set.read_reqs[0].length; LOGDBG("read: offset:%zu, len:%zu", offset, length); @@ -1923,7 +1910,7 @@ ssize_t UNIFYFS_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) size_t retcount = count; read_req_t tmp_req; - tmp_req.fid = fid; + tmp_req.gfid = unifyfs_gfid_from_fid(fid); tmp_req.offset = offset; tmp_req.length = count; tmp_req.errcode = UNIFYFS_SUCCESS; From 75d9713c6b37f3bfcaa4c34090c0f90584027c76 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 30 Oct 2019 20:43:40 -0700 Subject: [PATCH 038/168] client: style edits to chmod wrappers --- client/src/unifyfs-sysio.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 0911f48cd..f3d6196e9 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -2300,14 +2300,13 @@ int UNIFYFS_WRAP(close)(int fd) /* Helper function used by fchmod() and chmod() */ static int __chmod(int fid, mode_t mode) { - int gfid; - unifyfs_filemeta_t* meta; - const char* path; int ret; - path = unifyfs_path_from_fid(fid); + /* get path for printing debug messages */ + const char* path = unifyfs_path_from_fid(fid); - meta = unifyfs_get_meta_from_fid(fid); + /* lookup metadata for this file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if (!meta) { LOGDBG("chmod: %s no metadata info", path); errno = ENOENT; @@ -2321,7 +2320,12 @@ static int __chmod(int fid, mode_t mode) return -1; } - gfid = unifyfs_generate_gfid(path); + /* found file, and it's not yet laminated, + * get the global file id */ + int gfid = unifyfs_gfid_from_fid(fid); + + /* TODO: need to fetch global metadata in case + * another process has changed it */ /* * If the chmod clears all the existing write bits, then it's a laminate. @@ -2342,6 +2346,8 @@ static int __chmod(int fid, mode_t mode) errno = EIO; return -1; } + + /* record locally that this file is now laminated */ meta->is_laminated = 1; } @@ -2349,6 +2355,8 @@ static int __chmod(int fid, mode_t mode) meta->mode = meta->mode & ~0777; meta->mode = meta->mode | mode; + /* update the global meta data to reflect new permissions, + * size, and laminated flag */ ret = unifyfs_set_global_file_meta_from_fid(fid); if (ret) { LOGERR("chmod: can't set global meta entry for %s (fid:%d)", @@ -2356,15 +2364,14 @@ static int __chmod(int fid, mode_t mode) errno = EIO; return -1; } + return 0; } int UNIFYFS_WRAP(fchmod)(int fd, mode_t mode) { /* check whether we should intercept this file descriptor */ - int origfd = fd; if (unifyfs_intercept_fd(&fd)) { - /* TODO: what to do if underlying file has been deleted? */ /* check that fd is actually in use */ @@ -2373,9 +2380,9 @@ int UNIFYFS_WRAP(fchmod)(int fd, mode_t mode) errno = EBADF; return -1; } + LOGDBG("fchmod: setting fd %d to %o", fd, mode); return __chmod(fid, mode); - } else { MAP_OR_FAIL(fchmod); int ret = UNIFYFS_REAL(fchmod)(fd, mode); @@ -2383,16 +2390,12 @@ int UNIFYFS_WRAP(fchmod)(int fd, mode_t mode) } } - int UNIFYFS_WRAP(chmod)(const char* path, mode_t mode) { - int fid, gfid; - int ret; - unifyfs_filemeta_t* meta; /* determine whether we should intercept this path */ if (unifyfs_intercept_path(path)) { /* check if path exists */ - fid = unifyfs_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(path); if (fid < 0) { LOGDBG("chmod: unifyfs_get_id_from path failed, returning -1, %s", path); From 693b5d10d0743c934b0b0a9920afc178d3cd5c25 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Wed, 30 Oct 2019 10:22:21 -0700 Subject: [PATCH 039/168] Put local and log sizes in st_rdev, update bootstrap.sh - Previously we stored the local size in stat.st_rdev and the log size in stat.st_dev. However, HDF5 expects sane values in st_dev, so move log size into the upper 32-bits of st_rdev. Fixes: #396 - Update bootstrap.sh to export LEVELDB_ROOT and FLATCC_ROOT instead of having them as --with-leveldb and --with-flatcc, in accordance with #390. Also add --prefix line to point to install dir. --- bootstrap.sh | 9 +++++---- client/src/unifyfs-sysio.c | 14 +++++++------- t/std/size.c | 4 ++-- t/sys/write-read.c | 4 ++-- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index ff66a2e99..dd046ea57 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -104,11 +104,12 @@ cd .. cd "$ROOT" echo "*************************************************************************" -echo "Dependencies are all built. You can now build Unify with:" +echo "Dependencies are all built. You can now build UnifyFS with:" echo "" -echo " export PKG_CONFIG_PATH=$INSTALL_DIR/lib/pkgconfig" -echo " ./autogen.sh && ./configure --with-leveldb=$INSTALL_DIR" \ - "--with-gotcha=$INSTALL_DIR --with-flatcc=$INSTALL_DIR" +echo -n " export PKG_CONFIG_PATH=$INSTALL_DIR/lib/pkgconfig && " +echo "export LEVELDB_ROOT=$INSTALL_DIR && export FLATCC_ROOT=$INSTALL_DIR" +echo -n " ./autogen.sh && ./configure --with-gotcha=$INSTALL_DIR" +echo " --prefix=$INSTALL_DIR" echo " make" echo "" echo "*************************************************************************" diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index f3d6196e9..db2604eed 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -364,17 +364,17 @@ static int __stat(const char* path, struct stat* buf) unifyfs_file_attr_to_stat(&fattr, buf); /* - * For debugging purposes, we hijack st_rdev to store our local, - * non-laminated file size. Note: st_rdev is only a 32-bit number, - * so don't depend on it if the file is really big (we use the - * lower 32-bits of the size). + * For debugging and testing purposes, we hijack st_rdev to store our + * local size and log size. We also assume the stat struct is + * the 64-bit variant. The values are stored as: + * + * st_rdev = log_size << 32 | local_size; * - * We also hijack st_dev to store our log_size for debugging. */ buf->st_rdev = fid; if (fid >= 0) { /* If we have a local file */ - buf->st_rdev = unifyfs_fid_local_size(fid) & 0xFFFFFFFF; - buf->st_dev = unifyfs_fid_log_size(fid) & 0xFFFFFFFF; + buf->st_rdev = (unifyfs_fid_log_size(fid) << 32) | + (unifyfs_fid_local_size(fid) & 0xFFFFFFFF); } if (!fattr.is_laminated) { diff --git a/t/std/size.c b/t/std/size.c index 7140121c4..2367a510b 100644 --- a/t/std/size.c +++ b/t/std/size.c @@ -46,11 +46,11 @@ void get_size(char* path, size_t* global, size_t* local, size_t* log) } if (local) { - *local = sb.st_rdev; + *local = sb.st_rdev & 0xFFFFFFFF; } if (log) { - *log = sb.st_dev; + *log = (sb.st_rdev >> 32) & 0xFFFFFFFF; } } diff --git a/t/sys/write-read.c b/t/sys/write-read.c index 38e77d30f..c738f2c70 100644 --- a/t/sys/write-read.c +++ b/t/sys/write-read.c @@ -41,11 +41,11 @@ void get_size(char* path, size_t* global, size_t* local, size_t* log) } if (local) { - *local = sb.st_rdev; + *local = sb.st_rdev & 0xFFFFFFFF; } if (log) { - *log = sb.st_dev; + *log = (sb.st_rdev >> 32) & 0xFFFFFFFF; } } From a58c74c0e54911db4063e464526a8ddb7eb9ca8d Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sat, 2 Nov 2019 12:13:01 -0700 Subject: [PATCH 040/168] client: check for overflow when splitting read requests --- client/src/unifyfs-internal.h | 5 - client/src/unifyfs-sysio.c | 299 ++++++++++++++++------------------ 2 files changed, 139 insertions(+), 165 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 14eee542f..6c41fe054 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -310,11 +310,6 @@ typedef struct { unifyfs_index_t* index_entry; } unifyfs_index_buf_t; -typedef struct { - read_req_t read_reqs[UNIFYFS_MAX_READ_CNT]; - int count; -} read_req_set_t; - extern unifyfs_index_buf_t unifyfs_indices; extern unsigned long unifyfs_max_index_entries; extern long unifyfs_spillover_max_chunks; diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index db2604eed..c4b2db2f8 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -1312,158 +1312,141 @@ static int unifyfs_locate_req(read_req_t* read_reqs, int count, } } -/* - * given an read request, split it into multiple indices whose range +/* given a read request, split it into multiple requests whose range * is equal or smaller than slice_range size - * @param cur_read_req: the read request to split - * @param slice_range: the slice size of the key-value store - * @return out_set: the set of split read requests - * */ -static int unifyfs_split_read_requests(read_req_t* req, - read_req_set_t* out_set, - size_t slice_range) + * + * @param req: read request to be split + * @param slice_range: slice size of the key-value store + * @return out_set: output set of split read requests + * @param maxcount: number of entries in output array + * @return used_count: number of entries added to output array */ +static int unifyfs_split_read_request( + read_req_t* in_req, /* read request to split */ + long slice_range, /* number of bytes in each slice */ + read_req_t* out_set, /* output array to store new requests in */ + off_t maxcount, /* max number of items in output array */ + off_t* used_count) /* number of entries we added in split */ { - /* compute offset of first and last byte in request */ + /* check that we have at least one spot in buffer */ + if (maxcount == 0) { + *used_count = 0; + return UNIFYFS_FAILURE; + } + + /* make a copy of the input request so we can modify it */ + read_req_t tmp_req = *in_req; + read_req_t* req = &tmp_req; + + /* first byte offset this request will read from */ size_t req_start = req->offset; - size_t req_end = req->offset + req->length - 1; + + /* last byte offset this request will read from */ + size_t req_end = req->offset + req->length - 1; /* compute offset of first and last byte of slice * that contains first byte of request */ + + /* starting byte offset of slice that first read offset falls in */ size_t slice_start = (req->offset / slice_range) * slice_range; - size_t slice_end = slice_start + slice_range - 1; + + /* last byte offset of slice that first read offset falls in */ + size_t slice_end = slice_start + slice_range - 1; /* initialize request count in output set */ - memset(out_set, 0, sizeof(read_req_set_t)); int count = 0; + /* define new read requests in out_set by splitting request + * at slice boundaries */ if (req_end <= slice_end) { /* slice fully contains request * - * slice_start slice_end - * req_start req_end - * + * slice_start slice_end + * req_start req_end */ - out_set->read_reqs[count] = *req; + out_set[count] = *req; count++; } else { - /* read request spans multiple slices + /* ending offset of request is beyond last offset in first slice, + * so this request spans across multiple slices * - * slice_start slice_end next_slice_start next_slice_end - * req_start req_end + * slice_start slice_end next_slice_start next_slice_end + * req_start req_end * */ - /* account for leading bytes in read request in first slice */ - out_set->read_reqs[count] = *req; - out_set->read_reqs[count].length = slice_end - req_start + 1; - count++; + /* compute number of bytes until end of first slice */ + long length = slice_end - req_start + 1; - /* account for all middle slices */ - do { - /* advance to next slice */ - slice_start = slice_end + 1; - slice_end = slice_start + slice_range - 1; + out_set[count].gfid = req->gfid; + out_set[count].offset = req->offset; + out_set[count].length = length; + out_set[count].errcode = req->errcode; + count++; - if (req_end <= slice_end) { - /* found the slice that contains end byte in read request */ - break; - } + /* update write index to account for index we just added */ + req->offset += length; + req->length -= length; - /* full slice is contained in read request */ - out_set->read_reqs[count].gfid = req->gfid; - out_set->read_reqs[count].offset = slice_start; - out_set->read_reqs[count].length = slice_range; - out_set->read_reqs[count].errcode = UNIFYFS_SUCCESS; - count++; - } while (1); - - /* account for bytes in final slice */ - out_set->read_reqs[count].gfid = req->gfid; - out_set->read_reqs[count].offset = slice_start; - out_set->read_reqs[count].length = req_end - slice_start + 1; - out_set->read_reqs[count].errcode = UNIFYFS_SUCCESS; - count++; - } + /* check that we have room to write more requests */ + if (count >= maxcount) { + /* no room to write more requests, + * and we have at least one more, + * record number we wrote and return with error */ + *used_count = count; + return UNIFYFS_FAILURE; + } - /* set size of output set */ - out_set->count = count; + /* advance slice boundary offsets to next slice */ + slice_end += slice_range; - return 0; -} + /* loop until we find the slice that contains + * ending offset of read */ + while (req_end > slice_end) { + /* ending offset of read is beyond end of this slice, + * so read spans the full length of this slice */ + length = slice_range; -/* - * coalesce read requests referring to contiguous data within a given - * file id, and split read requests whose size is larger than - * unifyfs_key_slice_range into more requests that are smaller - * - * Note: a series of read requests that have overlapping spans - * will prevent merging of contiguous ranges, this should still - * function, but performance may be lost - * - * @param read_req: a list of read requests - * @param count: number of read requests - * @param tmp_set: a temporary read requests buffer - * to hold the intermediate result - * @param unifyfs_key_slice_range: slice size of distributed - * key-value store - * @return out_set: the coalesced read requests - * - * */ -static int unifyfs_coalesce_read_reqs(read_req_t* read_req, int count, - size_t slice_range, - read_req_set_t* out_set) -{ - read_req_set_t tmp_set; + /* full slice is contained in read request */ + out_set[count].gfid = req->gfid; + out_set[count].offset = req->offset; + out_set[count].length = length; + out_set[count].errcode = req->errcode; + count++; - /* initialize output and temporary sets */ - out_set->count = 0; - memset(&tmp_set, 0, sizeof(tmp_set)); + /* update read request to account for index we just added */ + req->offset += length; + req->length -= length; - int i; - int out_idx = 0; - for (i = 0; i < count; i++) { - /* index into temp set */ - int tmp_idx = 0; - - /* split this read request into parts based on slice range - * store resulting requests in tmp_set */ - unifyfs_split_read_requests(&read_req[i], &tmp_set, slice_range); - - /* look to merge last item in output set with first item - * in split requests */ - if (out_idx > 0) { - /* get pointer to last item in out_set */ - read_req_t* out_req = &(out_set->read_reqs[out_idx - 1]); - - /* get pointer to first item in tmp_set */ - read_req_t* tmp_req = &(tmp_set.read_reqs[0]); - - /* look to merge these items if they are contiguous */ - if (out_req->gfid == tmp_req->gfid && - out_req->offset + out_req->length == tmp_req->offset) { - /* refers to contiguous range in the same file, - * coalesce if also in the same slice */ - uint64_t cur_slice = out_req->offset / slice_range; - uint64_t tmp_slice = tmp_req->offset / slice_range; - if (cur_slice == tmp_slice) { - /* just increase length to merge */ - out_req->length += tmp_req->length; - - /* bump offset into tmp set array */ - tmp_idx++; - } + /* check that we have room to write more requests */ + if (count >= maxcount) { + /* no room to write more requests, + * and we have at least one more, + * record number we wrote and return with error */ + *used_count = count; + return UNIFYFS_FAILURE; } - } - /* tack on remaining items from tmp set into output set */ - for (; tmp_idx < tmp_set.count; tmp_idx++) { - out_set->read_reqs[out_idx] = tmp_set.read_reqs[tmp_idx]; - out_set->count++; - out_idx++; + /* advance slice boundary offsets to next slice */ + slice_end += slice_range; } + + /* this slice contains the remainder of read */ + length = req->length; + out_set[count].gfid = req->gfid; + out_set[count].offset = req->offset; + out_set[count].length = length; + out_set[count].errcode = req->errcode; + count++; + + /* update read request to account for index we just added */ + req->offset += length; + req->length -= length; } - return 0; + /* record number of entries we added */ + *used_count = count; + + return UNIFYFS_SUCCESS; } /* @@ -1758,33 +1741,9 @@ static int process_read_data(read_req_t* read_reqs, int count, int* done) * */ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) { - int i; - int tot_sz = 0; - int rc = UNIFYFS_SUCCESS; - int num = 0; - int* ptr_size = NULL; - int* ptr_num = NULL; - -#if 0 /* TODO: when meta has correct file size, we can use this code */ - /* Adjust length for fitting the EOF. */ - for (i = 0; i < count; i++) { - /* get pointer to read request */ - read_req_t* req = &read_reqs[i]; - - /* get metadata for this file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(req->fid); - if (meta == NULL) { - return UNIFYFS_ERROR_BADF; - } + int read_rc; - /* compute last byte of read request */ - size_t last_offset = req->offset + req->length; - if (last_offset > meta->size) { - /* shorten the request to read just up to end */ - req->length = meta->size - req->offset; - } - } -#endif + int rc = UNIFYFS_SUCCESS; /* * Todo: When the number of read requests exceed the @@ -1795,17 +1754,36 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) /* order read request by increasing file id, then increasing offset */ qsort(read_reqs, count, sizeof(read_req_t), compare_read_req); - /* coalesce the contiguous read requests */ - read_req_set_t read_req_set; - unifyfs_coalesce_read_reqs(read_reqs, count, - unifyfs_key_slice_range, - &read_req_set); + /* TODO: move this split code to server and then pass original + * read requests from client to server */ + /* split read requests at file offset boundaries used internally + * in the server key/value store */ + int i; + off_t req_count = 0; + read_req_t read_set[UNIFYFS_MAX_READ_CNT]; + for (i = 0; i < count; i++) { + /* remaining entries we have in our read requests array */ + off_t remaining = UNIFYFS_MAX_READ_CNT - req_count; + + /* split current request at key/value offsets */ + off_t used = 0; + read_rc = unifyfs_split_read_request(&read_reqs[i], + unifyfs_key_slice_range, &read_set[req_count], remaining, &used); + + /* bail out with error if we failed to process read requests */ + if (read_rc != UNIFYFS_SUCCESS) { + LOGERR("Failed to split read requests"); + return read_rc; + } + + /* account of request slots we used up */ + req_count += used; + } /* prepare our shared memory buffer for delegator */ delegator_signal(); - int read_rc; - if (read_req_set.count > 1) { + if (req_count > 1) { /* got multiple read requests, * build up a flat buffer to include them all */ flatcc_builder_t builder; @@ -1815,11 +1793,11 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) unifyfs_Extent_vec_start(&builder); /* fill in values for each request entry */ - for (i = 0; i < read_req_set.count; i++) { + for (i = 0; i < req_count; i++) { unifyfs_Extent_vec_push_create(&builder, - read_req_set.read_reqs[i].gfid, - read_req_set.read_reqs[i].offset, - read_req_set.read_reqs[i].length); + read_set[i].gfid, + read_set[i].offset, + read_set[i].length); } /* complete the array */ @@ -1832,24 +1810,25 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) void* buffer = flatcc_builder_finalize_buffer(&builder, &size); assert(buffer); LOGDBG("mread: n_reqs:%d, flatcc buffer (%p) sz:%zu", - read_req_set.count, buffer, size); + req_count, buffer, size); /* invoke read rpc here */ - read_rc = invoke_client_mread_rpc(read_req_set.count, size, buffer); + read_rc = invoke_client_mread_rpc(req_count, size, buffer); flatcc_builder_clear(&builder); free(buffer); } else { /* got a single read request */ - int gfid = read_req_set.read_reqs[0].gfid; - size_t offset = read_req_set.read_reqs[0].offset; - size_t length = read_req_set.read_reqs[0].length; + int gfid = read_set[0].gfid; + size_t offset = read_set[0].offset; + size_t length = read_set[0].length; LOGDBG("read: offset:%zu, len:%zu", offset, length); read_rc = invoke_client_read_rpc(gfid, offset, length); } /* bail out with error if we failed to even start the read */ if (read_rc != UNIFYFS_SUCCESS) { + LOGERR("Failed to issue read RPC to server"); return read_rc; } From 66448e0d791362bc303a31a07537d03c5a1c3635 Mon Sep 17 00:00:00 2001 From: CamStan Date: Mon, 4 Nov 2019 13:59:23 -0800 Subject: [PATCH 041/168] Update Travis CI for recent Spack changes Set the target architecture and update how the module environment is set up to accommodate for recent changes to Spack. --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f9de90d0c..4db600ed1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,8 @@ before_install: - | test -f $HOME/spack/etc/spack/packages.yaml || cat > $HOME/spack/etc/spack/packages.yaml << ' EOF' packages: + all: + target: [x86_64] autoconf: buildable: False paths: @@ -49,14 +51,13 @@ before_install: EOF install: - - $HOME/spack/bin/spack install environment-modules + - $HOME/spack/bin/spack bootstrap - . $HOME/spack/share/spack/setup-env.sh - spack install leveldb - spack install gotcha@0.0.2 - spack install flatcc - spack install margo^mercury+bmi~boostsys # prepare build environment - - spack load environment-modules - source <(spack module tcl loads leveldb gotcha@0.0.2 flatcc mercury argobots margo) - eval $(./scripts/git_log_test_env.sh) - export TEST_CHECKPATCH_SKIP_FILES From 34e9fb600eb99ebb684c69169749302dc86248b9 Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Mon, 11 Nov 2019 17:01:59 -0500 Subject: [PATCH 042/168] fix EOF for remote reads after laminate This fixes the writeread example in io_shuffle mode, where each process reads different data than it wrote. Previously, we were seeing EOFs on reads with offsets that exceeded the locally cached file size, which was based on what was written. The cached file size was being used because processes other than rank 0 did not mark the file as laminated. The fix was to add a stat() call for the non-zero ranks, and to make sure that the processing of stat() caused the local metadata to be correctly updated, which wasn't the case before. --- client/src/unifyfs-internal.h | 7 +- client/src/unifyfs-sysio.c | 45 ++++++------ client/src/unifyfs.c | 132 +++++++++++++++------------------- examples/src/testutil_rdwr.h | 17 ++++- 4 files changed, 98 insertions(+), 103 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 6c41fe054..d0d3c0f84 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -500,11 +500,8 @@ off_t unifyfs_fid_log_size(int fid); */ off_t unifyfs_fid_logical_size(int fid); -/* fill in limited amount of stat information for global file id */ -int unifyfs_gfid_stat(int gfid, struct stat* buf); - -/* fill in limited amount of stat information */ -int unifyfs_fid_stat(int fid, struct stat* buf); +/* Update local metadata for file from global metadata */ +int unifyfs_fid_update_file_meta(int fid, unifyfs_file_attr_t* gfattr); /* allocate a file id slot for a new file * return the fid or -1 on error */ diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index c4b2db2f8..f14a028aa 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -338,50 +338,48 @@ int UNIFYFS_WRAP(remove)(const char* path) /* The main stat call for all the *stat() functions */ static int __stat(const char* path, struct stat* buf) { - int gfid, fid; + int gfid, fid, ret; unifyfs_file_attr_t fattr; - int ret; - - gfid = unifyfs_generate_gfid(path); - fid = unifyfs_get_fid_from_path(path); /* check that caller gave us a buffer to write to */ if (!buf) { errno = EFAULT; return -1; } + memset(buf, 0, sizeof(*buf)); /* lookup stat data for global file id */ - ret = invoke_client_metaget_rpc(gfid, &fattr); + gfid = unifyfs_generate_gfid(path); + ret = unifyfs_get_global_file_meta(gfid, &fattr); if (ret != UNIFYFS_SUCCESS) { LOGDBG("metaget failed"); - return ret; + errno = ENOENT; + return -1; } - memset(buf, 0, sizeof(*buf)); + /* update local file metadata (if applicable) */ + fid = unifyfs_get_fid_from_path(path); + unifyfs_fid_update_file_meta(fid, &fattr); /* copy attributes to stat struct */ unifyfs_file_attr_to_stat(&fattr, buf); - /* - * For debugging and testing purposes, we hijack st_rdev to store our - * local size and log size. We also assume the stat struct is - * the 64-bit variant. The values are stored as: - * - * st_rdev = log_size << 32 | local_size; - * - */ - buf->st_rdev = fid; if (fid >= 0) { /* If we have a local file */ - buf->st_rdev = (unifyfs_fid_log_size(fid) << 32) | - (unifyfs_fid_local_size(fid) & 0xFFFFFFFF); + /* + * For debugging and testing purposes, we hijack st_rdev to store our + * local size and log size. We also assume the stat struct is + * the 64-bit variant. The values are stored as: + * + * st_rdev = log_size << 32 | local_size; + * + */ + buf->st_rdev = (unifyfs_fid_log_size(fid) << 32); + buf->st_rdev |= (unifyfs_fid_local_size(fid) & 0xFFFFFFFF); } + /* global filesize is zero for non-laminated files */ if (!fattr.is_laminated) { - /* - * It was decided that all non-laminated files would report a global - * filesize of zero. - */ + LOGDBG("file is NOT laminated") buf->st_size = 0; } @@ -544,6 +542,7 @@ ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) /* if we don't read any bytes, return success */ if (count == 0) { + LOGDBG("returning EOF"); return 0; } diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 1e9ba35ba..8f7655749 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -740,7 +740,10 @@ off_t unifyfs_fid_global_size(int fid) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - return meta->global_size; + if (NULL != meta) { + return meta->global_size; + } + return (off_t)-1; } /* Return the log size of the file */ @@ -748,7 +751,10 @@ off_t unifyfs_fid_local_size(int fid) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - return meta->local_size; + if (NULL != meta) { + return meta->local_size; + } + return (off_t)-1; } /* @@ -771,13 +777,41 @@ off_t unifyfs_fid_log_size(int fid) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - return meta->log_size; + if (NULL != meta) { + return meta->log_size; + } + return (off_t)-1; +} + +/* Update local metadata for file from global metadata */ +int unifyfs_fid_update_file_meta(int fid, unifyfs_file_attr_t* gfattr) +{ + if (NULL == gfattr) { + return UNIFYFS_FAILURE; + } + + /* lookup local metadata for file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if (NULL != meta) { + /* update lamination state */ + meta->is_laminated = gfattr->is_laminated; + if (meta->is_laminated) { + /* update file size */ + meta->global_size = (off_t)gfattr->size; + meta->local_size = meta->global_size; + LOGDBG("laminated file size is %zu bytes", + (size_t)meta->global_size); + } + return UNIFYFS_SUCCESS; + } + /* else, bad fid */ + return UNIFYFS_FAILURE; } int unifyfs_set_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) { /* check that we have an input buffer */ - if (!gfattr) { + if (NULL == gfattr) { return UNIFYFS_FAILURE; } @@ -789,7 +823,7 @@ int unifyfs_set_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) { /* check that we have an output buffer to write to */ - if (!gfattr) { + if (NULL == gfattr) { return UNIFYFS_FAILURE; } @@ -800,7 +834,6 @@ int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) /* found it, copy attributes to output struct */ *gfattr = fmeta; } - return ret; } @@ -854,35 +887,6 @@ int unifyfs_set_global_file_meta_from_fid(int fid) return ret; } -/* fill in limited amount of stat information for global file id */ -int unifyfs_gfid_stat(int gfid, struct stat* buf) -{ - /* check that we have an output buffer to write to */ - if (!buf) { - return UNIFYFS_ERROR_INVAL; - } - - /* zero out user's stat buffer */ - memset(buf, 0, sizeof(struct stat)); - - /* lookup stat data for global file id */ - unifyfs_file_attr_t fattr; - int ret = invoke_client_metaget_rpc(gfid, &fattr); - if (ret != UNIFYFS_SUCCESS) { - return ret; - } - - /* It was decided that non-laminated files return a file size of 0 */ - if (!fattr.is_laminated) { - fattr.size = 0; - } - - /* copy stat structure */ - unifyfs_file_attr_to_stat(&fattr, buf); - - return UNIFYFS_SUCCESS; -} - /* allocate a file id slot for a new file * return the fid or -1 on error */ int unifyfs_fid_alloc() @@ -975,15 +979,15 @@ int unifyfs_fid_create_directory(const char* path) found_global = 1; } - /* check whether we already have info for this item - * locally and globally */ - if (found_local && found_global) { - /* this directory already exists */ + /* can't create if it already exists */ + if (found_global) { return (int) UNIFYFS_ERROR_EXIST; } - if (found_local && !found_global) { - /* FIXME: so, we have detected the cache inconsistency here. + if (found_local) { + /* exists locally, but not globally + * + * FIXME: so, we have detected the cache inconsistency here. * we cannot simply unlink or remove the entry because then we also * need to check whether any subdirectories or files exist. * @@ -1000,12 +1004,6 @@ int unifyfs_fid_create_directory(const char* path) return (int) UNIFYFS_ERROR_IO; } - if (!found_local && found_global) { - /* populate the local cache, then return EEXIST */ - - return (int) UNIFYFS_ERROR_EXIST; - } - /* now, we need to create a new directory. */ fid = unifyfs_fid_create_file(path); if (fid < 0) { @@ -1173,26 +1171,6 @@ int unifyfs_fid_truncate(int fid, off_t length) return UNIFYFS_SUCCESS; } -/* - * hash a path to gfid - * @param path: file path - * return: error code, gfid - * */ -static int unifyfs_get_global_fid(const char* path, int* gfid) -{ - MD5_CTX ctx; - - unsigned char md[16]; - memset(md, 0, 16); - - MD5_Init(&ctx); - MD5_Update(&ctx, path, strlen(path)); - MD5_Final(md, &ctx); - - *gfid = *((int*)md); - return UNIFYFS_SUCCESS; -} - /* opens a new file id with specified path, access flags, and permissions, * fills outfid with file id and outpos with position for current file pointer, * returns UNIFYFS error code @@ -1242,8 +1220,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, if (gfattr.is_laminated && ((flags & (O_CREAT | O_TRUNC | O_APPEND | O_WRONLY)) || ((mode & 0222) && (flags != O_RDONLY)))) { - LOGDBG("Can't open %s with a writable flag on laminated file.", - path); + LOGDBG("Can't open laminated file %s with a writable flag.", path); return EROFS; } @@ -1255,10 +1232,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, if (found_local && !found_global) { LOGDBG("file found locally, but seems to be deleted globally. " "invalidating the local cache."); - return EROFS; - unifyfs_fid_unlink(fid); - return (int) UNIFYFS_ERROR_NOENT; } @@ -1288,8 +1262,11 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } /* initialize global size of file from global metadata */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - meta->global_size = gfattr.size; + ret = unifyfs_fid_update_file_meta(fid, &gfattr); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("failed to update file metadata from global"); + /* this is not a fatal error, continue */ + } } else if (found_local && found_global) { /* file exists and is valid. */ if ((flags & O_CREAT) && (flags & O_EXCL)) { @@ -1304,6 +1281,13 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, return (int)UNIFYFS_ERROR_NOTDIR; } + /* update local metadata from global metadata */ + ret = unifyfs_fid_update_file_meta(fid, &gfattr); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("failed to update file metadata from global"); + /* this is not a fatal error, continue */ + } + if ((flags & O_TRUNC) && (flags & (O_RDWR | O_WRONLY))) { unifyfs_fid_truncate(fid, 0); } diff --git a/examples/src/testutil_rdwr.h b/examples/src/testutil_rdwr.h index bc17667a0..ff9297ff3 100644 --- a/examples/src/testutil_rdwr.h +++ b/examples/src/testutil_rdwr.h @@ -15,6 +15,8 @@ #ifndef UNIFYFS_TESTUTIL_RDWR_H #define UNIFYFS_TESTUTIL_RDWR_H +#include "testutil.h" + /* -------- Write Helper Methods -------- */ static inline @@ -217,10 +219,23 @@ int write_laminate(test_cfg* cfg, const char* filepath) int chmod_rc = chmod(filepath, 0444); if (-1 == chmod_rc) { /* lamination failed */ - test_print(cfg, "chmod() lamination failed"); + test_print(cfg, "chmod() during lamination failed"); rc = -1; } } + if (cfg->io_pattern == IO_PATTERN_N1) { + test_barrier(cfg); + if (cfg->rank != 0) { + /* call stat() to update global metadata */ + struct stat st; + int stat_rc = stat(filepath, &st); + if (-1 == stat_rc) { + /* lamination failed */ + test_print(cfg, "stat() update during lamination failed"); + rc = -1; + } + } + } return rc; } From 5db0ded5396044de7ea40f937c12fb356d213a0a Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Mon, 25 Nov 2019 14:59:00 -0500 Subject: [PATCH 043/168] fixup - remove redundant error check --- client/src/unifyfs-sysio.c | 4 +++- client/src/unifyfs.c | 12 ++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index f14a028aa..c91bf15ae 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -359,7 +359,9 @@ static int __stat(const char* path, struct stat* buf) /* update local file metadata (if applicable) */ fid = unifyfs_get_fid_from_path(path); - unifyfs_fid_update_file_meta(fid, &fattr); + if (fid != -1) { + unifyfs_fid_update_file_meta(fid, &fattr); + } /* copy attributes to stat struct */ unifyfs_file_attr_to_stat(&fattr, buf); diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 8f7655749..003f9c7f3 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -1262,11 +1262,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } /* initialize global size of file from global metadata */ - ret = unifyfs_fid_update_file_meta(fid, &gfattr); - if (ret != UNIFYFS_SUCCESS) { - LOGERR("failed to update file metadata from global"); - /* this is not a fatal error, continue */ - } + unifyfs_fid_update_file_meta(fid, &gfattr); } else if (found_local && found_global) { /* file exists and is valid. */ if ((flags & O_CREAT) && (flags & O_EXCL)) { @@ -1282,11 +1278,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } /* update local metadata from global metadata */ - ret = unifyfs_fid_update_file_meta(fid, &gfattr); - if (ret != UNIFYFS_SUCCESS) { - LOGERR("failed to update file metadata from global"); - /* this is not a fatal error, continue */ - } + unifyfs_fid_update_file_meta(fid, &gfattr); if ((flags & O_TRUNC) && (flags & (O_RDWR | O_WRONLY))) { unifyfs_fid_truncate(fid, 0); From 0ee6452190364ef82d96295714c904cebc8bd4ad Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Wed, 4 Dec 2019 16:02:53 -0500 Subject: [PATCH 044/168] Fixes two errors about the system configuration file (/etc/unifyfs): - consider install prefix when reading /etc/unifyfs/unifyfs.conf - removing double quotations around strings from the unifyfs.conf template file --- common/src/Makefile.am | 2 ++ extras/unifyfs.conf.in | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/common/src/Makefile.am b/common/src/Makefile.am index 00dc6ae0a..41c07770b 100644 --- a/common/src/Makefile.am +++ b/common/src/Makefile.am @@ -59,6 +59,8 @@ libunifyfs_common_la_CPPFLAGS = \ $(MARGO_CFLAGS) \ $(FLATCC_CFLAGS) +libunifyfs_common_la_CPPFLAGS += -DSYSCONFDIR="$(sysconfdir)" + libunifyfs_common_la_LDFLAGS = \ -version-info $(LIBUNIFYFS_LT_VERSION) diff --git a/extras/unifyfs.conf.in b/extras/unifyfs.conf.in index 365c7194e..20402e027 100644 --- a/extras/unifyfs.conf.in +++ b/extras/unifyfs.conf.in @@ -1,6 +1,9 @@ # unifyfs.conf -# NOTE: settings with default values are commented out +# NOTE: +# - settings with default values are commented out +# - string values should not be quoted, e.g., /var/tmp is correct but +# "/var/tmp" is not. # # COMMENT STYLE: # '#' start of line comment character @@ -8,9 +11,9 @@ # SECTION: top-level configuration [unifyfs] -# consistency = "LAMINATED" ; NONE | LAMINATED | POSIX -# daemonize = on ; servers will become daemons -# mountpoint = "/unifyfs" ; mountpoint (i.e., prefix path) +# consistency = LAMINATED ; NONE | LAMINATED | POSIX +# daemonize = on ; servers will become daemons +# mountpoint = /unifyfs ; mountpoint (i.e., prefix path) # SECTION: client settings [client] @@ -24,8 +27,8 @@ verbosity = 5 ; logging verbosity level [0-5] (default: 0) # SECTION: metadata settings [meta] -# db_name = "unifyfs_metadb" ; metadata datbase name -db_path = "/var/tmp" ; metadata database directory path (default: /tmp) +# db_name = unifyfs_metadb ; metadata datbase name +db_path = /var/tmp ; metadata database directory path (default: /tmp) # SECTION: shared memory segment settings [shmem] @@ -34,7 +37,7 @@ chunk_mem = 67108864 ; segment size for data chunks (default: 256 MiB) # SECTION: spillover local to each node [spillover] -# enabled = on ; enable spillover to local storage -# data_dir = "/mnt/ssd" ; directory path for data spillover -# meta_dir = "/mnt/ssd" ; directory path for metadata spillover -size = 268435456 ; data spillover max size (default: 1 GiB) +# enabled = on ; enable spillover to local storage +# data_dir = /mnt/ssd ; directory path for data spillover +# meta_dir = /mnt/ssd ; directory path for metadata spillover +size = 268435456 ; data spillover max size (default: 1 GiB) From 3f0371ea455fb8d8eb48a1d0c7fcb48a206e24c4 Mon Sep 17 00:00:00 2001 From: CamStan Date: Tue, 10 Dec 2019 16:53:42 -0800 Subject: [PATCH 045/168] Fix -wrap configure bug When building on some systems, the configure check for HAVE_LD_WRAP would fail with an undefinded reference to __wrap_malloc. This would result in the static examples not being built and installed. This adds a definition for __wrap_malloc for this test to pass. --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 492932075..2a780637b 100755 --- a/configure.ac +++ b/configure.ac @@ -176,11 +176,11 @@ AC_TRY_COMPILE( AC_MSG_RESULT(no) ) -AC_MSG_CHECKING(if linker suppots -wrap) +AC_MSG_CHECKING(if linker supports -wrap) OLD_LDFLAGS=$LDFLAGS LDFLAGS=$LDFLAGS LDFLAGS+="-Wl,-wrap,malloc" -AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]],[[int *test = malloc(sizeof(int));]])], +AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]],[[void* __wrap_malloc(size_t size);]],[[int *test = malloc(sizeof(int));]])], [ AC_MSG_RESULT([yes]) AM_CONDITIONAL([HAVE_LD_WRAP],[true]) From 044b57c11d69193b37769b2b58c3b75e1053d576 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 3 Jan 2020 15:37:53 -0800 Subject: [PATCH 046/168] server: fix offset when reading from spillover file --- server/src/unifyfs_service_manager.c | 316 ++++++++++++++++++--------- 1 file changed, 208 insertions(+), 108 deletions(-) diff --git a/server/src/unifyfs_service_manager.c b/server/src/unifyfs_service_manager.c index 4adafe5ee..a1c7ea9db 100644 --- a/server/src/unifyfs_service_manager.c +++ b/server/src/unifyfs_service_manager.c @@ -36,7 +36,6 @@ #include "unifyfs_server_rpcs.h" #include "margo_server.h" - /* Service Manager (SM) state */ typedef struct { /* the SM thread */ @@ -74,7 +73,10 @@ do { \ pthread_mutex_unlock(&(sm->sync)); \ } while (0) -/* Decode and issue chunk-reads received from request manager +/* Decode and issue chunk-reads received from request manager. + * We get a list of read requests for data on our node. Read + * data for each request and construct a set of read replies + * that will be sent back to the request manager. * * @param src_rank : source delegator rank * @param src_app_id : app id at source delegator @@ -102,109 +104,158 @@ int sm_issue_chunk_reads(int src_rank, ptr += sizeof(int); assert(num == num_chks); + /* total data size we'll be reading */ size_t total_data_sz = *((size_t*)ptr); ptr += sizeof(size_t); - /* get pointer to read request */ + /* get pointer to read request array */ chunk_read_req_t* reqs = (chunk_read_req_t*)ptr; - remote_chunk_reads_t* rcr = (remote_chunk_reads_t*) - calloc(1, sizeof(remote_chunk_reads_t)); - if (NULL == rcr) { - LOGERR("failed to allocate remote_chunk_reads"); - return UNIFYFS_ERROR_NOMEM; - } - rcr->rank = src_rank; - rcr->app_id = src_app_id; - rcr->client_id = src_client_id; - rcr->rdreq_id = src_req_id; - rcr->num_chunks = num_chks; - rcr->reqs = NULL; + /* we'll allocate a buffer to hold a list of chunk read response + * structures, one for each chunk, followed by a data buffer + * to hold all data for all reads */ + /* compute the size of that buffer */ size_t resp_sz = sizeof(chunk_read_resp_t) * num_chks; - size_t buf_sz = resp_sz + total_data_sz; - rcr->total_sz = buf_sz; + size_t buf_sz = resp_sz + total_data_sz; + /* allocate the buffer */ // NOTE: calloc() is required here, don't use malloc char* crbuf = (char*) calloc(1, buf_sz); if (NULL == crbuf) { LOGERR("failed to allocate chunk_read_reqs"); - free(rcr); return UNIFYFS_ERROR_NOMEM; } - chunk_read_resp_t* resp = (chunk_read_resp_t*)crbuf; - rcr->resp = resp; + /* the chunk read response array starts as the first + * byte in our buffer and the data buffer follows + * the read response array */ + chunk_read_resp_t* resp = (chunk_read_resp_t*)crbuf; char* databuf = crbuf + resp_sz; + /* allocate a struct for the chunk read request */ + remote_chunk_reads_t* rcr = (remote_chunk_reads_t*) + calloc(1, sizeof(remote_chunk_reads_t)); + if (NULL == rcr) { + LOGERR("failed to allocate remote_chunk_reads"); + return UNIFYFS_ERROR_NOMEM; + } + + /* fill in chunk read request */ + rcr->rank = src_rank; + rcr->app_id = src_app_id; + rcr->client_id = src_client_id; + rcr->rdreq_id = src_req_id; + rcr->num_chunks = num_chks; + rcr->reqs = NULL; + rcr->total_sz = buf_sz; + rcr->resp = resp; + LOGDBG("issuing %d requests, total data size = %zu", num_chks, total_data_sz); - /* points to offset in read reply buffer */ + /* points to offset in read reply buffer to place + * data for next read */ size_t buf_cursor = 0; int i; int last_app = -1; app_config_t* app_config = NULL; for (i = 0; i < num_chks; i++) { + /* pointer to next read request */ chunk_read_req_t* rreq = reqs + i; + + /* pointer to next read response */ chunk_read_resp_t* rresp = resp + i; - /* get size of data we are to read */ - size_t size = rreq->nbytes; + /* get size and log offset of data we are to read */ + size_t size = rreq->nbytes; size_t offset = rreq->log_offset; /* record request metadata in response */ - rresp->nbytes = size; - rresp->offset = rreq->offset; - LOGDBG("reading chunk(offset=%zu, size=%zu)", rreq->offset, size); + rresp->read_rc = 0; + rresp->nbytes = size; + rresp->offset = rreq->offset; + LOGDBG("reading chunk(offset=%zu, size=%zu)", + rreq->offset, size); - /* get app id and client id for this read task, - * defines log files holding data */ + /* get app id and corresponding app_config struct */ int app_id = rreq->log_app_id; - int cli_id = rreq->log_client_id; if (app_id != last_app) { /* look up app config for given app id */ app_config = (app_config_t*) arraylist_get(app_config_list, app_id); assert(app_config); + + /* remember the current app_id to skip lookup if the next + * request is for the same app_id */ last_app = app_id; } - int spillfd = app_config->spill_log_fds[cli_id]; - char* log_ptr = app_config->shm_superblocks[cli_id] + - app_config->data_offset + offset; - char* buf_ptr = databuf + buf_cursor; + /* client id for this read task */ + int cli_id = rreq->log_client_id; + + /* get size of data region for this + * app_id and client_id */ + size_t data_size = app_config->data_size; /* prepare read opertions based on data location */ - size_t sz_from_mem = 0; + size_t sz_from_mem = 0; size_t sz_from_spill = 0; - if ((offset + size) <= app_config->data_size) { + if ((offset + size) <= data_size) { /* requested data is totally in shared memory */ sz_from_mem = size; - } else if (offset < app_config->data_size) { + } else if (offset < data_size) { /* part of the requested data is in shared memory */ - sz_from_mem = app_config->data_size - offset; + sz_from_mem = data_size - offset; sz_from_spill = size - sz_from_mem; } else { /* all requested data is in spillover file */ sz_from_spill = size; } + + /* get pointer to next position in buffer to store read data */ + char* buf_ptr = databuf + buf_cursor; + + /* read data from shared memory */ if (sz_from_mem > 0) { - /* read data from shared memory */ - memcpy(buf_ptr, log_ptr, sz_from_mem); + /* start of data within in superblock */ + char* super_addr = app_config->shm_superblocks[cli_id]; + char* data_addr = super_addr + app_config->data_offset; + char* data_ptr = data_addr + offset; + + /* copy data from superblock into read reply buffer */ + memcpy(buf_ptr, data_ptr, sz_from_mem); + + /* we assume memcpy copied everything */ rresp->read_rc = sz_from_mem; } + + /* read data from spillover file */ if (sz_from_spill > 0) { - /* read data from spillover file */ - ssize_t nread = pread(spillfd, (buf_ptr + sz_from_mem), - sz_from_spill, 0); + /* file descriptor for open spillover file for this + * app/client */ + int spill_fd = app_config->spill_log_fds[cli_id]; + + /* offset within spill over file, need to subtract off + * range of offsets that land in data region of + * superblock */ + off_t spill_offset = (off_t)(offset - data_size + + sz_from_mem); + + /* read data from the spillover file */ + ssize_t nread = pread(spill_fd, (buf_ptr + sz_from_mem), + sz_from_spill, spill_offset); if (-1 == nread) { + /* pread hit an error, return error code */ rresp->read_rc = (ssize_t)(-errno); } else { + /* add to byte counts we may have started from memcpy */ rresp->read_rc += nread; } } + + /* update to point to next slot in read reply buffer */ buf_cursor += size; /* update accounting for burst size */ @@ -212,15 +263,22 @@ int sm_issue_chunk_reads(int src_rank, } if (src_rank != glb_pmi_rank) { - /* add chunk_reads to svcmgr response list */ + /* we need to send these read responses to another rank, + * add chunk_reads to svcmgr response list and another + * thread will take care of that */ LOGDBG("adding to svcmgr chunk_reads"); assert(NULL != sm); + SM_LOCK(); arraylist_add(sm->chunk_reads, rcr); SM_UNLOCK(); + + /* rcr will be freed later by the sending thread */ + LOGDBG("done adding to svcmgr chunk_reads"); return UNIFYFS_SUCCESS; - } else { /* response is for myself */ + } else { + /* response is for myself, post it directly */ LOGDBG("responding to myself"); int rc = rm_post_chunk_read_responses(src_app_id, src_client_id, src_rank, src_req_id, @@ -228,8 +286,10 @@ int sm_issue_chunk_reads(int src_rank, if (rc != (int)UNIFYFS_SUCCESS) { LOGERR("failed to handle chunk read responses"); } + /* clean up allocated buffers */ free(rcr); + return rc; } } @@ -237,6 +297,8 @@ int sm_issue_chunk_reads(int src_rank, /* initialize and launch service manager thread */ int svcmgr_init(void) { + /* allocate a service manager struct, + * store in global variable */ sm = (svcmgr_state_t*)calloc(1, sizeof(svcmgr_state_t)); if (NULL == sm) { LOGERR("failed to allocate service manager state!"); @@ -282,6 +344,7 @@ int svcmgr_fini(void) sm->time_to_exit = 1; pthread_join(sm->thrd, NULL); } + if (sm->initialized) { SM_LOCK(); } @@ -293,6 +356,7 @@ int svcmgr_fini(void) pthread_mutex_destroy(&(sm->sync)); } + /* free the service manager struct allocated during init */ free(sm); sm = NULL; } @@ -302,25 +366,44 @@ int svcmgr_fini(void) /* iterate over list of chunk reads and send responses */ static int send_chunk_read_responses(void) { + /* assume we'll succeed */ int rc = (int)UNIFYFS_SUCCESS; + + /* this will hold a list of chunk read requests if we find any */ arraylist_t* chunk_reads = NULL; + + /* lock to access global service manager object */ pthread_mutex_lock(&(sm->sync)); + + /* if we have any chunk reads, take pointer to the list + * of chunk read requests and replace it with a newly allocated + * list on the service manager structure */ int num_chunk_reads = arraylist_size(sm->chunk_reads); if (num_chunk_reads) { + /* got some chunk read requets, take the list and replace + * it with an empty list */ LOGDBG("processing %d chunk read responses", num_chunk_reads); chunk_reads = sm->chunk_reads; sm->chunk_reads = arraylist_create(); } + + /* release lock on service manager object */ pthread_mutex_unlock(&(sm->sync)); + + /* iterate over each chunk read request */ for (int i = 0; i < num_chunk_reads; i++) { - /* get data structure */ + /* get next chunk read request */ remote_chunk_reads_t* rcr = (remote_chunk_reads_t*) arraylist_get(chunk_reads, i); + rc = invoke_chunk_read_response_rpc(rcr); } + + /* free the list if we have one */ if (NULL != chunk_reads) { arraylist_free(chunk_reads); } + return rc; } @@ -388,58 +471,71 @@ void* sm_service_reads(void* arg) /* BEGIN MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ -/* invokes the chunk_read_response rpc */ +/* invokes the chunk_read_response rpc, this sends a set of read + * reply headers and corresponding data back to a server that + * had requested we read data on its behalf, the headers and + * data are posted as a bulk transfer buffer */ int invoke_chunk_read_response_rpc(remote_chunk_reads_t* rcr) { + /* assume we'll succeed */ int rc = (int)UNIFYFS_SUCCESS; - int dst_srvr_rank = rcr->rank; - hg_handle_t handle; - chunk_read_response_in_t in; - chunk_read_response_out_t out; - hg_return_t hret; - hg_addr_t dst_srvr_addr; - hg_size_t bulk_sz = rcr->total_sz; - void* data_buf = (void*)rcr->resp; - assert(dst_srvr_rank < (int)glb_num_servers); - dst_srvr_addr = glb_servers[dst_srvr_rank].margo_svr_addr; + /* rank of destination server */ + int dst_rank = rcr->rank; + assert(dst_rank < (int)glb_num_servers); + + /* get address of destinaton server */ + hg_addr_t dst_addr = glb_servers[dst_rank].margo_svr_addr; - hret = margo_create(unifyfsd_rpc_context->svr_mid, dst_srvr_addr, - unifyfsd_rpc_context->rpcs.chunk_read_response_id, - &handle); + /* pointer to struct containing rpc context info, + * shorter name for convience */ + ServerRpcContext_t* ctx = unifyfsd_rpc_context; + + /* get handle to read response rpc on destination server */ + hg_handle_t handle; + hg_return_t hret = margo_create(ctx->svr_mid, dst_addr, + ctx->rpcs.chunk_read_response_id, &handle); assert(hret == HG_SUCCESS); + /* get address and size of our response buffer */ + void* data_buf = (void*)rcr->resp; + hg_size_t bulk_sz = rcr->total_sz; + /* fill in input struct */ - in.src_rank = (int32_t)glb_pmi_rank; - in.app_id = (int32_t)rcr->app_id; + chunk_read_response_in_t in; + in.src_rank = (int32_t)glb_pmi_rank; + in.app_id = (int32_t)rcr->app_id; in.client_id = (int32_t)rcr->client_id; - in.req_id = (int32_t)rcr->rdreq_id; - in.num_chks = (int32_t)rcr->num_chunks; + in.req_id = (int32_t)rcr->rdreq_id; + in.num_chks = (int32_t)rcr->num_chunks; in.bulk_size = bulk_sz; - /* register request buffer for bulk remote access */ - hret = margo_bulk_create(unifyfsd_rpc_context->svr_mid, 1, - &data_buf, &bulk_sz, - HG_BULK_READ_ONLY, &in.bulk_handle); + /* register our response buffer for bulk remote read access */ + hret = margo_bulk_create(ctx->svr_mid, 1, + &data_buf, &bulk_sz, HG_BULK_READ_ONLY, &in.bulk_handle); assert(hret == HG_SUCCESS); + /* call the read response rpc */ LOGDBG("invoking the chunk-read-response rpc function"); hret = margo_forward(handle, &in); if (hret != HG_SUCCESS) { + /* failed to invoke the rpc */ rc = (int)UNIFYFS_FAILURE; } else { - /* decode response */ + /* rpc executed, now decode response */ + chunk_read_response_out_t out; hret = margo_get_output(handle, &out); if (hret == HG_SUCCESS) { rc = (int)out.ret; LOGDBG("chunk-read-response rpc to %d - ret=%d", - dst_srvr_rank, rc); + dst_rank, rc); margo_free_output(handle, &out); } else { rc = (int)UNIFYFS_FAILURE; } } + /* free resources allocated for executing margo rpc */ margo_bulk_free(in.bulk_handle); margo_destroy(handle); @@ -457,27 +553,25 @@ int invoke_chunk_read_response_rpc(remote_chunk_reads_t* rcr) * print the message, and return my rank */ static void server_hello_rpc(hg_handle_t handle) { - int rc, src_rank; - hg_return_t hret; - char* msg; - server_hello_in_t in; - server_hello_out_t out; - /* get input params */ - rc = margo_get_input(handle, &in); + server_hello_in_t in; + int rc = margo_get_input(handle, &in); assert(rc == HG_SUCCESS); - src_rank = (int)in.src_rank; - msg = strdup(in.message_str); + + /* extract params from input struct */ + int src_rank = (int)in.src_rank; + char* msg = strdup(in.message_str); if (NULL != msg) { LOGDBG("got message '%s' from server %d", msg, src_rank); free(msg); } /* fill output structure to return to caller */ + server_hello_out_t out; out.ret = (int32_t)glb_pmi_rank; /* send output back to caller */ - hret = margo_respond(handle, &out); + hg_return_t hret = margo_respond(handle, &out); assert(hret == HG_SUCCESS); /* free margo resources */ @@ -491,21 +585,19 @@ DEFINE_MARGO_RPC_HANDLER(server_hello_rpc) * decode payload based on tag, and call appropriate svcmgr routine */ static void server_request_rpc(hg_handle_t handle) { - int rc, req_id, src_rank, tag; int32_t ret; hg_return_t hret; - hg_bulk_t bulk_handle; - size_t bulk_sz; - server_request_in_t in; - server_request_out_t out; /* get input params */ - rc = margo_get_input(handle, &in); + server_request_in_t in; + int rc = margo_get_input(handle, &in); assert(rc == HG_SUCCESS); - src_rank = (int)in.src_rank; - req_id = (int)in.req_id; - tag = (int)in.req_tag; - bulk_sz = (size_t)in.bulk_size; + + /* extract params from input struct */ + int src_rank = (int)in.src_rank; + int req_id = (int)in.req_id; + int tag = (int)in.req_tag; + size_t bulk_sz = (size_t)in.bulk_size; LOGDBG("handling request from server %d: tag=%d req=%d sz=%zu", src_rank, tag, req_id, bulk_sz); @@ -513,10 +605,11 @@ static void server_request_rpc(hg_handle_t handle) /* get margo info */ const struct hg_info* hgi = margo_get_info(handle); assert(NULL != hgi); + margo_instance_id mid = margo_hg_info_get_instance(hgi); assert(mid != MARGO_INSTANCE_NULL); - int reqcmd = 0; + hg_bulk_t bulk_handle; void* reqbuf = NULL; if (bulk_sz) { /* allocate and register local target buffer for bulk access */ @@ -534,7 +627,6 @@ static void server_request_rpc(hg_handle_t handle) in.bulk_handle, 0, bulk_handle, 0, in.bulk_size); assert(hret == HG_SUCCESS); - reqcmd = *(int*)reqbuf; } switch (tag) { @@ -545,10 +637,13 @@ static void server_request_rpc(hg_handle_t handle) } } + server_request_out_t out; request_out: - /* fill output structure and return to caller */ + /* fill output structure */ out.ret = ret; + + /* return to caller */ hret = margo_respond(handle, &out); assert(hret == HG_SUCCESS); @@ -567,24 +662,20 @@ DEFINE_MARGO_RPC_HANDLER(server_request_rpc) * decode payload based on tag, and call appropriate svcmgr routine */ static void chunk_read_request_rpc(hg_handle_t handle) { - int rc, req_id, num_chks; - int src_rank, app_id, client_id; - int32_t ret; hg_return_t hret; - hg_bulk_t bulk_handle; - size_t bulk_sz; - chunk_read_request_in_t in; - chunk_read_request_out_t out; /* get input params */ - rc = margo_get_input(handle, &in); + chunk_read_request_in_t in; + int rc = margo_get_input(handle, &in); assert(rc == HG_SUCCESS); - src_rank = (int)in.src_rank; - app_id = (int)in.app_id; - client_id = (int)in.client_id; - req_id = (int)in.req_id; - num_chks = (int)in.num_chks; - bulk_sz = (size_t)in.bulk_size; + + /* extract params from input struct */ + int src_rank = (int)in.src_rank; + int app_id = (int)in.app_id; + int client_id = (int)in.client_id; + int req_id = (int)in.req_id; + int num_chks = (int)in.num_chks; + size_t bulk_sz = (size_t)in.bulk_size; LOGDBG("handling chunk read request from server %d: " "req=%d num_chunks=%d bulk_sz=%zu", @@ -593,9 +684,11 @@ static void chunk_read_request_rpc(hg_handle_t handle) /* get margo info */ const struct hg_info* hgi = margo_get_info(handle); assert(NULL != hgi); + margo_instance_id mid = margo_hg_info_get_instance(hgi); assert(mid != MARGO_INSTANCE_NULL); + hg_bulk_t bulk_handle; int reqcmd = (int)SVC_CMD_INVALID; void* reqbuf = NULL; if (bulk_sz) { @@ -611,13 +704,17 @@ static void chunk_read_request_rpc(hg_handle_t handle) in.bulk_handle, 0, bulk_handle, 0, in.bulk_size); assert(hret == HG_SUCCESS); + + /* first int in request buffer is the command */ reqcmd = *(int*)reqbuf; } } + /* verify this is a request for data */ + int32_t ret; if (reqcmd == (int)SVC_CMD_RDREQ_CHK) { + /* chunk read request command */ LOGDBG("request command: SVC_CMD_RDREQ_CHK"); - /* chunk read request */ sm_issue_chunk_reads(src_rank, app_id, client_id, req_id, num_chks, (char*)reqbuf); ret = (int32_t)UNIFYFS_SUCCESS; @@ -627,8 +724,11 @@ static void chunk_read_request_rpc(hg_handle_t handle) ret = (int32_t)UNIFYFS_ERROR_INVAL; } - /* fill output structure and return to caller */ + /* fill output structure */ + chunk_read_request_out_t out; out.ret = ret; + + /* return output to caller */ hret = margo_respond(handle, &out); assert(hret == HG_SUCCESS); From 89d2097865a0a15d6531fa65b9bf89a7f5c755c0 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 3 Jan 2020 23:32:13 -0800 Subject: [PATCH 047/168] server: edits to request manager --- server/src/unifyfs_request_manager.c | 91 ++++++++++++++++++---------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 4f7ffad01..edde14a2f 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -1141,29 +1141,31 @@ int rm_post_chunk_read_responses(int app_id, size_t bulk_sz, char* resp_buf) { - int rc, thrd_id; - app_config_t* app_config = NULL; - reqmgr_thrd_t* thrd_ctrl = NULL; - server_read_req_t* rdreq = NULL; - remote_chunk_reads_t* del_reads = NULL; + int rc; /* lookup RM thread control structure for this app id */ - app_config = (app_config_t*) arraylist_get(app_config_list, app_id); + app_config_t* app_config = (app_config_t*) + arraylist_get(app_config_list, app_id); assert(NULL != app_config); - thrd_id = app_config->thrd_idxs[client_id]; - thrd_ctrl = rm_get_thread(thrd_id); + + int thrd_id = app_config->thrd_idxs[client_id]; + + reqmgr_thrd_t* thrd_ctrl = rm_get_thread(thrd_id); assert(NULL != thrd_ctrl); RM_LOCK(thrd_ctrl); + remote_chunk_reads_t* del_reads = NULL; + /* find read req associated with req_id */ - rdreq = thrd_ctrl->read_reqs + req_id; + server_read_req_t* rdreq = thrd_ctrl->read_reqs + req_id; for (int i = 0; i < rdreq->num_remote_reads; i++) { if (rdreq->remote_reads[i].rank == src_rank) { del_reads = rdreq->remote_reads + i; break; } } + if (NULL != del_reads) { LOGDBG("posting chunk responses for req %d from delegator %d", req_id, src_rank); @@ -1556,63 +1558,86 @@ int invoke_chunk_read_request_rpc(int dst_srvr_rank, /* handler for remote read request response */ static void chunk_read_response_rpc(hg_handle_t handle) { - int rc, src_rank, req_id; - int app_id, client_id, thrd_id; - int i, num_chks; int32_t ret; hg_return_t hret; - hg_bulk_t bulk_handle; - size_t bulk_sz; - chunk_read_response_in_t in; - chunk_read_response_out_t out; - void* resp_buf = NULL; /* get input params */ - rc = margo_get_input(handle, &in); + chunk_read_response_in_t in; + int rc = margo_get_input(handle, &in); assert(rc == HG_SUCCESS); - src_rank = (int)in.src_rank; - app_id = (int)in.app_id; - client_id = (int)in.client_id; - req_id = (int)in.req_id; - num_chks = (int)in.num_chks; - bulk_sz = (size_t)in.bulk_size; + /* extract params from input struct */ + int src_rank = (int)in.src_rank; + int app_id = (int)in.app_id; + int client_id = (int)in.client_id; + int req_id = (int)in.req_id; + int num_chks = (int)in.num_chks; + size_t bulk_sz = (size_t)in.bulk_size; + + /* The input parameters specify the info for a bulk transfer + * buffer on the sending process. We use that info to pull data + * from the sender into a local buffer. This buffer contains + * the read reply headers and associated read data for requests + * we had sent earlier. */ + + /* pull the remote data via bulk transfer */ if (0 == bulk_sz) { + /* sender is trying to send an empty buffer, + * don't think that should happen unless maybe + * we had sent a read request list that was empty? */ LOGERR("empty response buffer"); ret = (int32_t)UNIFYFS_ERROR_INVAL; } else { - resp_buf = malloc(bulk_sz); + /* allocate a buffer to hold the incoming data */ + char* resp_buf = (char*) malloc(bulk_sz); if (NULL == resp_buf) { + /* allocation failed, that's bad */ LOGERR("failed to allocate chunk read responses buffer"); ret = (int32_t)UNIFYFS_ERROR_NOMEM; } else { - /* pull response data */ + /* got a buffer, now pull response data */ ret = (int32_t)UNIFYFS_SUCCESS; + + /* get margo info */ const struct hg_info* hgi = margo_get_info(handle); assert(NULL != hgi); + margo_instance_id mid = margo_hg_info_get_instance(hgi); assert(mid != MARGO_INSTANCE_NULL); - hret = margo_bulk_create(mid, 1, &resp_buf, &in.bulk_size, - HG_BULK_WRITE_ONLY, &bulk_handle); + + /* pass along address of buffer we want to transfer + * data into to prepare it for a bulk write, + * get resulting margo handle */ + hg_bulk_t bulk_handle; + hret = margo_bulk_create(mid, 1, (void**)&resp_buf, &in.bulk_size, + HG_BULK_WRITE_ONLY, &bulk_handle); assert(hret == HG_SUCCESS); + + /* execute the transfer to pull data from remote side + * into our local bulk transfer buffer */ hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, - in.bulk_handle, 0, - bulk_handle, 0, in.bulk_size); + in.bulk_handle, 0, bulk_handle, 0, in.bulk_size); assert(hret == HG_SUCCESS); + /* process read replies (headers and data) we just + * received */ rc = rm_post_chunk_read_responses(app_id, client_id, - src_rank, req_id, num_chks, - bulk_sz, (char*)resp_buf); + src_rank, req_id, num_chks, bulk_sz, resp_buf); if (rc != (int)UNIFYFS_SUCCESS) { LOGERR("failed to handle chunk read responses") ret = rc; } + + /* deregister our bulk transfer buffer */ margo_bulk_free(bulk_handle); } } - /* fill output structure and return to caller */ + /* fill output structure */ + chunk_read_response_out_t out; out.ret = ret; + + /* return to caller */ hret = margo_respond(handle, &out); assert(hret == HG_SUCCESS); From 0151fc8bd7bb364a296dcccf992692f85c451650 Mon Sep 17 00:00:00 2001 From: Craig Philip Steffen Date: Mon, 16 Dec 2019 11:54:37 -0800 Subject: [PATCH 048/168] transfer API into unifyFS chmod()s to laminate --- client/src/unifyfs.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 003f9c7f3..498492114 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -2786,6 +2786,7 @@ int unifyfs_transfer_file(const char* src, const char* dst, int parallel) int ret = 0; int dir = 0; struct stat sb_src = { 0, }; + mode_t source_file_mode_write_removed; struct stat sb_dst = { 0, }; int unify_src = 0; int unify_dst = 0; @@ -2793,6 +2794,8 @@ int unifyfs_transfer_file(const char* src, const char* dst, int parallel) char* pos = dst_path; char* src_path = strdup(src); + int local_return_val; + if (!src_path) { return -ENOMEM; } @@ -2828,9 +2831,27 @@ int unifyfs_transfer_file(const char* src, const char* dst, int parallel) } if (parallel) { - return do_transfer_file_parallel(src_path, dst_path, &sb_src, dir); + local_return_val = + do_transfer_file_parallel(src_path, dst_path, &sb_src, dir); } else { - return do_transfer_file_serial(src_path, dst_path, &sb_src, dir); - } + local_return_val = + do_transfer_file_serial(src_path, dst_path, &sb_src, dir); + } + + // We know here that one (but not both) of the constituent files + // is in the unify FS. We just have to decide if the *destination* file is. + // If it is, then now that we've transferred it, we'll set it to be readable + // so that it will be laminated and will be readable by other processes. + if (unify_dst) { + // pull the source file's mode bits, remove all the write bits but leave + // the rest intact and store that new mode. Now that the file has been + // copied into the unify file system, chmod the file to the new + // permission. When unify senses all the write bits are removed it will + // laminate the file. + source_file_mode_write_removed = + (sb_src.st_mode) & ~(0222); + chmod(dst_path, source_file_mode_write_removed); + } + return local_return_val; } From b147efe91259556c0222a16de4dbbf45481d3ddd Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sat, 4 Jan 2020 13:24:25 -0800 Subject: [PATCH 049/168] client: add nread field to read request This adds an nread field to the client-side read request structure. We use this to track the max offset within each read request that has valid data. It is used to handle short reads and to deal with holes. --- client/src/unifyfs-internal.h | 21 +- client/src/unifyfs-stdio.c | 10 +- client/src/unifyfs-sysio.c | 507 +++++++++++++--------------------- 3 files changed, 214 insertions(+), 324 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index d0d3c0f84..f641a694c 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -297,12 +297,23 @@ typedef struct { } unifyfs_filename_t; /*unifyfs structures*/ + +/* This structure defines a client read request for a file. + * It is initialized by the client describing the global file id, + * offset, and length to be read and provides a pointer to + * the user buffer where the data should be placed. The + * server sets the errcode field to UNIFYFS_SUCCESS if the read + * succeeds and otherwise records an error code pertaining to + * why the read failed. The server records the number of bytes + * read in the nread field, which the client can use to detect + * short read operations. */ typedef struct { - int gfid; - int errcode; - size_t offset; - size_t length; - char* buf; + int gfid; /* global file id to be read */ + int errcode; /* error code for read operation if any */ + size_t offset; /* logical offset in file to read from */ + size_t length; /* number of bytes to read */ + size_t nread; /* number of bytes actually read */ + char* buf; /* pointer to user buffer to place data */ } read_req_t; typedef struct { diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index 9cfa6600f..be9fbe3b0 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -493,7 +493,7 @@ static int unifyfs_stream_flush(FILE* stream) /* reads count bytes from stream into buf, sets stream EOF and error * indicators as appropriate, sets errno if error, updates file * position, returns number of bytes read in retcount, returns UNIFYFS - * error codes*/ + * error codes */ static int unifyfs_stream_read( FILE* stream, void* buf, @@ -599,15 +599,15 @@ static int unifyfs_stream_read( /* read data from file into buffer */ size_t bufcount; - size_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, + ssize_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize); - if (read_rc == -1) { + if (read_rc == -1) { /* * ERROR: read error, set error indicator. errno is already set * by unifyfs_fd_read() */ s->err = 1; - return read_rc; + return UNIFYFS_ERROR_IO; } /* record new buffer range within file */ @@ -2270,7 +2270,7 @@ static int __srefill(unifyfs_stream_t* stream) /* read data from file into buffer */ size_t bufcount; - size_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize); + ssize_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize); if (read_rc < 0) { /* ERROR: read error, set error indicator */ s->err = 1; diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index c91bf15ae..9a2615b1e 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -548,32 +548,35 @@ ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) return 0; } - read_req_t tmp_req; - tmp_req.gfid = unifyfs_gfid_from_fid(fid); - tmp_req.offset = (size_t) pos; - tmp_req.length = count; - tmp_req.errcode = UNIFYFS_SUCCESS; - tmp_req.buf = buf; - - int ret = unifyfs_fd_logreadlist(&tmp_req, 1); - - /* - * FIXME: when we can get the global file size correctly, the following - * should be rewritten. currently, we cannot detect EOF reliably. - */ + /* fill in read request */ + read_req_t req; + req.gfid = unifyfs_gfid_from_fid(fid); + req.offset = (size_t) pos; + req.length = count; + req.nread = 0; + req.errcode = UNIFYFS_SUCCESS; + req.buf = buf; + + /* execute read operation */ + ssize_t retcount; + int ret = unifyfs_fd_logreadlist(&req, 1); if (ret != UNIFYFS_SUCCESS) { - if (tmp_req.errcode != UNIFYFS_SUCCESS) { - /* error reading data */ - errno = EIO; - count = -1; - } else { - count = 0; /* possible EOF */ - } + /* failed to issue read operation */ + errno = EIO; + retcount = -1; + } else if (req.errcode != UNIFYFS_SUCCESS) { + /* read executed, but failed */ + errno = EIO; + retcount = -1; } else { - /* success, update position */ - filedesc->pos += (off_t) count; + /* success, get number of bytes read from read request field */ + retcount = (ssize_t) req.nread; + + /* update file pointer position */ + filedesc->pos += (off_t) retcount; } - return count; + + return retcount; } /* @@ -974,21 +977,9 @@ ssize_t UNIFYFS_WRAP(read)(int fd, void* buf, size_t count) return (ssize_t)(-1); } -#if 0 // THIS IS BROKEN UNTIL WE HAVE GLOBAL SIZE - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (meta == NULL) { - /* ERROR: invalid file descriptor */ - errno = EBADF; - return (ssize_t)(-1); - } - - /* check for end of file */ - if (filedesc->pos >= meta->size) { - return 0; /* EOF */ - } -#endif - - return unifyfs_fd_read(fd, filedesc->pos, buf, count); + /* execute read */ + ssize_t ret = unifyfs_fd_read(fd, filedesc->pos, buf, count); + return ret; } else { MAP_OR_FAIL(read); ssize_t ret = UNIFYFS_REAL(read)(fd, buf, count); @@ -1142,6 +1133,7 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], reqs[reqcnt].gfid = unifyfs_gfid_from_fid(fid); reqs[reqcnt].offset = (size_t)(cbp->aio_offset); reqs[reqcnt].length = cbp->aio_nbytes; + reqs[reqcnt].nread = 0; reqs[reqcnt].errcode = EINPROGRESS; reqs[reqcnt].buf = (char*)(cbp->aio_buf); reqcnt++; @@ -1242,77 +1234,6 @@ static int compare_read_req(const void* a, const void* b) } } -/* returns index into read_req of item whose offset is - * just below offset of target item (if one exists) */ -static int unifyfs_locate_req(read_req_t* read_reqs, int count, - read_req_t* match_req) -{ - /* if list is empty, indicate that there is valid starting request */ - if (count == 0) { - return -1; - } - - /* if we only have one item, return its index */ - if (count == 1) { - return 0; - } - - /* if we have two items, return index to item that must come before */ - if (count == 2) { - if (compare_read_req(match_req, &read_reqs[1]) < 0) { - /* second item is clearly bigger, so try first */ - return 0; - } - - /* second item is less than or equal to target */ - return 1; - } - - /* execute binary search comparing target to list of requests */ - - int left = 0; - int right = count - 1; - int mid = (left + right) / 2; - - /* binary search until we find an exact match or have cut the list - * to just two items */ - int cmp; - while ((left + 1) < right) { - cmp = compare_read_req(match_req, &read_reqs[mid]); - if (cmp == 0) { - /* found exact match */ - return mid; - } else if (cmp > 0) { - /* if target if bigger than mid, set left bound to mid */ - left = mid; - } else { - /* if target is smaller than mid, set right bounds to mid */ - right = mid; - } - - /* update middle index */ - mid = (left + right) / 2; - } - - /* got two items, let's pick one */ - if (compare_read_req(match_req, &read_reqs[left]) < 0) { - /* target is smaller than left item, - * return index to left of left item if we can */ - if (left == 0) { - /* at left most item, so return this index */ - return 0; - } - return left - 1; - } else if (compare_read_req(match_req, &read_reqs[right]) < 0) { - /* target is smaller than right item, - * return index of item one less than right */ - return right - 1; - } else { - /* target is greater or equal to right item */ - return right; - } -} - /* given a read request, split it into multiple requests whose range * is equal or smaller than slice_range size * @@ -1450,165 +1371,6 @@ static int unifyfs_split_read_request( return UNIFYFS_SUCCESS; } -/* - * match the received read_requests with the - * client's read requests - * @param read_reqs: a list of read requests - * @param count: number of read requests - * @param match_req: received read request to match - * @return error code - * - * */ -static int unifyfs_match_received_ack(read_req_t* read_reqs, int count, - read_req_t* match_req) -{ - /* given fid, offset, and length of match_req that holds read reply, - * identify which read request this belongs to in read_req array, - * then copy data to user buffer */ - - /* create a request corresponding to the first byte in read reply */ - read_req_t match_start = *match_req; - - /* create a request corresponding to last byte in read reply */ - read_req_t match_end = *match_req; - match_end.offset += match_end.length - 1; - - /* find index of read request that contains our first byte */ - int start_pos = unifyfs_locate_req(read_reqs, count, &match_start); - - /* find index of read request that contains our last byte */ - int end_pos = unifyfs_locate_req(read_reqs, count, &match_end); - - /* could not find a valid read request in read_req array */ - if (start_pos == -1) { - return UNIFYFS_FAILURE; - } - - /* s: start of match_req, e: end of match_req */ - - if (start_pos == 0) { - if (compare_read_req(&match_start, &read_reqs[0]) < 0) { - /* starting offset in read reply comes before lowest - * offset in read requests, consider this to be an error - * - * ************ *********** ************* - * s - * - * */ - return UNIFYFS_FAILURE; - } - } - - /* create read request corresponding to first byte of first read request */ - read_req_t first_start = read_reqs[start_pos]; - - /* create read request corresponding to last byte of first read request */ - read_req_t first_end = read_reqs[start_pos]; - first_end.offset += first_end.length - 1; - - /* check whether read reply is fully contained by first read request */ - if (compare_read_req(&match_start, &first_start) >= 0 && - compare_read_req(&match_end, &first_end) <= 0) { - /* read reply is fully contained within first read request - * - * first_s first_e - * ***************** ************* - * s e - * - * */ - - /* copy data to user buffer if no error */ - if (match_req->errcode == UNIFYFS_SUCCESS) { - /* compute buffer location to copy data */ - size_t offset = (size_t)(match_start.offset - first_start.offset); - char* buf = first_start.buf + offset; - - /* copy data to user buffer */ - memcpy(buf, match_req->buf, match_req->length); - - return UNIFYFS_SUCCESS; - } else { - /* hit an error during read, so record this fact - * in user's original read request */ - read_reqs[start_pos].errcode = match_req->errcode; - return UNIFYFS_FAILURE; - } - } - - /* define read request for offset of first byte in last read request */ - read_req_t last_start = read_reqs[end_pos]; - - /* define read request for offset of last byte in last read request */ - read_req_t last_end = read_reqs[end_pos]; - last_end.offset += last_end.length - 1; - - /* determine whether read reply is contained in a range of read requests */ - if (compare_read_req(&match_start, &first_start) >= 0 && - compare_read_req(&match_end, &last_end) <= 0) { - /* read reply spans multiple read requests - * - * first_s first_e req_s req_e req_s req_e last_s last_e - * ***************** *********** *********** **************** - * s e - * - * */ - - /* check that read requests from start_pos to end_pos - * define a contiguous set of bytes */ - int i; - for (i = start_pos + 1; i <= end_pos; i++) { - if ((read_reqs[i - 1].offset + read_reqs[i - 1].length) - != read_reqs[i].offset) { - /* read requests are noncontiguous, error */ - return UNIFYFS_FAILURE; - } - } - - /* read requests are contiguous, fill all buffers in middle */ - if (match_req->errcode == UNIFYFS_SUCCESS) { - /* get pointer to start of read reply data */ - char* ptr = match_req->buf; - - /* compute position in user buffer to copy data */ - size_t offset = (size_t)(match_start.offset - first_start.offset); - char* buf = first_start.buf + offset; - - /* compute number of bytes to copy into first read request */ - size_t length = - (size_t)(first_end.offset - match_start.offset + 1); - - /* copy data into user buffer for first read request */ - memcpy(buf, ptr, length); - ptr += length; - - /* copy data for middle read requests */ - for (i = start_pos + 1; i < end_pos; i++) { - memcpy(read_reqs[i].buf, ptr, read_reqs[i].length); - ptr += read_reqs[i].length; - } - - /* compute bytes for last read request */ - length = (size_t)(match_end.offset - last_start.offset + 1); - - /* copy data into user buffer for last read request */ - memcpy(last_start.buf, ptr, length); - ptr += length; - - return UNIFYFS_SUCCESS; - } else { - /* hit an error during read, update errcode in user's - * original read request from start to end inclusive */ - for (i = start_pos; i <= end_pos; i++) { - read_reqs[i].errcode = match_req->errcode; - } - return UNIFYFS_FAILURE; - } - } - - /* could not find a matching read request, return an error */ - return UNIFYFS_FAILURE; -} - /* notify our delegator that the shared memory buffer * is now clear and ready to hold more read data */ static void delegator_signal(void) @@ -1689,38 +1451,112 @@ static int process_read_data(read_req_t* read_reqs, int count, int* done) shm_header_t* shm_hdr = (shm_header_t*)shm_recv_buf; char* shmptr = ((char*)shm_hdr) + sizeof(shm_header_t); + /* get number of read replies in shared memory */ size_t num = shm_hdr->meta_cnt; if (0 == num) { LOGDBG("no read responses available"); return rc; } - /* process each of our replies */ + /* process each of our read replies */ size_t i; for (i = 0; i < num; i++) { /* get pointer to current read reply header */ - shm_meta_t* msg = (shm_meta_t*)shmptr; + shm_meta_t* rep = (shm_meta_t*)shmptr; shmptr += sizeof(shm_meta_t); - /* define request object */ - read_req_t req; - req.gfid = msg->gfid; - req.offset = msg->offset; - req.length = msg->length; - req.errcode = msg->errcode; + /* get pointer to data */ + char* rep_buf = shmptr; + shmptr += rep->length; + + /* get start and end offset of reply */ + size_t rep_start = rep->offset; + size_t rep_end = rep->offset + rep->length; + + /* iterate over each of our read requests */ + size_t j; + for (j = 0; j < count; j++) { + /* get pointer to read request */ + read_req_t* req = &read_reqs[j]; + + /* skip if this request if not the same file */ + if (rep->gfid != req->gfid) { + /* request and reply are for different files */ + continue; + } - LOGDBG("read reply: gfid=%d offset=%zu size=%zu", - req.gfid, req.offset, req.length); + /* same file, now get start and end offsets + * of this read request */ + size_t req_start = req->offset; + size_t req_end = req->offset + req->length; + + /* test whether reply overlaps with request, + * overlap if: + * start of reply comes before the end of request + * AND + * end of reply comes after the start of request */ + int overlap = (rep_start < req_end && rep_end > req_start); + if (!overlap) { + /* reply does not overlap with this request */ + continue; + } - /* get pointer to data */ - req.buf = shmptr; - shmptr += msg->length; + /* this reply overlaps with the request, check that + * we didn't get an error */ + if (rep->errcode != UNIFYFS_SUCCESS) { + /* TODO: should we look for the reply with an errcode + * with the lowest start offset? */ - /* process this read reply, identify which application read - * request this reply goes to and copy data to user buffer */ - int tmp_rc = unifyfs_match_received_ack(read_reqs, count, &req); - if (tmp_rc != UNIFYFS_SUCCESS) { - rc = UNIFYFS_FAILURE; + /* read reply has an error, mark the read request + * as also having an error, then quit processing */ + req->errcode = rep->errcode; + continue; + } + + /* otherwise, we have an error-free, overlapping reply + * for this request, copy data into request buffer */ + + /* start of overlapping segment is the maximum of + * reply and request start offsets */ + size_t start = rep_start; + if (req_start > start) { + start = req_start; + } + + /* end of overlapping segment is the mimimum of + * reply and request end offsets */ + size_t end = rep_end; + if (req_end < end) { + end = req_end; + } + + /* compute length of overlapping segment */ + size_t length = end - start; + + /* get number of bytes from start of reply and request + * buffers to the start of the overlap region */ + size_t rep_offset = start - rep_start; + size_t req_offset = start - req_start; + + /* if we have a gap, fill with zeros */ + size_t gap_start = req_start + req->nread; + if (start > gap_start) { + size_t gap_length = start - gap_start; + char* req_ptr = req->buf + req->nread; + memset(req_ptr, 0, gap_length); + } + + /* copy data from reply buffer into request buffer */ + char* req_ptr = req->buf + req_offset; + char* rep_ptr = rep_buf + rep_offset; + memcpy(req_ptr, rep_ptr, length); + + /* update max number of bytes we have written to in the + * request buffer */ + size_t nread = end - req_start; + if (nread > req->nread) { + req->nread = nread; + } } } @@ -1853,6 +1689,60 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) } } + /* check for short reads, compare to file size, fill holes */ + for (i = 0; i < count; i++) { + /* get pointer to next read request */ + read_req_t* req = &read_reqs[i]; + + /* if we hit an error on our read, nothing else to do */ + if (req->errcode != UNIFYFS_SUCCESS) { + continue; + } + + /* if we read all of the bytes, we're done */ + if (req->nread == req->length) { + continue; + } + + /* otherwise, we have a short read, check whether there + * would be a hole after us, in which case we fill with + * zeros */ + + /* get file size for this file */ + size_t filesize; + int ret = invoke_client_filesize_rpc(req->gfid, &filesize); + if (ret != UNIFYFS_SUCCESS) { + /* failed to get file size */ + req->errcode = ret; + continue; + } + + /* get offset of where hole starts */ + size_t gap_start = req->offset + req->nread; + + /* get last offset of the read request */ + size_t req_end = req->offset + req->length; + + /* if file size is larger than last offset we wrote to in + * read request, then there is a hole we can fill */ + if (filesize > gap_start) { + /* assume we can fill the full request with zero */ + size_t gap_length = req_end - gap_start; + if (req_end > filesize) { + /* request is trying to read past end of file, + * so only fill zeros up to end of file */ + gap_length = filesize - gap_start; + } + + /* copy zeros into request buffer */ + char* req_ptr = req->buf + req->nread; + memset(req_ptr, 0, gap_length); + + /* update number of bytes read */ + req->nread += gap_length; + } + } + return rc; } @@ -1871,44 +1761,33 @@ ssize_t UNIFYFS_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) return (ssize_t)(-1); } -#if 0 // THIS IS BROKEN UNTIL WE HAVE GLOBAL SIZE - /* get pointer to file descriptor structure */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (meta == NULL) { - /* ERROR: invalid file descriptor */ - errno = EBADF; - return (ssize_t)(-1); - } - - /* check for end of file */ - if (offset >= meta->size) { - return 0; - } -#endif - - /* assume we'll succeed in read */ - size_t retcount = count; - - read_req_t tmp_req; - tmp_req.gfid = unifyfs_gfid_from_fid(fid); - tmp_req.offset = offset; - tmp_req.length = count; - tmp_req.errcode = UNIFYFS_SUCCESS; - tmp_req.buf = buf; - - int ret = unifyfs_fd_logreadlist(&tmp_req, 1); + /* fill in read request */ + read_req_t req; + req.gfid = unifyfs_gfid_from_fid(fid); + req.offset = offset; + req.length = count; + req.nread = 0; + req.errcode = UNIFYFS_SUCCESS; + req.buf = buf; + + /* execute read operation */ + ssize_t retcount; + int ret = unifyfs_fd_logreadlist(&req, 1); if (ret != UNIFYFS_SUCCESS) { /* error reading data */ errno = EIO; retcount = -1; - } else if (tmp_req.errcode != UNIFYFS_SUCCESS) { + } else if (req.errcode != UNIFYFS_SUCCESS) { /* error reading data */ errno = EIO; retcount = -1; + } else { + /* read succeeded, get number of bytes from nread field */ + retcount = (ssize_t) req.nread; } /* return number of bytes read */ - return (ssize_t) retcount; + return retcount; } else { MAP_OR_FAIL(pread); ssize_t ret = UNIFYFS_REAL(pread)(fd, buf, count, offset); From 5108a60a60df8af37b65d3e94018d7473bc080aa Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sat, 4 Jan 2020 13:26:54 -0800 Subject: [PATCH 050/168] server: handle read rpcs that have no write index values In the case that a incoming client read request lands within a hole or past the end of a file, no write index entries will be returned. The client can now handle the case where the server returns no data. This extends the server code to continue through the normal response path even when no read replies can be generated. --- client/src/unifyfs-sysio.c | 4 - server/src/unifyfs_global.h | 3 +- server/src/unifyfs_request_manager.c | 244 +++++++++++++++++---------- 3 files changed, 160 insertions(+), 91 deletions(-) diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 9a2615b1e..deeb3b359 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -1453,10 +1453,6 @@ static int process_read_data(read_req_t* read_reqs, int count, int* done) /* get number of read replies in shared memory */ size_t num = shm_hdr->meta_cnt; - if (0 == num) { - LOGDBG("no read responses available"); - return rc; - } /* process each of our read replies */ size_t i; diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index 2ffc9f746..23a60d99f 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -72,7 +72,8 @@ typedef enum { // NEW READ REQUEST STRUCTURES typedef enum { - READREQ_INIT = 0, + READREQ_NULL = 0, /* request not initialized */ + READREQ_READY, /* request ready to be issued */ READREQ_STARTED, /* chunk requests issued */ READREQ_PARTIAL_COMPLETE, /* some reads completed */ READREQ_COMPLETE /* all reads completed */ diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index edde14a2f..d05a77517 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -352,10 +352,10 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, int num_vals, unifyfs_keyval_t* keyvals) { - int thrd_id = thrd_ctrl->thrd_ndx; + /* get app id for this request batch */ int app_id = thrd_ctrl->app_id; - int client_id = thrd_ctrl->client_id; + /* allocate read request structures */ chunk_read_req_t* all_chunk_reads = (chunk_read_req_t*) calloc((size_t)num_vals, sizeof(chunk_read_req_t)); if (NULL == all_chunk_reads) { @@ -363,36 +363,45 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, return UNIFYFS_ERROR_NOMEM; } + /* wait on lock before we attach new array to global variable */ RM_LOCK(thrd_ctrl); LOGDBG("creating chunk requests for rdreq %d", rdreq->req_ndx); + /* attach read request array to global request mananger struct */ rdreq->chunks = all_chunk_reads; - int i, curr_del; + /* iterate over write index values and create read requests + * for each one, also count up number of delegators that we'll + * forward read requests to */ + int i; int prev_del = -1; - int del_ndx = 0; - chunk_read_req_t* chk_read; + int num_del = 0; for (i = 0; i < num_vals; i++) { - /* count the delegators */ - curr_del = keyvals[i].val.delegator_rank; - if ((prev_del != -1) && (curr_del != prev_del)) { - del_ndx++; + /* get target delegator for this request */ + int curr_del = keyvals[i].val.delegator_rank; + + /* if target delegator is different from last target, + * increment our delegator count */ + if ((prev_del == -1) || (curr_del != prev_del)) { + num_del++; } prev_del = curr_del; - /* create chunk-reads */ + /* get pointer to next read request structure */ debug_log_key_val(__func__, &keyvals[i].key, &keyvals[i].val); - chk_read = all_chunk_reads + i; - chk_read->nbytes = keyvals[i].val.len; - chk_read->offset = keyvals[i].key.offset; - chk_read->log_offset = keyvals[i].val.addr; - chk_read->log_app_id = keyvals[i].val.app_id; - chk_read->log_client_id = keyvals[i].val.rank; + chunk_read_req_t* chk = all_chunk_reads + i; + + /* fill in chunk read request */ + chk->nbytes = keyvals[i].val.len; + chk->offset = keyvals[i].key.offset; + chk->log_offset = keyvals[i].val.addr; + chk->log_app_id = keyvals[i].val.app_id; + chk->log_client_id = keyvals[i].val.rank; } /* allocate per-delgator chunk-reads */ - int num_dels = del_ndx + 1; + int num_dels = num_del; rdreq->num_remote_reads = num_dels; rdreq->remote_reads = (remote_chunk_reads_t*) calloc((size_t)num_dels, sizeof(remote_chunk_reads_t)); @@ -402,46 +411,126 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, return UNIFYFS_ERROR_NOMEM; } - /* populate per-delegator chunk-reads info */ - size_t del_data_sz = 0; - remote_chunk_reads_t* del_reads; + /* get pointer to start of chunk read request array */ + remote_chunk_reads_t* reads = rdreq->remote_reads; + + /* iterate over write index values again and now create + * per-delegator chunk-reads info, for each delegator + * that we'll request data from, this totals up the number + * of read requests and total read data size from that + * delegator */ prev_del = -1; - del_ndx = 0; + size_t del_data_sz = 0; for (i = 0; i < num_vals; i++) { - curr_del = keyvals[i].val.delegator_rank; + /* get target delegator for this request */ + int curr_del = keyvals[i].val.delegator_rank; + + /* if target delegator is different from last target, + * close out the total number of bytes for the last + * delegator, note this assumes our write index values are + * sorted by delegator rank */ if ((prev_del != -1) && (curr_del != prev_del)) { /* record total data for previous delegator */ - del_reads = rdreq->remote_reads + del_ndx; - del_reads->total_sz = del_data_sz; - /* advance to next delegator */ - del_ndx++; + reads->total_sz = del_data_sz; + + /* advance to read request for next delegator */ + reads += 1; + + /* reset our running tally of bytes to 0 */ del_data_sz = 0; } prev_del = curr_del; - /* update total data size for current delegator */ + /* update total read data size for current delegator */ del_data_sz += keyvals[i].val.len; - del_reads = rdreq->remote_reads + del_ndx; - if (0 == del_reads->num_chunks) { - /* initialize structure */ - del_reads->rank = curr_del; - del_reads->rdreq_id = rdreq->req_ndx; - del_reads->reqs = all_chunk_reads + i; - del_reads->resp = NULL; + /* if this is the first read request for this delegator, + * initialize fields on the per-delegator read request + * structure */ + if (0 == reads->num_chunks) { + /* TODO: let's describe what these fields are for */ + reads->rank = curr_del; + reads->rdreq_id = rdreq->req_ndx; + reads->reqs = all_chunk_reads + i; + reads->resp = NULL; } - del_reads->num_chunks++; + + /* increment number of read requests we're sending + * to this delegator */ + reads->num_chunks++; + } + + /* record total data size for final delegator (if any), + * would have missed doing this in the above loop */ + if (num_vals > 0) { + reads->total_sz = del_data_sz; } - del_reads = rdreq->remote_reads + del_ndx; - del_reads->total_sz = del_data_sz; + + /* mark request as ready to be started */ + rdreq->status = READREQ_READY; /* wake up the request manager thread for the requesting client */ signal_new_requests(thrd_ctrl); + RM_UNLOCK(thrd_ctrl); return UNIFYFS_SUCCESS; } +/* signal the client process for it to start processing read + * data in shared memory */ +static int client_signal(shm_header_t* hdr, + shm_region_state_e flag) +{ + if (flag == SHMEM_REGION_DATA_READY) { + LOGDBG("setting data-ready"); + } else if (flag == SHMEM_REGION_DATA_COMPLETE) { + LOGDBG("setting data-complete"); + } + + /* we signal the client by setting a flag value within + * a shared memory segment that the client is monitoring */ + hdr->state = flag; + + /* TODO: MEM_FLUSH */ + + return UNIFYFS_SUCCESS; +} + +/* wait until client has processed all read data in shared memory */ +static int client_wait(shm_header_t* hdr) +{ + int rc = (int)UNIFYFS_SUCCESS; + + /* specify time to sleep between checking flag in shared + * memory indicating client has processed data */ + struct timespec shm_wait_tm; + shm_wait_tm.tv_sec = 0; + shm_wait_tm.tv_nsec = SHM_WAIT_INTERVAL; + + /* wait for client to set flag to 0 */ + int max_sleep = 10000000; // 10s + volatile int* vip = (volatile int*)&(hdr->state); + while (*vip != SHMEM_REGION_EMPTY) { + /* not there yet, sleep for a while */ + nanosleep(&shm_wait_tm, NULL); + + /* TODO: MEM_FETCH */ + + max_sleep--; + if (0 == max_sleep) { + LOGERR("timed out waiting for empty"); + rc = (int)UNIFYFS_ERROR_SHMEM; + break; + } + } + + /* reset header to reflect empty state */ + hdr->meta_cnt = 0; + hdr->bytes = 0; + return rc; +} + /************************ * These functions are called by the rpc handler to assign work * to the request manager thread @@ -543,15 +632,13 @@ int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, return UNIFYFS_ERROR_NOMEM; } - /* TODO: if there are file extents not accounted for we should - * either return 0 for that data (holes) or EOF if reading past - * the end of the file */ - if (UNIFYFS_SUCCESS != rc || num_vals == 0) { + if (UNIFYFS_SUCCESS != rc) { /* failed to find any key / value pairs */ rc = UNIFYFS_FAILURE; } else { + /* if we get more than one write index entry + * sort them by file id and then by delegator rank */ if (num_vals > 1) { - /* sort keyvals by delegator */ qsort(keyvals, (size_t)num_vals, sizeof(unifyfs_keyval_t), compare_kv_gfid_rank); } @@ -564,6 +651,7 @@ int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, rdreq->client_id = client_id; rdreq->extent.gfid = gfid; rdreq->extent.errcode = EINPROGRESS; + rc = create_chunk_requests(thrd_ctrl, rdreq, num_vals, keyvals); if (rc != (int)UNIFYFS_SUCCESS) { @@ -971,7 +1059,7 @@ static int rm_request_remote_chunks(reqmgr_thrd_t* thrd_ctrl) if (req->num_remote_reads > 0) { LOGDBG("read req %d is active", i); debug_print_read_req(req); - if (req->status == READREQ_INIT) { + if (req->status == READREQ_READY) { req->status = READREQ_STARTED; /* iterate over each delegator we need to send requests to */ remote_chunk_reads_t* remote_reads; @@ -1004,6 +1092,10 @@ static int rm_request_remote_chunks(reqmgr_thrd_t* thrd_ctrl) /* already started */ LOGDBG("read req %d already processed", i); } + } else if (req->num_remote_reads == 0) { + if (req->status == READREQ_READY) { + req->status = READREQ_STARTED; + } } } @@ -1042,57 +1134,37 @@ static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) ret = rc; } } - } - } + } else if ((req->num_remote_reads == 0) && + (req->status == READREQ_STARTED)) { + /* look up client shared memory region */ + app_config_t* app_config = + (app_config_t*) arraylist_get(app_config_list, req->app_id); + assert(NULL != app_config); + shm_header_t* client_shm = + (shm_header_t*) app_config->shm_recv_bufs[req->client_id]; - return ret; -} + RM_LOCK(thrd_ctrl); -/* signal the client process for it to start processing read - * data in shared memory */ -static int client_signal(shm_header_t* hdr, - shm_region_state_e flag) -{ - if (flag == SHMEM_REGION_DATA_READY) { - LOGDBG("setting data-ready"); - } else if (flag == SHMEM_REGION_DATA_COMPLETE) { - LOGDBG("setting data-complete"); - } - hdr->state = flag; - /* TODO: MEM_FLUSH */ - return UNIFYFS_SUCCESS; -} + /* mark request as complete */ + req->status = READREQ_COMPLETE; -/* wait until client has processed all read data in shared memory */ -static int client_wait(shm_header_t* hdr) -{ - int rc = (int)UNIFYFS_SUCCESS; + /* signal client that we're now done writing data */ + client_signal(client_shm, SHMEM_REGION_DATA_COMPLETE); - /* specify time to sleep between checking flag in shared - * memory indicating client has processed data */ - struct timespec shm_wait_tm; - shm_wait_tm.tv_sec = 0; - shm_wait_tm.tv_nsec = SHM_WAIT_INTERVAL; + /* wait for client to read data */ + client_wait(client_shm); - /* wait for client to set flag to 0 */ - int max_sleep = 10000000; // 10s - volatile int* vip = (volatile int*)&(hdr->state); - while (*vip != SHMEM_REGION_EMPTY) { - /* not there yet, sleep for a while */ - nanosleep(&shm_wait_tm, NULL); - /* TODO: MEM_FETCH */ - max_sleep--; - if (0 == max_sleep) { - LOGERR("timed out waiting for empty"); - rc = (int)UNIFYFS_ERROR_SHMEM; - break; + rc = release_read_req(thrd_ctrl, req); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to release server_read_req_t"); + ret = rc; + } + + RM_UNLOCK(thrd_ctrl); } } - /* reset header to reflect empty state */ - hdr->meta_cnt = 0; - hdr->bytes = 0; - return rc; + return ret; } static shm_meta_t* reserve_shmem_meta(app_config_t* app_config, From 0151150667e7d944e543a0192eaa20f49c7efbe6 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 6 Jan 2020 19:41:06 -0800 Subject: [PATCH 051/168] tests: check read from file with holes --- t/Makefile.am | 6 +- t/sys/sysio_suite.c | 2 + t/sys/sysio_suite.h | 3 + t/sys/write-read-hole.c | 209 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 t/sys/write-read-hole.c diff --git a/t/Makefile.am b/t/Makefile.am index 840333ddb..bb6b17ba9 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -100,7 +100,8 @@ sys_sysio_gotcha_t_SOURCES = sys/sysio_suite.h \ sys/mkdir-rmdir.c \ sys/open.c \ sys/open64.c \ - sys/write-read.c + sys/write-read.c \ + sys/write-read-hole.c sys_sysio_gotcha_t_CPPFLAGS = $(test_cppflags) sys_sysio_gotcha_t_LDADD = $(test_ldadd) @@ -113,7 +114,8 @@ sys_sysio_static_t_SOURCES = sys/sysio_suite.h \ sys/mkdir-rmdir.c \ sys/open.c \ sys/open64.c \ - sys/write-read.c + sys/write-read.c \ + sys/write-read-hole.c sys_sysio_static_t_CPPFLAGS = $(test_cppflags) sys_sysio_static_t_LDADD = $(test_static_ldadd) diff --git a/t/sys/sysio_suite.c b/t/sys/sysio_suite.c index 2c656144e..6cebb83f4 100644 --- a/t/sys/sysio_suite.c +++ b/t/sys/sysio_suite.c @@ -83,6 +83,8 @@ int main(int argc, char* argv[]) write_read_test(unifyfs_root); + write_read_hole_test(unifyfs_root); + MPI_Finalize(); done_testing(); diff --git a/t/sys/sysio_suite.h b/t/sys/sysio_suite.h index 3ca8a327c..d4f90ddb3 100644 --- a/t/sys/sysio_suite.h +++ b/t/sys/sysio_suite.h @@ -47,4 +47,7 @@ int open64_test(char* unifyfs_root); int write_read_test(char* unifyfs_root); +/* test reading from file with holes */ +int write_read_hole_test(char* unifyfs_root); + #endif /* SYSIO_SUITE_H */ diff --git a/t/sys/write-read-hole.c b/t/sys/write-read-hole.c new file mode 100644 index 000000000..fe703e13e --- /dev/null +++ b/t/sys/write-read-hole.c @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + + /* + * Test reading from file with holes + */ +#include +#include +#include +#include +#include +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +/* Get global, local, or log sizes (or all) */ +static +void get_size(char* path, size_t* global, size_t* local, size_t* log) +{ + struct stat sb = {0}; + int rc; + + rc = stat(path, &sb); + if (rc != 0) { + printf("Error: %s\n", strerror(errno)); + exit(1); /* die on failure */ + } + if (global) { + *global = sb.st_size; + } + + if (local) { + *local = sb.st_rdev & 0xFFFFFFFF; + } + + if (log) { + *log = (sb.st_rdev >> 32) & 0xFFFFFFFF; + } +} + +static int check_contents(char* buf, size_t len, char c) +{ + int valid = 1; + size_t i; + for (i = 0; i < len; i++) { + if (buf[i] != c) { + valid = 0; + } + } + return valid; +} + +int write_read_hole_test(char* unifyfs_root) +{ + char path[64]; + int rc; + int fd; + size_t global, local, log; + + size_t bufsize = 1024*1024; + char* buf = (char*) malloc(bufsize); + int i; + for (i = 0; i < bufsize; i++) { + buf[i] = 1; + } + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* create a file that contains: + * [0, 1MB) - data = "1" + * [1MB, 2MB) - hole = "0" implied + * [2MB, 3MB) - data = "1" */ + + /* Write to the file */ + fd = open(path, O_WRONLY | O_CREAT, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + rc = write(fd, buf, bufsize); + ok(rc == bufsize, "%s:%d write() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* Test writing to a different offset */ + rc = lseek(fd, 2*bufsize, SEEK_SET); + ok(rc == 2*bufsize, "%s:%d lseek() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + rc = write(fd, buf, bufsize); + ok(rc == bufsize, "%s:%d write() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* Check global and local size on our un-laminated file */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s:%d global size is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + ok(local == 3*bufsize, "%s:%d local size is %d: %s", + __FILE__, __LINE__, local, strerror(errno)); + ok(log == 2*bufsize, "%s:%d log size is %d: %s", + __FILE__, __LINE__, log, strerror(errno)); + + /* flush writes */ + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* Laminate */ + rc = chmod(path, 0444); + ok(rc == 0, "%s:%d chmod(0444) (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* Check global and local size on our un-laminated file */ + get_size(path, &global, &local, &log); + ok(global == 3*bufsize, "%s:%d global size is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + ok(local == 3*bufsize, "%s:%d local size is %d: %s", + __FILE__, __LINE__, local, strerror(errno)); + ok(log == 2*bufsize, "%s:%d log size is %d: %s", + __FILE__, __LINE__, log, strerror(errno)); + + close(fd); + + /*************** + * open file for reading + ***************/ + + fd = open(path, O_RDONLY); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + + /* read segment [0, 1MB) -- should be all "1" + * this should be a full read, all from actual data */ + memset(buf, 2, bufsize); + ssize_t nread = pread(fd, buf, bufsize, 0*bufsize); + ok(nread == bufsize, "%s:%d pread(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* check that full buffer is "1" */ + int valid = check_contents(buf, bufsize, 1); + ok(valid == 1, "%s:%d data check", + __FILE__, __LINE__); + + + /* read segment [1MB, 2MB) -- should be all "0" + * this should be a full read, all from a hole */ + memset(buf, 2, bufsize); + nread = pread(fd, buf, bufsize, 1*bufsize); + ok(nread == bufsize, "%s:%d pread(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* check that full buffer is "0" */ + valid = check_contents(buf, bufsize, 0); + ok(valid == 1, "%s:%d data check", + __FILE__, __LINE__); + + + /* read segment [0.5MB, 1.5MB) + * should be a full read, half data, half hole */ + memset(buf, 2, bufsize); + nread = pread(fd, buf, bufsize, bufsize/2); + ok(nread == bufsize, "%s:%d pread(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* check that data portion is "1" */ + valid = check_contents(buf, bufsize/2, 1); + ok(valid == 1, "%s:%d data check", + __FILE__, __LINE__); + + /* check that hole portion is "0" */ + valid = check_contents(buf + bufsize/2, bufsize/2, 0); + ok(valid == 1, "%s:%d data check", + __FILE__, __LINE__); + + + /* read segment [2.5MB, 3.5MB) + * should read only half of requested amount, + * half data, half past end of file */ + memset(buf, 2, bufsize); + nread = pread(fd, buf, bufsize, 2*bufsize + bufsize/2); + ok(nread == bufsize/2, "%s:%d pread(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* first half of buffer should be "1" */ + valid = check_contents(buf, bufsize/2, 1); + ok(valid == 1, "%s:%d data check", + __FILE__, __LINE__); + + /* second half of buffer should not be changed, still "2" */ + valid = check_contents(buf + bufsize/2, bufsize/2, 2); + ok(valid == 1, "%s:%d data check", + __FILE__, __LINE__); + + + close(fd); + + free(buf); + + return 0; +} From 31aeccf0eb7b3655009463174fb5acc7715464d4 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 9 Jan 2020 20:00:30 -0800 Subject: [PATCH 052/168] client: always use logical_size to get global/local_size --- client/src/unifyfs-sysio.c | 2 +- client/src/unifyfs.c | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index deeb3b359..e7ef87080 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -1010,7 +1010,7 @@ ssize_t UNIFYFS_WRAP(write)(int fd, const void* buf, size_t count) * file position. */ int fid = unifyfs_get_fid_from_fd(fd); - pos = unifyfs_fid_local_size(fid); + pos = unifyfs_fid_logical_size(fid); } else { pos = filedesc->pos; } diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 498492114..751a83fef 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -1286,9 +1286,8 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, if (flags & O_APPEND) { /* We only support O_APPEND on non-laminated (local) files, so - * use local_size here. */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - pos = meta->local_size; + * this will use local_size here. */ + pos = unifyfs_fid_logical_size(fid); } } else { /* !found_local && !found_global From 6a4f566ba04e0288e5a6a9c263553eb8ff4fef43 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Fri, 10 Jan 2020 13:59:46 -0800 Subject: [PATCH 053/168] Add write flattening This flattens all the writes on an fsync to reduce the number of extents transferred. For example, let's say you did five writes: [w1---------------] [w2-----------] [w3-------------] [w4-------------] [w5----------------] After flattening: [w1][w5----------------][w4-----] This also means that reads are faster, since there's fewer extents to iterate over. You can disable write flattening via the UNIFYFS_CLIENT_FLATTEN_WRITES config value (enabled by default). Other notes: - The red-black tree implementation (tree.h) included in this commit was taken from OpenBSD: http://cvsweb.openbsd.org/cgi-bin/cvsweb/~checkout~/src/sys/sys/tree.h tree.h's macros are valid but checkpatch still flags them. Exempt it: TEST_CHECKPATCH_SKIP_FILES=common/src/tree.h - Cameron's travis.yml fix is included. - Fixed a bug in size.c --- .travis.yml | 1 + client/src/unifyfs-fixed.c | 277 +++++-- client/src/unifyfs-fixed.h | 2 + client/src/unifyfs-internal.h | 4 + client/src/unifyfs-sysio.c | 7 +- client/src/unifyfs.c | 33 +- common/src/LICENSE.tree_dot_h | 22 + common/src/Makefile.am | 5 +- common/src/seg_tree.c | 354 +++++++++ common/src/seg_tree.h | 48 ++ common/src/tree.h | 1007 ++++++++++++++++++++++++++ common/src/unifyfs_configurator.h | 1 + configure.ac | 3 + docs/configuration.rst | 11 +- examples/src/Makefile.am | 16 +- examples/src/multi-write.c | 193 +++++ server/src/unifyfs_request_manager.c | 6 + t/std/size.c | 7 +- 18 files changed, 1916 insertions(+), 81 deletions(-) create mode 100644 common/src/LICENSE.tree_dot_h create mode 100644 common/src/seg_tree.c create mode 100644 common/src/seg_tree.h create mode 100644 common/src/tree.h create mode 100644 examples/src/multi-write.c diff --git a/.travis.yml b/.travis.yml index 4db600ed1..736e2bd0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ addons: update: true packages: - autoconf + - autoconf-archive - automake - build-essential - cmake diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 0f78bc02d..c0318055b 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -44,6 +44,7 @@ #include "unifyfs-fixed.h" #include "unifyfs_log.h" #include "margo_client.h" +#include "seg_tree.h" static inline unifyfs_chunkmeta_t* filemeta_get_chunkmeta(const unifyfs_filemeta_t* meta, @@ -336,12 +337,22 @@ static int unifyfs_coalesce_index( return UNIFYFS_SUCCESS; } -/* given an index, split it into multiple indices whose range is equal or - * smaller than slice_range size - * @param cur_idx: the index to split - * @param slice_range: the slice size of the key-value store - * @return index_set: the set of split indices - * @param maxcount: number of entries in output array */ +/* + * Given an index, split it into multiple indices whose range is equal or + * smaller than slice_range size. For example, if you passed a cur_index + * for a 3.5MB write, and the slice size was 1MB, it would split it into + * four indexes, and update cur_index.length to be zero. This also takes + * in a 'maxcount' field, so you can limit the number of indexes you + * create. Using our above example, if 'maxcount=2', then this would + * create two indexes, and update cur_index.length to 1.5MB (for the remaining + * data). + * + * @param cur_idx: The index to split + * @param slice_range: The slice size of the key-value store + * @return index_set: The set of split indices + * @param maxcount: Number of entries in output array + * @param used_count: Number of entries we actually added in the split + */ static int unifyfs_split_index( unifyfs_index_t* cur_idx, /* write index to split (offset and length) */ long slice_range, /* number of bytes in each slice */ @@ -471,6 +482,152 @@ static int unifyfs_split_index( return UNIFYFS_SUCCESS; } +/* + * Clear all entries in the log index. This only clears the metadata, + * not the data itself. + */ +static void +unifyfs_clear_index(void) +{ + *unifyfs_indices.ptr_num_entries = 0; +} + +/* + * Sync all the extents to the server. Clears the metadata index afterwards. + * + * Returns 0 on success, nonzero otherwise. + */ +int +unifyfs_sync(int gfid) +{ + if (unifyfs_flatten_writes) { + unifyfs_rewrite_index_from_seg_tree(); + } + + int ret = invoke_client_fsync_rpc(gfid); + if (ret != UNIFYFS_SUCCESS) { + /* something went wrong when trying to flush key/values */ + LOGERR("failed to flush key/value index to server"); + return UNIFYFS_ERROR_IO; + } + + /* + * flushed, clear buffer and refresh number of entries + * and number remaining + */ + unifyfs_clear_index(); + + return UNIFYFS_SUCCESS; +} + +void +unifyfs_add_index_entry_to_seg_tree(unifyfs_filemeta_t* meta, + unifyfs_index_t* index) +{ + if (!unifyfs_flatten_writes) { + /* We're not flattening writes. Nothing to do */ + return; + } + + /* + * Store the write in our segment tree. We will later use this for + * flattening writes. + */ + seg_tree_add(&meta->seg_tree, index->file_pos, + index->file_pos + index->length - 1, + index->log_pos); + + /* + * We want to make sure the next write following this wont overflow the + * max number of index entries (if it were synced). A write can at most + * create two new nodes in the seg_tree. If we're close to potentially + * filling up the index, sync it out. + */ + if (seg_tree_count(&meta->seg_tree) >= (unifyfs_max_index_entries - 2)) { + unifyfs_sync(meta->gfid); + } +} + +/* Add the metadata for a single write to the index */ +static int unifyfs_logio_add_write_meta_to_index(unifyfs_filemeta_t* meta, + off_t file_pos, off_t log_pos, size_t length) +{ + /* define an new index entry for this write operation */ + int gfid = meta->gfid; + unifyfs_index_t cur_idx; + cur_idx.gfid = gfid; + cur_idx.file_pos = file_pos; + cur_idx.log_pos = log_pos; + cur_idx.length = length; + + /* lookup number of existing index entries */ + off_t num_entries = *(unifyfs_indices.ptr_num_entries); + + /* get pointer to index array */ + unifyfs_index_t* idxs = unifyfs_indices.index_entry; + + /* attempt to coalesce contiguous index entries if we + * have an existing index in the buffer */ + if (num_entries > 0) { + /* get pointer to last element in index array */ + unifyfs_index_t* prev_idx = &idxs[num_entries - 1]; + + /* attempt to coalesce current index with last index, + * updates fields in last index and current index + * accordingly */ + unifyfs_coalesce_index(prev_idx, &cur_idx, + unifyfs_key_slice_range); + if (cur_idx.length == 0) { + /* We were able to coalesce this write into prev_idx */ + unifyfs_add_index_entry_to_seg_tree(meta, prev_idx); + } + } + + /* add new index entries if needed */ + while (cur_idx.length > 0) { + /* remaining entries we can fit in the shared memory region */ + off_t remaining_entries = unifyfs_max_index_entries - num_entries; + + /* if we have filled the key/value buffer, flush it to server */ + if (0 == remaining_entries) { + /* index buffer is full, flush it */ + int ret = unifyfs_sync(cur_idx.gfid); + if (ret != UNIFYFS_SUCCESS) { + /* something went wrong when trying to flush key/values */ + LOGERR("failed to flush key/value index to server"); + return UNIFYFS_ERROR_IO; + } + } + + /* split any remaining write index at boundaries of + * unifyfs_key_slice_range */ + off_t used_entries = 0; + int split_rc = unifyfs_split_index(&cur_idx, + unifyfs_key_slice_range, &idxs[num_entries], + remaining_entries, &used_entries); + if (split_rc != UNIFYFS_SUCCESS) { + /* in this case, we have copied data to the log, + * but we failed to generate index entries, + * we're returning with an error and leaving the data + * in the log */ + LOGERR("failed to split write index"); + return UNIFYFS_ERROR_IO; + } + + /* Add our split index entries to our seg_tree */ + for (int i = 0; i < used_entries; i++) { + unifyfs_add_index_entry_to_seg_tree(meta, &idxs[num_entries + i]); + } + + /* account for entries we just added */ + num_entries += used_entries; + + /* update number of entries in index array */ + (*unifyfs_indices.ptr_num_entries) = num_entries; + } + return UNIFYFS_SUCCESS; +} + /* write count bytes from user buffer into specified chunk id at chunk offset, * count should fit within chunk starting from specified offset */ static int unifyfs_logio_chunk_write( @@ -543,78 +700,64 @@ static int unifyfs_logio_chunk_write( /* get global file id for this file */ int gfid = unifyfs_gfid_from_fid(fid); - /* define an new index entry for this write operation */ - unifyfs_index_t cur_idx; - cur_idx.gfid = gfid; - cur_idx.file_pos = pos; - cur_idx.log_pos = log_offset; - cur_idx.length = count; + /* Update our write metadata with the new write */ + int rc = unifyfs_logio_add_write_meta_to_index(meta, pos, log_offset, + count); + return rc; +} - /* lookup number of existing index entries */ - off_t num_entries = *(unifyfs_indices.ptr_num_entries); +/* + * Remove all entries in the current index and re-write it using the write + * metadata stored in all the file's seg_trees. This only re-writes the + * metadata in the index. All the actual data is still kept in the log and + * will be referenced correctly by the new metadata. + * + * After this function is done 'unifyfs_indices' will have been totally + * re-written. The writes in the index will be flattened, non-overlapping, + * and sequential, for each file, one after another. All seg_trees will be + * cleared. + * + * This function is called when we sync our extents. + */ +void unifyfs_rewrite_index_from_seg_tree(void) +{ + struct seg_tree_node* node; + unifyfs_index_t* indexes = unifyfs_indices.index_entry; + unsigned long idx = 0; - /* get pointer to index array */ - unifyfs_index_t* idxs = unifyfs_indices.index_entry; - /* attempt to coalesce contiguous index entries if we - * have an existing index in the buffer */ - if (num_entries > 0) { - /* get pointer to last element in index array */ - unifyfs_index_t* prev_idx = &idxs[num_entries - 1]; + /* Erase the index before we re-write it */ + unifyfs_clear_index(); - /* attempt to coalesce current index with last index, - * updates fields in last index and current index - * accordingly */ - unifyfs_coalesce_index(prev_idx, &cur_idx, - unifyfs_key_slice_range); - } + /* For each fid .. */ + for (int i = 0; i < UNIFYFS_MAX_FILEDESCS; i++) { + int fid, gfid; - /* add new index entries if needed */ - while (cur_idx.length > 0) { - /* remaining entries we can fit in the shared memory region */ - off_t remaining_entries = unifyfs_max_index_entries - num_entries; + fid = unifyfs_fds[i].fid; + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if (!meta) { + continue; + } - /* if we have filled the key/value buffer, flush it to server */ - if (0 == remaining_entries) { - /* index buffer is full, flush it */ - int ret = invoke_client_fsync_rpc(cur_idx.gfid); - if (ret != UNIFYFS_SUCCESS) { - /* something went wrong when trying to flush key/values */ - LOGERR("failed to flush key/value index to server"); - return UNIFYFS_ERROR_IO; - } + gfid = unifyfs_gfid_from_fid(fid); - /* flushed, clear buffer and refresh number of entries - * and number remaining */ - num_entries = 0; - *(unifyfs_indices.ptr_num_entries) = num_entries; - remaining_entries = unifyfs_max_index_entries - num_entries; - } + node = NULL; + seg_tree_rdlock(&meta->seg_tree); - /* split any remaining write index at boundaries of - * unifyfs_key_slice_range */ - off_t used_entries = 0; - int split_rc = unifyfs_split_index(&cur_idx, - unifyfs_key_slice_range, &idxs[num_entries], - remaining_entries, &used_entries); - if (split_rc != UNIFYFS_SUCCESS) { - /* in this case, we have copied data to the log, - * but we failed to generate index entries, - * we're returning with an error and leaving the data - * in the log */ - LOGERR("failed to split write index"); - return UNIFYFS_ERROR_IO; + /* For each write in this file's seg_tree ... */ + while ((node = seg_tree_iter(&meta->seg_tree, node))) { + indexes[idx].file_pos = node->start; + indexes[idx].log_pos = node->ptr; + indexes[idx].length = node->end - node->start + 1; + indexes[idx].gfid = gfid; + idx++; } + seg_tree_unlock(&meta->seg_tree); - /* account for entries we just added */ - num_entries += used_entries; - - /* update number of entries in index array */ - (*unifyfs_indices.ptr_num_entries) = num_entries; + /* All done processing this files writes. Clear its seg_tree */ + seg_tree_clear(&meta->seg_tree); } - - /* assume write was successful if we get to here */ - return UNIFYFS_SUCCESS; + *unifyfs_indices.ptr_num_entries = idx; } /* write count bytes from user buffer into specified chunk id at chunk offset, diff --git a/client/src/unifyfs-fixed.h b/client/src/unifyfs-fixed.h index dbd46b4e9..8e9bd6929 100644 --- a/client/src/unifyfs-fixed.h +++ b/client/src/unifyfs-fixed.h @@ -81,4 +81,6 @@ int unifyfs_fid_store_fixed_write( size_t count /* number of bytes to write */ ); +void unifyfs_rewrite_index_from_seg_tree(void); +int unifyfs_sync(int gfid); #endif /* UNIFYFS_FIXED_H */ diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index f641a694c..e342d2d0d 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -96,6 +96,7 @@ #include "unifyfs_log.h" #include "unifyfs_meta.h" #include "unifyfs_shm.h" +#include "seg_tree.h" // client headers #include "unifyfs.h" @@ -283,6 +284,8 @@ typedef struct { uint32_t mode; /* st_mode bits. This has file * permission info and will tell you if this * is a regular file or directory. */ + struct seg_tree seg_tree; /* Segment tree containing our coalesced + * writes */ } unifyfs_filemeta_t; /* struct used to map a full path to its local file id, @@ -387,6 +390,7 @@ extern int unifyfs_use_memfs; extern int unifyfs_use_spillover; extern int unifyfs_max_files; /* maximum number of files to store */ +extern bool unifyfs_flatten_writes; /* enable write flattening */ extern size_t unifyfs_chunk_mem; /* number of bytes in memory to be used for chunk storage */ extern int unifyfs_chunk_bits; /* we set chunk size = 2^unifyfs_chunk_bits */ diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index e7ef87080..d41552b54 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -1885,6 +1885,11 @@ int UNIFYFS_WRAP(fsync)(int fd) { /* check whether we should intercept this file descriptor */ if (unifyfs_intercept_fd(&fd)) { + if (*unifyfs_indices.ptr_num_entries == 0) { + /* Nothing to sync */ + return 0; + } + /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { @@ -1911,7 +1916,7 @@ int UNIFYFS_WRAP(fsync)(int fd) /* invoke fsync rpc to register index metadata with server */ int gfid = unifyfs_gfid_from_fid(fid); - int ret = invoke_client_fsync_rpc(gfid); + int ret = unifyfs_sync(gfid); if (ret != UNIFYFS_SUCCESS) { /* sync failed for some reason, set errno and return error */ errno = unifyfs_err_map_to_errno(ret); diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 751a83fef..05681c771 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -62,6 +62,7 @@ #include "unifyfs_server_rpcs.h" #include "unifyfs_rpc_util.h" #include "margo_client.h" +#include "seg_tree.h" /* avoid duplicate mounts (for now) */ static int unifyfs_mounted = -1; @@ -118,6 +119,7 @@ static off_t unifyfs_min_long; /* TODO: moved these to fixed file */ int unifyfs_max_files; /* maximum number of files to store */ +bool unifyfs_flatten_writes; /* flatten our writes (true = enabled) */ size_t unifyfs_chunk_mem; /* number of bytes in memory to be used for chunk storage */ int unifyfs_chunk_bits; /* we set chunk size = 2^unifyfs_chunk_bits */ off_t unifyfs_chunk_size; /* chunk size in bytes */ @@ -644,6 +646,11 @@ static int unifyfs_fid_store_free(int fid) /* set storage type back to NULL */ meta->storage = FILE_STORAGE_NULL; + /* Free our write seg_tree */ + if (unifyfs_flatten_writes) { + seg_tree_destroy(&meta->seg_tree); + } + return UNIFYFS_SUCCESS; } @@ -916,6 +923,8 @@ int unifyfs_fid_free(int fid) * returns the new fid, or negative value on error */ int unifyfs_fid_create_file(const char* path) { + int rc; + /* check that pathname is within bounds */ size_t pathlen = strlen(path) + 1; if (pathlen > UNIFYFS_MAX_FILENAME) { @@ -951,6 +960,15 @@ int unifyfs_fid_create_file(const char* path) meta->is_laminated = 0; meta->mode = UNIFYFS_STAT_DEFAULT_FILE_MODE; + if (unifyfs_flatten_writes) { + /* Initialize our segment tree that will record our writes */ + rc = seg_tree_init(&meta->seg_tree); + if (rc != 0) { + errno = rc; + return -rc; + } + } + /* PTHREAD_PROCESS_SHARED allows Process-Shared Synchronization */ pthread_spin_init(&meta->fspinlock, PTHREAD_PROCESS_SHARED); @@ -1028,7 +1046,10 @@ int unifyfs_fid_create_directory(const char* path) /* write count bytes from buf into file starting at offset pos, * all bytes are assumed to be allocated to file, so file should - * be extended before calling this routine */ + * be extended before calling this routine. + * + * Returns a UNIFYFS_ERROR on error, 0 on success. + */ int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count) { int rc; @@ -1760,6 +1781,16 @@ static int unifyfs_init(int rank) } } + /* Determine if we should flatten writes or not */ + unifyfs_flatten_writes = 1; + cfgval = client_cfg.client_flatten_writes; + if (cfgval != NULL) { + rc = configurator_bool_val(cfgval, &b); + if (rc == 0) { + unifyfs_flatten_writes = (bool)b; + } + } + /* determine number of bits for chunk size */ unifyfs_chunk_bits = UNIFYFS_CHUNK_BITS; cfgval = client_cfg.shmem_chunk_bits; diff --git a/common/src/LICENSE.tree_dot_h b/common/src/LICENSE.tree_dot_h new file mode 100644 index 000000000..ae4b4fe2f --- /dev/null +++ b/common/src/LICENSE.tree_dot_h @@ -0,0 +1,22 @@ +Copyright 2002 Niels Provos +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/common/src/Makefile.am b/common/src/Makefile.am index 41c07770b..92c0a6a38 100644 --- a/common/src/Makefile.am +++ b/common/src/Makefile.am @@ -34,7 +34,10 @@ BASE_SRCS = \ unifyfs_runstate.h \ unifyfs_runstate.c \ unifyfs_shm.h \ - unifyfs_shm.c + unifyfs_shm.c \ + tree.h \ + seg_tree.h \ + seg_tree.c OPT_FLAGS = OPT_LIBS = diff --git a/common/src/seg_tree.c b/common/src/seg_tree.c new file mode 100644 index 000000000..1f530c306 --- /dev/null +++ b/common/src/seg_tree.c @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * LLNL-CODE-741539 + * All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + /* + * This file is a simple, thread-safe, segment tree implementation. The + * segments in the tree are non-overlapping. Added segments overwrite the old + * segments in the tree. This is used to coalesce writes before an fsync. + */ +#include +#include +#include +#include +#include +#include +#include "seg_tree.h" +#include "tree.h" + +#define MIN(a, b) (a < b ? a : b) +#define MAX(a, b) (a > b ? a : b) + +int +compare_func(struct seg_tree_node* node1, struct seg_tree_node* node2) +{ + if (node1->start > node2->end) { + return 1; + } else if (node1->end < node2->start) { + return -1; + } else { + return 0; + } +} + +RB_PROTOTYPE(inttree, seg_tree_node, entry, compare_func) +RB_GENERATE(inttree, seg_tree_node, entry, compare_func) + +/* Returns 0 on success, positive non-zero error code otherwise */ +int seg_tree_init(struct seg_tree* seg_tree) +{ + memset(seg_tree, 0, sizeof(*seg_tree)); + pthread_rwlock_init(&seg_tree->rwlock, NULL); + RB_INIT(&seg_tree->head); + + return 0; +}; + +/* + * Remove and free all nodes in the seg_tree. + */ +void seg_tree_destroy(struct seg_tree* seg_tree) +{ + seg_tree_clear(seg_tree); +}; + +/* Allocate a range tree. Free it with free() when finished */ +static struct seg_tree_node* +seg_tree_node_alloc(unsigned long start, unsigned long end, unsigned long ptr) +{ + struct seg_tree_node* node; + node = calloc(1, sizeof(*node)); + if (!node) { + return NULL; + } + + node->start = start; + node->end = end; + node->ptr = ptr; + return node; +} + +/* + * Given two start/end ranges, return a new range from start1/end1 that + * does not overlap start2/end2. The non-overlapping range is stored + * in new_start/new_end. If there are no non-overlapping ranges, + * return 1 from this function, else return 0. If there are two + * non-overlapping ranges, return the first one in new_start/new_end. + */ +static int +get_non_overlapping_range(unsigned long start1, unsigned long end1, + long start2, long end2, long* new_start, long* new_end) +{ + if (start1 >= start2 && end1 <= end2) { + /* Completely overlapping */ + return 1; + } else if (start1 < start2) { + /* + * s1 ------- e1 + * s2--------e2 + * ---- non-overlap + * + * also: + * + * s1 -------------------e1 + * s2--------e2 + * ---- non-overlap + */ + *new_start = start1; + *new_end = MIN(end1, start2 - 1); + } else if (start1 > start2 && end1 > end2) { + /* + * s1 ----- e1 + * s2------- e2 + */ + *new_start = MAX(start1, end2 + 1); + *new_end = end1; + } else if (start1 == start2 && end1 > end2) { + *new_start = end2 + 1; + *new_end = end1; + } + return 0; +} + +/* + * Add an entry to the range tree. Returns 0 on success, nonzero otherwise. + */ +int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, + unsigned long end, unsigned long ptr) +{ + struct seg_tree_node* node; + struct seg_tree_node* overlap = NULL; + struct seg_tree_node* resized; + struct seg_tree_node* remaining; + long new_start = 0, new_end = 0; + int rc; + + /* Create our range */ + node = seg_tree_node_alloc(start, end, ptr); + if (!node) { + return ENOMEM; + } + + seg_tree_wrlock(seg_tree); + /* + * Try to insert our range into the RB tree. If it overlaps with any other + * range, then it is not inserted, and the overlapping range node is + * returned in 'overlap'. If 'overlap' is NULL, then there were no + * overlaps, and our range was successfully inserted. + */ + while ((overlap = RB_INSERT(inttree, &seg_tree->head, node))) { + /* + * Our range overlaps with another range (in 'overlap'). Is there any + * any part of 'overlap' that does not overlap our range? If so, + * delete the old 'overlap' and insert the smaller, non-overlapping + * range. + */ + rc = get_non_overlapping_range(overlap->start, overlap->end, start, end, + &new_start, &new_end); + if (rc) { + /* We can't find a non-overlapping range. Delete the old range. */ + RB_REMOVE(inttree, &seg_tree->head, overlap); + seg_tree->count--; + free(overlap); + } else { + /* + * Part of the old range was non-overlapping. Split the old range + * into two ranges: one for the non-overlapping section, and one for + * the remaining section. The non-overlapping section gets + * inserted without issue. The remaining section will be processed + * on the next pass of this while() loop. + */ + resized = seg_tree_node_alloc(new_start, new_end, + overlap->ptr + (new_start - overlap->start)); + if (!resized) { + return ENOMEM; + } + /* Remove our old range */ + RB_REMOVE(inttree, &seg_tree->head, overlap); + + /* Insert the non-overlapping part of the new range */ + RB_INSERT(inttree, &seg_tree->head, resized); + + if (resized->end + 1 >= overlap->start && + resized->end +1 <= overlap->end) { + /* + * There's still a remaining section after the non-overlapping + * part. Add it in. + */ + remaining = seg_tree_node_alloc(resized->end + 1, overlap->end, + overlap->ptr + (resized->end + 1 - overlap->start)); + if (!resized) { + free(overlap); + return ENOMEM; + } + RB_INSERT(inttree, &seg_tree->head, remaining); + seg_tree->count++; + } + free(overlap); + } + } + if (!overlap) { + seg_tree->count++; + } + + seg_tree->max = MAX(seg_tree->max, end); + seg_tree_unlock(seg_tree); + + return 0; +} + +/* + * Given a range tree and a starting node, iterate though all the nodes + * in the tree, returning the next one each time. If start is NULL, then + * start with the first node in the tree. + * + * This is meant to be called in a loop, like: + * + * seg_tree_rdlock(seg_tree); + * + * struct seg_tree_node *node = NULL; + * while ((node = seg_tree_iter(seg_tree, node))) { + * printf("[%d-%d]", node->start, node->end); + * } + * + * seg_tree_unlock(seg_tree); + * + * Note: this function does no locking, and assumes you're properly locking + * and unlocking the seg_tree before doing the iteration (see + * seg_tree_rdlock()/seg_tree_wrlock()/seg_tree_unlock()). + */ +struct seg_tree_node* +seg_tree_iter(struct seg_tree* seg_tree, struct seg_tree_node* start) +{ + struct seg_tree_node* next = NULL; + if (start == NULL) { + /* Initial case, no starting node */ + next = RB_MIN(inttree, &seg_tree->head); + return next; + } + + /* + * We were given a valid start node. Look it up to start our traversal + * from there. + */ + next = RB_FIND(inttree, &seg_tree->head, start); + if (!next) { + /* Some kind of error */ + return NULL; + } + + /* Look up our next node */ + next = RB_NEXT(inttree, &seg_tree->head, start); + + return next; +} + +/* + * Lock a seg_tree for reading. This should only be used for calling + * seg_tree_iter(). All the other seg_tree functions provide their + * own locking. + */ +void +seg_tree_rdlock(struct seg_tree* seg_tree) +{ + assert(pthread_rwlock_rdlock(&seg_tree->rwlock) == 0); +} + +/* + * Lock a seg_tree for read/write. This should only be used for calling + * seg_tree_iter(). All the other seg_tree functions provide their + * own locking. + */ +void +seg_tree_wrlock(struct seg_tree* seg_tree) +{ + assert(pthread_rwlock_wrlock(&seg_tree->rwlock) == 0); +} + +/* + * Unlock a seg_tree for read/write. This should only be used for calling + * seg_tree_iter(). All the other seg_tree functions provide their + * own locking. + */ +void +seg_tree_unlock(struct seg_tree* seg_tree) +{ + assert(pthread_rwlock_unlock(&seg_tree->rwlock) == 0); +} + +/* + * Remove all nodes in seg_tree, but keep it initialized so you can + * seg_tree_add() to it. + */ +void seg_tree_clear(struct seg_tree* seg_tree) +{ + struct seg_tree_node* node = NULL; + struct seg_tree_node* oldnode = NULL; + + seg_tree_wrlock(seg_tree); + + if (RB_EMPTY(&seg_tree->head)) { + /* seg_tree is empty, nothing to do */ + seg_tree_unlock(seg_tree); + return; + } + + /* Remove and free each node in the tree */ + while ((node = seg_tree_iter(seg_tree, node))) { + if (oldnode) { + RB_REMOVE(inttree, &seg_tree->head, oldnode); + free(oldnode); + } + oldnode = node; + } + if (oldnode) { + RB_REMOVE(inttree, &seg_tree->head, oldnode); + free(oldnode); + } + + seg_tree->count = 0; + seg_tree->max = 0; + seg_tree_unlock(seg_tree); +} + +/* Return the number of segments in the segment tree */ +unsigned long seg_tree_count(struct seg_tree* seg_tree) +{ + unsigned long count; + + seg_tree_wrlock(seg_tree); + count = seg_tree->count; + seg_tree_unlock(seg_tree); + return count; +} + +/* Return the maximum segment value in the tree */ +unsigned long seg_tree_max(struct seg_tree* seg_tree) +{ + unsigned long max; + + seg_tree_wrlock(seg_tree); + max = seg_tree->max; + seg_tree_unlock(seg_tree); + return max; +} diff --git a/common/src/seg_tree.h b/common/src/seg_tree.h new file mode 100644 index 000000000..042e39ff2 --- /dev/null +++ b/common/src/seg_tree.h @@ -0,0 +1,48 @@ +#ifndef __SEG_TREE_H__ +#define __SEG_TREE_H__ +#include +#include "tree.h" + +struct seg_tree_node { + RB_ENTRY(seg_tree_node) entry; + unsigned long start, end; /* our range */ + unsigned long ptr; /* pointer to our data buffer */ +}; + +struct seg_tree { + RB_HEAD(inttree, seg_tree_node) head; + pthread_rwlock_t rwlock; + unsigned long count; /* number of segments */ + long max; /* maximum segment value in the tree */ +}; + +int seg_tree_init(struct seg_tree* seg_tree); +void seg_tree_clear(struct seg_tree* seg_tree); +void seg_tree_destroy(struct seg_tree* seg_tree); +int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, + unsigned long end, unsigned long ptr); + +struct seg_tree_node* seg_tree_iter(struct seg_tree* seg_tree, + struct seg_tree_node* start); + +unsigned long seg_tree_count(struct seg_tree* seg_tree); +unsigned long seg_tree_max(struct seg_tree* seg_tree); + +/* + * Locking functions for use with seg_tree_iter(). They allow you to lock the + * tree to iterate over it: + * + * seg_tree_rdlock(&seg_tree); + * + * struct seg_tree_node *node = NULL; + * while ((node = seg_tree_iter(seg_tree, node))) { + * printf("[%d-%d]", node->start, node->end); + * } + * + * seg_tree_unlock(&seg_tree); + */ +void seg_tree_rdlock(struct seg_tree* seg_tree); +void seg_tree_wrlock(struct seg_tree* seg_tree); +void seg_tree_unlock(struct seg_tree* seg_tree); + +#endif diff --git a/common/src/tree.h b/common/src/tree.h new file mode 100644 index 000000000..fb190e7e2 --- /dev/null +++ b/common/src/tree.h @@ -0,0 +1,1007 @@ +/* $OpenBSD: tree.h,v 1.29 2017/07/30 19:27:20 deraadt Exp $ */ +/* + * Copyright 2002 Niels Provos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_TREE_H_ +#define _SYS_TREE_H_ + +/* OpenBSD only #include - commenting it out */ +/* #include */ + +/* + * This file defines data structures for different types of trees: + * splay trees and red-black trees. + * + * A splay tree is a self-organizing data structure. Every operation + * on the tree causes a splay to happen. The splay moves the requested + * node to the root of the tree and partly rebalances it. + * + * This has the benefit that request locality causes faster lookups as + * the requested nodes move to the top of the tree. On the other hand, + * every lookup causes memory writes. + * + * The Balance Theorem bounds the total access time for m operations + * and n inserts on an initially empty tree as O((m + n)lg n). The + * amortized cost for a sequence of m accesses to a splay tree is O(lg n); + * + * A red-black tree is a binary search tree with the node color as an + * extra attribute. It fulfills a set of conditions: + * - every search path from the root to a leaf consists of the + * same number of black nodes, + * - each red node (except for the root) has a black parent, + * - each leaf node is black. + * + * Every operation on a red-black tree is bounded as O(lg n). + * The maximum height of a red-black tree is 2lg (n+1). + */ + +#define SPLAY_HEAD(name, type) \ +struct name { \ + struct type *sph_root; /* root of the tree */ \ +} + +#define SPLAY_INITIALIZER(root) \ + { NULL } + +#define SPLAY_INIT(root) do { \ + (root)->sph_root = NULL; \ +} while (0) + +#define SPLAY_ENTRY(type) \ +struct { \ + struct type *spe_left; /* left element */ \ + struct type *spe_right; /* right element */ \ +} + +#define SPLAY_LEFT(elm, field) (elm)->field.spe_left +#define SPLAY_RIGHT(elm, field) (elm)->field.spe_right +#define SPLAY_ROOT(head) (head)->sph_root +#define SPLAY_EMPTY(head) (SPLAY_ROOT(head) == NULL) + +/* SPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold SPLAY_{RIGHT,LEFT} */ +#define SPLAY_ROTATE_RIGHT(head, tmp, field) do { \ + SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(tmp, field); \ + SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ + (head)->sph_root = tmp; \ +} while (0) + +#define SPLAY_ROTATE_LEFT(head, tmp, field) do { \ + SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(tmp, field); \ + SPLAY_LEFT(tmp, field) = (head)->sph_root; \ + (head)->sph_root = tmp; \ +} while (0) + +#define SPLAY_LINKLEFT(head, tmp, field) do { \ + SPLAY_LEFT(tmp, field) = (head)->sph_root; \ + tmp = (head)->sph_root; \ + (head)->sph_root = SPLAY_LEFT((head)->sph_root, field); \ +} while (0) + +#define SPLAY_LINKRIGHT(head, tmp, field) do { \ + SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ + tmp = (head)->sph_root; \ + (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field); \ +} while (0) + +#define SPLAY_ASSEMBLE(head, node, left, right, field) do { \ + SPLAY_RIGHT(left, field) = SPLAY_LEFT((head)->sph_root, field); \ + SPLAY_LEFT(right, field) = SPLAY_RIGHT((head)->sph_root, field);\ + SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(node, field); \ + SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(node, field); \ +} while (0) + +/* Generates prototypes and inline functions */ + +#define SPLAY_PROTOTYPE(name, type, field, cmp) \ +void name##_SPLAY(struct name *, struct type *); \ +void name##_SPLAY_MINMAX(struct name *, int); \ +struct type *name##_SPLAY_INSERT(struct name *, struct type *); \ +struct type *name##_SPLAY_REMOVE(struct name *, struct type *); \ + \ +/* Finds the node with the same key as elm */ \ +static __unused __inline struct type * \ +name##_SPLAY_FIND(struct name *head, struct type *elm) \ +{ \ + if (SPLAY_EMPTY(head)) \ + return(NULL); \ + name##_SPLAY(head, elm); \ + if ((cmp)(elm, (head)->sph_root) == 0) \ + return (head->sph_root); \ + return (NULL); \ +} \ + \ +static __unused __inline struct type * \ +name##_SPLAY_NEXT(struct name *head, struct type *elm) \ +{ \ + name##_SPLAY(head, elm); \ + if (SPLAY_RIGHT(elm, field) != NULL) { \ + elm = SPLAY_RIGHT(elm, field); \ + while (SPLAY_LEFT(elm, field) != NULL) { \ + elm = SPLAY_LEFT(elm, field); \ + } \ + } else \ + elm = NULL; \ + return (elm); \ +} \ + \ +static __unused __inline struct type * \ +name##_SPLAY_MIN_MAX(struct name *head, int val) \ +{ \ + name##_SPLAY_MINMAX(head, val); \ + return (SPLAY_ROOT(head)); \ +} + +/* Main splay operation. + * Moves node close to the key of elm to top + */ +#define SPLAY_GENERATE(name, type, field, cmp) \ +struct type * \ +name##_SPLAY_INSERT(struct name *head, struct type *elm) \ +{ \ + if (SPLAY_EMPTY(head)) { \ + SPLAY_LEFT(elm, field) = SPLAY_RIGHT(elm, field) = NULL; \ + } else { \ + int __comp; \ + name##_SPLAY(head, elm); \ + __comp = (cmp)(elm, (head)->sph_root); \ + if(__comp < 0) { \ + SPLAY_LEFT(elm, field) = SPLAY_LEFT((head)->sph_root, field);\ + SPLAY_RIGHT(elm, field) = (head)->sph_root; \ + SPLAY_LEFT((head)->sph_root, field) = NULL; \ + } else if (__comp > 0) { \ + SPLAY_RIGHT(elm, field) = SPLAY_RIGHT((head)->sph_root, field);\ + SPLAY_LEFT(elm, field) = (head)->sph_root; \ + SPLAY_RIGHT((head)->sph_root, field) = NULL; \ + } else \ + return ((head)->sph_root); \ + } \ + (head)->sph_root = (elm); \ + return (NULL); \ +} \ + \ +struct type * \ +name##_SPLAY_REMOVE(struct name *head, struct type *elm) \ +{ \ + struct type *__tmp; \ + if (SPLAY_EMPTY(head)) \ + return (NULL); \ + name##_SPLAY(head, elm); \ + if ((cmp)(elm, (head)->sph_root) == 0) { \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL) { \ + (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);\ + } else { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + (head)->sph_root = SPLAY_LEFT((head)->sph_root, field);\ + name##_SPLAY(head, elm); \ + SPLAY_RIGHT((head)->sph_root, field) = __tmp; \ + } \ + return (elm); \ + } \ + return (NULL); \ +} \ + \ +void \ +name##_SPLAY(struct name *head, struct type *elm) \ +{ \ + struct type __node, *__left, *__right, *__tmp; \ + int __comp; \ +\ + SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ + __left = __right = &__node; \ +\ + while ((__comp = (cmp)(elm, (head)->sph_root))) { \ + if (__comp < 0) { \ + __tmp = SPLAY_LEFT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if ((cmp)(elm, __tmp) < 0){ \ + SPLAY_ROTATE_RIGHT(head, __tmp, field); \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKLEFT(head, __right, field); \ + } else if (__comp > 0) { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if ((cmp)(elm, __tmp) > 0){ \ + SPLAY_ROTATE_LEFT(head, __tmp, field); \ + if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKRIGHT(head, __left, field); \ + } \ + } \ + SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ +} \ + \ +/* Splay with either the minimum or the maximum element \ + * Used to find minimum or maximum element in tree. \ + */ \ +void name##_SPLAY_MINMAX(struct name *head, int __comp) \ +{ \ + struct type __node, *__left, *__right, *__tmp; \ +\ + SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ + __left = __right = &__node; \ +\ + while (1) { \ + if (__comp < 0) { \ + __tmp = SPLAY_LEFT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if (__comp < 0){ \ + SPLAY_ROTATE_RIGHT(head, __tmp, field); \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKLEFT(head, __right, field); \ + } else if (__comp > 0) { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if (__comp > 0) { \ + SPLAY_ROTATE_LEFT(head, __tmp, field); \ + if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKRIGHT(head, __left, field); \ + } \ + } \ + SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ +} + +#define SPLAY_NEGINF -1 +#define SPLAY_INF 1 + +#define SPLAY_INSERT(name, x, y) name##_SPLAY_INSERT(x, y) +#define SPLAY_REMOVE(name, x, y) name##_SPLAY_REMOVE(x, y) +#define SPLAY_FIND(name, x, y) name##_SPLAY_FIND(x, y) +#define SPLAY_NEXT(name, x, y) name##_SPLAY_NEXT(x, y) +#define SPLAY_MIN(name, x) (SPLAY_EMPTY(x) ? NULL \ + : name##_SPLAY_MIN_MAX(x, SPLAY_NEGINF)) +#define SPLAY_MAX(name, x) (SPLAY_EMPTY(x) ? NULL \ + : name##_SPLAY_MIN_MAX(x, SPLAY_INF)) + +#define SPLAY_FOREACH(x, name, head) \ + for ((x) = SPLAY_MIN(name, head); \ + (x) != NULL; \ + (x) = SPLAY_NEXT(name, head, x)) + +/* Macros that define a red-black tree */ +#define RB_HEAD(name, type) \ +struct name { \ + struct type *rbh_root; /* root of the tree */ \ +} + +#define RB_INITIALIZER(root) \ + { NULL } + +#define RB_INIT(root) do { \ + (root)->rbh_root = NULL; \ +} while (0) + +#define RB_BLACK 0 +#define RB_RED 1 +#define RB_ENTRY(type) \ +struct { \ + struct type *rbe_left; /* left element */ \ + struct type *rbe_right; /* right element */ \ + struct type *rbe_parent; /* parent element */ \ + int rbe_color; /* node color */ \ +} + +#define RB_LEFT(elm, field) (elm)->field.rbe_left +#define RB_RIGHT(elm, field) (elm)->field.rbe_right +#define RB_PARENT(elm, field) (elm)->field.rbe_parent +#define RB_COLOR(elm, field) (elm)->field.rbe_color +#define RB_ROOT(head) (head)->rbh_root +#define RB_EMPTY(head) (RB_ROOT(head) == NULL) + +#define RB_SET(elm, parent, field) do { \ + RB_PARENT(elm, field) = parent; \ + RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL; \ + RB_COLOR(elm, field) = RB_RED; \ +} while (0) + +#define RB_SET_BLACKRED(black, red, field) do { \ + RB_COLOR(black, field) = RB_BLACK; \ + RB_COLOR(red, field) = RB_RED; \ +} while (0) + +#ifndef RB_AUGMENT +#define RB_AUGMENT(x) do {} while (0) +#endif + +#define RB_ROTATE_LEFT(head, elm, tmp, field) do { \ + (tmp) = RB_RIGHT(elm, field); \ + if ((RB_RIGHT(elm, field) = RB_LEFT(tmp, field))) { \ + RB_PARENT(RB_LEFT(tmp, field), field) = (elm); \ + } \ + RB_AUGMENT(elm); \ + if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field))) { \ + if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ + RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ + else \ + RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ + } else \ + (head)->rbh_root = (tmp); \ + RB_LEFT(tmp, field) = (elm); \ + RB_PARENT(elm, field) = (tmp); \ + RB_AUGMENT(tmp); \ + if ((RB_PARENT(tmp, field))) \ + RB_AUGMENT(RB_PARENT(tmp, field)); \ +} while (0) + +#define RB_ROTATE_RIGHT(head, elm, tmp, field) do { \ + (tmp) = RB_LEFT(elm, field); \ + if ((RB_LEFT(elm, field) = RB_RIGHT(tmp, field))) { \ + RB_PARENT(RB_RIGHT(tmp, field), field) = (elm); \ + } \ + RB_AUGMENT(elm); \ + if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field))) { \ + if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ + RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ + else \ + RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ + } else \ + (head)->rbh_root = (tmp); \ + RB_RIGHT(tmp, field) = (elm); \ + RB_PARENT(elm, field) = (tmp); \ + RB_AUGMENT(tmp); \ + if ((RB_PARENT(tmp, field))) \ + RB_AUGMENT(RB_PARENT(tmp, field)); \ +} while (0) + +/* Generates prototypes and inline functions */ +#define RB_PROTOTYPE(name, type, field, cmp) \ + RB_PROTOTYPE_INTERNAL(name, type, field, cmp,) +#define RB_PROTOTYPE_STATIC(name, type, field, cmp) \ + RB_PROTOTYPE_INTERNAL(name, type, field, cmp, __attribute__((__unused__)) static) +#define RB_PROTOTYPE_INTERNAL(name, type, field, cmp, attr) \ +attr void name##_RB_INSERT_COLOR(struct name *, struct type *); \ +attr void name##_RB_REMOVE_COLOR(struct name *, struct type *, struct type *);\ +attr struct type *name##_RB_REMOVE(struct name *, struct type *); \ +attr struct type *name##_RB_INSERT(struct name *, struct type *); \ +attr struct type *name##_RB_FIND(struct name *, struct type *); \ +attr struct type *name##_RB_NFIND(struct name *, struct type *); \ +attr struct type *name##_RB_NEXT(struct type *); \ +attr struct type *name##_RB_PREV(struct type *); \ +attr struct type *name##_RB_MINMAX(struct name *, int); \ + \ + +/* Main rb operation. + * Moves node close to the key of elm to top + */ +#define RB_GENERATE(name, type, field, cmp) \ + RB_GENERATE_INTERNAL(name, type, field, cmp,) +#define RB_GENERATE_STATIC(name, type, field, cmp) \ + RB_GENERATE_INTERNAL(name, type, field, cmp, __attribute__((__unused__)) static) +#define RB_GENERATE_INTERNAL(name, type, field, cmp, attr) \ +attr void \ +name##_RB_INSERT_COLOR(struct name *head, struct type *elm) \ +{ \ + struct type *parent, *gparent, *tmp; \ + while ((parent = RB_PARENT(elm, field)) && \ + RB_COLOR(parent, field) == RB_RED) { \ + gparent = RB_PARENT(parent, field); \ + if (parent == RB_LEFT(gparent, field)) { \ + tmp = RB_RIGHT(gparent, field); \ + if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ + RB_COLOR(tmp, field) = RB_BLACK; \ + RB_SET_BLACKRED(parent, gparent, field);\ + elm = gparent; \ + continue; \ + } \ + if (RB_RIGHT(parent, field) == elm) { \ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + tmp = parent; \ + parent = elm; \ + elm = tmp; \ + } \ + RB_SET_BLACKRED(parent, gparent, field); \ + RB_ROTATE_RIGHT(head, gparent, tmp, field); \ + } else { \ + tmp = RB_LEFT(gparent, field); \ + if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ + RB_COLOR(tmp, field) = RB_BLACK; \ + RB_SET_BLACKRED(parent, gparent, field);\ + elm = gparent; \ + continue; \ + } \ + if (RB_LEFT(parent, field) == elm) { \ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + tmp = parent; \ + parent = elm; \ + elm = tmp; \ + } \ + RB_SET_BLACKRED(parent, gparent, field); \ + RB_ROTATE_LEFT(head, gparent, tmp, field); \ + } \ + } \ + RB_COLOR(head->rbh_root, field) = RB_BLACK; \ +} \ + \ +attr void \ +name##_RB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \ +{ \ + struct type *tmp; \ + while ((elm == NULL || RB_COLOR(elm, field) == RB_BLACK) && \ + elm != RB_ROOT(head)) { \ + if (RB_LEFT(parent, field) == elm) { \ + tmp = RB_RIGHT(parent, field); \ + if (RB_COLOR(tmp, field) == RB_RED) { \ + RB_SET_BLACKRED(tmp, parent, field); \ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + tmp = RB_RIGHT(parent, field); \ + } \ + if ((RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ + (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ + RB_COLOR(tmp, field) = RB_RED; \ + elm = parent; \ + parent = RB_PARENT(elm, field); \ + } else { \ + if (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK) {\ + struct type *oleft; \ + if ((oleft = RB_LEFT(tmp, field)))\ + RB_COLOR(oleft, field) = RB_BLACK;\ + RB_COLOR(tmp, field) = RB_RED; \ + RB_ROTATE_RIGHT(head, tmp, oleft, field);\ + tmp = RB_RIGHT(parent, field); \ + } \ + RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ + RB_COLOR(parent, field) = RB_BLACK; \ + if (RB_RIGHT(tmp, field)) \ + RB_COLOR(RB_RIGHT(tmp, field), field) = RB_BLACK;\ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + elm = RB_ROOT(head); \ + break; \ + } \ + } else { \ + tmp = RB_LEFT(parent, field); \ + if (RB_COLOR(tmp, field) == RB_RED) { \ + RB_SET_BLACKRED(tmp, parent, field); \ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + tmp = RB_LEFT(parent, field); \ + } \ + if ((RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ + (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ + RB_COLOR(tmp, field) = RB_RED; \ + elm = parent; \ + parent = RB_PARENT(elm, field); \ + } else { \ + if (RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) {\ + struct type *oright; \ + if ((oright = RB_RIGHT(tmp, field)))\ + RB_COLOR(oright, field) = RB_BLACK;\ + RB_COLOR(tmp, field) = RB_RED; \ + RB_ROTATE_LEFT(head, tmp, oright, field);\ + tmp = RB_LEFT(parent, field); \ + } \ + RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ + RB_COLOR(parent, field) = RB_BLACK; \ + if (RB_LEFT(tmp, field)) \ + RB_COLOR(RB_LEFT(tmp, field), field) = RB_BLACK;\ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + elm = RB_ROOT(head); \ + break; \ + } \ + } \ + } \ + if (elm) \ + RB_COLOR(elm, field) = RB_BLACK; \ +} \ + \ +attr struct type * \ +name##_RB_REMOVE(struct name *head, struct type *elm) \ +{ \ + struct type *child, *parent, *old = elm; \ + int color; \ + if (RB_LEFT(elm, field) == NULL) \ + child = RB_RIGHT(elm, field); \ + else if (RB_RIGHT(elm, field) == NULL) \ + child = RB_LEFT(elm, field); \ + else { \ + struct type *left; \ + elm = RB_RIGHT(elm, field); \ + while ((left = RB_LEFT(elm, field))) \ + elm = left; \ + child = RB_RIGHT(elm, field); \ + parent = RB_PARENT(elm, field); \ + color = RB_COLOR(elm, field); \ + if (child) \ + RB_PARENT(child, field) = parent; \ + if (parent) { \ + if (RB_LEFT(parent, field) == elm) \ + RB_LEFT(parent, field) = child; \ + else \ + RB_RIGHT(parent, field) = child; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = child; \ + if (RB_PARENT(elm, field) == old) \ + parent = elm; \ + (elm)->field = (old)->field; \ + if (RB_PARENT(old, field)) { \ + if (RB_LEFT(RB_PARENT(old, field), field) == old)\ + RB_LEFT(RB_PARENT(old, field), field) = elm;\ + else \ + RB_RIGHT(RB_PARENT(old, field), field) = elm;\ + RB_AUGMENT(RB_PARENT(old, field)); \ + } else \ + RB_ROOT(head) = elm; \ + RB_PARENT(RB_LEFT(old, field), field) = elm; \ + if (RB_RIGHT(old, field)) \ + RB_PARENT(RB_RIGHT(old, field), field) = elm; \ + if (parent) { \ + left = parent; \ + do { \ + RB_AUGMENT(left); \ + } while ((left = RB_PARENT(left, field))); \ + } \ + goto color; \ + } \ + parent = RB_PARENT(elm, field); \ + color = RB_COLOR(elm, field); \ + if (child) \ + RB_PARENT(child, field) = parent; \ + if (parent) { \ + if (RB_LEFT(parent, field) == elm) \ + RB_LEFT(parent, field) = child; \ + else \ + RB_RIGHT(parent, field) = child; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = child; \ +color: \ + if (color == RB_BLACK) \ + name##_RB_REMOVE_COLOR(head, parent, child); \ + return (old); \ +} \ + \ +/* Inserts a node into the RB tree */ \ +attr struct type * \ +name##_RB_INSERT(struct name *head, struct type *elm) \ +{ \ + struct type *tmp; \ + struct type *parent = NULL; \ + int comp = 0; \ + tmp = RB_ROOT(head); \ + while (tmp) { \ + parent = tmp; \ + comp = (cmp)(elm, parent); \ + if (comp < 0) \ + tmp = RB_LEFT(tmp, field); \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + RB_SET(elm, parent, field); \ + if (parent != NULL) { \ + if (comp < 0) \ + RB_LEFT(parent, field) = elm; \ + else \ + RB_RIGHT(parent, field) = elm; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = elm; \ + name##_RB_INSERT_COLOR(head, elm); \ + return (NULL); \ +} \ + \ +/* Finds the node with the same key as elm */ \ +attr struct type * \ +name##_RB_FIND(struct name *head, struct type *elm) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + int comp; \ + while (tmp) { \ + comp = cmp(elm, tmp); \ + if (comp < 0) \ + tmp = RB_LEFT(tmp, field); \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + return (NULL); \ +} \ + \ +/* Finds the first node greater than or equal to the search key */ \ +attr struct type * \ +name##_RB_NFIND(struct name *head, struct type *elm) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + struct type *res = NULL; \ + int comp; \ + while (tmp) { \ + comp = cmp(elm, tmp); \ + if (comp < 0) { \ + res = tmp; \ + tmp = RB_LEFT(tmp, field); \ + } \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + return (res); \ +} \ + \ +/* ARGSUSED */ \ +attr struct type * \ +name##_RB_NEXT(struct type *elm) \ +{ \ + if (RB_RIGHT(elm, field)) { \ + elm = RB_RIGHT(elm, field); \ + while (RB_LEFT(elm, field)) \ + elm = RB_LEFT(elm, field); \ + } else { \ + if (RB_PARENT(elm, field) && \ + (elm == RB_LEFT(RB_PARENT(elm, field), field))) \ + elm = RB_PARENT(elm, field); \ + else { \ + while (RB_PARENT(elm, field) && \ + (elm == RB_RIGHT(RB_PARENT(elm, field), field)))\ + elm = RB_PARENT(elm, field); \ + elm = RB_PARENT(elm, field); \ + } \ + } \ + return (elm); \ +} \ + \ +/* ARGSUSED */ \ +attr struct type * \ +name##_RB_PREV(struct type *elm) \ +{ \ + if (RB_LEFT(elm, field)) { \ + elm = RB_LEFT(elm, field); \ + while (RB_RIGHT(elm, field)) \ + elm = RB_RIGHT(elm, field); \ + } else { \ + if (RB_PARENT(elm, field) && \ + (elm == RB_RIGHT(RB_PARENT(elm, field), field))) \ + elm = RB_PARENT(elm, field); \ + else { \ + while (RB_PARENT(elm, field) && \ + (elm == RB_LEFT(RB_PARENT(elm, field), field)))\ + elm = RB_PARENT(elm, field); \ + elm = RB_PARENT(elm, field); \ + } \ + } \ + return (elm); \ +} \ + \ +attr struct type * \ +name##_RB_MINMAX(struct name *head, int val) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + struct type *parent = NULL; \ + while (tmp) { \ + parent = tmp; \ + if (val < 0) \ + tmp = RB_LEFT(tmp, field); \ + else \ + tmp = RB_RIGHT(tmp, field); \ + } \ + return (parent); \ +} + +#define RB_NEGINF -1 +#define RB_INF 1 + +#define RB_INSERT(name, x, y) name##_RB_INSERT(x, y) +#define RB_REMOVE(name, x, y) name##_RB_REMOVE(x, y) +#define RB_FIND(name, x, y) name##_RB_FIND(x, y) +#define RB_NFIND(name, x, y) name##_RB_NFIND(x, y) +#define RB_NEXT(name, x, y) name##_RB_NEXT(y) +#define RB_PREV(name, x, y) name##_RB_PREV(y) +#define RB_MIN(name, x) name##_RB_MINMAX(x, RB_NEGINF) +#define RB_MAX(name, x) name##_RB_MINMAX(x, RB_INF) + +#define RB_FOREACH(x, name, head) \ + for ((x) = RB_MIN(name, head); \ + (x) != NULL; \ + (x) = name##_RB_NEXT(x)) + +#define RB_FOREACH_SAFE(x, name, head, y) \ + for ((x) = RB_MIN(name, head); \ + ((x) != NULL) && ((y) = name##_RB_NEXT(x), 1); \ + (x) = (y)) + +#define RB_FOREACH_REVERSE(x, name, head) \ + for ((x) = RB_MAX(name, head); \ + (x) != NULL; \ + (x) = name##_RB_PREV(x)) + +#define RB_FOREACH_REVERSE_SAFE(x, name, head, y) \ + for ((x) = RB_MAX(name, head); \ + ((x) != NULL) && ((y) = name##_RB_PREV(x), 1); \ + (x) = (y)) + + +/* + * Copyright (c) 2016 David Gwynne + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +struct rb_type { + int (*t_compare)(const void *, const void *); + void (*t_augment)(void *); + unsigned int t_offset; /* offset of rb_entry in type */ +}; + +struct rb_tree { + struct rb_entry *rbt_root; +}; + +struct rb_entry { + struct rb_entry *rbt_parent; + struct rb_entry *rbt_left; + struct rb_entry *rbt_right; + unsigned int rbt_color; +}; + +#define RBT_HEAD(_name, _type) \ +struct _name { \ + struct rb_tree rbh_root; \ +} + +#define RBT_ENTRY(_type) struct rb_entry + +static inline void +_rb_init(struct rb_tree *rbt) +{ + rbt->rbt_root = NULL; +} + +static inline int +_rb_empty(struct rb_tree *rbt) +{ + return (rbt->rbt_root == NULL); +} + +void *_rb_insert(const struct rb_type *, struct rb_tree *, void *); +void *_rb_remove(const struct rb_type *, struct rb_tree *, void *); +void *_rb_find(const struct rb_type *, struct rb_tree *, const void *); +void *_rb_nfind(const struct rb_type *, struct rb_tree *, const void *); +void *_rb_root(const struct rb_type *, struct rb_tree *); +void *_rb_min(const struct rb_type *, struct rb_tree *); +void *_rb_max(const struct rb_type *, struct rb_tree *); +void *_rb_next(const struct rb_type *, void *); +void *_rb_prev(const struct rb_type *, void *); +void *_rb_left(const struct rb_type *, void *); +void *_rb_right(const struct rb_type *, void *); +void *_rb_parent(const struct rb_type *, void *); +void _rb_set_left(const struct rb_type *, void *, void *); +void _rb_set_right(const struct rb_type *, void *, void *); +void _rb_set_parent(const struct rb_type *, void *, void *); +void _rb_poison(const struct rb_type *, void *, unsigned long); +int _rb_check(const struct rb_type *, void *, unsigned long); + +#define RBT_INITIALIZER(_head) { { NULL } } + +#define RBT_PROTOTYPE(_name, _type, _field, _cmp) \ +extern const struct rb_type *const _name##_RBT_TYPE; \ + \ +__unused static inline void \ +_name##_RBT_INIT(struct _name *head) \ +{ \ + _rb_init(&head->rbh_root); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_INSERT(struct _name *head, struct _type *elm) \ +{ \ + return _rb_insert(_name##_RBT_TYPE, &head->rbh_root, elm); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_REMOVE(struct _name *head, struct _type *elm) \ +{ \ + return _rb_remove(_name##_RBT_TYPE, &head->rbh_root, elm); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_FIND(struct _name *head, const struct _type *key) \ +{ \ + return _rb_find(_name##_RBT_TYPE, &head->rbh_root, key); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_NFIND(struct _name *head, const struct _type *key) \ +{ \ + return _rb_nfind(_name##_RBT_TYPE, &head->rbh_root, key); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_ROOT(struct _name *head) \ +{ \ + return _rb_root(_name##_RBT_TYPE, &head->rbh_root); \ +} \ + \ +__unused static inline int \ +_name##_RBT_EMPTY(struct _name *head) \ +{ \ + return _rb_empty(&head->rbh_root); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_MIN(struct _name *head) \ +{ \ + return _rb_min(_name##_RBT_TYPE, &head->rbh_root); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_MAX(struct _name *head) \ +{ \ + return _rb_max(_name##_RBT_TYPE, &head->rbh_root); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_NEXT(struct _type *elm) \ +{ \ + return _rb_next(_name##_RBT_TYPE, elm); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_PREV(struct _type *elm) \ +{ \ + return _rb_prev(_name##_RBT_TYPE, elm); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_LEFT(struct _type *elm) \ +{ \ + return _rb_left(_name##_RBT_TYPE, elm); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_RIGHT(struct _type *elm) \ +{ \ + return _rb_right(_name##_RBT_TYPE, elm); \ +} \ + \ +__unused static inline struct _type * \ +_name##_RBT_PARENT(struct _type *elm) \ +{ \ + return _rb_parent(_name##_RBT_TYPE, elm); \ +} \ + \ +__unused static inline void \ +_name##_RBT_SET_LEFT(struct _type *elm, struct _type *left) \ +{ \ + return _rb_set_left(_name##_RBT_TYPE, elm, left); \ +} \ + \ +__unused static inline void \ +_name##_RBT_SET_RIGHT(struct _type *elm, struct _type *right) \ +{ \ + return _rb_set_right(_name##_RBT_TYPE, elm, right); \ +} \ + \ +__unused static inline void \ +_name##_RBT_SET_PARENT(struct _type *elm, struct _type *parent) \ +{ \ + return _rb_set_parent(_name##_RBT_TYPE, elm, parent); \ +} \ + \ +__unused static inline void \ +_name##_RBT_POISON(struct _type *elm, unsigned long poison) \ +{ \ + return _rb_poison(_name##_RBT_TYPE, elm, poison); \ +} \ + \ +__unused static inline int \ +_name##_RBT_CHECK(struct _type *elm, unsigned long poison) \ +{ \ + return _rb_check(_name##_RBT_TYPE, elm, poison); \ +} + +#define RBT_GENERATE_INTERNAL(_name, _type, _field, _cmp, _aug) \ +static int \ +_name##_RBT_COMPARE(const void *lptr, const void *rptr) \ +{ \ + const struct _type *l = lptr, *r = rptr; \ + return _cmp(l, r); \ +} \ +static const struct rb_type _name##_RBT_INFO = { \ + _name##_RBT_COMPARE, \ + _aug, \ + offsetof(struct _type, _field), \ +}; \ +const struct rb_type *const _name##_RBT_TYPE = &_name##_RBT_INFO + +#define RBT_GENERATE_AUGMENT(_name, _type, _field, _cmp, _aug) \ +static void \ +_name##_RBT_AUGMENT(void *ptr) \ +{ \ + struct _type *p = ptr; \ + return _aug(p); \ +} \ +RBT_GENERATE_INTERNAL(_name, _type, _field, _cmp, _name##_RBT_AUGMENT) + +#define RBT_GENERATE(_name, _type, _field, _cmp) \ + RBT_GENERATE_INTERNAL(_name, _type, _field, _cmp, NULL) + +#define RBT_INIT(_name, _head) _name##_RBT_INIT(_head) +#define RBT_INSERT(_name, _head, _elm) _name##_RBT_INSERT(_head, _elm) +#define RBT_REMOVE(_name, _head, _elm) _name##_RBT_REMOVE(_head, _elm) +#define RBT_FIND(_name, _head, _key) _name##_RBT_FIND(_head, _key) +#define RBT_NFIND(_name, _head, _key) _name##_RBT_NFIND(_head, _key) +#define RBT_ROOT(_name, _head) _name##_RBT_ROOT(_head) +#define RBT_EMPTY(_name, _head) _name##_RBT_EMPTY(_head) +#define RBT_MIN(_name, _head) _name##_RBT_MIN(_head) +#define RBT_MAX(_name, _head) _name##_RBT_MAX(_head) +#define RBT_NEXT(_name, _elm) _name##_RBT_NEXT(_elm) +#define RBT_PREV(_name, _elm) _name##_RBT_PREV(_elm) +#define RBT_LEFT(_name, _elm) _name##_RBT_LEFT(_elm) +#define RBT_RIGHT(_name, _elm) _name##_RBT_RIGHT(_elm) +#define RBT_PARENT(_name, _elm) _name##_RBT_PARENT(_elm) +#define RBT_SET_LEFT(_name, _elm, _l) _name##_RBT_SET_LEFT(_elm, _l) +#define RBT_SET_RIGHT(_name, _elm, _r) _name##_RBT_SET_RIGHT(_elm, _r) +#define RBT_SET_PARENT(_name, _elm, _p) _name##_RBT_SET_PARENT(_elm, _p) +#define RBT_POISON(_name, _elm, _p) _name##_RBT_POISON(_elm, _p) +#define RBT_CHECK(_name, _elm, _p) _name##_RBT_CHECK(_elm, _p) + +#define RBT_FOREACH(_e, _name, _head) \ + for ((_e) = RBT_MIN(_name, (_head)); \ + (_e) != NULL; \ + (_e) = RBT_NEXT(_name, (_e))) + +#define RBT_FOREACH_SAFE(_e, _name, _head, _n) \ + for ((_e) = RBT_MIN(_name, (_head)); \ + (_e) != NULL && ((_n) = RBT_NEXT(_name, (_e)), 1); \ + (_e) = (_n)) + +#define RBT_FOREACH_REVERSE(_e, _name, _head) \ + for ((_e) = RBT_MAX(_name, (_head)); \ + (_e) != NULL; \ + (_e) = RBT_PREV(_name, (_e))) + +#define RBT_FOREACH_REVERSE_SAFE(_e, _name, _head, _n) \ + for ((_e) = RBT_MAX(_name, (_head)); \ + (_e) != NULL && ((_n) = RBT_PREV(_name, (_e)), 1); \ + (_e) = (_n)) + +#endif /* _SYS_TREE_H_ */ diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index 1ea97aeba..32e02accf 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -70,6 +70,7 @@ UNIFYFS_CFG_CLI(unifyfs, daemonize, BOOL, on, "enable server daemonization", NULL, 'D', "on|off") \ UNIFYFS_CFG_CLI(unifyfs, mountpoint, STRING, /unifyfs, "mountpoint directory", NULL, 'm', "specify full path to desired mountpoint") \ UNIFYFS_CFG(client, max_files, INT, UNIFYFS_MAX_FILES, "client max file count", NULL) \ + UNIFYFS_CFG(client, flatten_writes, BOOL, on, "flatten writes", NULL) \ UNIFYFS_CFG_CLI(log, verbosity, INT, 0, "log verbosity level", NULL, 'v', "specify logging verbosity level") \ UNIFYFS_CFG_CLI(log, file, STRING, unifyfsd.log, "log file name", NULL, 'l', "specify log file name") \ UNIFYFS_CFG_CLI(log, dir, STRING, LOGDIR, "log file directory", configurator_directory_check, 'L', "specify full path to directory to contain log file") \ diff --git a/configure.ac b/configure.ac index 2a780637b..efc8ca147 100755 --- a/configure.ac +++ b/configure.ac @@ -196,6 +196,9 @@ AC_CHECK_HEADERS(mntent.h sys/mount.h) AX_LIB_HDF5 AM_CONDITIONAL([HAVE_HDF5], [test x$with_hdf5 = xyes]) +# Look for pthreads +AX_PTHREAD([],[AC_MSG_ERROR([pthreads are required])]) + # libc functions wrapped by unifyfs CP_WRAPPERS+="-Wl,-wrap,access" diff --git a/docs/configuration.rst b/docs/configuration.rst index 2a4f55ec1..24813cdee 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -63,11 +63,12 @@ a given section and key. .. table:: ``[client]`` section - client settings :widths: auto - ============= ====== ===================================================== - Key Type Description - ============= ====== ===================================================== - max_files INT maximum number of open files per client process - ============= ====== ===================================================== + ============== ====== ================================================================= + Key Type Description + ============== ====== ================================================================= + max_files INT maximum number of open files per client process + flatten_writes BOOL enable flattening writes (optimization for overwrite-heavy codes) + ============== ====== ================================================================= .. table:: ``[log]`` section - logging settings :widths: auto diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index eec7ff892..1f7f87777 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -23,7 +23,8 @@ if HAVE_LD_WRAP app-tileio-static \ transfer-static \ size-static \ - chmod-static + chmod-static \ + multi-write-static endif if HAVE_GOTCHA @@ -44,7 +45,8 @@ if HAVE_GOTCHA app-tileio-gotcha \ transfer-gotcha \ size-gotcha \ - chmod-gotcha + chmod-gotcha \ + multi-write-gotcha endif if HAVE_FORTRAN @@ -310,3 +312,13 @@ chmod_static_CPPFLAGS = $(test_cppflags) chmod_static_LDADD = $(test_static_ldadd) chmod_static_LDFLAGS = $(test_static_ldflags) +multi_write_gotcha_SOURCES = multi-write.c testutil.c +multi_write_gotcha_CPPFLAGS = $(test_cppflags) +multi_write_gotcha_LDADD = $(test_gotcha_ldadd) +multi_write_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +multi_write_static_SOURCES = multi-write.c testutil.c +multi_write_static_CPPFLAGS = $(test_cppflags) +multi_write_static_LDADD = $(test_static_ldadd) +multi_write_static_LDFLAGS = $(test_static_ldflags) + diff --git a/examples/src/multi-write.c b/examples/src/multi-write.c new file mode 100644 index 000000000..3483f3cbb --- /dev/null +++ b/examples/src/multi-write.c @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + * + * Test doing lots of writes to many open files and verify the data is written + * correctly. This can be used to exercise bugs. + * + * Test description: + * 1. Fill bigbuf[] with repeating A-Z + * 2. Do a bunch of writes with random offsets and lengths to multiple files, + * using bigbuf[] as the data. + * 3. Laminate the files. + * 4. Read them back, and verify the portions that did get written match the + * data from bigbuf[]. + */ +#include +#include +#include +#include +#include +#include + +#include "testutil.h" + +#define NUM_FILES 10 +#define NUM_WRITES 100 + +/* This is large enough write size to periodically cross 1MB slice boundaries */ +#define MAX_WRITE (1024*1024) +#define SEED 1 + +char bigbuf[1024*1024*10]; +char tmpbuf[1024*1024*10]; + +void fill_bigbuf(void) +{ + char r; + int i; + + /* Fill bigbuf[] repeating A-Z chars */ + for (i = 0; i < sizeof(bigbuf); i++) { + bigbuf[i] = 'A'+ (i % 26); + } +} + +/* Compare a file with the data in bigbuf[] */ +int check_file(char* file) +{ + int fd; + int rc; + int matched = 0; + fd = open(file, O_RDONLY, 0222); + + memset(tmpbuf, 0, sizeof(tmpbuf)); + rc = read(fd, tmpbuf, sizeof(tmpbuf)); + printf("%s: read %d bytes\n", file, rc); + + for (int i = 0; i < rc; i++) { + if (tmpbuf[i] == bigbuf[i]) { + matched++; + } + + if (tmpbuf[i] != bigbuf[i] && tmpbuf[i] != 0) { + printf("%s failed at offset %d (tmpbuf['%c'] != bigbuf['%c'])\n", + file, i, tmpbuf[i], bigbuf[i]); + printf("Comparing last 10 bytes before/after:\n"); + printf("expected: "); + for (int j = i - 10; j < i; j++) { + printf("%c", bigbuf[j] ? bigbuf[j] : ' '); + } + + printf("|%c|", bigbuf[i]); + + for (int j = i + 1; j < i + 11; j++) { + printf("%c", bigbuf[j] ? bigbuf[j] : ' '); + } + printf("\n"); + + printf("got: "); + + for (int j = i - 10; j < i; j++) { + printf("%c", tmpbuf[j] ? tmpbuf[j] : ' '); + } + + printf("|%c|", tmpbuf[i]); + + for (int j = i + 1; j < i + 11; j++) { + printf("%c", tmpbuf[j] ? tmpbuf[j] : ' '); + } + + printf("\n"); + + + return 1; + } + } + if (rc > 0 && matched == 0) { + printf("%s: No matches with file %s\n", __func__, file); + return 1; + } + return 0; +} + +int do_test(test_cfg* cfg) +{ + int rc; + int fds[NUM_FILES], fd; + char* file[NUM_FILES]; + char buf[40] = {0}; + int i; + int rnd; + int start, count; + fill_bigbuf(); + srand(SEED); + + /* Create our files */ + for (i = 0; i < NUM_FILES; i++) { + file[i] = mktemp_cmd(cfg, "/unifyfs"); + fds[i] = open(file[i], O_WRONLY | O_CREAT, 0222); + } + + /* Write our files */ + for (i = 0; i < NUM_WRITES; i++) { + /* Randomly pick on of our files to write to */ + rnd = rand() % NUM_FILES; + fd = fds[rnd]; + + /* Pick a random offset and count */ + start = rand() % (sizeof(bigbuf) - MAX_WRITE); + + /* + 1 so we always write at least 1 byte */ + count = (rand() % (MAX_WRITE-1)) + 1; + lseek(fd, start, SEEK_SET); + write(fd, &bigbuf[start], count); + } + + /* Sync extents of all our files and laminate them */ + for (i = 0; i < NUM_FILES; i++) { + rc = fsync(fds[i]); + if (rc != 0) { + printf("%s %d/%d failed to sync, rc = %d, (errno %d %s)\n", + file[i], i+1, NUM_FILES, rc, errno, strerror(errno)); + exit(1); + } + close(fds[i]); + + rc = chmod(file[i], 0444); + if (rc != 0) { + printf("%s failed to chmod, rc = %d\n", file[i], rc); + exit(1); + } + + } + + /* Verify the writes to the files match the values in bigbuf[] */ + for (i = 0; i < NUM_FILES; i++) { + if (check_file(file[i]) != 0) { + printf("file %d/%d failed\n", i+1, NUM_FILES); + exit(1); /* Error */ + } + + free(file[i]); + } + printf("Passed!\n"); +} + +int main(int argc, char* argv[]) +{ + test_cfg test_config; + test_cfg* cfg = &test_config; + int rc; + + rc = test_init(argc, argv, cfg); + if (rc) { + test_print(cfg, "ERROR - Test %s initialization failed!", argv[0]); + fflush(NULL); + return rc; + } + do_test(cfg); + + test_fini(cfg); + + return 0; +} diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index d05a77517..4f7925ca1 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -915,6 +915,12 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) * created by the client, these are stored as index_t * structs starting one page size offset into meta region */ char* ptr_extents = meta + page_sz; + + if (extent_num_entries == 0) { + /* Nothing to do */ + return UNIFYFS_SUCCESS; + } + unifyfs_index_t* meta_payload = (unifyfs_index_t*)(ptr_extents); /* allocate storage for file extent key/values */ diff --git a/t/std/size.c b/t/std/size.c index 2367a510b..b37861752 100644 --- a/t/std/size.c +++ b/t/std/size.c @@ -82,7 +82,7 @@ int size_test(char* unifyfs_root) strerror(errno)); ok(local == 12, "%s: local size is %d: %s", __FILE__, local, strerror(errno)); - ok(log == 12, "%s: log size is %d: %s", __FILE__, local, + ok(log == 12, "%s: log size is %d: %s", __FILE__, log, strerror(errno)); /* Open the file again with append, write to it. */ @@ -96,7 +96,6 @@ int size_test(char* unifyfs_root) rc = ftell(fp); ok(rc == 24, "%s: ftell() (rc=%d) %s", __FILE__, rc, strerror(errno)); - /* * Set our position to somewhere in the middle of the file. Since the file * is in append mode, this new position should be ignored, and writes @@ -124,7 +123,7 @@ int size_test(char* unifyfs_root) strerror(errno)); ok(local == 30, "%s: local size is %d: %s", __FILE__, local, strerror(errno)); - ok(log == 30, "%s: log size is %d: %s", __FILE__, local, + ok(log == 30, "%s: log size is %d: %s", __FILE__, log, strerror(errno)); @@ -147,7 +146,7 @@ int size_test(char* unifyfs_root) strerror(errno)); ok(local == 30, "%s: local size is %d: %s", __FILE__, local, strerror(errno)); - ok(local == 30, "%s: log size is %d: %s", __FILE__, local, + ok(log == 30, "%s: log size is %d: %s", __FILE__, log, strerror(errno)); /* Read it back */ From b9bf15bfb11a607d2bc534a25250652a6e2ebb29 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 2 Jan 2020 15:30:17 -0800 Subject: [PATCH 054/168] update size field in global meta on truncate This adds support for truncate and ftruncate for extending and shrinking a file. A new truncate rpc is defined on the server. The implementation internally updates the file size field on the global meta data on a truncate call. The filesize rpc has been updated to take the maximum of this file size field and the maximum offset of any write index as the true file size. To shrink a file, there are two additional steps done in the truncate rpc in that we delete any index entries that extend beyond the new file size, and we rewrite any entries that happen to overlap (start before and end after) the new file size. Note that unlinking a file current truncates the file to size=0, which will now delete any write index entries for the file. If the file is not laminated, the stat wrappers have been updated to query the filesize rpc to get the current size whereas they used to always return 0. --- client/src/margo_client.c | 41 ++ client/src/margo_client.h | 3 + client/src/unifyfs-sysio.c | 110 +++-- client/src/unifyfs.c | 60 ++- common/src/unifyfs_client_rpcs.h | 13 + server/src/margo_server.c | 4 + server/src/unifyfs_cmd_handler.c | 27 ++ server/src/unifyfs_metadata.c | 42 ++ server/src/unifyfs_metadata.h | 10 + server/src/unifyfs_request_manager.c | 353 ++++++++++++++ server/src/unifyfs_request_manager.h | 3 + t/Makefile.am | 6 +- t/sys/sysio_suite.c | 9 + t/sys/sysio_suite.h | 6 + t/sys/truncate.c | 667 +++++++++++++++++++++++++++ t/sys/write-read-hole.c | 66 ++- t/sys/write-read.c | 4 +- 17 files changed, 1335 insertions(+), 89 deletions(-) create mode 100644 t/sys/truncate.c diff --git a/client/src/margo_client.c b/client/src/margo_client.c index a4e6fbf3a..62b2a984d 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -40,6 +40,11 @@ static void register_client_rpcs(client_rpc_context_t* ctx) unifyfs_filesize_out_t, NULL); + ctx->rpcs.truncate_id = MARGO_REGISTER(mid, "unifyfs_truncate_rpc", + unifyfs_truncate_in_t, + unifyfs_truncate_out_t, + NULL); + ctx->rpcs.fsync_id = MARGO_REGISTER(mid, "unifyfs_fsync_rpc", unifyfs_fsync_in_t, unifyfs_fsync_out_t, @@ -396,6 +401,42 @@ int invoke_client_filesize_rpc(int gfid, size_t* outsize) return (int)ret; } +/* invokes the client truncate rpc function */ +int invoke_client_truncate_rpc(int gfid, size_t filesize) +{ + /* check that we have initialized margo */ + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + /* get handle to rpc function */ + hg_handle_t handle = create_handle(client_rpc_context->rpcs.truncate_id); + + /* fill in input struct */ + unifyfs_truncate_in_t in; + in.app_id = (int32_t)app_id; + in.local_rank_idx = (int32_t)local_rank_idx; + in.gfid = (int32_t)gfid; + in.filesize = (hg_size_t)filesize; + + /* call rpc function */ + LOGDBG("invoking the truncate rpc function in client"); + hg_return_t hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* decode response */ + unifyfs_filesize_out_t out; + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + int32_t ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + /* free resources */ + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} + /* invokes the client fsync rpc function */ int invoke_client_fsync_rpc(int gfid) { diff --git a/client/src/margo_client.h b/client/src/margo_client.h index 6fc76fb65..74fb91780 100644 --- a/client/src/margo_client.h +++ b/client/src/margo_client.h @@ -15,6 +15,7 @@ typedef struct ClientRpcIds { hg_id_t metaset_id; hg_id_t metaget_id; hg_id_t filesize_id; + hg_id_t truncate_id; hg_id_t fsync_id; hg_id_t read_id; hg_id_t mread_id; @@ -45,6 +46,8 @@ int invoke_client_metaget_rpc(int gfid, unifyfs_file_attr_t* f_meta); int invoke_client_filesize_rpc(int gfid, size_t* filesize); +int invoke_client_truncate_rpc(int gfid, size_t filesize); + int invoke_client_fsync_rpc(int gfid); int invoke_client_read_rpc(int gfid, size_t offset, size_t length); diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index d41552b54..a3c643f05 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -233,22 +233,35 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) { /* determine whether we should intercept this path or not */ if (unifyfs_intercept_path(path)) { - /* lookup the fid for the path */ + /* get file id for path name */ int fid = unifyfs_get_fid_from_path(path); - if (fid < 0) { + if (fid >= 0) { + /* got the file locally, use fid_truncate the file */ + int rc = unifyfs_fid_truncate(fid, length); + if (rc != UNIFYFS_SUCCESS) { + errno = EIO; + return -1; + } + } else { + /* otherwise call gfid truncate to attempt to + * truncate the global file */ + int gfid = unifyfs_generate_gfid(path); + + /* invoke truncate rpc */ + int rc = invoke_client_truncate_rpc(gfid, length); + if (rc != UNIFYFS_SUCCESS) { + LOGDBG("truncate rpc failed %s in UNIFYFS", path); + errno = EIO; + return -1; + } + } + +#if 0 /* ERROR: file does not exist */ LOGDBG("Couldn't find entry for %s in UNIFYFS", path); errno = ENOENT; return -1; - } - - /* truncate the file */ - int rc = unifyfs_fid_truncate(fid, length); - if (rc != UNIFYFS_SUCCESS) { - LOGDBG("unifyfs_fid_truncate failed for %s in UNIFYFS", path); - errno = EIO; - return -1; - } +#endif /* success */ return 0; @@ -335,30 +348,63 @@ int UNIFYFS_WRAP(remove)(const char* path) } } +/* Get global file meta data with accurate file size */ +static int unifyfs_get_meta_with_size(int gfid, unifyfs_file_attr_t* pfattr) +{ + /* lookup global meta data for this file */ + int ret = unifyfs_get_global_file_meta(gfid, pfattr); + if (ret != UNIFYFS_SUCCESS) { + LOGDBG("get metadata rpc failed"); + return ret; + } + + /* if file is laminated, we assume the file size in the meta + * data is already accurate, if not, look up the current file + * size with an rpc */ + if (!pfattr->is_laminated) { + /* lookup current global file size */ + size_t filesize; + ret = invoke_client_filesize_rpc(gfid, &filesize); + if (ret == UNIFYFS_SUCCESS) { + /* success, we have a file size value */ + pfattr->size = (uint64_t) filesize; + } else { + /* failed to get file size for some reason */ + LOGDBG("filesize rpc failed"); + return ret; + } + } + + return UNIFYFS_SUCCESS; +} + /* The main stat call for all the *stat() functions */ static int __stat(const char* path, struct stat* buf) { - int gfid, fid, ret; - unifyfs_file_attr_t fattr; - /* check that caller gave us a buffer to write to */ if (!buf) { - errno = EFAULT; + /* forgot buffer for stat */ + LOGDBG("invalid stat buffer"); + errno = EINVAL; return -1; } + + /* clear the user buffer */ memset(buf, 0, sizeof(*buf)); - /* lookup stat data for global file id */ - gfid = unifyfs_generate_gfid(path); - ret = unifyfs_get_global_file_meta(gfid, &fattr); + /* get global file id for given path */ + int gfid = unifyfs_generate_gfid(path); + + /* get stat information for file */ + unifyfs_file_attr_t fattr; + int ret = unifyfs_get_meta_with_size(gfid, &fattr); if (ret != UNIFYFS_SUCCESS) { - LOGDBG("metaget failed"); - errno = ENOENT; + errno = EIO; return -1; } /* update local file metadata (if applicable) */ - fid = unifyfs_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(path); if (fid != -1) { unifyfs_fid_update_file_meta(fid, &fattr); } @@ -379,12 +425,6 @@ static int __stat(const char* path, struct stat* buf) buf->st_rdev |= (unifyfs_fid_local_size(fid) & 0xFFFFFFFF); } - /* global filesize is zero for non-laminated files */ - if (!fattr.is_laminated) { - LOGDBG("file is NOT laminated") - buf->st_size = 0; - } - return 0; } @@ -401,14 +441,12 @@ int UNIFYFS_WRAP(stat)(const char* path, struct stat* buf) int UNIFYFS_WRAP(fstat)(int fd, struct stat* buf) { - int fid; - const char* path; LOGDBG("fstat was called for fd: %d", fd); /* check whether we should intercept this file descriptor */ if (unifyfs_intercept_fd(&fd)) { - fid = unifyfs_get_fid_from_fd(fd); - path = unifyfs_path_from_fid(fid); + int fid = unifyfs_get_fid_from_fd(fd); + const char* path = unifyfs_path_from_fid(fid); return __stat(path, buf); } else { MAP_OR_FAIL(fstat); @@ -466,8 +504,6 @@ int UNIFYFS_WRAP(__lxstat)(int vers, const char* path, struct stat* buf) int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) { LOGDBG("fxstat was called for fd %d", fd); - int fid; - const char* path; /* check whether we should intercept this file descriptor */ if (unifyfs_intercept_fd(&fd)) { @@ -476,8 +512,8 @@ int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) return -1; } - fid = unifyfs_get_fid_from_fd(fd); - path = unifyfs_path_from_fid(fid); + int fid = unifyfs_get_fid_from_fd(fd); + const char* path = unifyfs_path_from_fid(fid); return __stat(path, buf); } else { MAP_OR_FAIL(__fxstat); @@ -1850,6 +1886,7 @@ ssize_t UNIFYFS_WRAP(pwrite64)(int fd, const void* buf, size_t count, int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) { /* check whether we should intercept this file descriptor */ + int origfd = fd; if (unifyfs_intercept_fd(&fd)) { /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); @@ -1866,6 +1903,9 @@ int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) return -1; } + /* sync any pending writes if we have any before we truncate */ + UNIFYFS_WRAP(fsync)(origfd); + /* truncate the file */ int rc = unifyfs_fid_truncate(fid, length); if (rc != UNIFYFS_SUCCESS) { diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 05681c771..e80b50698 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -1157,38 +1157,28 @@ int unifyfs_fid_truncate(int fid, off_t length) /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if (meta->is_laminated) { - return EINVAL; /* Can't truncate a laminated file */ + /* Can't truncate a laminated file */ + return (int)UNIFYFS_ERROR_INVAL; } - /* get current size of file */ - off_t size = meta->local_size; - - /* drop data if length is less than current size, - * allocate new space and zero fill it if bigger */ - if (length < size) { - /* determine the number of chunks to leave after truncating */ - int shrink_rc = unifyfs_fid_shrink(fid, length); - if (shrink_rc != UNIFYFS_SUCCESS) { - return shrink_rc; - } - } else if (length > size) { - /* file size has been extended, allocate space */ - int extend_rc = unifyfs_fid_extend(fid, length); - if (extend_rc != UNIFYFS_SUCCESS) { - return (int)UNIFYFS_ERROR_NOSPC; + /* determine file storage type */ + if (meta->storage == FILE_STORAGE_LOGIO) { + /* invoke truncate rpc */ + int gfid = unifyfs_gfid_from_fid(fid); + int rc = invoke_client_truncate_rpc(gfid, length); + if (rc != UNIFYFS_SUCCESS) { + return rc; } - /* write zero values to new bytes */ - off_t gap_size = length - size; - int zero_rc = unifyfs_fid_write_zero(fid, size, gap_size); - if (zero_rc != UNIFYFS_SUCCESS) { - return (int)UNIFYFS_ERROR_IO; - } + /* truncate succeeded, update global and local size to + * reflect truncated size, note log size is not affected */ + meta->global_size = length; + meta->local_size = length; + } else { + /* unknown storage type */ + return (int)UNIFYFS_ERROR_IO; } - /* set the new size */ - meta->local_size = length; - return UNIFYFS_SUCCESS; } @@ -1371,14 +1361,20 @@ int unifyfs_fid_close(int fid) /* delete a file id and return file its resources to free pools */ int unifyfs_fid_unlink(int fid) { - /* return data to free pools */ - int rc = unifyfs_fid_truncate(fid, 0); - if (rc != UNIFYFS_SUCCESS) { - /* failed to release storage for the file, - * so bail out to keep its file id active */ - return rc; + int rc; + + /* if we have a file, return any data to free pools */ + if (!unifyfs_fid_is_dir(fid)) { + rc = unifyfs_fid_truncate(fid, 0); + if (rc != UNIFYFS_SUCCESS) { + /* failed to release storage for the file, + * so bail out to keep its file id active */ + return rc; + } } + /* TODO: delete global file meta data */ + /* finalize the storage we're using for this file */ rc = unifyfs_fid_store_free(fid); if (rc != UNIFYFS_SUCCESS) { diff --git a/common/src/unifyfs_client_rpcs.h b/common/src/unifyfs_client_rpcs.h index c4176e4d8..dc1c98f1f 100644 --- a/common/src/unifyfs_client_rpcs.h +++ b/common/src/unifyfs_client_rpcs.h @@ -116,6 +116,19 @@ MERCURY_GEN_PROC(unifyfs_filesize_out_t, ((hg_size_t)(filesize))) DECLARE_MARGO_RPC_HANDLER(unifyfs_filesize_rpc) +/* unifyfs_truncate_rpc (client => server) + * + * given an app_id, client_id, global file id, + * and a filesize, truncate file to that size */ +MERCURY_GEN_PROC(unifyfs_truncate_in_t, + ((int32_t)(app_id)) + ((int32_t)(local_rank_idx)) + ((int32_t)(gfid)) + ((hg_size_t)(filesize))) +MERCURY_GEN_PROC(unifyfs_truncate_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_truncate_rpc) + /* unifyfs_read_rpc (client => server) * * given an app_id, client_id, global file id, an offset, and a length, diff --git a/server/src/margo_server.c b/server/src/margo_server.c index dd4b208c9..1536a323e 100644 --- a/server/src/margo_server.c +++ b/server/src/margo_server.c @@ -171,6 +171,10 @@ static void register_client_server_rpcs(margo_instance_id mid) unifyfs_filesize_in_t, unifyfs_filesize_out_t, unifyfs_filesize_rpc); + MARGO_REGISTER(mid, "unifyfs_truncate_rpc", + unifyfs_truncate_in_t, unifyfs_truncate_out_t, + unifyfs_truncate_rpc); + MARGO_REGISTER(mid, "unifyfs_read_rpc", unifyfs_read_in_t, unifyfs_read_out_t, unifyfs_read_rpc) diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index 879b05729..080dcfd2e 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -495,6 +495,33 @@ static void unifyfs_filesize_rpc(hg_handle_t handle) } DEFINE_MARGO_RPC_HANDLER(unifyfs_filesize_rpc) +/* given an app_id, client_id, global file id, + * and file size, truncate file to that size */ +static void unifyfs_truncate_rpc(hg_handle_t handle) +{ + /* get input params */ + unifyfs_truncate_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); + + /* truncate file to specified size */ + int ret = rm_cmd_truncate(in.app_id, in.local_rank_idx, + in.gfid, in.filesize); + + /* build our output values */ + unifyfs_truncate_out_t out; + out.ret = (int32_t) ret; + + /* return to caller */ + hret = margo_respond(handle, &out); + assert(hret == HG_SUCCESS); + + /* free margo resources */ + margo_free_input(handle, &in); + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(unifyfs_truncate_rpc) + /* given an app_id, client_id, global file id, an offset, and a length, * initiate read operation to lookup and return data, * client synchronizes with server again later when data is available diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index 6e6f113eb..808f30947 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -441,3 +441,45 @@ int unifyfs_set_file_extents(int num_entries, return rc; } +/* delete the listed keys from the file extents */ +int unifyfs_delete_file_extents( + int num_entries, /* number of items in keys list */ + unifyfs_key_t** keys, /* list of keys to be deleted */ + int* key_lens) /* list of byte sizes for keys list items */ +{ + /* assume we'll succeed */ + int rc = UNIFYFS_SUCCESS; + + /* select index for file extents */ + md->primary_index = unifyfs_indexes[IDX_FILE_EXTENTS]; + + /* delete list of key/value pairs */ + struct mdhim_brm_t* brm = mdhimBDelete(md, md->primary_index, + (void**)(keys), key_lens, num_entries); + + /* check for errors and free resources */ + if (!brm) { + LOGERR("Error deleting file extents from MDHIM"); + rc = (int)UNIFYFS_ERROR_MDHIM; + } else { + /* step through linked list of messages, + * scan for any error and free messages */ + struct mdhim_brm_t* brmp = brm; + while (brmp) { + /* check current item for error */ + if (brmp->error < 0) { + LOGERR("Error deleting file extents from MDHIM"); + rc = (int)UNIFYFS_ERROR_MDHIM; + } + + /* record pointer to current item, + * advance loop pointer to next item in list, + * free resources for current item */ + brm = brmp; + brmp = brmp->next; + mdhim_full_release_msg(brm); + } + } + + return rc; +} diff --git a/server/src/unifyfs_metadata.h b/server/src/unifyfs_metadata.h index a326ef10e..9cf7b7e52 100644 --- a/server/src/unifyfs_metadata.h +++ b/server/src/unifyfs_metadata.h @@ -128,6 +128,16 @@ int unifyfs_get_file_extents(int num_keys, unifyfs_key_t** keys, int* key_lens, int* num_values, unifyfs_keyval_t** keyval); +/** + * Delete File extents from the KV-Store. + * + * @param[in] num_entries number of key value pairs to delete + * @param[in] keys array storing the keys + * @param[in] key_lens array with the length of the elements in \p keys + */ +int unifyfs_delete_file_extents(int num_entries, + unifyfs_key_t** keys, int* key_lens); + /** * Store File extents in the KV-Store. * diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 4f7925ca1..48c9e101e 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -606,10 +606,363 @@ int rm_cmd_filesize( keyvals = NULL; } + /* get filesize as recorded in metadata, which may be bigger if + * user issued an ftruncate on the file to extend it past the + * last write */ + size_t filesize_meta = filesize; + + /* given the global file id, look up file attributes + * from key/value store */ + unifyfs_file_attr_t fattr; + int ret = unifyfs_get_file_attribute(gfid, &fattr); + if (ret == UNIFYFS_SUCCESS) { + /* found file attribute for this file, now get its size */ + filesize_meta = fattr.size; + } else { + /* failed to find file attributes for this file */ + return UNIFYFS_FAILURE; + } + + /* take maximum of last write and file size from metadata */ + if (filesize_meta > filesize) { + filesize = filesize_meta; + } + *outsize = filesize; return rc; } +/* delete any key whose last byte is beyond the specified + * file size */ +static int truncate_delete_keys( + size_t filesize, /* new file size */ + int num, /* number of entries in keyvals */ + unifyfs_keyval_t* keyvals) /* list of existing key/values */ +{ + /* assume we'll succeed */ + int ret = (int) UNIFYFS_SUCCESS; + + /* pointers to memory we'll dynamically allocate for file extents */ + unifyfs_key_t** unifyfs_keys = NULL; + unifyfs_val_t** unifyfs_vals = NULL; + int* unifyfs_key_lens = NULL; + int* unifyfs_val_lens = NULL; + + /* in the worst case, we'll have to delete all existing keys */ + /* allocate storage for file extent key/values */ + /* TODO: possibly get this from memory pool */ + unifyfs_keys = alloc_key_array(num); + unifyfs_vals = alloc_value_array(num); + unifyfs_key_lens = calloc(num, sizeof(int)); + unifyfs_val_lens = calloc(num, sizeof(int)); + if ((NULL == unifyfs_keys) || + (NULL == unifyfs_vals) || + (NULL == unifyfs_key_lens) || + (NULL == unifyfs_val_lens)) { + LOGERR("failed to allocate memory for file extents"); + ret = (int)UNIFYFS_ERROR_NOMEM; + goto truncate_delete_exit; + } + + /* counter for number of key/values we need to delete */ + int delete_count = 0; + + /* iterate over each key, and if this index extends beyond desired + * file size, create an entry to delete that key */ + int i; + for (i = 0; i < num; i++) { + /* get pointer to next key value pair */ + unifyfs_keyval_t* kv = &keyvals[i]; + + /* get last byte offset for this segment of the file */ + size_t last_offset = kv->key.offset + kv->val.len; + + /* if this segment extends beyond the new file size, + * we need to delete this index entry */ + if (last_offset > filesize) { + /* found an index that extends past end of desired + * file size, get next empty key entry from the pool */ + unifyfs_key_t* key = unifyfs_keys[delete_count]; + + /* define the key to be deleted */ + key->gfid = kv->key.gfid; + key->offset = kv->key.offset; + + /* MDHIM needs to know the byte size of each key and value */ + unifyfs_key_lens[delete_count] = sizeof(unifyfs_key_t); + //unifyfs_val_lens[delete_count] = sizeof(unifyfs_val_t); + + /* increment the number of keys we're deleting */ + delete_count++; + } + } + + /* batch delete file extent key/values from MDHIM */ + if (delete_count > 0) { + ret = unifyfs_delete_file_extents(delete_count, + unifyfs_keys, unifyfs_key_lens); + if (ret != UNIFYFS_SUCCESS) { + /* TODO: need proper error handling */ + LOGERR("unifyfs_delete_file_extents() failed"); + goto truncate_delete_exit; + } + } + +truncate_delete_exit: + /* clean up memory */ + + if (NULL != unifyfs_keys) { + free_key_array(unifyfs_keys); + } + + if (NULL != unifyfs_vals) { + free_value_array(unifyfs_vals); + } + + if (NULL != unifyfs_key_lens) { + free(unifyfs_key_lens); + } + + if (NULL != unifyfs_val_lens) { + free(unifyfs_val_lens); + } + + return ret; +} + +/* rewrite any key that overlaps with new file size, + * we assume the existing key has already been deleted */ +static int truncate_rewrite_keys( + size_t filesize, /* new file size */ + int num, /* number of entries in keyvals */ + unifyfs_keyval_t* keyvals) /* list of existing key/values */ +{ + /* assume we'll succeed */ + int ret = (int) UNIFYFS_SUCCESS; + + /* pointers to memory we'll dynamically allocate for file extents */ + unifyfs_key_t** unifyfs_keys = NULL; + unifyfs_val_t** unifyfs_vals = NULL; + int* unifyfs_key_lens = NULL; + int* unifyfs_val_lens = NULL; + + /* in the worst case, we'll have to rewrite all existing keys */ + /* allocate storage for file extent key/values */ + /* TODO: possibly get this from memory pool */ + unifyfs_keys = alloc_key_array(num); + unifyfs_vals = alloc_value_array(num); + unifyfs_key_lens = calloc(num, sizeof(int)); + unifyfs_val_lens = calloc(num, sizeof(int)); + if ((NULL == unifyfs_keys) || + (NULL == unifyfs_vals) || + (NULL == unifyfs_key_lens) || + (NULL == unifyfs_val_lens)) { + LOGERR("failed to allocate memory for file extents"); + ret = (int)UNIFYFS_ERROR_NOMEM; + goto truncate_rewrite_exit; + } + + /* counter for number of key/values we need to rewrite */ + int count = 0; + + /* iterate over each key, and if this index starts before + * and ends after the desired file size, create an entry + * that ends at new file size */ + int i; + for (i = 0; i < num; i++) { + /* get pointer to next key value pair */ + unifyfs_keyval_t* kv = &keyvals[i]; + + /* get first byte offset for this segment of the file */ + size_t first_offset = kv->key.offset; + + /* get last byte offset for this segment of the file */ + size_t last_offset = kv->key.offset + kv->val.len; + + /* if this segment extends beyond the new file size, + * we need to rewrite this index entry */ + if (first_offset < filesize && + last_offset > filesize) { + /* found an index that overlaps end of desired + * file size, get next empty key entry from the pool */ + unifyfs_key_t* key = unifyfs_keys[count]; + + /* define the key to be rewritten */ + key->gfid = kv->key.gfid; + key->offset = kv->key.offset; + + /* compute new length of this entry */ + size_t newlen = (size_t)(filesize - first_offset); + + /* for the value, we store the log position, the length, + * the host server (delegator rank), the mount point id + * (app id), and the client id (rank) */ + unifyfs_val_t* val = unifyfs_vals[count]; + val->addr = kv->val.addr; + val->len = newlen; + val->delegator_rank = kv->val.delegator_rank; + val->app_id = kv->val.app_id; + val->rank = kv->val.rank; + + /* MDHIM needs to know the byte size of each key and value */ + unifyfs_key_lens[count] = sizeof(unifyfs_key_t); + unifyfs_val_lens[count] = sizeof(unifyfs_val_t); + + /* increment the number of keys we're deleting */ + count++; + } + } + + /* batch set file extent key/values from MDHIM */ + if (count > 0) { + ret = unifyfs_set_file_extents(count, + unifyfs_keys, unifyfs_key_lens, + unifyfs_vals, unifyfs_val_lens); + if (ret != UNIFYFS_SUCCESS) { + /* TODO: need proper error handling */ + LOGERR("unifyfs_set_file_extents() failed"); + goto truncate_rewrite_exit; + } + } + +truncate_rewrite_exit: + /* clean up memory */ + + if (NULL != unifyfs_keys) { + free_key_array(unifyfs_keys); + } + + if (NULL != unifyfs_vals) { + free_value_array(unifyfs_vals); + } + + if (NULL != unifyfs_key_lens) { + free(unifyfs_key_lens); + } + + if (NULL != unifyfs_val_lens) { + free(unifyfs_val_lens); + } + + return ret; +} + +/* given an app_id, client_id, global file id, + * and file size, truncate file to specified size + */ +int rm_cmd_truncate( + int app_id, /* app_id for requesting client */ + int client_id, /* client_id for requesting client */ + int gfid, /* global file id */ + size_t newsize) /* desired file size */ +{ + /* set offset and length to request *all* key/value pairs + * for this file */ + size_t offset = 0; + + /* want to pick the highest integer offset value a file + * could have here */ + size_t length = (SIZE_MAX >> 1) - 1; + + /* get the locations of all the read requests from the + * key-value store*/ + unifyfs_key_t key1, key2; + + /* create key to describe first byte we'll read */ + key1.gfid = gfid; + key1.offset = offset; + + /* create key to describe last byte we'll read */ + key2.gfid = gfid; + key2.offset = offset + length - 1; + + /* set up input params to specify range lookup */ + unifyfs_key_t* unifyfs_keys[2] = {&key1, &key2}; + int key_lens[2] = {sizeof(unifyfs_key_t), sizeof(unifyfs_key_t)}; + + /* look up all entries in this range */ + int num_vals = 0; + unifyfs_keyval_t* keyvals = NULL; + int rc = unifyfs_get_file_extents(2, unifyfs_keys, key_lens, + &num_vals, &keyvals); + if (UNIFYFS_SUCCESS != rc) { + /* failed to look up extents, bail with error */ + return UNIFYFS_FAILURE; + } + + /* compute our file size by iterating over each file + * segment and taking the max logical offset */ + int i; + size_t filesize = 0; + for (i = 0; i < num_vals; i++) { + /* get pointer to next key value pair */ + unifyfs_keyval_t* kv = &keyvals[i]; + + /* get last byte offset for this segment of the file */ + size_t last_offset = kv->key.offset + kv->val.len; + + /* update our filesize if this offset is bigger than the current max */ + if (last_offset > filesize) { + filesize = last_offset; + } + } + + /* get filesize as recorded in metadata, which may be bigger if + * user issued an ftruncate on the file to extend it past the + * last write */ + size_t filesize_meta = filesize; + + /* given the global file id, look up file attributes + * from key/value store */ + unifyfs_file_attr_t fattr; + rc = unifyfs_get_file_attribute(gfid, &fattr); + if (rc == UNIFYFS_SUCCESS) { + /* found file attribute for this file, now get its size */ + filesize_meta = fattr.size; + } else { + /* failed to find file attributes for this file */ + goto truncate_exit; + } + + /* take maximum of last write and file size from metadata */ + if (filesize_meta > filesize) { + filesize = filesize_meta; + } + + /* may need to throw away and rewrite keys if shrinking file */ + if (newsize < filesize) { + /* delete any key that extends beyond new file size */ + rc = truncate_delete_keys(newsize, num_vals, keyvals); + if (rc != UNIFYFS_SUCCESS) { + goto truncate_exit; + } + + /* rewrite any key that overlaps new file size */ + rc = truncate_rewrite_keys(newsize, num_vals, keyvals); + if (rc != UNIFYFS_SUCCESS) { + goto truncate_exit; + } + } + + /* update file size field with latest size */ + fattr.size = newsize; + rc = unifyfs_set_file_attribute(&fattr); + if (rc != UNIFYFS_SUCCESS) { + /* failed to update file attributes with new file size */ + goto truncate_exit; + } + +truncate_exit: + + /* free off key/value buffer returned from get_file_extents */ + if (NULL != keyvals) { + free(keyvals); + keyvals = NULL; + } + + return rc; +} + int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, int gfid, int app_id, int client_id, int num_keys, unifyfs_key_t** keys, int* keylens) diff --git a/server/src/unifyfs_request_manager.h b/server/src/unifyfs_request_manager.h index 6ce140b84..db1d0e1e0 100644 --- a/server/src/unifyfs_request_manager.h +++ b/server/src/unifyfs_request_manager.h @@ -110,6 +110,9 @@ int rm_cmd_read(int app_id, int client_id, int gfid, int rm_cmd_filesize(int app_id, int client_id, int gfid, size_t* outsize); +/* truncate file to specified size */ +int rm_cmd_truncate(int app_id, int client_id, int gfid, size_t size); + /* function called by main thread to instruct * resource manager thread to exit, * returns UNIFYFS_SUCCESS on success */ diff --git a/t/Makefile.am b/t/Makefile.am index bb6b17ba9..0c0dd941d 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -101,7 +101,8 @@ sys_sysio_gotcha_t_SOURCES = sys/sysio_suite.h \ sys/open.c \ sys/open64.c \ sys/write-read.c \ - sys/write-read-hole.c + sys/write-read-hole.c \ + sys/truncate.c sys_sysio_gotcha_t_CPPFLAGS = $(test_cppflags) sys_sysio_gotcha_t_LDADD = $(test_ldadd) @@ -115,7 +116,8 @@ sys_sysio_static_t_SOURCES = sys/sysio_suite.h \ sys/open.c \ sys/open64.c \ sys/write-read.c \ - sys/write-read-hole.c + sys/write-read-hole.c \ + sys/truncate.c sys_sysio_static_t_CPPFLAGS = $(test_cppflags) sys_sysio_static_t_LDADD = $(test_static_ldadd) diff --git a/t/sys/sysio_suite.c b/t/sys/sysio_suite.c index 6cebb83f4..58f4d9084 100644 --- a/t/sys/sysio_suite.c +++ b/t/sys/sysio_suite.c @@ -85,6 +85,15 @@ int main(int argc, char* argv[]) write_read_hole_test(unifyfs_root); + truncate_test(unifyfs_root); + truncate_bigempty(unifyfs_root); + truncate_eof(unifyfs_root); + truncate_truncsync(unifyfs_root); + truncate_pattern_size(unifyfs_root, 0); + truncate_pattern_size(unifyfs_root, 2020); + truncate_empty_read(unifyfs_root, 0); + truncate_empty_read(unifyfs_root, 2020); + MPI_Finalize(); done_testing(); diff --git a/t/sys/sysio_suite.h b/t/sys/sysio_suite.h index d4f90ddb3..a59a98910 100644 --- a/t/sys/sysio_suite.h +++ b/t/sys/sysio_suite.h @@ -50,4 +50,10 @@ int write_read_test(char* unifyfs_root); /* test reading from file with holes */ int write_read_hole_test(char* unifyfs_root); +/* Tests for UNIFYFS_WRAP(ftruncate) and UNIFYFS_WRAP(truncate) */ +int truncate_test(char* unifyfs_root); +int truncate_bigempty(char* unifyfs_root); +int truncate_eof(char* unifyfs_root); +int truncate_truncsync(char* unifyfs_root); + #endif /* SYSIO_SUITE_H */ diff --git a/t/sys/truncate.c b/t/sys/truncate.c new file mode 100644 index 000000000..ed9a73fe6 --- /dev/null +++ b/t/sys/truncate.c @@ -0,0 +1,667 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + + /* + * Test truncate and ftruncate + */ +#include +#include +#include +#include +#include +#include +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +/* Get global, local, or log sizes (or all) */ +static +void get_size(char* path, size_t* global, size_t* local, size_t* log) +{ + struct stat sb = {0}; + int rc; + + rc = stat(path, &sb); + if (rc != 0) { + printf("Error: %s\n", strerror(errno)); + exit(1); /* die on failure */ + } + if (global) { + *global = sb.st_size; + } + + if (local) { + *local = sb.st_rdev & 0xFFFFFFFF; + } + + if (log) { + *log = (sb.st_rdev >> 32) & 0xFFFFFFFF; + } +} + +int truncate_test(char* unifyfs_root) +{ + char path[64]; + int rc; + int fd; + size_t global, local, log; + + size_t bufsize = 1024*1024; + char* buf = (char*) malloc(bufsize); + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Open a new file for writing */ + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* file should be 0 bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 0); + ok(local == 0, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 0); + ok(log == 0, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 0); + + /* write 1MB and fsync, expect 1MB */ + rc = write(fd, buf, bufsize); + ok(rc == bufsize, "%s:%d write(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize, rc, strerror(errno)); + + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + get_size(path, &global, &local, &log); + ok(global == 1*bufsize, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 1*bufsize); + ok(local == 1*bufsize, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 1*bufsize); + ok(log == 1*bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 1*bufsize); + + /* skip a 1MB hole, write another 1MB, and fsync expect 3MB */ + rc = lseek(fd, 2*bufsize, SEEK_SET); + ok(rc == 2*bufsize, "%s:%d lseek(%d) (rc=%d): %s", + __FILE__, __LINE__, 2*bufsize, rc, strerror(errno)); + + rc = write(fd, buf, bufsize); + ok(rc == bufsize, "%s:%d write(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize, rc, strerror(errno)); + + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + get_size(path, &global, &local, &log); + ok(global == 3*bufsize, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 3*bufsize); + ok(local == 3*bufsize, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 3*bufsize); + ok(log == 2*bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 2*bufsize); + + /* ftruncate at 5MB, expect 5MB */ + rc = ftruncate(fd, 5*bufsize); + ok(rc == 0, "%s:%d ftruncate(%d) (rc=%d): %s", + __FILE__, __LINE__, 5*bufsize, rc, strerror(errno)); + + get_size(path, &global, &local, &log); + ok(global == 5*bufsize, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 5*bufsize); + ok(local == 5*bufsize, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 5*bufsize); + ok(log == 2*bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 2*bufsize); + + close(fd); + + /* truncate at 0.5 MB, expect 0.5MB */ + rc = truncate(path, bufsize/2); + ok(rc == 0, "%s:%d truncate(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize/2, rc, strerror(errno)); + + get_size(path, &global, &local, &log); + ok(global == bufsize/2, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, bufsize/2); + ok(local == bufsize/2, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, bufsize/2); + ok(log == 2*bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 2*bufsize); + + /* truncate to 0, expect 0 */ + rc = truncate(path, 0); + ok(rc == 0, "%s:%d truncate(%d) (rc=%d): %s", + __FILE__, __LINE__, 0, rc, strerror(errno)); + + get_size(path, &global, &local, &log); + ok(global == 0, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 0); + ok(local == 0, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 0); + ok(log == 2*bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 2*bufsize); + + free(buf); + + return 0; +} + +int truncate_bigempty(char* unifyfs_root) +{ + char path[64]; + int rc; + int fd; + size_t global, local, log; + + size_t bufsize = 1024*1024; + char* buf = (char*) malloc(bufsize); + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Open a new file for writing */ + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + get_size(path, &global, &local, &log); + ok(global == 0, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 0); + ok(local == 0, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 0); + ok(log == 0, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 0); + + /* ftruncate at 1TB, expect 1TB */ + off_t bigempty = 1024*1024*1024*1024ULL; + rc = ftruncate(fd, bigempty); + ok(rc == 0, "%s:%d ftruncate(%llu) (rc=%d): %s", + __FILE__, __LINE__, (unsigned long long) bigempty, + rc, strerror(errno)); + + get_size(path, &global, &local, &log); + ok(global == (size_t)bigempty, "%s:%d global size is %llu expected %llu", + __FILE__, __LINE__, global, (unsigned long long)bigempty, + strerror(errno)); + + close(fd); + + free(buf); + + return 0; +} + +int truncate_eof(char* unifyfs_root) +{ + char path[64]; + int rc; + int fd; + size_t global, local, log; + + size_t bufsize = 1024*1024; + char* buf = (char*) malloc(bufsize); + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Open a new file for writing */ + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* file should be 0 bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 0); + ok(local == 0, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 0); + ok(log == 0, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 0); + + /* write 1MB */ + rc = write(fd, buf, bufsize); + ok(rc == bufsize, "%s:%d write(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize, rc, strerror(errno)); + + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* ftruncate at 0.5MB */ + rc = ftruncate(fd, bufsize/2); + ok(rc == 0, "%s:%d ftruncate(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize/2, rc, strerror(errno)); + + close(fd); + + /* Open a file for reading */ + fd = open(path, O_RDONLY); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* ask for 1MB, should only get 0.5MB back */ + rc = read(fd, buf, bufsize); + ok(rc == bufsize/2, "%s:%d read(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize, rc, strerror(errno)); + + /* then should get 0 since at EOF */ + rc = read(fd, buf, bufsize); + ok(rc == 0, "%s:%d read(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize, rc, strerror(errno)); + + close(fd); + + /* truncate to 0 */ + rc = truncate(path, 0); + ok(rc == 0, "%s:%d truncate(%d) (rc=%d): %s", + __FILE__, __LINE__, 0, rc, strerror(errno)); + + /* Open a file for reading */ + fd = open(path, O_RDONLY); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* should get 0 since immediately at EOF */ + rc = read(fd, buf, bufsize); + ok(rc == 0, "%s:%d read(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize, rc, strerror(errno)); + + close(fd); + + free(buf); + + return 0; +} + +int truncate_truncsync(char* unifyfs_root) +{ + char path[64]; + int rc; + int fd; + size_t global, local, log; + + size_t bufsize = 1024*1024; + char* buf = (char*) malloc(bufsize); + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Open a new file for writing */ + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* file should be 0 bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 0); + ok(local == 0, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 0); + ok(log == 0, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 0); + + /* write 1MB */ + rc = write(fd, buf, bufsize); + ok(rc == bufsize, "%s:%d write(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize, rc, strerror(errno)); + + /* ftruncate to 0.5MB */ + rc = ftruncate(fd, bufsize/2); + ok(rc == 0, "%s:%d ftruncate(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize/2, rc, strerror(errno)); + + /* file should be 0.5MB bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == bufsize/2, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, bufsize/2); + ok(local == bufsize/2, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, bufsize/2); + ok(log == bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, bufsize); + + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* file should still be 0.5MB bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == bufsize/2, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, bufsize/2); + ok(local == bufsize/2, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, bufsize/2); + ok(log == bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, bufsize); + + close(fd); + + free(buf); + + return 0; +} + +/* fill buffer with known pattern based on file offset */ +int fill_pattern(char* buf, size_t size, size_t start) +{ + size_t i; + for (i = 0; i < size; i++) { + char expected = ((i + start) % 26) + 'A'; + buf[i] = expected; + } + return 0; +} + +/* fill buffer with known pattern based on file offset */ +int check_pattern(char* buf, size_t size, size_t start) +{ + size_t i; + for (i = 0; i < size; i++) { + char expected = ((i + start) % 26) + 'A'; + if (buf[i] != expected) { + return (int)(i+1); + } + } + return 0; +} + +/* check that buffer is all zero */ +int check_zeros(char* buf, size_t size) +{ + size_t i; + for (i = 0; i < size; i++) { + if (buf[i] != (char)0) { + return (int)(i+1); + } + } + return 0; +} + +/* write a known pattern of a known size, truncate to something smaller, + * read until EOF and verify contents along the way */ +int truncate_pattern_size(char* unifyfs_root, off_t seekpos) +{ + char path[64]; + int rc; + int fd; + size_t global, local, log; + int i; + + size_t bufsize = 1024*1024; + char* buf = (char*) malloc(bufsize); + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Open a new file for writing */ + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* file should be 0 bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 0); + ok(local == 0, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 0); + ok(log == 0, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 0); + + /* write pattern out of 20 MB in size */ + size_t nwritten = 0; + for (i = 0; i < 20; i++) { + /* fill buffer with known pattern based on file offset */ + fill_pattern(buf, bufsize, nwritten); + + /* write data to file */ + rc = write(fd, buf, bufsize); + ok(rc == bufsize, "%s:%d write(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize, rc, strerror(errno)); + + /* track number of bytes written so far */ + nwritten += (size_t)rc; + } + + /* set size we'll truncate file to */ + off_t truncsize = 5*bufsize + 42; + + /* ftruncate to 5MB + 42 bytes */ + rc = ftruncate(fd, truncsize); + ok(rc == 0, "%s:%d ftruncate(%d) (rc=%d): %s", + __FILE__, __LINE__, (int)truncsize, rc, strerror(errno)); + + /* file should be of size 5MB + 42 at this point */ + get_size(path, &global, &local, &log); + ok(global == truncsize, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, (int)truncsize); + ok(local == truncsize, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, (int)truncsize); + ok(log == 20*bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 20*bufsize); + + /* this kind of tests that the ftruncate above implied an fsync, + * can't really since the writes may have gone to disk on their + * own before ftruncate call */ + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* file should still be 5MB + 42 bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == truncsize, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, (int)truncsize); + ok(local == truncsize, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, (int)truncsize); + ok(log == 20*bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 20*bufsize); + + close(fd); + + /* read file back from offset 0 and verify size and contents */ + fd = open(path, O_RDONLY); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* see to position if file if seekpos is set */ + if (seekpos > 0) { + off_t pos = lseek(fd, seekpos, SEEK_SET); + ok(pos == seekpos, "%s:%d lseek(%d) (rc=%d): %s", + __FILE__, __LINE__, pos, rc, strerror(errno)); + } + + off_t numread = 0; + while (1) { + /* compute number of bytes we expect to read on next attempt */ + ssize_t expected = bufsize; + ssize_t remaining = (ssize_t)(truncsize - numread - seekpos); + if (expected > remaining) { + expected = remaining; + } + + /* ask for 1MB, should only get 0.5MB back */ + rc = read(fd, buf, bufsize); + ok(rc == expected, "%s:%d read(%d) (rc=%d) expected=%d %s", + __FILE__, __LINE__, bufsize, rc, expected, strerror(errno)); + + /* check that contents we read are correct */ + if (rc > 0) { + size_t start = numread + seekpos; + int check = check_pattern(buf, rc, start); + ok(check == 0, "%s:%d pattern check of bytes [%d, %d) rc=%d", + __FILE__, __LINE__, (int)start, (int)(start + rc), check); + + /* add to number of bytes read so far */ + numread += rc; + } + + /* break if we hit end of file */ + if (rc == 0) { + /* check that total read is expected size */ + ok(numread == (truncsize - seekpos), + "%s:%d read %d bytes, expected %d", + __FILE__, __LINE__, (int)numread, (int)(truncsize - seekpos)); + break; + } + + /* check that we don't run past expected + * end of file (and hang the test) */ + if (numread > (truncsize - seekpos)) { + ok(numread <= (truncsize - seekpos), + "%s:%d read %d bytes, expected %d", + __FILE__, __LINE__, (int)numread, (int)(truncsize - seekpos)); + break; + } + + /* break if we hit an error, would have been + * reported in read above */ + if (rc < 0) { + break; + } + } + + close(fd); + + free(buf); + + return 0; +} + +/* truncate an empty file to something and read until EOF, + * check size and contents of buffer */ +int truncate_empty_read(char* unifyfs_root, off_t seekpos) +{ + char path[64]; + int rc; + int fd; + size_t global, local, log; + int i; + + size_t bufsize = 1024*1024; + char* buf = (char*) malloc(bufsize); + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Open a new file for writing */ + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* file should be 0 bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 0); + ok(local == 0, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 0); + ok(log == 0, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 0); + + /* set size we'll truncate file to */ + off_t truncsize = 5*bufsize + 42; + + /* ftruncate to 5MB + 42 bytes */ + rc = ftruncate(fd, truncsize); + ok(rc == 0, "%s:%d ftruncate(%d) (rc=%d): %s", + __FILE__, __LINE__, (int)truncsize, rc, strerror(errno)); + + /* file should be of size 5MB + 42 at this point */ + get_size(path, &global, &local, &log); + ok(global == truncsize, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, (int)truncsize); + ok(local == truncsize, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, (int)truncsize); + ok(log == 0, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 0); + + /* this kind of tests that the ftruncate above implied an fsync, + * can't really since the writes may have gone to disk on their + * own before ftruncate call */ + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* file should still be 5MB + 42 bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == truncsize, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, (int)truncsize); + ok(local == truncsize, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, (int)truncsize); + ok(log == 0, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 0); + + close(fd); + + /* read file back from offset 0 and verify size and contents */ + fd = open(path, O_RDONLY); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* see to position if file if seekpos is set */ + if (seekpos > 0) { + off_t pos = lseek(fd, seekpos, SEEK_SET); + ok(pos == seekpos, "%s:%d lseek(%d) (rc=%d): %s", + __FILE__, __LINE__, pos, rc, strerror(errno)); + } + + off_t numread = 0; + while (1) { + /* compute number of bytes we expect to read on next attempt */ + ssize_t expected = bufsize; + ssize_t remaining = (ssize_t)(truncsize - numread - seekpos); + if (expected > remaining) { + expected = remaining; + } + + /* ask for 1MB, should only get 0.5MB back */ + rc = read(fd, buf, bufsize); + ok(rc == expected, "%s:%d read(%d) (rc=%d) expected=%d %s", + __FILE__, __LINE__, bufsize, rc, expected, strerror(errno)); + + /* check that contents we read are correct */ + if (rc > 0) { + size_t start = numread + seekpos; + int check = check_zeros(buf, rc); + ok(check == 0, "%s:%d pattern check of bytes [%d, %d) rc=%d", + __FILE__, __LINE__, (int)start, (int)(start + rc), check); + + /* add to number of bytes read so far */ + numread += rc; + } + + /* break if we hit end of file */ + if (rc == 0) { + /* check that total read is expected size */ + ok(numread == (truncsize - seekpos), + "%s:%d read %d bytes, expected %d", + __FILE__, __LINE__, (int)numread, (int)(truncsize - seekpos)); + break; + } + + /* check that we don't run past expected + * end of file (and hang the test) */ + if (numread > (truncsize - seekpos)) { + ok(numread <= (truncsize - seekpos), + "%s:%d read %d bytes, expected %d", + __FILE__, __LINE__, (int)numread, (int)(truncsize - seekpos)); + break; + } + + /* break if we hit an error, would have been + * reported in read above */ + if (rc < 0) { + break; + } + } + + close(fd); + + free(buf); + + return 0; +} diff --git a/t/sys/write-read-hole.c b/t/sys/write-read-hole.c index fe703e13e..59680872b 100644 --- a/t/sys/write-read-hole.c +++ b/t/sys/write-read-hole.c @@ -79,22 +79,25 @@ int write_read_hole_test(char* unifyfs_root) /* create a file that contains: * [0, 1MB) - data = "1" * [1MB, 2MB) - hole = "0" implied - * [2MB, 3MB) - data = "1" */ + * [2MB, 3MB) - data = "1" + * [3MB, 4MB) - hole = "0" implied */ /* Write to the file */ fd = open(path, O_WRONLY | O_CREAT, 0222); ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", __FILE__, __LINE__, path, fd, strerror(errno)); + /* write "1" to [0MB, 1MB) */ rc = write(fd, buf, bufsize); ok(rc == bufsize, "%s:%d write() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); - /* Test writing to a different offset */ + /* skip over [1MB, 2MB) for implied "0" */ rc = lseek(fd, 2*bufsize, SEEK_SET); ok(rc == 2*bufsize, "%s:%d lseek() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); + /* write "1" to [2MB, 3MB) */ rc = write(fd, buf, bufsize); ok(rc == bufsize, "%s:%d write() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); @@ -113,6 +116,21 @@ int write_read_hole_test(char* unifyfs_root) ok(rc == 0, "%s:%d fsync() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); + /* Check global and local size on our un-laminated file */ + get_size(path, &global, &local, &log); + ok(global == 3*bufsize, "%s:%d global size is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + ok(local == 3*bufsize, "%s:%d local size is %d: %s", + __FILE__, __LINE__, local, strerror(errno)); + ok(log == 2*bufsize, "%s:%d log size is %d: %s", + __FILE__, __LINE__, log, strerror(errno)); + + /* truncate file at 4MB, extends file so that + * [3MB, 4MB) is implied "0" */ + rc = ftruncate(fd, 4*bufsize); + ok(rc == 0, "%s:%d ftruncate() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + /* Laminate */ rc = chmod(path, 0444); ok(rc == 0, "%s:%d chmod(0444) (rc=%d): %s", @@ -120,9 +138,9 @@ int write_read_hole_test(char* unifyfs_root) /* Check global and local size on our un-laminated file */ get_size(path, &global, &local, &log); - ok(global == 3*bufsize, "%s:%d global size is %d: %s", + ok(global == 4*bufsize, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(local == 3*bufsize, "%s:%d local size is %d: %s", + ok(local == 4*bufsize, "%s:%d local size is %d: %s", __FILE__, __LINE__, local, strerror(errno)); ok(log == 2*bufsize, "%s:%d log size is %d: %s", __FILE__, __LINE__, log, strerror(errno)); @@ -142,8 +160,11 @@ int write_read_hole_test(char* unifyfs_root) * this should be a full read, all from actual data */ memset(buf, 2, bufsize); ssize_t nread = pread(fd, buf, bufsize, 0*bufsize); - ok(nread == bufsize, "%s:%d pread(%s) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + ok(nread == bufsize, + "%s:%d pread expected=%llu got=%llu: errno=%s", + __FILE__, __LINE__, + (unsigned long long) bufsize, (unsigned long long) nread, + strerror(errno)); /* check that full buffer is "1" */ int valid = check_contents(buf, bufsize, 1); @@ -155,8 +176,11 @@ int write_read_hole_test(char* unifyfs_root) * this should be a full read, all from a hole */ memset(buf, 2, bufsize); nread = pread(fd, buf, bufsize, 1*bufsize); - ok(nread == bufsize, "%s:%d pread(%s) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + ok(nread == bufsize, + "%s:%d pread expected=%llu got=%llu: errno=%s", + __FILE__, __LINE__, + (unsigned long long) bufsize, (unsigned long long) nread, + strerror(errno)); /* check that full buffer is "0" */ valid = check_contents(buf, bufsize, 0); @@ -168,8 +192,11 @@ int write_read_hole_test(char* unifyfs_root) * should be a full read, half data, half hole */ memset(buf, 2, bufsize); nread = pread(fd, buf, bufsize, bufsize/2); - ok(nread == bufsize, "%s:%d pread(%s) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + ok(nread == bufsize, + "%s:%d pread expected=%llu got=%llu: errno=%s", + __FILE__, __LINE__, + (unsigned long long) bufsize, (unsigned long long) nread, + strerror(errno)); /* check that data portion is "1" */ valid = check_contents(buf, bufsize/2, 1); @@ -182,16 +209,19 @@ int write_read_hole_test(char* unifyfs_root) __FILE__, __LINE__); - /* read segment [2.5MB, 3.5MB) + /* read segment [3.5MB, 4.5MB) * should read only half of requested amount, - * half data, half past end of file */ + * half hole, half past end of file */ memset(buf, 2, bufsize); - nread = pread(fd, buf, bufsize, 2*bufsize + bufsize/2); - ok(nread == bufsize/2, "%s:%d pread(%s) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); - - /* first half of buffer should be "1" */ - valid = check_contents(buf, bufsize/2, 1); + nread = pread(fd, buf, bufsize, 3*bufsize + bufsize/2); + ok(nread == bufsize/2, + "%s:%d pread expected=%llu got=%llu: errno=%s", + __FILE__, __LINE__, + (unsigned long long) bufsize/2, (unsigned long long) nread, + strerror(errno)); + + /* first half of buffer should be "0" */ + valid = check_contents(buf, bufsize/2, 0); ok(valid == 1, "%s:%d data check", __FILE__, __LINE__); diff --git a/t/sys/write-read.c b/t/sys/write-read.c index c738f2c70..a41cd7456 100644 --- a/t/sys/write-read.c +++ b/t/sys/write-read.c @@ -88,7 +88,7 @@ int write_read_test(char* unifyfs_root) /* Check global and local size on our un-laminated file */ get_size(path, &global, &local, &log); - ok(global == 0, "%s: global size is %d: %s", __FILE__, global, + ok(global == 15, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); ok(local == 15, "%s: local size is %d: %s", __FILE__, local, strerror(errno)); @@ -116,7 +116,7 @@ int write_read_test(char* unifyfs_root) /* Check global and local size on our un-laminated file */ get_size(path, &global, &local, &log); - ok(global == 0, "%s: global size is %d: %s", __FILE__, global, + ok(global == 21, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); ok(local == 21, "%s: local size is %d: %s", __FILE__, local, strerror(errno)); From fce1d1e5ce1eeb9571658272eea71b9dbb349790 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 13 Jan 2020 17:24:05 -0800 Subject: [PATCH 055/168] update truncate/ftruncate/fsync to all call unifyfs_sync Use the unifyfs_sync(gfid) function to fsync data before truncating a file. It updates the truncate(), ftruncate() and fsync() wrappers to call unifyfs_sync(gfid). Ensures that the spillover file is synced to disk in unifyfs_fsync() before it invokes the client-to-server fsync rpc. Without this, the data that was meant to be written to the spillover file might still be cached in memory and not on disk when another process attempts to read it. We should be sure it's on disk before it's registered with the server. Since the index rewrite that happens during sync rewrites entries for all files, this uses the global segment count across all files for the test that triggers a sync. --- client/src/unifyfs-fixed.c | 79 +++++++++++++++------ client/src/unifyfs-internal.h | 3 + client/src/unifyfs-sysio.c | 55 +++++++-------- client/src/unifyfs.c | 3 + t/sys/sysio_suite.c | 2 + t/sys/sysio_suite.h | 4 ++ t/sys/truncate.c | 126 ++++++++++++++++++++++++++++++++++ 7 files changed, 218 insertions(+), 54 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index c0318055b..c55f9fa81 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -486,8 +486,7 @@ static int unifyfs_split_index( * Clear all entries in the log index. This only clears the metadata, * not the data itself. */ -static void -unifyfs_clear_index(void) +static void unifyfs_clear_index(void) { *unifyfs_indices.ptr_num_entries = 0; } @@ -497,13 +496,33 @@ unifyfs_clear_index(void) * * Returns 0 on success, nonzero otherwise. */ -int -unifyfs_sync(int gfid) +int unifyfs_sync(int gfid) { + /* write contents from segment tree to index buffer + * if we're using that optimization */ if (unifyfs_flatten_writes) { unifyfs_rewrite_index_from_seg_tree(); } + /* if there are no index entries, we've got nothing to sync */ + if (*unifyfs_indices.ptr_num_entries == 0) { + return UNIFYFS_SUCCESS; + } + + /* ensure any data written to the spill over file is flushed */ + /* if using spill over, fsync spillover data to disk */ + if (unifyfs_use_spillover) { + int ret = __real_fsync(unifyfs_spilloverblock); + if (ret != 0) { + /* error, need to set errno appropriately, + * we called the real fsync which should + * have already set errno to something reasonable */ + LOGERR("failed to flush data to spill over file"); + return UNIFYFS_ERROR_IO; + } + } + + /* tell the server to grab our new extents */ int ret = invoke_client_fsync_rpc(gfid); if (ret != UNIFYFS_SUCCESS) { /* something went wrong when trying to flush key/values */ @@ -511,17 +530,15 @@ unifyfs_sync(int gfid) return UNIFYFS_ERROR_IO; } - /* - * flushed, clear buffer and refresh number of entries - * and number remaining - */ + /* flushed, clear buffer and refresh number of entries + * and number remaining */ unifyfs_clear_index(); return UNIFYFS_SUCCESS; } -void -unifyfs_add_index_entry_to_seg_tree(unifyfs_filemeta_t* meta, +void unifyfs_add_index_entry_to_seg_tree( + unifyfs_filemeta_t* meta, unifyfs_index_t* index) { if (!unifyfs_flatten_writes) { @@ -529,6 +546,11 @@ unifyfs_add_index_entry_to_seg_tree(unifyfs_filemeta_t* meta, return; } + /* to update the global running segment count, we need to capture + * the count in this tree before adding and the count after to + * add the difference */ + unsigned long count_before = seg_tree_count(&meta->seg_tree); + /* * Store the write in our segment tree. We will later use this for * flattening writes. @@ -543,8 +565,15 @@ unifyfs_add_index_entry_to_seg_tree(unifyfs_filemeta_t* meta, * create two new nodes in the seg_tree. If we're close to potentially * filling up the index, sync it out. */ - if (seg_tree_count(&meta->seg_tree) >= (unifyfs_max_index_entries - 2)) { + if (unifyfs_segment_count >= (unifyfs_max_index_entries - 2)) { + /* this will flush our segments, sync them, and set the running + * segment count back to 0 */ unifyfs_sync(meta->gfid); + } else { + /* increase the running global segment count by the number of + * new entries we added to this tree */ + unsigned long count_after = seg_tree_count(&meta->seg_tree); + unifyfs_segment_count += (count_after - count_before); } } @@ -721,42 +750,48 @@ static int unifyfs_logio_chunk_write( */ void unifyfs_rewrite_index_from_seg_tree(void) { - struct seg_tree_node* node; + /* get pointer to index buffer */ unifyfs_index_t* indexes = unifyfs_indices.index_entry; - unsigned long idx = 0; - /* Erase the index before we re-write it */ unifyfs_clear_index(); + /* count up number of entries we wrote to buffer */ + unsigned long idx = 0; + /* For each fid .. */ for (int i = 0; i < UNIFYFS_MAX_FILEDESCS; i++) { - int fid, gfid; - - fid = unifyfs_fds[i].fid; + /* get file id for each file descriptor */ + int fid = unifyfs_fds[i].fid; unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if (!meta) { continue; } - gfid = unifyfs_gfid_from_fid(fid); + int gfid = unifyfs_gfid_from_fid(fid); - node = NULL; seg_tree_rdlock(&meta->seg_tree); /* For each write in this file's seg_tree ... */ + struct seg_tree_node* node = NULL; while ((node = seg_tree_iter(&meta->seg_tree, node))) { indexes[idx].file_pos = node->start; - indexes[idx].log_pos = node->ptr; - indexes[idx].length = node->end - node->start + 1; - indexes[idx].gfid = gfid; + indexes[idx].log_pos = node->ptr; + indexes[idx].length = node->end - node->start + 1; + indexes[idx].gfid = gfid; idx++; } + seg_tree_unlock(&meta->seg_tree); /* All done processing this files writes. Clear its seg_tree */ seg_tree_clear(&meta->seg_tree); } + + /* reset our segment count since we just dumped them all */ + unifyfs_segment_count = 0; + + /* record total number of entries in index buffer */ *unifyfs_indices.ptr_num_entries = idx; } diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index e342d2d0d..7c04d6a4e 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -328,6 +328,9 @@ extern unifyfs_index_buf_t unifyfs_indices; extern unsigned long unifyfs_max_index_entries; extern long unifyfs_spillover_max_chunks; +/* tracks total number of unsync'd segments for all files */ +extern unsigned long unifyfs_segment_count; + extern int local_rank_cnt; extern int local_rank_idx; extern int local_del_cnt; diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index a3c643f05..da76cc909 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -233,6 +233,17 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) { /* determine whether we should intercept this path or not */ if (unifyfs_intercept_path(path)) { + /* get global file id for path */ + int gfid = unifyfs_generate_gfid(path); + + /* before we truncate, sync any data cached this file id */ + int ret = unifyfs_sync(gfid); + if (ret != UNIFYFS_SUCCESS) { + /* sync failed for some reason, set errno and return error */ + errno = unifyfs_err_map_to_errno(ret); + return -1; + } + /* get file id for path name */ int fid = unifyfs_get_fid_from_path(path); if (fid >= 0) { @@ -243,10 +254,6 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) return -1; } } else { - /* otherwise call gfid truncate to attempt to - * truncate the global file */ - int gfid = unifyfs_generate_gfid(path); - /* invoke truncate rpc */ int rc = invoke_client_truncate_rpc(gfid, length); if (rc != UNIFYFS_SUCCESS) { @@ -256,13 +263,6 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) } } -#if 0 - /* ERROR: file does not exist */ - LOGDBG("Couldn't find entry for %s in UNIFYFS", path); - errno = ENOENT; - return -1; -#endif - /* success */ return 0; } else { @@ -1903,8 +1903,16 @@ int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) return -1; } - /* sync any pending writes if we have any before we truncate */ - UNIFYFS_WRAP(fsync)(origfd); + /* get global file id for fid */ + int gfid = unifyfs_gfid_from_fid(fid); + + /* before we truncate, sync any data cached this file id */ + int ret = unifyfs_sync(gfid); + if (ret != UNIFYFS_SUCCESS) { + /* sync failed for some reason, set errno and return error */ + errno = unifyfs_err_map_to_errno(ret); + return -1; + } /* truncate the file */ int rc = unifyfs_fid_truncate(fid, length); @@ -1925,11 +1933,6 @@ int UNIFYFS_WRAP(fsync)(int fd) { /* check whether we should intercept this file descriptor */ if (unifyfs_intercept_fd(&fd)) { - if (*unifyfs_indices.ptr_num_entries == 0) { - /* Nothing to sync */ - return 0; - } - /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { @@ -1938,22 +1941,12 @@ int UNIFYFS_WRAP(fsync)(int fd) return -1; } + /* skip this file if no new data has been written to it */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if (!meta->needs_sync) { return 0; } - /* if using spill over, fsync spillover data to disk */ - if (unifyfs_use_spillover) { - int ret = __real_fsync(unifyfs_spilloverblock); - if (ret != 0) { - /* error, need to set errno appropriately, - * we called the real fsync which should - * have already set errno to something reasonable */ - return -1; - } - } - /* invoke fsync rpc to register index metadata with server */ int gfid = unifyfs_gfid_from_fid(fid); int ret = unifyfs_sync(gfid); @@ -1963,9 +1956,7 @@ int UNIFYFS_WRAP(fsync)(int fd) return -1; } - /* server has processed entries in index buffer, reset it */ - *(unifyfs_indices.ptr_num_entries) = 0; - + /* update metadata to indicate that data has been synced */ meta->needs_sync = 0; return 0; } else { diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index e80b50698..37d59bf50 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -77,6 +77,9 @@ static size_t unifyfs_fattr_buf_size; unsigned long unifyfs_max_index_entries; /* max metadata log entries */ int unifyfs_spillmetablock; +/* tracks total number of unsync'd segments for all files */ +unsigned long unifyfs_segment_count; + int global_rank_cnt; /* count of world ranks */ int local_rank_cnt; int local_rank_idx; diff --git a/t/sys/sysio_suite.c b/t/sys/sysio_suite.c index 58f4d9084..9a3b5a741 100644 --- a/t/sys/sysio_suite.c +++ b/t/sys/sysio_suite.c @@ -93,6 +93,8 @@ int main(int argc, char* argv[]) truncate_pattern_size(unifyfs_root, 2020); truncate_empty_read(unifyfs_root, 0); truncate_empty_read(unifyfs_root, 2020); + truncate_ftrunc_before_sync(unifyfs_root); + truncate_trunc_before_sync(unifyfs_root); MPI_Finalize(); diff --git a/t/sys/sysio_suite.h b/t/sys/sysio_suite.h index a59a98910..e1cfa7085 100644 --- a/t/sys/sysio_suite.h +++ b/t/sys/sysio_suite.h @@ -55,5 +55,9 @@ int truncate_test(char* unifyfs_root); int truncate_bigempty(char* unifyfs_root); int truncate_eof(char* unifyfs_root); int truncate_truncsync(char* unifyfs_root); +int truncate_pattern_size(char* unifyfs_root, int pos); +int truncate_empty_read(char* unifyfs_root, int pos); +int truncate_ftrunc_before_sync(char* unifyfs_root); +int truncate_trunc_before_sync(char* unifyfs_root); #endif /* SYSIO_SUITE_H */ diff --git a/t/sys/truncate.c b/t/sys/truncate.c index ed9a73fe6..ddb580758 100644 --- a/t/sys/truncate.c +++ b/t/sys/truncate.c @@ -665,3 +665,129 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) return 0; } + +int truncate_ftrunc_before_sync(char* unifyfs_root) +{ + char path[64]; + int rc; + int fd; + size_t global, local, log; + + size_t bufsize = 1024; + char* buf = (char*) malloc(bufsize); + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Open a new file for writing */ + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* file should be 0 bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 0); + ok(local == 0, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 0); + ok(log == 0, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 0); + + /* write a small amount, intended to be small enough that + * the write itself does not cause an implicit fsync */ + + /* write data to file */ + rc = write(fd, buf, bufsize); + ok(rc == bufsize, "%s:%d write(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize, rc, strerror(errno)); + + /* then truncate the file to 0 */ + off_t truncsize = 0; + rc = ftruncate(fd, truncsize); + ok(rc == 0, "%s:%d ftruncate(%d) (rc=%d): %s", + __FILE__, __LINE__, (int)truncsize, rc, strerror(errno)); + + /* then fsync the file */ + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* finally, check that the file is 0 bytes, + * i.e., check that the writes happened before the truncate + * and not at the fsync */ + get_size(path, &global, &local, &log); + ok(global == truncsize, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, (int)truncsize); + ok(local == truncsize, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, (int)truncsize); + ok(log == bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, bufsize); + + close(fd); + + free(buf); + + return 0; +} + +int truncate_trunc_before_sync(char* unifyfs_root) +{ + char path[64]; + int rc; + int fd; + size_t global, local, log; + + size_t bufsize = 1024; + char* buf = (char*) malloc(bufsize); + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Open a new file for writing */ + fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* file should be 0 bytes at this point */ + get_size(path, &global, &local, &log); + ok(global == 0, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, 0); + ok(local == 0, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, 0); + ok(log == 0, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, 0); + + /* write a small amount, intended to be small enough that + * the write itself does not cause an implicit fsync */ + + /* write data to file */ + rc = write(fd, buf, bufsize); + ok(rc == bufsize, "%s:%d write(%d) (rc=%d): %s", + __FILE__, __LINE__, bufsize, rc, strerror(errno)); + + /* then truncate the file to 0 */ + off_t truncsize = 0; + rc = truncate(path, truncsize); + ok(rc == 0, "%s:%d truncate(%d) (rc=%d): %s", + __FILE__, __LINE__, (int)truncsize, rc, strerror(errno)); + + /* then fsync the file */ + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* finally, check that the file is 0 bytes, + * i.e., check that the writes happened before the truncate + * and not at the fsync */ + get_size(path, &global, &local, &log); + ok(global == truncsize, "%s:%d global size is %d expected %d", + __FILE__, __LINE__, global, (int)truncsize); + ok(local == truncsize, "%s:%d local size is %d expected %d", + __FILE__, __LINE__, local, (int)truncsize); + ok(log == bufsize, "%s:%d log size is %d expected %d", + __FILE__, __LINE__, log, bufsize); + + close(fd); + + free(buf); + + return 0; +} From b9a37537ce935780b64867e7f314b22247037e32 Mon Sep 17 00:00:00 2001 From: Craig Philip Steffen Date: Mon, 13 Jan 2020 11:57:55 -0800 Subject: [PATCH 056/168] clarify indent statement in check-patch --- scripts/linux_kernel_checkpatch/checkpatch.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/linux_kernel_checkpatch/checkpatch.pl b/scripts/linux_kernel_checkpatch/checkpatch.pl index 49af5338b..2db3b969b 100755 --- a/scripts/linux_kernel_checkpatch/checkpatch.pl +++ b/scripts/linux_kernel_checkpatch/checkpatch.pl @@ -3514,7 +3514,7 @@ sub process { ($s !~ /^\s*(?:\}|\{|else\b)/)) || ($sindent > $indent + 8))) { WARN("SUSPECT_CODE_INDENT", - "suspect code indent for conditional statements ($indent, $sindent)\n" . $herecurr . "$stat_real\n"); + "suspect (wrong # of spaces?) code indent for cond. stmts ($indent, $sindent)\n" . $herecurr . "$stat_real\n"); } } From cddc9f9a45406ffd504a02b680c9608a62836f13 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 10 Jan 2020 12:43:42 -0800 Subject: [PATCH 057/168] add client-to-server rpcs for unlink and laminate This adds client-to-server rpcs for "create", unlink, and laminate operations. We borrow the existing metaset rpc to serve double duty as a "create" rpc. We add a flag to the rpc parameters to indicate whether the size and laminate fields should be set or not when updating the metadata from a client. We set both size and is_laminated to 0 when first creating the file, otherwise changing file meta data from the client will not update either the size or is_laminated field. For unlink, the rpc will remove the file metadata, and if the file is a regular file, it also removes any write extent keys for that file. For laminate, the rpc gets the current file size and updates the metadata with the file size and it sets the is_laminated flag to 1. --- client/src/margo_client.c | 95 +++++++++++++++++++- client/src/margo_client.h | 8 +- client/src/unifyfs-internal.h | 19 ++-- client/src/unifyfs-sysio.c | 29 +++--- client/src/unifyfs.c | 81 ++++++++++------- common/src/unifyfs_client_rpcs.h | 25 ++++++ server/src/margo_server.c | 8 ++ server/src/unifyfs_cmd_handler.c | 94 +++++++++++++++----- server/src/unifyfs_metadata.c | 74 +++++++++++++++- server/src/unifyfs_metadata.h | 15 +++- server/src/unifyfs_request_manager.c | 84 +++++++++++++++++- server/src/unifyfs_request_manager.h | 6 ++ t/Makefile.am | 6 +- t/server/unifyfs_meta_get_test.c | 2 +- t/sys/sysio_suite.c | 2 + t/sys/sysio_suite.h | 3 + t/sys/unlink.c | 127 +++++++++++++++++++++++++++ 17 files changed, 597 insertions(+), 81 deletions(-) create mode 100644 t/sys/unlink.c diff --git a/client/src/margo_client.c b/client/src/margo_client.c index 62b2a984d..58d9d36ad 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -45,6 +45,16 @@ static void register_client_rpcs(client_rpc_context_t* ctx) unifyfs_truncate_out_t, NULL); + ctx->rpcs.unlink_id = MARGO_REGISTER(mid, "unifyfs_unlink_rpc", + unifyfs_unlink_in_t, + unifyfs_unlink_out_t, + NULL); + + ctx->rpcs.laminate_id = MARGO_REGISTER(mid, "unifyfs_laminate_rpc", + unifyfs_laminate_in_t, + unifyfs_laminate_out_t, + NULL); + ctx->rpcs.fsync_id = MARGO_REGISTER(mid, "unifyfs_fsync_rpc", unifyfs_fsync_in_t, unifyfs_fsync_out_t, @@ -273,8 +283,18 @@ int invoke_client_unmount_rpc(void) return (int)ret; } -/* invokes the client metaset rpc function */ -int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta) +/* + * Set the metadata values for a file (after optionally creating it). + * The gfid for the file is in f_meta->gfid. + * + * create: If set to 1, attempt to create the file first. If the file + * already exists, then update its metadata with the values in + * f_meta. If set to 0, and the file does not exist, then + * the server will return an error. + * + * f_meta: The metadata values to update. + */ +int invoke_client_metaset_rpc(int create, unifyfs_file_attr_t* f_meta) { /* check that we have initialized margo */ if (NULL == client_rpc_context) { @@ -286,6 +306,7 @@ int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta) /* fill in input struct */ unifyfs_metaset_in_t in; + in.create = (int32_t) create; in.gfid = f_meta->gfid; in.filename = f_meta->filename; in.mode = f_meta->mode; @@ -437,6 +458,76 @@ int invoke_client_truncate_rpc(int gfid, size_t filesize) return (int)ret; } +/* invokes the client unlink rpc function */ +int invoke_client_unlink_rpc(int gfid) +{ + /* check that we have initialized margo */ + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + /* get handle to rpc function */ + hg_handle_t handle = create_handle(client_rpc_context->rpcs.unlink_id); + + /* fill in input struct */ + unifyfs_unlink_in_t in; + in.app_id = (int32_t)app_id; + in.local_rank_idx = (int32_t)local_rank_idx; + in.gfid = (int32_t)gfid; + + /* call rpc function */ + LOGDBG("invoking the unlink rpc function in client"); + hg_return_t hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* decode response */ + unifyfs_filesize_out_t out; + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + int32_t ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + /* free resources */ + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} + +/* invokes the client-to-server laminate rpc function */ +int invoke_client_laminate_rpc(int gfid) +{ + /* check that we have initialized margo */ + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + /* get handle to rpc function */ + hg_handle_t handle = create_handle(client_rpc_context->rpcs.laminate_id); + + /* fill in input struct */ + unifyfs_unlink_in_t in; + in.app_id = (int32_t)app_id; + in.local_rank_idx = (int32_t)local_rank_idx; + in.gfid = (int32_t)gfid; + + /* call rpc function */ + LOGDBG("invoking the laminate rpc function in client"); + hg_return_t hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* decode response */ + unifyfs_filesize_out_t out; + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + int32_t ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + /* free resources */ + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} + /* invokes the client fsync rpc function */ int invoke_client_fsync_rpc(int gfid) { diff --git a/client/src/margo_client.h b/client/src/margo_client.h index 74fb91780..694770640 100644 --- a/client/src/margo_client.h +++ b/client/src/margo_client.h @@ -16,6 +16,8 @@ typedef struct ClientRpcIds { hg_id_t metaget_id; hg_id_t filesize_id; hg_id_t truncate_id; + hg_id_t unlink_id; + hg_id_t laminate_id; hg_id_t fsync_id; hg_id_t read_id; hg_id_t mread_id; @@ -40,7 +42,7 @@ int invoke_client_mount_rpc(void); int invoke_client_unmount_rpc(void); -int invoke_client_metaset_rpc(unifyfs_file_attr_t* f_meta); +int invoke_client_metaset_rpc(int create, unifyfs_file_attr_t* f_meta); int invoke_client_metaget_rpc(int gfid, unifyfs_file_attr_t* f_meta); @@ -48,6 +50,10 @@ int invoke_client_filesize_rpc(int gfid, size_t* filesize); int invoke_client_truncate_rpc(int gfid, size_t filesize); +int invoke_client_unlink_rpc(int gfid); + +int invoke_client_laminate_rpc(int gfid); + int invoke_client_fsync_rpc(int gfid); int invoke_client_read_rpc(int gfid, size_t offset, size_t length); diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 7c04d6a4e..c58f744d5 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -576,13 +576,18 @@ int unifyfs_fid_unlink(int fid); int unifyfs_generate_gfid(const char* path); -int unifyfs_set_global_file_meta_from_fid(int fid); - -int unifyfs_set_global_file_meta(int gfid, - unifyfs_file_attr_t* gfattr); - -int unifyfs_get_global_file_meta(int gfid, - unifyfs_file_attr_t* gfattr); +int unifyfs_set_global_file_meta_from_fid( + int fid, + int create); + +int unifyfs_set_global_file_meta( + int gfid, + int create, + unifyfs_file_attr_t* gfattr); + +int unifyfs_get_global_file_meta( + int gfid, + unifyfs_file_attr_t* gfattr); // These require types/structures defined above #include "unifyfs-fixed.h" diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index da76cc909..5008d6a4f 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -2226,29 +2226,21 @@ static int __chmod(int fid, mode_t mode) */ if ((meta->mode & 0222) && (((meta->mode & 0222) & mode) == 0)) { - - /* - * We're laminating. Calculate the file size so we can cache it - * (both locally and on the server). - */ - ret = invoke_client_filesize_rpc(gfid, &meta->global_size); + /* We're laminating. */ + ret = invoke_client_laminate_rpc(gfid); if (ret) { LOGERR("chmod: couldn't get the global file size on laminate"); errno = EIO; return -1; } - - /* record locally that this file is now laminated */ - meta->is_laminated = 1; } /* Clear out our old permission bits, and set the new ones in */ meta->mode = meta->mode & ~0777; meta->mode = meta->mode | mode; - /* update the global meta data to reflect new permissions, - * size, and laminated flag */ - ret = unifyfs_set_global_file_meta_from_fid(fid); + /* update the global meta data to reflect new permissions */ + ret = unifyfs_set_global_file_meta_from_fid(fid, 0); if (ret) { LOGERR("chmod: can't set global meta entry for %s (fid:%d)", path, fid); @@ -2256,6 +2248,19 @@ static int __chmod(int fid, mode_t mode) return -1; } + /* read metadata back to pick up file size and laminated flag */ + unifyfs_file_attr_t attr = {0}; + ret = unifyfs_get_global_file_meta(gfid, &attr); + if (ret) { + LOGERR("chmod: can't get global meta entry for %s (fid:%d)", + path, fid); + errno = EIO; + return -1; + } + + /* update global size of file from global metadata */ + unifyfs_fid_update_file_meta(fid, &attr); + return 0; } diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 37d59bf50..e75bfe1fd 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -818,15 +818,35 @@ int unifyfs_fid_update_file_meta(int fid, unifyfs_file_attr_t* gfattr) return UNIFYFS_FAILURE; } -int unifyfs_set_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) +/* + * Set the metadata values for a file (after optionally creating it). + * The gfid for the file is in f_meta->gfid. + * + * gfid: The global file id on which to set metadata. + * + * create: If set to 1, attempt to create the file first. If the file + * already exists, then update its metadata with the values in + * gfattr. If set to 0, and the file does not exist, then + * the server will return an error. + * + * gfattr: The metadata values to store. + */ +int unifyfs_set_global_file_meta( + int gfid, /* file id to set meta data for */ + int create, /* whether to set size/laminated fields (1) or not (0) */ + unifyfs_file_attr_t* gfattr) /* meta data to store for file */ { /* check that we have an input buffer */ if (NULL == gfattr) { return UNIFYFS_FAILURE; } + /* force the gfid field value to match the gfid we're + * submitting this under */ + gfattr->gfid = gfid; + /* submit file attributes to global key/value store */ - int ret = invoke_client_metaset_rpc(gfattr); + int ret = invoke_client_metaset_rpc(create, gfattr); return ret; } @@ -847,7 +867,18 @@ int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) return ret; } -int unifyfs_set_global_file_meta_from_fid(int fid) +/* + * Set the metadata values for a file (after optionally creating it), + * using metadata associated with a given local file id. + * + * fid: The local file id on which to base global metadata values. + * + * create: If set to 1, attempt to create the file first. If the file + * already exists, then update its metadata with the values in + * gfattr. If set to 0, and the file does not exist, then + * the server will return an error. + */ +int unifyfs_set_global_file_meta_from_fid(int fid, int create) { /* initialize an empty file attributes structure */ unifyfs_file_attr_t fattr = {0}; @@ -870,30 +901,19 @@ int unifyfs_set_global_file_meta_from_fid(int fid) fattr.ctime = tp; /* copy file mode bits and lamination flag */ - fattr.mode = meta->mode; - fattr.is_laminated = meta->is_laminated; + fattr.mode = meta->mode; - /* set file size, we use the global size field if laminated - * set size to 0 otherwise */ - if (meta->is_laminated) { - /* - * If is_laminated is set, we're either laminating for the first time, - * in which case meta->global_size will have already been calculated and - * filled in with the global file size, or, the file was already - * laminated. In either case, we write meta->global_size to the server. - */ - fattr.size = meta->global_size; - LOGDBG("setting fattr to %d", fattr.size); - } else { - fattr.size = 0; - } + /* these fields are set by server, except when we're creating a + * new file in which case, we should initialize them both to 0 */ + fattr.is_laminated = 0; + fattr.size = 0; /* capture current uid and gid */ fattr.uid = getuid(); fattr.gid = getgid(); /* submit file attributes to global key/value store */ - int ret = unifyfs_set_global_file_meta(meta->gfid, &fattr); + int ret = unifyfs_set_global_file_meta(meta->gfid, create, &fattr); return ret; } @@ -1037,7 +1057,7 @@ int unifyfs_fid_create_directory(const char* path) meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; /* insert global meta data for directory */ - int ret = unifyfs_set_global_file_meta_from_fid(fid); + int ret = unifyfs_set_global_file_meta_from_fid(fid, 1); if (ret != UNIFYFS_SUCCESS) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); @@ -1333,7 +1353,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } /* insert file attribute for file in key-value store */ - ret = unifyfs_set_global_file_meta_from_fid(fid); + ret = unifyfs_set_global_file_meta_from_fid(fid, 1); if (ret != UNIFYFS_SUCCESS) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); @@ -1366,18 +1386,15 @@ int unifyfs_fid_unlink(int fid) { int rc; - /* if we have a file, return any data to free pools */ - if (!unifyfs_fid_is_dir(fid)) { - rc = unifyfs_fid_truncate(fid, 0); - if (rc != UNIFYFS_SUCCESS) { - /* failed to release storage for the file, - * so bail out to keep its file id active */ - return rc; - } + /* invoke unlink rpc */ + int gfid = unifyfs_gfid_from_fid(fid); + rc = invoke_client_unlink_rpc(gfid); + if (rc != UNIFYFS_SUCCESS) { + /* TODO: if item does not exist globally, but just locally, + * we still want to delete item locally */ + return rc; } - /* TODO: delete global file meta data */ - /* finalize the storage we're using for this file */ rc = unifyfs_fid_store_free(fid); if (rc != UNIFYFS_SUCCESS) { diff --git a/common/src/unifyfs_client_rpcs.h b/common/src/unifyfs_client_rpcs.h index dc1c98f1f..7458da9a2 100644 --- a/common/src/unifyfs_client_rpcs.h +++ b/common/src/unifyfs_client_rpcs.h @@ -58,6 +58,7 @@ MERCURY_GEN_STRUCT_PROC(sys_timespec_t, * given a global file id and a file name, * record key/value entry for this file */ MERCURY_GEN_PROC(unifyfs_metaset_in_t, + ((int32_t)(create)) ((hg_const_string_t)(filename)) ((int32_t)(gfid)) ((uint32_t)(mode)) @@ -129,6 +130,30 @@ MERCURY_GEN_PROC(unifyfs_truncate_out_t, ((int32_t)(ret))) DECLARE_MARGO_RPC_HANDLER(unifyfs_truncate_rpc) +/* unifyfs_unlink_rpc (client => server) + * + * given an app_id, client_id, and global file id, + * unlink the file */ +MERCURY_GEN_PROC(unifyfs_unlink_in_t, + ((int32_t)(app_id)) + ((int32_t)(local_rank_idx)) + ((int32_t)(gfid))) +MERCURY_GEN_PROC(unifyfs_unlink_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_unlink_rpc) + +/* unifyfs_laminate_rpc (client => server) + * + * given an app_id, client_id, and global file id, + * laminate the file */ +MERCURY_GEN_PROC(unifyfs_laminate_in_t, + ((int32_t)(app_id)) + ((int32_t)(local_rank_idx)) + ((int32_t)(gfid))) +MERCURY_GEN_PROC(unifyfs_laminate_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_laminate_rpc) + /* unifyfs_read_rpc (client => server) * * given an app_id, client_id, global file id, an offset, and a length, diff --git a/server/src/margo_server.c b/server/src/margo_server.c index 1536a323e..812987b66 100644 --- a/server/src/margo_server.c +++ b/server/src/margo_server.c @@ -175,6 +175,14 @@ static void register_client_server_rpcs(margo_instance_id mid) unifyfs_truncate_in_t, unifyfs_truncate_out_t, unifyfs_truncate_rpc); + MARGO_REGISTER(mid, "unifyfs_unlink_rpc", + unifyfs_unlink_in_t, unifyfs_unlink_out_t, + unifyfs_unlink_rpc); + + MARGO_REGISTER(mid, "unifyfs_laminate_rpc", + unifyfs_laminate_in_t, unifyfs_laminate_out_t, + unifyfs_laminate_rpc); + MARGO_REGISTER(mid, "unifyfs_read_rpc", unifyfs_read_in_t, unifyfs_read_out_t, unifyfs_read_rpc) diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index 080dcfd2e..7449409ef 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -370,22 +370,21 @@ static void unifyfs_metaget_rpc(hg_handle_t handle) /* given the global file id, look up file attributes * from key/value store */ unifyfs_file_attr_t attr_val; - int ret = unifyfs_get_file_attribute(in.gfid, &attr_val); /* build our output values */ unifyfs_metaget_out_t out; - out.gfid = attr_val.gfid; - out.mode = attr_val.mode; - out.uid = attr_val.uid; - out.gid = attr_val.gid; - out.size = attr_val.size; - out.atime = attr_val.atime; - out.mtime = attr_val.mtime; - out.ctime = attr_val.ctime; - out.filename = attr_val.filename; + out.gfid = attr_val.gfid; + out.mode = attr_val.mode; + out.uid = attr_val.uid; + out.gid = attr_val.gid; + out.size = attr_val.size; + out.atime = attr_val.atime; + out.mtime = attr_val.mtime; + out.ctime = attr_val.ctime; + out.filename = attr_val.filename; out.is_laminated = attr_val.is_laminated; - out.ret = ret; + out.ret = ret; /* send output back to caller */ hret = margo_respond(handle, &out); @@ -409,18 +408,21 @@ static void unifyfs_metaset_rpc(hg_handle_t handle) /* store file name for given global file id */ unifyfs_file_attr_t fattr; memset(&fattr, 0, sizeof(fattr)); - fattr.gfid = in.gfid; + int create = (int) in.create; + fattr.gfid = in.gfid; strncpy(fattr.filename, in.filename, sizeof(fattr.filename)); - fattr.mode = in.mode; - fattr.uid = in.uid; - fattr.gid = in.gid; - fattr.size = in.size; - fattr.atime = in.atime; - fattr.mtime = in.mtime; - fattr.ctime = in.ctime; + fattr.mode = in.mode; + fattr.uid = in.uid; + fattr.gid = in.gid; + fattr.size = in.size; + fattr.atime = in.atime; + fattr.mtime = in.mtime; + fattr.ctime = in.ctime; fattr.is_laminated = in.is_laminated; - int ret = unifyfs_set_file_attribute(&fattr); + /* if we're creating the file, + * we initialize both the size and laminate flags */ + int ret = unifyfs_set_file_attribute(create, create, &fattr); /* build our output values */ unifyfs_metaset_out_t out; @@ -522,6 +524,58 @@ static void unifyfs_truncate_rpc(hg_handle_t handle) } DEFINE_MARGO_RPC_HANDLER(unifyfs_truncate_rpc) +/* given an app_id, client_id, and global file id, + * remove file from system */ +static void unifyfs_unlink_rpc(hg_handle_t handle) +{ + /* get input params */ + unifyfs_truncate_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); + + /* truncate file to specified size */ + int ret = rm_cmd_unlink(in.app_id, in.local_rank_idx, in.gfid); + + /* build our output values */ + unifyfs_truncate_out_t out; + out.ret = (int32_t) ret; + + /* return to caller */ + hret = margo_respond(handle, &out); + assert(hret == HG_SUCCESS); + + /* free margo resources */ + margo_free_input(handle, &in); + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(unifyfs_unlink_rpc) + +/* given an app_id, client_id, and global file id, + * laminate file */ +static void unifyfs_laminate_rpc(hg_handle_t handle) +{ + /* get input params */ + unifyfs_truncate_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); + + /* truncate file to specified size */ + int ret = rm_cmd_laminate(in.app_id, in.local_rank_idx, in.gfid); + + /* build our output values */ + unifyfs_truncate_out_t out; + out.ret = (int32_t) ret; + + /* return to caller */ + hret = margo_respond(handle, &out); + assert(hret == HG_SUCCESS); + + /* free margo resources */ + margo_free_input(handle, &in); + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(unifyfs_laminate_rpc) + /* given an app_id, client_id, global file id, an offset, and a length, * initiate read operation to lookup and return data, * client synchronizes with server again later when data is available diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index 808f30947..2715c6a5f 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -209,15 +209,46 @@ int meta_sanitize(void) /* * */ -int unifyfs_set_file_attribute(unifyfs_file_attr_t* fattr_ptr) +int unifyfs_set_file_attribute( + int set_size, + int set_laminate, + unifyfs_file_attr_t* fattr_ptr) { int rc = UNIFYFS_SUCCESS; /* select index for file attributes */ md->primary_index = unifyfs_indexes[IDX_FILE_ATTR]; - /* insert file attribute for given global file id */ int gfid = fattr_ptr->gfid; + + /* if we want to preserve some settings, + * we copy those fields from attributes + * on the existing entry, if there is one */ + int preserve = (!set_size || !set_laminate); + if (preserve) { + /* lookup existing attributes for the file */ + unifyfs_file_attr_t attr; + int get_rc = unifyfs_get_file_attribute(gfid, &attr); + if (get_rc == UNIFYFS_SUCCESS) { + /* found the attributes for this file, + * if size flag is not set, preserve existing size value */ + if (!set_size) { + fattr_ptr->size = attr.size; + } + + /* if laminate flag is not set, + * preserve existing is_laminated state */ + if (!set_laminate) { + fattr_ptr->is_laminated = attr.is_laminated; + } + } else { + /* otherwise, trying to update attributes for a file that + * we can't find */ + return get_rc; + } + } + + /* insert file attribute for given global file id */ struct mdhim_brm_t* brm = mdhimPut(md, &gfid, sizeof(int), fattr_ptr, sizeof(unifyfs_file_attr_t), @@ -310,6 +341,45 @@ int unifyfs_get_file_attribute( return rc; } +/* given a global file id, delete file attributes */ +int unifyfs_delete_file_attribute( + int gfid) +{ + int rc = UNIFYFS_SUCCESS; + + /* select index holding file attributes, + * delete entry for given file id */ + md->primary_index = unifyfs_indexes[IDX_FILE_ATTR]; + struct mdhim_brm_t* brm = mdhimDelete(md, md->primary_index, + &gfid, sizeof(int)); + + /* check for errors and free resources */ + if (!brm) { + LOGERR("Error deleting file attributes from MDHIM"); + rc = (int)UNIFYFS_ERROR_MDHIM; + } else { + /* step through linked list of messages, + * scan for any error and free messages */ + struct mdhim_brm_t* brmp = brm; + while (brmp) { + /* check current item for error */ + if (brmp->error < 0) { + LOGERR("Error deleting file attributes from MDHIM"); + rc = (int)UNIFYFS_ERROR_MDHIM; + } + + /* record pointer to current item, + * advance loop pointer to next item in list, + * free resources for current item */ + brm = brmp; + brmp = brmp->next; + mdhim_full_release_msg(brm); + } + } + + return rc; +} + /* * */ diff --git a/server/src/unifyfs_metadata.h b/server/src/unifyfs_metadata.h index 9cf7b7e52..188cc8ea2 100644 --- a/server/src/unifyfs_metadata.h +++ b/server/src/unifyfs_metadata.h @@ -93,13 +93,26 @@ void print_fsync_indices(unifyfs_key_t** unifyfs_keys, int unifyfs_get_file_attribute(int gfid, unifyfs_file_attr_t* ptr_attr_val); +/** + * Delete file attribute from the KV-Store. + * + * @param [in] gfid + * @return UNIFYFS_SUCCESS on success + */ +int unifyfs_delete_file_attribute(int gfid); + /** * Store a File attribute to the KV-Store. * + * @param[in] size_flag + * @param[in] laminate_flag * @param[in] *ptr_attr_val * @return UNIFYFS_SUCCESS on success */ -int unifyfs_set_file_attribute(unifyfs_file_attr_t* ptr_attr_val); +int unifyfs_set_file_attribute( + int size_flag, + int laminate_flag, + unifyfs_file_attr_t* ptr_attr_val); /** * Store File attributes to the KV-Store. diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 48c9e101e..c663f6679 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -946,7 +946,7 @@ int rm_cmd_truncate( /* update file size field with latest size */ fattr.size = newsize; - rc = unifyfs_set_file_attribute(&fattr); + rc = unifyfs_set_file_attribute(1, 0, &fattr); if (rc != UNIFYFS_SUCCESS) { /* failed to update file attributes with new file size */ goto truncate_exit; @@ -963,6 +963,88 @@ int rm_cmd_truncate( return rc; } +/* given an app_id, client_id, and global file id, + * remove file */ +int rm_cmd_unlink( + int app_id, /* app_id for requesting client */ + int client_id, /* client_id for requesting client */ + int gfid) /* global file id */ +{ + int rc = UNIFYFS_SUCCESS; + + /* given the global file id, look up file attributes + * from key/value store */ + unifyfs_file_attr_t attr; + int ret = unifyfs_get_file_attribute(gfid, &attr); + if (ret != UNIFYFS_SUCCESS) { + /* failed to find attributes for the file */ + return ret; + } + + /* if item is a file, call truncate to free space */ + mode_t mode = (mode_t) attr.mode; + if ((mode & S_IFMT) == S_IFREG) { + /* item is regular file, truncate to 0 */ + ret = rm_cmd_truncate(app_id, client_id, gfid, 0); + if (ret != UNIFYFS_SUCCESS) { + /* failed to delete write extents for file, + * let's leave the file attributes in place */ + return ret; + } + } + + /* delete metadata */ + ret = unifyfs_delete_file_attribute(gfid); + if (ret != UNIFYFS_SUCCESS) { + rc = ret; + } + + return rc; +} + +/* given an app_id, client_id, and global file id, + * laminate file */ +int rm_cmd_laminate( + int app_id, /* app_id for requesting client */ + int client_id, /* client_id for requesting client */ + int gfid) /* global file id */ +{ + int rc = UNIFYFS_SUCCESS; + + /* given the global file id, look up file attributes + * from key/value store */ + unifyfs_file_attr_t attr; + int ret = unifyfs_get_file_attribute(gfid, &attr); + if (ret != UNIFYFS_SUCCESS) { + /* failed to find attributes for the file */ + return ret; + } + + /* if item is not a file, bail with error */ + mode_t mode = (mode_t) attr.mode; + if ((mode & S_IFMT) != S_IFREG) { + /* item is not a regular file */ + return UNIFYFS_ERROR_INVAL; + } + + /* lookup current file size */ + size_t filesize; + ret = rm_cmd_filesize(app_id, client_id, gfid, &filesize); + if (ret != UNIFYFS_SUCCESS) { + /* failed to get file size for file */ + return ret; + } + + /* update fields in metadata */ + attr.size = filesize; + attr.is_laminated = 1; + + /* update metadata, set size and laminate */ + rc = unifyfs_set_file_attribute(1, 1, &attr); + + return rc; +} + int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, int gfid, int app_id, int client_id, int num_keys, unifyfs_key_t** keys, int* keylens) diff --git a/server/src/unifyfs_request_manager.h b/server/src/unifyfs_request_manager.h index db1d0e1e0..c6c4f7891 100644 --- a/server/src/unifyfs_request_manager.h +++ b/server/src/unifyfs_request_manager.h @@ -113,6 +113,12 @@ int rm_cmd_filesize(int app_id, int client_id, int gfid, size_t* outsize); /* truncate file to specified size */ int rm_cmd_truncate(int app_id, int client_id, int gfid, size_t size); +/* delete file */ +int rm_cmd_unlink(int app_id, int client_id, int gfid); + +/* laminate file */ +int rm_cmd_laminate(int app_id, int client_id, int gfid); + /* function called by main thread to instruct * resource manager thread to exit, * returns UNIFYFS_SUCCESS on success */ diff --git a/t/Makefile.am b/t/Makefile.am index 0c0dd941d..6bbf4427f 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -102,7 +102,8 @@ sys_sysio_gotcha_t_SOURCES = sys/sysio_suite.h \ sys/open64.c \ sys/write-read.c \ sys/write-read-hole.c \ - sys/truncate.c + sys/truncate.c \ + sys/unlink.c sys_sysio_gotcha_t_CPPFLAGS = $(test_cppflags) sys_sysio_gotcha_t_LDADD = $(test_ldadd) @@ -117,7 +118,8 @@ sys_sysio_static_t_SOURCES = sys/sysio_suite.h \ sys/open64.c \ sys/write-read.c \ sys/write-read-hole.c \ - sys/truncate.c + sys/truncate.c \ + sys/unlink.c sys_sysio_static_t_CPPFLAGS = $(test_cppflags) sys_sysio_static_t_LDADD = $(test_static_ldadd) diff --git a/t/server/unifyfs_meta_get_test.c b/t/server/unifyfs_meta_get_test.c index d1ecc180f..7bd7e8be5 100644 --- a/t/server/unifyfs_meta_get_test.c +++ b/t/server/unifyfs_meta_get_test.c @@ -20,7 +20,7 @@ int unifyfs_set_file_attribute_test(void) snprintf(fattr.filename, sizeof(fattr.filename), TEST_META_FILE); fflush(NULL); - rc = unifyfs_set_file_attribute(&fattr); + rc = unifyfs_set_file_attribute(1, 1, &fattr); ok(UNIFYFS_SUCCESS == rc, "Stored file attribute"); fflush(NULL); return 0; diff --git a/t/sys/sysio_suite.c b/t/sys/sysio_suite.c index 9a3b5a741..6e735c508 100644 --- a/t/sys/sysio_suite.c +++ b/t/sys/sysio_suite.c @@ -96,6 +96,8 @@ int main(int argc, char* argv[]) truncate_ftrunc_before_sync(unifyfs_root); truncate_trunc_before_sync(unifyfs_root); + unlink_test(unifyfs_root); + MPI_Finalize(); done_testing(); diff --git a/t/sys/sysio_suite.h b/t/sys/sysio_suite.h index e1cfa7085..9fb32d1c8 100644 --- a/t/sys/sysio_suite.h +++ b/t/sys/sysio_suite.h @@ -60,4 +60,7 @@ int truncate_empty_read(char* unifyfs_root, int pos); int truncate_ftrunc_before_sync(char* unifyfs_root); int truncate_trunc_before_sync(char* unifyfs_root); +/* Test for UNIFYFS_WRAP(unlink) */ +int unlink_test(char* unifyfs_root); + #endif /* SYSIO_SUITE_H */ diff --git a/t/sys/unlink.c b/t/sys/unlink.c new file mode 100644 index 000000000..47ff40325 --- /dev/null +++ b/t/sys/unlink.c @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + + /* + * Test unlink + */ +#include +#include +#include +#include +#include +#include +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +static int unlink_after_sync_test(char* unifyfs_root) +{ + char path[64]; + int rc; + int fd; + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + fd = open(path, O_WRONLY | O_CREAT, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + rc = write(fd, "hello world", 12); + ok(rc == 12, "%s:%d write() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + rc = close(fd); + ok(rc == 0, "%s:%d close() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + struct stat sb = {0}; + rc = stat(path, &sb); + ok(rc == 0, "%s:%d stat() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + rc = unlink(path); + ok(rc == 0, "%s:%d unlink() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + rc = stat(path, &sb); + ok(rc == -1, "%s:%d stat() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + return 0; +} + +static int unlink_after_sync_laminate_test(char* unifyfs_root) +{ + char path[64]; + int rc; + int fd; + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + fd = open(path, O_WRONLY | O_CREAT, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + rc = write(fd, "hello world", 12); + ok(rc == 12, "%s:%d write() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + rc = fsync(fd); + ok(rc == 0, "%s:%d fsync() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + rc = close(fd); + ok(rc == 0, "%s:%d close() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + /* Laminate */ + rc = chmod(path, 0444); + ok(rc == 0, "%s:%d chmod(0444) (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + struct stat sb = {0}; + rc = stat(path, &sb); + ok(rc == 0, "%s:%d stat() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + rc = unlink(path); + ok(rc == 0, "%s:%d unlink() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + rc = stat(path, &sb); + ok(rc == -1, "%s:%d stat() (rc=%d): %s", + __FILE__, __LINE__, rc, strerror(errno)); + + return 0; +} + +int unlink_test(char* unifyfs_root) +{ + int rc = 0; + + int ret = unlink_after_sync_test(unifyfs_root); + if (ret != 0) { + rc = ret; + } + + ret = unlink_after_sync_laminate_test(unifyfs_root); + if (ret != 0) { + rc = ret; + } + + return rc; +} From ce78c0dd4100962da45e3bd5b7e008a4390c0606 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 13 Jan 2020 23:39:21 -0800 Subject: [PATCH 058/168] client: satisfy read requests from local extent info if possible This adds a second segment tree to the client metadata of each file (fid) that tracks all extents written to the file. We use this to satisfy later read requests by having the client refer to this segment tree and then copy data directly from its data logs. Any request that cannot be fully satisfied on the client is sent to the server for processing. The goal is to make reads as fast as writes in the case that a client process reads back the same data it wrote. This optimization cannot be used as currently written for apps that call truncate, because the server currently has no way to inform the client that it should truncate its extent list. To let users opt-in, a new UNIFYFS_CLIENT_LOCAL_EXTENTS=1 option is added to enable local read back. This optimization is off by default. This moves read request splitting from the client to the server. Splitting the read requests into 1MB chunks was limiting performance, and this splitting is not necessary when the client is reading its own data. This also moves the write index splitting from the client to the server. TEST_CHECKPATCH_SKIP_FILES=common/src/unifyfs_configurator.h --- client/src/unifyfs-fixed.c | 199 ++-------- client/src/unifyfs-internal.h | 48 +-- client/src/unifyfs-sysio.c | 539 ++++++++++++++++++--------- client/src/unifyfs.c | 51 ++- common/src/seg_tree.c | 36 ++ common/src/seg_tree.h | 6 + common/src/unifyfs_configurator.h | 1 + docs/configuration.rst | 7 + server/src/unifyfs_request_manager.c | 388 ++++++++++++++----- 9 files changed, 794 insertions(+), 481 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index c55f9fa81..6be3d3d93 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -337,151 +337,6 @@ static int unifyfs_coalesce_index( return UNIFYFS_SUCCESS; } -/* - * Given an index, split it into multiple indices whose range is equal or - * smaller than slice_range size. For example, if you passed a cur_index - * for a 3.5MB write, and the slice size was 1MB, it would split it into - * four indexes, and update cur_index.length to be zero. This also takes - * in a 'maxcount' field, so you can limit the number of indexes you - * create. Using our above example, if 'maxcount=2', then this would - * create two indexes, and update cur_index.length to 1.5MB (for the remaining - * data). - * - * @param cur_idx: The index to split - * @param slice_range: The slice size of the key-value store - * @return index_set: The set of split indices - * @param maxcount: Number of entries in output array - * @param used_count: Number of entries we actually added in the split - */ -static int unifyfs_split_index( - unifyfs_index_t* cur_idx, /* write index to split (offset and length) */ - long slice_range, /* number of bytes in each slice */ - unifyfs_index_t* index_set, /* output array to store new indexes in */ - off_t maxcount, /* max number of items in output array */ - off_t* used_count) /* number of entries we added in split */ -{ - /* first byte offset this write will write to */ - long idx_start = cur_idx->file_pos; - - /* last byte offset this write will write to */ - long idx_end = cur_idx->file_pos + cur_idx->length - 1; - - /* starting byte offset of slice that first write offset falls in */ - long slice_start = (idx_start / slice_range) * slice_range; - - /* last byte offset of slice that first write offset falls in */ - long slice_end = slice_start + slice_range - 1; - - /* get pointer to first output index structure */ - unifyfs_index_t* set = index_set; - - /* initialize count of output index entries */ - off_t count = 0; - - /* define new index entries in index_set by splitting write index - * at slice boundaries */ - if (idx_end <= slice_end) { - /* index falls fully within one slice - * - * slice_start slice_end - * idx_start idx_end - */ - set[count] = *cur_idx; - count++; - - /* update write index to account for index we just added */ - long length = cur_idx->length; - cur_idx->file_pos += length; - cur_idx->log_pos += length; - cur_idx->length -= length; - } else { - /* ending offset of index is beyond last offset in first slice, - * so this index spans across multiple slices - * - * slice_start slice_end next_slice_start next_slice_end - * idx_start idx_end - */ - - /* compute number of bytes until end of first slice */ - long length = slice_end - idx_start + 1; - - /* copy over all fields in current index, - * update length field to adjust for boundary of first slice */ - set[count].gfid = cur_idx->gfid; - set[count].file_pos = cur_idx->file_pos; - set[count].length = length; - set[count].log_pos = cur_idx->log_pos; - count++; - - /* update write index to account for index we just added */ - cur_idx->file_pos += length; - cur_idx->log_pos += length; - cur_idx->length -= length; - - /* check that we have room to write more index values */ - if (count >= maxcount) { - /* no room to write more index values, - * and we have at least one more, - * record number we wrote and return with success */ - *used_count = count; - return UNIFYFS_SUCCESS; - } - - /* advance slice boundary offsets to next slice */ - slice_end += slice_range; - - /* loop until we find the slice that contains - * ending offset of write */ - while (idx_end > slice_end) { - /* ending offset of write is beyond end of this slice, - * so write spans the full length of this slice */ - length = slice_range; - - /* define index for this slice */ - set[count].gfid = cur_idx->gfid; - set[count].file_pos = cur_idx->file_pos; - set[count].length = length; - set[count].log_pos = cur_idx->log_pos; - count++; - - /* update write index to account for index we just added */ - cur_idx->file_pos += length; - cur_idx->log_pos += length; - cur_idx->length -= length; - - /* check that we have room to write more index values */ - if (count >= maxcount) { - /* no room to write more index values, - * and we have at least one more, - * record number we wrote and return with success */ - *used_count = count; - return UNIFYFS_SUCCESS; - } - - /* advance slice boundary offsets to next slice */ - slice_end += slice_range; - } - - /* this slice contains the remainder of write */ - length = cur_idx->length; - set[count].gfid = cur_idx->gfid; - set[count].file_pos = cur_idx->file_pos; - set[count].length = length; - set[count].log_pos = cur_idx->log_pos; - count++; - - /* update write index to account for index we just added */ - cur_idx->file_pos += length; - cur_idx->log_pos += length; - cur_idx->length -= length; - } - - /* record number of entires we added */ - *used_count = count; - - return UNIFYFS_SUCCESS; -} - /* * Clear all entries in the log index. This only clears the metadata, * not the data itself. @@ -541,6 +396,13 @@ void unifyfs_add_index_entry_to_seg_tree( unifyfs_filemeta_t* meta, unifyfs_index_t* index) { + /* add index to our local log */ + if (unifyfs_local_extents) { + seg_tree_add(&meta->extents, index->file_pos, + index->file_pos + index->length - 1, + index->log_pos); + } + if (!unifyfs_flatten_writes) { /* We're not flattening writes. Nothing to do */ return; @@ -549,13 +411,13 @@ void unifyfs_add_index_entry_to_seg_tree( /* to update the global running segment count, we need to capture * the count in this tree before adding and the count after to * add the difference */ - unsigned long count_before = seg_tree_count(&meta->seg_tree); + unsigned long count_before = seg_tree_count(&meta->extents_sync); /* * Store the write in our segment tree. We will later use this for * flattening writes. */ - seg_tree_add(&meta->seg_tree, index->file_pos, + seg_tree_add(&meta->extents_sync, index->file_pos, index->file_pos + index->length - 1, index->log_pos); @@ -572,7 +434,7 @@ void unifyfs_add_index_entry_to_seg_tree( } else { /* increase the running global segment count by the number of * new entries we added to this tree */ - unsigned long count_after = seg_tree_count(&meta->seg_tree); + unsigned long count_after = seg_tree_count(&meta->extents_sync); unifyfs_segment_count += (count_after - count_before); } } @@ -581,8 +443,10 @@ void unifyfs_add_index_entry_to_seg_tree( static int unifyfs_logio_add_write_meta_to_index(unifyfs_filemeta_t* meta, off_t file_pos, off_t log_pos, size_t length) { - /* define an new index entry for this write operation */ + /* global file id for this entry */ int gfid = meta->gfid; + + /* define an new index entry for this write operation */ unifyfs_index_t cur_idx; cur_idx.gfid = gfid; cur_idx.file_pos = file_pos; @@ -612,8 +476,8 @@ static int unifyfs_logio_add_write_meta_to_index(unifyfs_filemeta_t* meta, } } - /* add new index entries if needed */ - while (cur_idx.length > 0) { + /* add new index entry if needed */ + if (cur_idx.length > 0) { /* remaining entries we can fit in the shared memory region */ off_t remaining_entries = unifyfs_max_index_entries - num_entries; @@ -628,32 +492,19 @@ static int unifyfs_logio_add_write_meta_to_index(unifyfs_filemeta_t* meta, } } - /* split any remaining write index at boundaries of - * unifyfs_key_slice_range */ - off_t used_entries = 0; - int split_rc = unifyfs_split_index(&cur_idx, - unifyfs_key_slice_range, &idxs[num_entries], - remaining_entries, &used_entries); - if (split_rc != UNIFYFS_SUCCESS) { - /* in this case, we have copied data to the log, - * but we failed to generate index entries, - * we're returning with an error and leaving the data - * in the log */ - LOGERR("failed to split write index"); - return UNIFYFS_ERROR_IO; - } + /* copy entry into index buffer */ + idxs[num_entries] = cur_idx; - /* Add our split index entries to our seg_tree */ - for (int i = 0; i < used_entries; i++) { - unifyfs_add_index_entry_to_seg_tree(meta, &idxs[num_entries + i]); - } + /* Add index entry to our seg_tree */ + unifyfs_add_index_entry_to_seg_tree(meta, &idxs[num_entries]); /* account for entries we just added */ - num_entries += used_entries; + num_entries += 1; /* update number of entries in index array */ (*unifyfs_indices.ptr_num_entries) = num_entries; } + return UNIFYFS_SUCCESS; } @@ -770,11 +621,11 @@ void unifyfs_rewrite_index_from_seg_tree(void) int gfid = unifyfs_gfid_from_fid(fid); - seg_tree_rdlock(&meta->seg_tree); + seg_tree_rdlock(&meta->extents_sync); /* For each write in this file's seg_tree ... */ struct seg_tree_node* node = NULL; - while ((node = seg_tree_iter(&meta->seg_tree, node))) { + while ((node = seg_tree_iter(&meta->extents_sync, node))) { indexes[idx].file_pos = node->start; indexes[idx].log_pos = node->ptr; indexes[idx].length = node->end - node->start + 1; @@ -782,10 +633,10 @@ void unifyfs_rewrite_index_from_seg_tree(void) idx++; } - seg_tree_unlock(&meta->seg_tree); + seg_tree_unlock(&meta->extents_sync); /* All done processing this files writes. Clear its seg_tree */ - seg_tree_clear(&meta->seg_tree); + seg_tree_clear(&meta->extents_sync); } /* reset our segment count since we just dumped them all */ diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index c58f744d5..511756e20 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -266,26 +266,27 @@ typedef struct { } unifyfs_chunkmeta_t; typedef struct { - off_t global_size; /* Global size of the file */ - off_t local_size; /* Local size of the file */ - off_t log_size; /* Log size. This is the sum of all the - * write counts. */ - pthread_spinlock_t fspinlock; /* file lock variable */ - enum flock_enum flock_status; /* file lock status */ - - int storage; /* FILE_STORAGE type */ - - int gfid; /* global file id for this file */ - int needs_sync; /* have unsynced writes */ - - off_t chunks; /* number of chunks allocated to file */ - off_t chunkmeta_idx; /* starting index in unifyfs_chunkmeta */ - int is_laminated; /* Is this file laminated */ - uint32_t mode; /* st_mode bits. This has file - * permission info and will tell you if this - * is a regular file or directory. */ - struct seg_tree seg_tree; /* Segment tree containing our coalesced - * writes */ + off_t global_size; /* Global size of the file */ + off_t local_size; /* Local size of the file */ + off_t log_size; /* Log size. This is the sum of all the + * write counts. */ + pthread_spinlock_t fspinlock; /* file lock variable */ + enum flock_enum flock_status; /* file lock status */ + + int storage; /* FILE_STORAGE type */ + + int gfid; /* global file id for this file */ + int needs_sync; /* have unsynced writes */ + + off_t chunks; /* number of chunks allocated to file */ + off_t chunkmeta_idx; /* starting index in unifyfs_chunkmeta */ + int is_laminated; /* Is this file laminated */ + uint32_t mode; /* st_mode bits. This has file + * permission info and will tell you if this + * is a regular file or directory. */ + struct seg_tree extents_sync; /* Segment tree containing our coalesced + * writes between sync operations */ + struct seg_tree extents; /* Segment tree of all local data extents */ } unifyfs_filemeta_t; /* struct used to map a full path to its local file id, @@ -393,7 +394,8 @@ extern int unifyfs_use_memfs; extern int unifyfs_use_spillover; extern int unifyfs_max_files; /* maximum number of files to store */ -extern bool unifyfs_flatten_writes; /* enable write flattening */ +extern bool unifyfs_flatten_writes; /* enable write flattening */ +extern bool unifyfs_local_extents; /* enable tracking of local extents */ extern size_t unifyfs_chunk_mem; /* number of bytes in memory to be used for chunk storage */ extern int unifyfs_chunk_bits; /* we set chunk size = 2^unifyfs_chunk_bits */ @@ -484,6 +486,10 @@ const char* unifyfs_path_from_fid(int fid); /* Given a fid, return a gfid */ int unifyfs_gfid_from_fid(const int fid); +/* returns fid for corresponding gfid, if one is active, + * returns -1 otherwise */ +int unifyfs_fid_from_gfid(const int gfid); + /* given an UNIFYFS error code, return corresponding errno code */ int unifyfs_err_map_to_errno(int rc); diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 5008d6a4f..9c37bce83 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -44,6 +44,7 @@ #include "unifyfs-sysio.h" #include "margo_client.h" #include "ucr_read_builder.h" +#include "seg_tree.h" /* ------------------- * define external variables @@ -1270,143 +1271,6 @@ static int compare_read_req(const void* a, const void* b) } } -/* given a read request, split it into multiple requests whose range - * is equal or smaller than slice_range size - * - * @param req: read request to be split - * @param slice_range: slice size of the key-value store - * @return out_set: output set of split read requests - * @param maxcount: number of entries in output array - * @return used_count: number of entries added to output array */ -static int unifyfs_split_read_request( - read_req_t* in_req, /* read request to split */ - long slice_range, /* number of bytes in each slice */ - read_req_t* out_set, /* output array to store new requests in */ - off_t maxcount, /* max number of items in output array */ - off_t* used_count) /* number of entries we added in split */ -{ - /* check that we have at least one spot in buffer */ - if (maxcount == 0) { - *used_count = 0; - return UNIFYFS_FAILURE; - } - - /* make a copy of the input request so we can modify it */ - read_req_t tmp_req = *in_req; - read_req_t* req = &tmp_req; - - /* first byte offset this request will read from */ - size_t req_start = req->offset; - - /* last byte offset this request will read from */ - size_t req_end = req->offset + req->length - 1; - - /* compute offset of first and last byte of slice - * that contains first byte of request */ - - /* starting byte offset of slice that first read offset falls in */ - size_t slice_start = (req->offset / slice_range) * slice_range; - - /* last byte offset of slice that first read offset falls in */ - size_t slice_end = slice_start + slice_range - 1; - - /* initialize request count in output set */ - int count = 0; - - /* define new read requests in out_set by splitting request - * at slice boundaries */ - if (req_end <= slice_end) { - /* slice fully contains request - * - * slice_start slice_end - * req_start req_end - */ - out_set[count] = *req; - count++; - } else { - /* ending offset of request is beyond last offset in first slice, - * so this request spans across multiple slices - * - * slice_start slice_end next_slice_start next_slice_end - * req_start req_end - * - */ - - /* compute number of bytes until end of first slice */ - long length = slice_end - req_start + 1; - - out_set[count].gfid = req->gfid; - out_set[count].offset = req->offset; - out_set[count].length = length; - out_set[count].errcode = req->errcode; - count++; - - /* update write index to account for index we just added */ - req->offset += length; - req->length -= length; - - /* check that we have room to write more requests */ - if (count >= maxcount) { - /* no room to write more requests, - * and we have at least one more, - * record number we wrote and return with error */ - *used_count = count; - return UNIFYFS_FAILURE; - } - - /* advance slice boundary offsets to next slice */ - slice_end += slice_range; - - /* loop until we find the slice that contains - * ending offset of read */ - while (req_end > slice_end) { - /* ending offset of read is beyond end of this slice, - * so read spans the full length of this slice */ - length = slice_range; - - /* full slice is contained in read request */ - out_set[count].gfid = req->gfid; - out_set[count].offset = req->offset; - out_set[count].length = length; - out_set[count].errcode = req->errcode; - count++; - - /* update read request to account for index we just added */ - req->offset += length; - req->length -= length; - - /* check that we have room to write more requests */ - if (count >= maxcount) { - /* no room to write more requests, - * and we have at least one more, - * record number we wrote and return with error */ - *used_count = count; - return UNIFYFS_FAILURE; - } - - /* advance slice boundary offsets to next slice */ - slice_end += slice_range; - } - - /* this slice contains the remainder of read */ - length = req->length; - out_set[count].gfid = req->gfid; - out_set[count].offset = req->offset; - out_set[count].length = length; - out_set[count].errcode = req->errcode; - count++; - - /* update read request to account for index we just added */ - req->offset += length; - req->length -= length; - } - - /* record number of entries we added */ - *used_count = count; - - return UNIFYFS_SUCCESS; -} - /* notify our delegator that the shared memory buffer * is now clear and ready to hold more read data */ static void delegator_signal(void) @@ -1600,59 +1464,336 @@ static int process_read_data(read_req_t* read_reqs, int count, int* done) return rc; } +/* This uses information in the extent map for a file on the client to + * complete any read requests. It only complets a request if it contains + * all of the data. Otherwise the request is copied to the list of + * requests to be handled by the server. */ +static void service_local_reqs( + read_req_t* read_reqs, /* list of input read requests */ + int count, /* number of input read requests */ + read_req_t* local_reqs, /* list to copy requests completed by client */ + read_req_t* server_reqs, /* list to copy requests to be handled by server */ + int* out_count) /* number of items copied to server list */ +{ + /* this will track the total number of requests we're passing + * on to the server */ + int local_count = 0; + int server_count = 0; + + /* iterate over each input read request, satisfy it locally if we can + * otherwise copy request into output list that the server will handle + * for us */ + int i; + for (i = 0; i < count; i++) { + /* get current read request */ + read_req_t* req = &read_reqs[i]; + + /* skip any request that's already completed or errored out, + * we pass those requests on to server */ + if (req->nread >= req->length || req->errcode != UNIFYFS_SUCCESS) { + /* copy current request into list of requests + * that we'll ask server for */ + memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); + server_count++; + continue; + } + + /* get gfid, start, and length of this request */ + int gfid = req->gfid; + size_t req_start = req->offset; + size_t req_end = req->offset + req->length; + + /* lookup local extents if we have them */ + int fid = unifyfs_fid_from_gfid(gfid); + + /* move to next request if we can't find the matching fid */ + if (fid < 0) { + /* copy current request into list of requests + * that we'll ask server for */ + memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); + server_count++; + continue; + } + + /* get pointer to extents for this file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + struct seg_tree* extents = &meta->extents; + + /* lock the extent tree for reading */ + seg_tree_rdlock(extents); + + /* identify whether we can satisfy this full request + * or not, assume we can */ + int have_local = 1; + + /* this will point to the offset of the next byte we + * need to account for */ + size_t expected_start = req_start; + + /* iterate over extents we have for this file, + * and check that there are no holes in coverage, + * we search for a starting extent using a range + * of just the very first byte that we need */ + struct seg_tree_node* first; + first = seg_tree_find_nolock(extents, req_start, req_start); + struct seg_tree_node* next = first; + while (next != NULL && next->start < req_end) { + if (expected_start >= next->start) { + /* this extent has the next byte we expect, + * bump up to the first byte past the end + * of this extent */ + expected_start = next->end + 1; + } else { + /* there is a gap between extents so we're missing + * some bytes */ + have_local = 0; + break; + } + + /* get the next element in the tree */ + next = seg_tree_iter(extents, next); + } + + /* check that we account for the full request + * up until the last byte */ + if (expected_start < req_end) { + /* missing some bytes at the end of the request */ + have_local = 0; + } + + /* if we can't fully satisfy the request, copy request to + * output array, so it can be passed on to server */ + if (!have_local) { + /* copy current request into list of requests + * that we'll ask server for */ + memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); + server_count++; + + /* release lock before we go to next request */ + seg_tree_unlock(extents); + + continue; + } + + /* otherwise we can copy the data locally, iterate + * over the extents and copy data into request buffer, + * again search for a starting extent using a range + * of just the very first byte that we need */ + next = first; + while (next != NULL && next->start < req_end) { + /* get start and end of this extent (reply) */ + size_t rep_start = next->start; + size_t rep_end = next->end + 1; + + /* get the offset into the log */ + size_t pos = next->ptr; + + /* start of overlapping segment is the maximum of + * reply and request start offsets */ + size_t start = rep_start; + if (req_start > start) { + start = req_start; + } + + /* end of overlapping segment is the mimimum of + * reply and request end offsets */ + size_t end = rep_end; + if (req_end < end) { + end = req_end; + } + + /* compute length of overlapping segment */ + size_t length = end - start; + + /* get number of bytes from start of reply and request + * buffers to the start of the overlap region */ + size_t rep_offset = start - rep_start; + size_t req_offset = start - req_start; + + /* if we have a gap, fill with zeros */ + size_t gap_start = req_start + req->nread; + if (start > gap_start) { + size_t gap_length = start - gap_start; + char* req_ptr = req->buf + req->nread; + memset(req_ptr, 0, gap_length); + } + + /* copy data from reply buffer into request buffer */ + char* req_ptr = req->buf + req_offset; + + /* compute total size of shared memory data region */ + size_t chunk_mem_size = unifyfs_max_chunks * unifyfs_chunk_size; + + /* compute number of bytes we'll read from shared memory */ + size_t mem_length = 0; + if (pos < chunk_mem_size) { + /* we need to start from memory, assume we'll get it all */ + mem_length = length; + if (pos + length > chunk_mem_size) { + /* amount to read extends past end of memory segment, + * compute the number of bytes in memory */ + mem_length = chunk_mem_size - pos; + } + } + + /* any remainder comes from the spill over file */ + size_t spill_length = length - mem_length; + + /* copy data from memory into request buffer */ + if (mem_length > 0) { + /* pointer to data is starting address of superblock + * data region (unify_chunks) + offset within the + * superblock to the start of the segment in + * reply (pos) + any offset from start of that segment + * until the first byte we are asking for (rep_offset) */ + char* mem_ptr = unifyfs_chunks + pos + rep_offset; + memcpy(req_ptr, mem_ptr, mem_length); + } + + /* copy data from spill over into request buffer */ + if (spill_length > 0) { + /* advance in user buffer past any bytes we copied in + * from memory */ + char* reqbuf = req_ptr + mem_length; + + /* offset to start of data in spill over file is + * position within log (pos) + offset from start of + * segment to the first byte in segment that overlaps + * with request (rep_offset) + any bytes copied from + * memory (mem_length) - the size of the data region + * in memory */ + off_t spill_offset = pos + rep_offset + mem_length + - chunk_mem_size; + + /* TODO: loop on this if we get a short read + * or EIO/EAGAIN */ + + /* read data from file into request buffer */ + ssize_t rc = pread(unifyfs_spilloverblock, + reqbuf, spill_length, spill_offset); + if (rc != length) { + /* had a problem reading, + * set the request error code */ + req->errcode = UNIFYFS_ERROR_IO; + } + } + + /* update max number of bytes we have written to in the + * request buffer */ + size_t nread = end - req_start; + if (nread > req->nread) { + req->nread = nread; + } + + /* get the next element in the tree */ + next = seg_tree_iter(extents, next); + } + + /* copy request data to list we completed locally */ + memcpy(&local_reqs[local_count], req, sizeof(read_req_t)); + local_count++; + + /* done reading the tree */ + seg_tree_unlock(extents); + } + + /* return to user the number of key/values we set */ + *out_count = server_count; + + return; +} + /* * get data for a list of read requests from the * delegator + * * @param read_reqs: a list of read requests * @param count: number of read requests * @return error code - * * */ -int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) +int unifyfs_fd_logreadlist(read_req_t* in_reqs, int in_count) { + int i; int read_rc; + /* assume we'll succeed */ int rc = UNIFYFS_SUCCESS; - /* - * Todo: When the number of read requests exceed the - * request buffer, split list io into multiple bulk - * sends and transfer in bulks - * */ + /* assume we'll service all requests from the server */ + int count = in_count; + read_req_t* read_reqs = in_reqs; - /* order read request by increasing file id, then increasing offset */ - qsort(read_reqs, count, sizeof(read_req_t), compare_read_req); + /* TODO: if the file is laminated and we know the file size, + * we could adjust some reads to not try reading past the EOF */ - /* TODO: move this split code to server and then pass original - * read requests from client to server */ - /* split read requests at file offset boundaries used internally - * in the server key/value store */ - int i; - off_t req_count = 0; - read_req_t read_set[UNIFYFS_MAX_READ_CNT]; - for (i = 0; i < count; i++) { - /* remaining entries we have in our read requests array */ - off_t remaining = UNIFYFS_MAX_READ_CNT - req_count; + /* if the option is enabled to service requests locally, try it, + * in this case we'll allocate a large array which we split into + * two, the first half will record requests we completed locally + * and the second half will store requests to be sent to the server */ + + /* this records the pointer to the temp request array if + * we allocate one, we should free this later if not NULL */ + read_req_t* reqs = NULL; - /* split current request at key/value offsets */ - off_t used = 0; - read_rc = unifyfs_split_read_request(&read_reqs[i], - unifyfs_key_slice_range, &read_set[req_count], remaining, &used); + /* this will point to the start of the array of requests we + * complete locally */ + read_req_t* local_reqs = NULL; - /* bail out with error if we failed to process read requests */ - if (read_rc != UNIFYFS_SUCCESS) { - LOGERR("Failed to split read requests"); - return read_rc; + /* attempt to complete requests locally if enabled */ + if (unifyfs_local_extents) { + /* allocate space to make local and server copies of the requests, + * each list will be at most in_count long */ + size_t reqs_size = 2 * in_count * sizeof(read_req_t); + reqs = (read_req_t*) malloc(reqs_size); + if (reqs == NULL) { + return UNIFYFS_ERROR_NOMEM; } - /* account of request slots we used up */ - req_count += used; + /* define pointers to space where we can build our list + * of requests handled on the client and those left + * for the server */ + local_reqs = &reqs[0]; + read_reqs = &reqs[in_count]; + + /* service reads from local extent info if we can, this copies + * completed requests from in_reqs into local_reqs, and it copies + * any requests that can't be completed locally into the read_reqs + * to be processed by the server */ + service_local_reqs(in_reqs, in_count, local_reqs, read_reqs, &count); + + /* bail early if we satisfied all requests locally */ + if (count == 0) { + /* copy completed requests back into user's array */ + memcpy(in_reqs, local_reqs, in_count * sizeof(read_req_t)); + + /* free the temporary array */ + free(reqs); + return rc; + } } + /* TODO: When the number of read requests exceed the + * request buffer, split list io into multiple bulk + * sends and transfer in bulks */ + + /* check that we have enough slots for all read requests */ + if (count > UNIFYFS_MAX_READ_CNT) { + LOGERR("Too many requests to pass to server"); + if (reqs != NULL) { + free(reqs); + } + return read_rc; + } + + /* order read request by increasing file id, then increasing offset */ + qsort(read_reqs, count, sizeof(read_req_t), compare_read_req); + /* prepare our shared memory buffer for delegator */ delegator_signal(); - if (req_count > 1) { + /* we select different rpcs depending on the number of + * read requests */ + if (count > 1) { /* got multiple read requests, * build up a flat buffer to include them all */ flatcc_builder_t builder; @@ -1662,11 +1803,9 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) unifyfs_Extent_vec_start(&builder); /* fill in values for each request entry */ - for (i = 0; i < req_count; i++) { + for (i = 0; i < count; i++) { unifyfs_Extent_vec_push_create(&builder, - read_set[i].gfid, - read_set[i].offset, - read_set[i].length); + read_reqs[i].gfid, read_reqs[i].offset, read_reqs[i].length); } /* complete the array */ @@ -1678,26 +1817,34 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) size_t size = 0; void* buffer = flatcc_builder_finalize_buffer(&builder, &size); assert(buffer); + LOGDBG("mread: n_reqs:%d, flatcc buffer (%p) sz:%zu", - req_count, buffer, size); + count, buffer, size); - /* invoke read rpc here */ - read_rc = invoke_client_mread_rpc(req_count, size, buffer); + /* invoke multi-read rpc */ + read_rc = invoke_client_mread_rpc(count, size, buffer); + /* free flat buffer resources */ flatcc_builder_clear(&builder); free(buffer); } else { /* got a single read request */ - int gfid = read_set[0].gfid; - size_t offset = read_set[0].offset; - size_t length = read_set[0].length; + int gfid = read_reqs[0].gfid; + size_t offset = read_reqs[0].offset; + size_t length = read_reqs[0].length; + LOGDBG("read: offset:%zu, len:%zu", offset, length); + + /* invoke single read rpc */ read_rc = invoke_client_read_rpc(gfid, offset, length); } /* bail out with error if we failed to even start the read */ if (read_rc != UNIFYFS_SUCCESS) { LOGERR("Failed to issue read RPC to server"); + if (reqs != NULL) { + free(reqs); + } return read_rc; } @@ -1706,6 +1853,9 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) * are missed * */ + /* spin waiting for read data to come back from the server, + * we process it in batches as it comes in, eventually the + * server will tell us it's sent us everything it can */ int done = 0; while (!done) { int tmp_rc = delegator_wait(); @@ -1721,7 +1871,9 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) } } - /* check for short reads, compare to file size, fill holes */ + /* got all of the data we'll get from the server, + * check for short reads and whether those short + * reads are from errors, holes, or the end of the file */ for (i = 0; i < count; i++) { /* get pointer to next read request */ read_req_t* req = &read_reqs[i]; @@ -1737,8 +1889,8 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) } /* otherwise, we have a short read, check whether there - * would be a hole after us, in which case we fill with - * zeros */ + * would be a hole after us, in which case we fill the + * request buffer with zeros */ /* get file size for this file */ size_t filesize; @@ -1775,6 +1927,33 @@ int unifyfs_fd_logreadlist(read_req_t* read_reqs, int count) } } + /* if we attempted to service requests from our local extent map, + * then we need to copy the resulting read requests from the local + * and server arrays back into the user's original array */ + if (unifyfs_local_extents) { + /* TODO: would be nice to copy these back into the same order + * in which we received them. */ + + /* copy locally completed requests back into user's array */ + int local_count = in_count - count; + if (local_count > 0) { + memcpy(in_reqs, local_reqs, local_count * sizeof(read_req_t)); + } + + /* copy sever completed requests back into user's array */ + if (count > 0) { + /* skip past any items we copied in from the local requests */ + char* in_ptr = in_reqs + local_count * sizeof(read_req_t); + memcpy(in_ptr, read_reqs, count * sizeof(read_req_t)); + } + + /* free storage we used for copies of requests */ + if (reqs != NULL) { + free(reqs); + reqs = NULL; + } + } + return rc; } diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index e75bfe1fd..ff21f692d 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -122,7 +122,8 @@ static off_t unifyfs_min_long; /* TODO: moved these to fixed file */ int unifyfs_max_files; /* maximum number of files to store */ -bool unifyfs_flatten_writes; /* flatten our writes (true = enabled) */ +bool unifyfs_flatten_writes; /* flatten our writes (true = enabled) */ +bool unifyfs_local_extents; /* track data extents in client to read local */ size_t unifyfs_chunk_mem; /* number of bytes in memory to be used for chunk storage */ int unifyfs_chunk_bits; /* we set chunk size = 2^unifyfs_chunk_bits */ off_t unifyfs_chunk_size; /* chunk size in bytes */ @@ -651,7 +652,12 @@ static int unifyfs_fid_store_free(int fid) /* Free our write seg_tree */ if (unifyfs_flatten_writes) { - seg_tree_destroy(&meta->seg_tree); + seg_tree_destroy(&meta->extents_sync); + } + + /* Free our extent seg_tree */ + if (unifyfs_local_extents) { + seg_tree_destroy(&meta->extents); } return UNIFYFS_SUCCESS; @@ -703,6 +709,22 @@ int unifyfs_gfid_from_fid(const int fid) return meta->gfid; } +/* scan list of files and return fid corresponding to target gfid, + * returns -1 if not found */ +int unifyfs_fid_from_gfid(int gfid) +{ + int i; + for (i = 0; i < unifyfs_max_files; i++) { + if (unifyfs_filelist[i].in_use && + unifyfs_filemetas[i].gfid == gfid) { + /* found a file id that's in use and it matches + * the target fid, this is the one */ + return i; + } + } + return -1; +} + /* Given a fid, return the path. */ const char* unifyfs_path_from_fid(int fid) { @@ -985,10 +1007,20 @@ int unifyfs_fid_create_file(const char* path) if (unifyfs_flatten_writes) { /* Initialize our segment tree that will record our writes */ - rc = seg_tree_init(&meta->seg_tree); + rc = seg_tree_init(&meta->extents_sync); if (rc != 0) { errno = rc; - return -rc; + fid = -1; + } + } + + /* Initialize our segment tree to track extents for all writes + * by this process, can be used to read back local data */ + if (unifyfs_local_extents) { + rc = seg_tree_init(&meta->extents); + if (rc != 0) { + errno = rc; + fid = -1; } } @@ -1807,6 +1839,17 @@ static int unifyfs_init(int rank) } } + /* Determine if we should track all write extents and use them + * to service read requests if all data is local */ + unifyfs_local_extents = 0; + cfgval = client_cfg.client_local_extents; + if (cfgval != NULL) { + rc = configurator_bool_val(cfgval, &b); + if (rc == 0) { + unifyfs_local_extents = (bool)b; + } + } + /* determine number of bits for chunk size */ unifyfs_chunk_bits = UNIFYFS_CHUNK_BITS; cfgval = client_cfg.shmem_chunk_bits; diff --git a/common/src/seg_tree.c b/common/src/seg_tree.c index 1f530c306..2c11f138c 100644 --- a/common/src/seg_tree.c +++ b/common/src/seg_tree.c @@ -217,6 +217,42 @@ int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, return 0; } +/* Search tree for an entry that overlaps with given range of + * [start, end]. Returns the first overlapping entry if found, + * which is the overlapping entry having the lowest starting + * offset, and returns NULL otherwise. Assumes caller has lock + * on tree. */ +struct seg_tree_node* seg_tree_find_nolock( + struct seg_tree* seg_tree, + unsigned long start, + unsigned long end) +{ + /* Create a range of just our starting byte offset */ + struct seg_tree_node* node = seg_tree_node_alloc(start, start, 0); + if (!node) { + return NULL; + } + + /* Search tree for either a range that overlaps with + * the target range (starting byte), or otherwise the + * node for the next biggest starting byte. */ + struct seg_tree_node* next = RB_NFIND(inttree, &seg_tree->head, node); + + free(node); + + /* We may have found a node that doesn't include our starting + * byte offset, but it would be the range with the lowest + * starting offset after the target starting offset. Check whether + * this overlaps our end offset */ + if (next && next->start <= end) { + return next; + } + + /* Otherwise, there is not element that overlaps with the + * target range of [start, end]. */ + return NULL; +} + /* * Given a range tree and a starting node, iterate though all the nodes * in the tree, returning the next one each time. If start is NULL, then diff --git a/common/src/seg_tree.h b/common/src/seg_tree.h index 042e39ff2..d392e8916 100644 --- a/common/src/seg_tree.h +++ b/common/src/seg_tree.h @@ -22,6 +22,12 @@ void seg_tree_destroy(struct seg_tree* seg_tree); int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, unsigned long end, unsigned long ptr); +struct seg_tree_node* seg_tree_find_nolock( + struct seg_tree* seg_tree, + unsigned long start, + unsigned long end +); + struct seg_tree_node* seg_tree_iter(struct seg_tree* seg_tree, struct seg_tree_node* start); diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index 32e02accf..f9fa68955 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -71,6 +71,7 @@ UNIFYFS_CFG_CLI(unifyfs, mountpoint, STRING, /unifyfs, "mountpoint directory", NULL, 'm', "specify full path to desired mountpoint") \ UNIFYFS_CFG(client, max_files, INT, UNIFYFS_MAX_FILES, "client max file count", NULL) \ UNIFYFS_CFG(client, flatten_writes, BOOL, on, "flatten writes", NULL) \ + UNIFYFS_CFG(client, local_extents, BOOL, off, "track extents to service reads of local data", NULL) \ UNIFYFS_CFG_CLI(log, verbosity, INT, 0, "log verbosity level", NULL, 'v', "specify logging verbosity level") \ UNIFYFS_CFG_CLI(log, file, STRING, unifyfsd.log, "log file name", NULL, 'l', "specify log file name") \ UNIFYFS_CFG_CLI(log, dir, STRING, LOGDIR, "log file directory", configurator_directory_check, 'L', "specify full path to directory to contain log file") \ diff --git a/docs/configuration.rst b/docs/configuration.rst index 24813cdee..d4289ab24 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -68,8 +68,15 @@ a given section and key. ============== ====== ================================================================= max_files INT maximum number of open files per client process flatten_writes BOOL enable flattening writes (optimization for overwrite-heavy codes) + local_extents BOOL service reads from local data if possible (default: off) ============== ====== ================================================================= +Enabling the ``local_extents`` optimization may significantly improve read +performance. However, it should not be used by applications +in which different processes write to a given byte offset within +the file, nor should it be used with applications that truncate +files. + .. table:: ``[log]`` section - logging settings :widths: auto diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index c663f6679..eb365a578 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -1104,6 +1104,150 @@ int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, return rc; } +/* return number of slice ranges needed to cover range */ +static size_t num_slices(size_t offset, size_t length) +{ + size_t start = offset / max_recs_per_slice; + size_t end = (offset + length - 1) / max_recs_per_slice; + size_t count = end - start + 1; + return count; +} + +/* given a global file id, an offset, and a length to read from that + * file, create keys needed to query MDHIM for location of data + * corresponding to that extent, returns the number of keys inserted + * into key array provided by caller */ +static int split_request( + unifyfs_key_t** keys, /* list to add newly created keys into */ + int* keylens, /* list to add byte size of each key */ + int gfid, /* target global file id to read from */ + size_t offset, /* starting offset of read */ + size_t length) /* number of bytes to read */ +{ + /* offset of first byte in request */ + size_t pos = offset; + + /* offset of last byte in request */ + size_t last_offset = offset + length - 1; + + /* iterate over slice ranges and generate a start/end + * pair of keys for each */ + int count = 0; + while (pos <= last_offset) { + /* compute offset for first byte in this segment */ + size_t start = pos; + + /* offset for last byte in this segment, + * assume that's the last byte of the same segment + * containing start, unless that happens to be + * beyond the last byte of the actual request */ + size_t start_slice = start / max_recs_per_slice; + size_t end = (start_slice + 1) * max_recs_per_slice - 1; + if (end > last_offset) { + end = last_offset; + } + + /* create key to describe first byte we'll read + * in this slice */ + keys[count]->gfid = gfid; + keys[count]->offset = start; + keylens[count] = sizeof(unifyfs_key_t); + count++; + + /* create key to describe last byte we'll read + * in this slice */ + keys[count]->gfid = gfid; + keys[count]->offset = end; + keylens[count] = sizeof(unifyfs_key_t); + count++; + + /* advance to first byte offset of next slice */ + pos = end + 1; + } + + /* return number of keys we generated */ + return count; +} + +/* given an extent corresponding to a write index, create new key/value + * pairs for that extent, splitting into multiple keys at the slice + * range boundaries (max_recs_per_slice), it returns the number of + * newly created key/values inserted into the given key and value + * arrays */ +static int split_index( + unifyfs_key_t** keys, /* list to add newly created keys into */ + unifyfs_val_t** vals, /* list to add newly created values into */ + int* keylens, /* list for size of each key */ + int* vallens, /* list for size of each value */ + int gfid, /* global file id of write */ + size_t offset, /* starting byte offset of extent */ + size_t length, /* number of bytes in extent */ + size_t log_offset, /* offset within data log */ + int server_rank, /* rank of server hosting data */ + int app_id, /* app_id holding data */ + int client_rank) /* client rank holding data */ +{ + /* offset of first byte in request */ + size_t pos = offset; + + /* offset of last byte in request */ + size_t last_offset = offset + length - 1; + + /* this will track the current offset within the log + * where the data starts, we advance it with each key + * we generate depending on the data associated with + * each key */ + size_t logpos = log_offset; + + /* iterate over slice ranges and generate a start/end + * pair of keys for each */ + int count = 0; + while (pos <= last_offset) { + /* compute offset for first byte in this slice */ + size_t start = pos; + + /* offset for last byte in this slice, + * assume that's the last byte of the same slice + * containing start, unless that happens to be + * beyond the last byte of the actual request */ + size_t start_slice = start / max_recs_per_slice; + size_t end = (start_slice + 1) * max_recs_per_slice - 1; + if (end > last_offset) { + end = last_offset; + } + + /* length of extent in this slice */ + size_t len = end - start + 1; + + /* create key to describe this log entry */ + unifyfs_key_t* k = keys[count]; + k->gfid = gfid; + k->offset = start; + keylens[count] = sizeof(unifyfs_key_t); + + /* create value to store address of data */ + unifyfs_val_t* v = vals[count]; + v->addr = logpos; + v->len = len; + v->app_id = app_id; + v->rank = client_rank; + v->delegator_rank = server_rank; + vallens[count] = sizeof(unifyfs_val_t); + + /* advance to next slot in key/value arrays */ + count++; + + /* advance offset into log */ + logpos += len; + + /* advance to first byte offset of next slice */ + pos = end + 1; + } + + /* return number of keys we generated */ + return count; +} + /* read function for one requested extent, * called from rpc handler to fill shared data structures * with read requests to be handled by the delegator thread @@ -1137,34 +1281,56 @@ int rm_cmd_read( * other mechanism to retrieve all relevant key-value pairs from the * KV-store. */ - unifyfs_key_t key1, key2; - /* create key to describe first byte we'll read */ - key1.gfid = gfid; - key1.offset = offset; + /* count number of slices this range covers */ + size_t slices = num_slices(offset, length); + if (slices >= UNIFYFS_MAX_SPLIT_CNT) { + LOGERR("Error allocating buffers"); + return (int)UNIFYFS_ERROR_NOMEM; + } - /* create key to describe last byte we'll read */ - key2.gfid = gfid; - key2.offset = offset + length - 1; + /* allocate key storage */ + size_t key_cnt = slices * 2; + unifyfs_key_t** keys = alloc_key_array(key_cnt); + int* key_lens = (int*) calloc(key_cnt, sizeof(int)); + if ((NULL == keys) || + (NULL == key_lens)) { + // this is a fatal error + // TODO: we need better error handling + LOGERR("Error allocating buffers"); + return (int)UNIFYFS_ERROR_NOMEM; + } - unifyfs_key_t* unifyfs_keys[2] = {&key1, &key2}; - int key_lens[2] = {sizeof(unifyfs_key_t), sizeof(unifyfs_key_t)}; + /* split range of read request at boundaries used for + * MDHIM range query */ + split_request(keys, key_lens, gfid, offset, length); + + /* queue up the read operations */ + int rc = create_gfid_chunk_reads(thrd_ctrl, gfid, + app_id, client_id, key_cnt, keys, key_lens); - return create_gfid_chunk_reads(thrd_ctrl, gfid, app_id, client_id, - 2, unifyfs_keys, key_lens); + /* free memory allocated for key storage */ + free_key_array(keys); + free(key_lens); + + return rc; } /* send the read requests to the remote delegators * * @param app_id: application id * @param client_id: client id for requesting process - * @param gfid: global file id * @param req_num: number of read requests * @param reqbuf: read requests buffer * @return success/error code */ -int rm_cmd_mread(int app_id, int client_id, - size_t req_num, void* reqbuf) +int rm_cmd_mread( + int app_id, + int client_id, + size_t req_num, + void* reqbuf) { + int rc = UNIFYFS_SUCCESS; + /* get pointer to app structure for this app id */ app_config_t* app_config = (app_config_t*)arraylist_get(app_config_list, app_id); @@ -1185,13 +1351,27 @@ int rm_cmd_mread(int app_id, int client_id, size_t extents_len = unifyfs_Extent_vec_len(extents); assert(extents_len == req_num); - // allocate key storage - unifyfs_key_t** unifyfs_keys; - int* key_lens; - size_t key_cnt = req_num * 2; - unifyfs_keys = alloc_key_array(key_cnt); - key_lens = (int*) calloc(key_cnt, sizeof(int)); - if ((NULL == unifyfs_keys) || + /* count up number of slices these request cover */ + int j; + size_t slices = 0; + for (j = 0; j < req_num; j++) { + /* get offset and length of next request */ + size_t off = unifyfs_Extent_offset(unifyfs_Extent_vec_at(extents, j)); + size_t len = unifyfs_Extent_length(unifyfs_Extent_vec_at(extents, j)); + + /* add in number of slices this request needs */ + slices += num_slices(off, len); + } + if (slices >= UNIFYFS_MAX_SPLIT_CNT) { + LOGERR("Error allocating buffers"); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* allocate key storage */ + size_t key_cnt = slices * 2; + unifyfs_key_t** keys = alloc_key_array(key_cnt); + int* key_lens = (int*) calloc(key_cnt, sizeof(int)); + if ((NULL == keys) || (NULL == key_lens)) { // this is a fatal error // TODO: we need better error handling @@ -1200,29 +1380,32 @@ int rm_cmd_mread(int app_id, int client_id, } /* get chunks corresponding to requested client read extents */ - int rc, num_keys; - int gfid = -1; + int ret; + int num_keys = 0; int last_gfid = -1; - int ndx = 0; - size_t j, eoff, elen; for (j = 0; j < req_num; j++) { - gfid = unifyfs_Extent_fid(unifyfs_Extent_vec_at(extents, j)); + /* get the file id for this request */ + int gfid = unifyfs_Extent_fid(unifyfs_Extent_vec_at(extents, j)); + + /* if we have switched to a different file, create chunk reads + * for the previous file */ if (j && (gfid != last_gfid)) { - // create requests for all extents of last_gfid - num_keys = ndx; - rc = create_gfid_chunk_reads(thrd_ctrl, last_gfid, app_id, - client_id, num_keys, - unifyfs_keys, key_lens); - if (rc != UNIFYFS_SUCCESS) { + /* create requests for all extents of last_gfid */ + ret = create_gfid_chunk_reads(thrd_ctrl, last_gfid, + app_id, client_id, num_keys, keys, key_lens); + if (ret != UNIFYFS_SUCCESS) { LOGERR("Error creating chunk reads for gfid=%d", last_gfid); + rc = ret; } - // reset ndx for current gfid - ndx = 0; + + /* reset key counter for the current gfid */ + num_keys = 0; } - eoff = unifyfs_Extent_offset(unifyfs_Extent_vec_at(extents, j)); - elen = unifyfs_Extent_length(unifyfs_Extent_vec_at(extents, j)); - LOGDBG("gfid:%d, offset:%zu, length:%zu", gfid, eoff, elen); + /* get offset and length of current read request */ + size_t off = unifyfs_Extent_offset(unifyfs_Extent_vec_at(extents, j)); + size_t len = unifyfs_Extent_length(unifyfs_Extent_vec_at(extents, j)); + LOGDBG("gfid:%d, offset:%zu, length:%zu", gfid, off, len); /* Generate a pair of keys for each read request, representing * the start and end offsets. MDHIM returns all key-value pairs that @@ -1233,32 +1416,27 @@ int rm_cmd_mread(int app_id, int client_id, * utilize some other mechanism to retrieve all relevant KV * pairs from the KV-store. */ - key_lens[ndx] = sizeof(unifyfs_key_t); - key_lens[ndx + 1] = sizeof(unifyfs_key_t); - /* create key to describe first byte we'll read */ - unifyfs_keys[ndx]->gfid = gfid; - unifyfs_keys[ndx]->offset = eoff; + /* split range of read request at boundaries used for + * MDHIM range query */ + int used = split_request(&keys[num_keys], &key_lens[num_keys], + gfid, off, len); + num_keys += used; - /* create key to describe last byte we'll read */ - unifyfs_keys[ndx + 1]->gfid = gfid; - unifyfs_keys[ndx + 1]->offset = eoff + elen - 1; - - ndx += 2; + /* keep track of the last gfid value that we processed */ last_gfid = gfid; } - // create requests for all extents of last_gfid - num_keys = ndx; - rc = create_gfid_chunk_reads(thrd_ctrl, last_gfid, app_id, - client_id, num_keys, - unifyfs_keys, key_lens); - if (rc != UNIFYFS_SUCCESS) { + /* create requests for all extents of final gfid */ + ret = create_gfid_chunk_reads(thrd_ctrl, last_gfid, + app_id, client_id, num_keys, keys, key_lens); + if (ret != UNIFYFS_SUCCESS) { LOGERR("Error creating chunk reads for gfid=%d", last_gfid); + rc = ret; } - // cleanup - free_key_array(unifyfs_keys); + /* free memory allocated for key storage */ + free_key_array(keys); free(key_lens); return rc; @@ -1323,12 +1501,6 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) /* assume we'll succeed */ int ret = (int)UNIFYFS_SUCCESS; - /* pointers to memory we'll dynamically allocate for file extents */ - unifyfs_key_t** unifyfs_keys = NULL; - unifyfs_val_t** unifyfs_vals = NULL; - int* unifyfs_key_lens = NULL; - int* unifyfs_val_lens = NULL; - /* get memory page size on this machine */ int page_sz = getpagesize(); @@ -1358,51 +1530,63 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) unifyfs_index_t* meta_payload = (unifyfs_index_t*)(ptr_extents); + /* total up number of key/value pairs we'll need for this + * set of index values */ + size_t slices = 0; + for (i = 0; i < extent_num_entries; i++) { + size_t offset = meta_payload[i].file_pos; + size_t length = meta_payload[i].length; + slices += num_slices(offset, length); + } + if (slices >= UNIFYFS_MAX_SPLIT_CNT) { + LOGERR("Error allocating buffers"); + return (int)UNIFYFS_ERROR_NOMEM; + } + + /* pointers to memory we'll dynamically allocate for file extents */ + unifyfs_key_t** keys = NULL; + unifyfs_val_t** vals = NULL; + int* key_lens = NULL; + int* val_lens = NULL; + /* allocate storage for file extent key/values */ /* TODO: possibly get this from memory pool */ - unifyfs_keys = alloc_key_array(extent_num_entries); - unifyfs_vals = alloc_value_array(extent_num_entries); - unifyfs_key_lens = calloc(extent_num_entries, sizeof(int)); - unifyfs_val_lens = calloc(extent_num_entries, sizeof(int)); - if ((NULL == unifyfs_keys) || - (NULL == unifyfs_vals) || - (NULL == unifyfs_key_lens) || - (NULL == unifyfs_val_lens)) { + keys = alloc_key_array(slices); + vals = alloc_value_array(slices); + key_lens = calloc(slices, sizeof(int)); + val_lens = calloc(slices, sizeof(int)); + if ((NULL == keys) || + (NULL == vals) || + (NULL == key_lens) || + (NULL == val_lens)) { LOGERR("failed to allocate memory for file extents"); ret = (int)UNIFYFS_ERROR_NOMEM; goto rm_cmd_fsync_exit; } /* create file extent key/values for insertion into MDHIM */ + int count = 0; for (i = 0; i < extent_num_entries; i++) { - /* for a key, we store the global file id and logical file offset */ - unifyfs_key_t* key = unifyfs_keys[i]; - key->gfid = meta_payload[i].gfid; - key->offset = meta_payload[i].file_pos; - - /* for the value, we store the log position, the length, - * the host server (delegator rank), the mount point id (app id), - * and the client id (rank) */ - unifyfs_val_t* val = unifyfs_vals[i]; - val->addr = meta_payload[i].log_pos; - val->len = meta_payload[i].length; - val->delegator_rank = glb_pmi_rank; - val->app_id = app_id; - val->rank = client_side_id; - - LOGDBG("extent - gfid:%d, offset:%zu, length:%zu, app:%d, clid:%d", - key->gfid, key->offset, - val->len, val->app_id, val->rank); - - /* MDHIM needs to know the byte size of each key and value */ - unifyfs_key_lens[i] = sizeof(unifyfs_key_t); - unifyfs_val_lens[i] = sizeof(unifyfs_val_t); + /* get file offset, length, and log offset for this entry */ + unifyfs_index_t* meta = &meta_payload[i]; + int gfid = meta->gfid; + size_t offset = meta->file_pos; + size_t length = meta->length; + size_t logpos = meta->log_pos; + + /* split this entry at the offset boundaries */ + int used = split_index( + &keys[count], &vals[count], &key_lens[count], &val_lens[count], + gfid, offset, length, logpos, + glb_pmi_rank, app_id, client_side_id); + + /* count up the number of keys we used for this index */ + count += used; } /* batch insert file extent key/values into MDHIM */ - ret = unifyfs_set_file_extents((int)extent_num_entries, - unifyfs_keys, unifyfs_key_lens, - unifyfs_vals, unifyfs_val_lens); + ret = unifyfs_set_file_extents((int)count, + keys, key_lens, vals, val_lens); if (ret != UNIFYFS_SUCCESS) { /* TODO: need proper error handling */ LOGERR("unifyfs_set_file_extents() failed"); @@ -1412,20 +1596,20 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) rm_cmd_fsync_exit: /* clean up memory */ - if (NULL != unifyfs_keys) { - free_key_array(unifyfs_keys); + if (NULL != keys) { + free_key_array(keys); } - if (NULL != unifyfs_vals) { - free_value_array(unifyfs_vals); + if (NULL != vals) { + free_value_array(vals); } - if (NULL != unifyfs_key_lens) { - free(unifyfs_key_lens); + if (NULL != key_lens) { + free(key_lens); } - if (NULL != unifyfs_val_lens) { - free(unifyfs_val_lens); + if (NULL != val_lens) { + free(val_lens); } return ret; From 43e8fb863ec16ecccfd0c9e89304a0b13817cdad Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Thu, 16 Jan 2020 17:08:10 -0800 Subject: [PATCH 059/168] Add seg_tree test, fix argobots in bootstrap.sh - Add a test case to exercise our segment tree library. - Choose a specific version of argobots in bootstrap.sh (fixes #439) - Have unifyfs_unmount correctly use the static libraries. This fixes one of the 'make check' errors for me when libgotcha isn't in LD_LIBRARY_PATH. --- .gitignore | 1 + bootstrap.sh | 1 + t/9006-seg-tree-test.t | 8 +++ t/Makefile.am | 15 ++++- t/seg_tree_test.c | 147 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 169 insertions(+), 3 deletions(-) create mode 100755 t/9006-seg-tree-test.t create mode 100644 t/seg_tree_test.c diff --git a/.gitignore b/.gitignore index c851036c6..9da40ac40 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ examples/src/*-static t/sys/open.t t/test-results/ t/unifyfs_unmount.t +t/seg_tree_test.t t/test_run_env.sh deps install diff --git a/bootstrap.sh b/bootstrap.sh index dd046ea57..563f23041 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -63,6 +63,7 @@ cd .. echo "### building argobots ###" cd argobots +git checkout v1.0rc2 ./autogen.sh && CC=gcc ./configure --prefix="$INSTALL_DIR" make -j $(nproc) && make install cd .. diff --git a/t/9006-seg-tree-test.t b/t/9006-seg-tree-test.t new file mode 100755 index 000000000..a765eec35 --- /dev/null +++ b/t/9006-seg-tree-test.t @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Source sharness environment scripts to pick up test environment +# and UnifyFS runtime settings. +# +. $(dirname $0)/sharness.d/00-test-env.sh +. $(dirname $0)/sharness.d/01-unifyfs-settings.sh +$UNIFYFS_BUILD_DIR/t/seg_tree_test.t diff --git a/t/Makefile.am b/t/Makefile.am index 6bbf4427f..4fd33f130 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -10,6 +10,7 @@ TESTS = \ 0500-sysio-static.t \ 0600-stdio-static.t \ 9005-unifyfs-unmount.t \ + 9006-seg-tree-test.t \ 9010-stop-unifyfsd.t \ 9020-mountpoint-empty.t \ 9100-metadata-api.t \ @@ -22,6 +23,7 @@ check_SCRIPTS = \ 0500-sysio-static.t \ 0600-stdio-static.t \ 9005-unifyfs-unmount.t \ + 9006-seg-tree-test.t \ 9010-stop-unifyfsd.t \ 9020-mountpoint-empty.t \ 9100-metadata-api.t \ @@ -44,7 +46,9 @@ libexec_PROGRAMS = \ sys/sysio-static.t \ std/stdio-static.t \ server/metadata.t \ - unifyfs_unmount.t + unifyfs_unmount.t \ + seg_tree_test.t + test_ldadd = \ $(top_builddir)/t/lib/libtap.la \ @@ -158,5 +162,10 @@ server_metadata_t_LDFLAGS = $(AM_LDFLAGS) unifyfs_unmount_t_SOURCES = unifyfs_unmount.c unifyfs_unmount_t_CPPFLAGS = $(test_cppflags) -unifyfs_unmount_t_LDADD = $(test_ldadd) -unifyfs_unmount_t_LDFLAGS = $(AM_LDFLAGS) +unifyfs_unmount_t_LDADD = $(test_static_ldadd) +unifyfs_unmount_t_LDFLAGS = $(test_static_ldflags) + +seg_tree_test_t_SOURCES = seg_tree_test.c +seg_tree_test_t_CPPFLAGS = $(test_cppflags) +seg_tree_test_t_LDADD = $(test_static_ldadd) +seg_tree_test_t_LDFLAGS = $(test_static_ldflags) diff --git a/t/seg_tree_test.c b/t/seg_tree_test.c new file mode 100644 index 000000000..b6ed850e7 --- /dev/null +++ b/t/seg_tree_test.c @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include +#include "seg_tree.h" +#include "t/lib/tap.h" +#include "t/lib/testutil.h" +/* + * Test our Segment Tree library + */ + +/* + * Print the seg_tree to a buffer. Returns dst so we can directly print the + * result. + */ +char* print_tree(char* dst, struct seg_tree* seg_tree) +{ + int ptr = 0; + struct seg_tree_node* node = NULL; + + /* In case we don't actually print anything */ + dst[0] = '\0'; + + seg_tree_rdlock(seg_tree); + while ((node = seg_tree_iter(seg_tree, node))) { + ptr += sprintf(&dst[ptr], "[%lu-%lu:%c]", node->start, node->end, + *((char*)node->ptr)); + } + seg_tree_unlock(seg_tree); + return dst; +} + +int main(int argc, char** argv) +{ + struct seg_tree seg_tree; + char buf[500]; + char tmp[255]; + int i; + unsigned long max, count; + + plan(NO_PLAN); + + /* + * Initialize our buffer with the character for the buffer pos (0-9). + * We'll use this to make sure the node.ptr value is getting updated + * correctly by the seg_tree. + */ + for (i = 0; i < sizeof(buf); i++) { + buf[i] = (i % 10) + '0'; + } + + seg_tree_init(&seg_tree); + + /* Initial insert */ + seg_tree_add(&seg_tree, 5, 10, (unsigned long) (buf + 5)); + is("[5-10:5]", print_tree(tmp, &seg_tree), "Initial insert works"); + + /* Non-overlapping insert */ + seg_tree_add(&seg_tree, 100, 150, (unsigned long) (buf + 100)); + is("[5-10:5][100-150:0]", print_tree(tmp, &seg_tree), + "Non-overlapping works"); + + /* Add range overlapping part of the left size */ + seg_tree_add(&seg_tree, 2, 7, (unsigned long) (buf + 2)); + is("[2-7:2][8-10:8][100-150:0]", print_tree(tmp, &seg_tree), + "Left size overlap works"); + + /* Add range overlapping part of the right size */ + seg_tree_add(&seg_tree, 9, 12, (unsigned long) (buf + 9)); + is("[2-7:2][8-8:8][9-12:9][100-150:0]", print_tree(tmp, &seg_tree), + "Right size overlap works"); + + /* Add range totally within another range */ + seg_tree_add(&seg_tree, 3, 4, (unsigned long) (buf + 3)); + is("[2-2:2][3-4:3][5-7:5][8-8:8][9-12:9][100-150:0]", + print_tree(tmp, &seg_tree), "Inside range works"); + + /* Test counts */ + max = seg_tree_max(&seg_tree); + count = seg_tree_count(&seg_tree); + ok(max == 150, "max is 150 (got %lu)", max); + ok(count == 6, "count is 6 (got %lu)", count); + + /* Add a range that blows away multiple ranges, and overlaps */ + seg_tree_add(&seg_tree, 4, 120, (unsigned long) (buf + 4)); + is("[2-2:2][3-3:3][4-120:4][121-150:1]", print_tree(tmp, &seg_tree), + "Blow away multiple ranges works"); + + /* Test counts */ + max = seg_tree_max(&seg_tree); + count = seg_tree_count(&seg_tree); + ok(max == 150, "max is 150 (got %lu)", max); + ok(count == 4, "count is 4 (got %lu)", count); + + seg_tree_clear(&seg_tree); + is("", print_tree(tmp, &seg_tree), "seg_tree_clear() works"); + + max = seg_tree_max(&seg_tree); + count = seg_tree_count(&seg_tree); + ok(max == 0, "max 0 (got %lu)", max); + ok(count == 0, "count is 0 (got %lu)", count); + + /* + * Now let's write a long extent, and then sawtooth over it with 1 byte + * extents. + */ + seg_tree_add(&seg_tree, 0, 50, (unsigned long) (buf + 50)); + seg_tree_add(&seg_tree, 0, 0, (unsigned long) (buf + 0)); + seg_tree_add(&seg_tree, 2, 2, (unsigned long) (buf + 2)); + seg_tree_add(&seg_tree, 4, 4, (unsigned long) (buf + 4)); + seg_tree_add(&seg_tree, 6, 6, (unsigned long) (buf + 6)); + is("[0-0:0][1-1:1][2-2:2][3-3:3][4-4:4][5-5:5][6-6:6][7-50:7]", + print_tree(tmp, &seg_tree), "Sawtooth extents works"); + + max = seg_tree_max(&seg_tree); + count = seg_tree_count(&seg_tree); + ok(max == 50, "max 50 (got %lu)", max); + ok(count == 8, "count is 8 (got %lu)", count); + + /* + * Write a range, then completely overwrite it with the + * same range. Use a different buf value to verify it changed. + */ + seg_tree_clear(&seg_tree); + seg_tree_add(&seg_tree, 20, 30, (unsigned long) (buf + 0)); + is("[20-30:0]", print_tree(tmp, &seg_tree), "Initial [20-30] write works"); + + seg_tree_add(&seg_tree, 20, 30, (unsigned long) (buf + 8)); + is("[20-30:8]", print_tree(tmp, &seg_tree), "Same range overwrite works"); + + seg_tree_destroy(&seg_tree); + + done_testing(); + + return 0; +} From cf74a49fc2ec7e058e73664ae4fd29c07bed5233 Mon Sep 17 00:00:00 2001 From: CamStan Date: Mon, 27 Jan 2020 11:53:17 -0800 Subject: [PATCH 060/168] Replace UnifyCR logo with temporary UnifyFS logo This replaces the image used for our docs with a temporary version of the same logo but with a name change. The image has the same name and so the docs should need to be updated to account for it. TEST_CHECKPATCH_ALLOW_FAILURE=yes --- docs/images/unify-logo.png | Bin 106269 -> 184178 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/images/unify-logo.png b/docs/images/unify-logo.png index c6a1a6049b2507e2c2ab2e135d676068fc9e7b62..bbd68966feb10a9360da4ea5dea051b27c3ad219 100644 GIT binary patch literal 184178 zcmeFZ^JbE_B`4iX=^8^h(jhH1 zx?^;YeJ=3zzJ33N?+*tD&H>Nsc|GF(xIgZXYkqjDu1I&5`78uMbjnJPwIGQ0*U{(c zQ{a_Yv41@wi0ZB7qeoAbA3eI^;AC%ZX=4UKT%kUpl1jBtue_tAKauANaJ%5x@HFAx z&O_F;0Qs*^2hQe6d>-1MQDY9f!l?MYcy!d{+x+NwZjOJ>jngu6@vI3~3{(Gg4u2SU zU88Jrf-plV-!_oTjE=E#`$(N_&eZr^yTw%JXAR7IHUr0S`I6A4hm`Q9ckV^u$Od3AKx;}70_k1q)um&kkZ@7KS@E?xL5 zCE!xi*9%Dmg@6yYmlUNB9tF!j;EOT1bKmLSmlIdRCTKDKBZVUC&%*7r?w@t3{=HRy z`oeD-W|6@|nUbZtGozSO`9)C#En0I~Wn6L5%Ee!mbOr{c!k!FeLsX}*E{&g98#PoK zW?pX#q0{f(cR1&QD|fbDp4gyL>dK9>_w#>bTqM`Oz`ng~g&ibFx9*ame+HbsehHz$9bQjOH$aM#fp@msjGsFGQIa{vGSG9Y=m zx^Aw{Sv84XKdJOeiF@X+>ED$`rs>x|3K*Yp+VQS(dK(c&vw-EgwRKpxE+vmZP=Uq4 zhhKBP`N=;|p&MVQ|2&hsM8)>M*TPQE{_okR$8xlPp8DNBA@RQ#Z(scNzYE>?;0HE# z{Goi(<$uqVZvJ=4|7;2U7evs1*#rFt8xZs#UPAvB3JChILPGyh1L!}R1^xe!8iWad zgv9K#vzr_`HFVX4VuF1IxH)CxqQX_vCn9=;C+6-WA}8v#vAdKgvzUVTgGQ`4cRhAL zK^dVgl$Va#8c9J}r*g^c%cLeNG0U%O5c+%L>IpmC#mR{Zc`+(u~q%3trzAiamemD9m)w^-7|}L$C#W zHI-=ckZ3DnHPcfE_n?e0(^GqfKr~s!X7fBlBC@kpfh|<_=_)hRltPm;(@@>vH?#WY zl@X~Z`&6aXxzkt$e2h6h!5klnOaWiR-BQ-|ayrTfty~T=&Oi_*t;6)wSSn>=tc(&j zmWn-k>=cHiG~NJ@DaR`s;PRHPtUow81-yls(htJmE2vRef=lf16?xt~g&E<3jQ&?M zVg>G6SBhO?g*f{}hl}Edi&!{0S}j5dHt-5rl5nMX8yWN?BP%@eaW0?*adego-Tgs_P?aXjEV1`nHqsP|m3d)Vm2^`uASq z%*zShzOwIY@#i&srLJ$E|7ao68q-q3*lE{hnveh7zxAVW=9*9+N-B62yzUxxw%(Qs zs`+{QODzou7-zAe!DYJbXkgF{1w|&cJ#0U-gPvM$#Hju=*nCz`aJ!f6FtU1hHhnO* zRn=NXXld7|t%T8+m?=Vc4)T+=)(UnO`@P>YX1vWw>z}}S-VOXy&ECaf&GGYLBds@! zUXYu){<*o5=}y~LixO#U|JKWE70of${AT%u5xZ?p9acr)LQva9Dz>)JMwfWvD+(j6 z|1fq_QL(**j#ceFliI2JoS*ztF`fqhaHb}^c@tmYH*K1Kx)sKiZ@sL@D{m=|9`ZN= zm3;|zZ;)u}a3BmU*e2#)A8F6PRE>peK#hxq3qYm*l{;x zS(npfC7xhBK7CKcT8fL_*s;cThjm-E7BO=U{tLGd>OwUbSO};HEyx!V1#d%sjK+HE z!1}vAD9i$9CBAiWv$UXZumxo=G^nT{m`G1|=I}}3(%alN!}iV=Jz$L^8;rN;MVa@; z*Ms)IJ=BlBz2A7K`MYt31nXiNd`JW1?NZgcXUY{;>rPJ9>F&%e?X+^~v}`+y1_LjF z^@KDdJ{KK~-9rd6^35yk8_8C$-9ces8Q;7KgoJD!x--?c5~f=$lJ{P^ZwePt&`Fhh zUCCdcF+z|x;)H}Zwg0N7o0moo{?H9w>wmruMBpi==!t(vxW%v<|Fn;z;ltA{t8O2L zufqDM{8WxoXa7~ofQ)ziQf+ptnC zwb_^+;vPm3u&GMTTh_f1Kt#@+h< zvf09lX0hvz*wF&6J3IZmlMqH7-`mjHGB8t%Mw;WU$!iMTfFM3$ssxpk+@{4X(Ng!2 zL_Q9B_bwJTbFpPJLAJV)6Qkua<=5n6H{B{)`&;7w-kgeoQSOfz9$Iu!L8!mUhg-5T z(jF@sgYQEMU5i4G%qKI@@JjGL>u7maAG2;EJ}k1|8rUuN=&v|JD6pcu+H+cH^v#7| z<)0M7Uu^Bg^`Puw`{X>9JZX z6N+yM1Bb=>ie}>CaC<5%3%vs$z(6P0F=Rep^+DFsDDG--)d^59FaHvU2bUUS^lUlo zCLPM64_9uGS2dfgti)>G0>ra~hv?++yXX`Ox{EXm=>#M@q@02Hl559M;KDf3Ez(r@ zdK3H-tvVH>YGp_3{J~~!t6_W7P_cobhqw--U4INfMWIJ>Gpex>6EN4fZBqxd4OXyrHN!Nmi_HD7s`Z` zw8t!(wyfJqUOLCQub|pNNaq@=ZS&`#u8~L9tM13H5LE4RLc-#QFJ^&JV~gWpvEOrQ z!Z)`F+BvH+$vKnVUNVi%(>991Tg6BKh{K|EtG~I+GPsk>?=Lf*uQ7TGvjTo!TQt~= zYsO7A9jLfX0TaEw{A;(Djefh(XD4%D$#(kH*J$F-eFTE}e9b!AiQj5Y*g|kbw+%qZ z|Gf4csyQbbdwzXmbmGVn?8`%~{t4U8!b9-uelw}M_7*;4({m-H42vAtJOoMUoRHw} zekItl)sbQ~GiynIs&bBQ8_<(e)GNF>S2tUVW(t2BwyrIrd#CLrOfEo&XU!1iGp|x4 z3=xOMGh`bM+kZtwYf7z4@GJMyjX(^sG~=!%Oy_&9q$0+W-}GB$Z1Gzb0%|^=+-(Yt1}<$R9xkLLFdTMG;6v6tNGVd% zJ7Qf+rzjOeO%BMoS`Ter<3Bvde}|CLaWd zEd6xJOIoo$;IZLpn`;#ShX_%ybNr0O$#(Z40|{_v0KU*)o#71b%hQKj9M1QD3tabe z&epXz^BGGUGVtXs)fs6BN|SV(1fFQW^lP_Cy?uR9!aLsoyZWh=6z$a!h8h~*m`|Yk zf#L}qFggE&7ajwZeFCg5gO(x3D-P4(pHS&p<^}h6Y&Z63!{u6W2c#l%GL9*plH&nP zo>$)Sz+-RyihLrOG}#x}P99jTeu#D5v+(JdA?%D~?ff8I#_Aw?7>U$eIcplhd|Uw& z?bt(f_WaH_r6woz+P;QNG*6dPY%-U+o2%I2j5fk^?Rnb0=5S1j*99q#;^e5!-MI2^ z-kk7gqvpJxKHGhIsD`so_=E(etz+ee#uiCOQjrCeM?i$GpRKvmXXPT;hiX|{>@hH^ zSh{-|n&|`qGJ$8vp_L@fxJ2%Pj6RP+YIcY5Wc65aYpLLo8hJdQA0!m=B>?%m2F(%Zk{V_$DTs(+zb@5E-!cC zgh!Dkw+7+}28;QJL)rji=S05vwPleHd-)R=Wq11RNS49xyqF{jtbT+qC~NcT!icmO z09+os9MhG}6dN8WUPQ5$Y_m9=!@fvt#ka~%Hh!;n3TBRCE_06~Jm%O<~>Ety_hYua|U-lWs7riKiP z7M@a($LB-n0nR)q_Sv|-Pv*7wOudo-xO}(DzQu?3;gg zq6V9pEvP}yuDvZv`E<1N+JwrO2oryCT7U+?|3P{}n**yRg+#O)u`F_8epP65~G_hMUFJUtYufGUqF7&kL6_Rv$I>K$5Q4SiR8sCcozrTPhEW| zevt8we5ks!sLF7$fv&eWfTVct(+(e&HbpU;9lesW<)xeI%eMIN?`0NT7MtQy#lkFk z!dBR`Q=8_o#mM>McU0e;pFzX>|h(jK3OiCwDc zw;oZAYFYK_11j5o^VwFa)$lPSR3uPlwNml|IynKrOZph zR(avZMAaPPwZ^eGV#Cab9$q_+byHkGAPCPg5vW!%GK3o1zhgf+Ua z{={B?;mF8gy`mS#VOLv6I4k|u_}xp8_cD#DCTJ)AYA)1S>s;H;Q>PW9&( z0Incn)-bt(l>9_RjS>UYC{k$ z$JjK_;jy~y5#O!xf~;#thGAty7xI4pq}j=C%05yEiegQLnWQ8we6)gD zVLQA>O~H}tEy(uHUQ4aXDC_HIL4m*f1*3>g@s}Ff{W2is{W)dKm&f8MRgJSI@V{j{ z;BX2k)UI!bof{p^sJ?JwuoGrscz(I7bp3l5rv1Bd-&54aTXJLX!jb}Ko&skO8kUFF z=b}Y@=IL=%#hwo-{hR+iQ}90V_U0Rr`Mjet>z;pLTaDS2J*f4z*u|>RK%+DU7UyP1 zI^W74*Z0j=+qH{ZiF#)A&`20ECHx4NjnH>MD&(rudQfel;|OTV+@Ngv{^XL>Z%_U% zEjiB@jH_8`J+Cr)W>a-#eQmm9OHj0Je`~Zo=P(ByxkrK3u;>bZ9cYP`b17=5%C*|6 zqBZ;vmJA3=>?$txI~@Ob*WbHp9kuB~c>sEEWDIqUdoCyoNtN+A=SrPSb`kK;P<2v4 z?DSzTN6vQQfxe8#9;u&X&(gF`FwjULipgNB+*q3igMtxLHRV8yx!WU_(m1O7oI~yX za=5fb!nCR#j(h`WxclPh7CNk4&V!A|p+e8iw)Yu(M}(A);1ZWH@hKe-s%o!6ZL@)2 zk{eW=WfsoyPxa2T$mev3Uh(4>hzWMKUr3lAt|zqaYfv=aO;jgI)z72?DF=xUN8%F{ zj-DhWf6Zns`<$&E6IDNx9n#$4sEs{uLq!=%)UhhB-gtr{Y^7BBE-C&r@alL9sAk&& zHx(J@{%d4+wO*1$22AD_xg+T{RO9LOAmV01uMKEbra>|NscKDp3Wc4o!w>naZ8dsM z6Pi6Fgd=9YXxgV6l6(v3yzsZ8sBhP6`bf0A>TGxDEY%`=8Z2Cq&u<|_Na`Gl+M_6y z384LerJsOS7*UVT%#HxEsId8J#@}W;(Nt`i=`>_;V`*9yX`Vls)2lbcr@wC2e<3~o>s`s|VryTNa*<`YbJ_mzU`#Yum65cj9j3`bLd3BBDQ$fMc=~vh! zh9)O*D?t_Tf2g47MRq?w z->S}ov1QsGKwibK+tjNam=Azu6k{I zOD}eFB64cU9KM8x6YtSD_^LInHZskV&)4}3R%7Hr;VnfH8}#868IV+%0Cb)m7ts0K zEveB6ME`F}Cc13>ftyzxH;nX<20BD<>fH()?g_Ug8fohJk^Y_AS|nh0aQ#ctBA;+O zGuQV<{zQeqw1Gs0n~lE#}Ze-V3pt$S4ckbnn3b+MdULGh@bI*C%A{4oD+M zk>gqB+$VSVV|xs+ilw@b>&;XP2~}M1OHbEcV3*)z9zBi&^@^51k@Jk%<*eKrPjM_ zci(D#^u@dIY8*&9yVjx^XWLabySyq*;d$hfA7=2F;B{g+CdZeuVzkyVEye4infoE4_NN` zlaN?fo0%LosteR99J*}(LGYc5k_RaM{>uF1ju&-ZNlCYl(qLi$)52hPFax^U z*2;09icGZ7Zq$zXF5BauNsD`Y`d0WiSTwh1c5;3^4}cQm6Z8K26%(tS!a zejgn*Rk0>X8JufocPB~rL47|ivfn;^$+SyE4c?!J3{kLEVCz&`zrV-0!TIs7uz*lV z3jFQe?l6gs8$#MJC2lTPc#x84zp&by+foioj#CAlBu}xKH_&P4)+~t_#E8bp6DMo* z5#>TKYm$Li-X)$7eW?c@E!_An9+Ut-xW=ey_Xng4dh^Y)E@9+T z=o#Vn#mMMw)K}T^16%UsntOv>_@qSsOv#4 zDJIus#BiaTvPCZ+lSxB2H2qUAC%P14h1nQ@ZyaoLWakE?u?5xyc1z5fv3rD`*ENBl zp~&PWk=DA{>C52en#;x8XNjBp8x1ib^RL4-nA$hX?}H`B+Qp`stKCOeu}+GE}yJ-*24Qsw?G0SyfIt3}EWg?x+ZKa9aw zZNEwH|9lY)^TclEeaY#{VRSwG{iyExdn&5suwo=``Jm-+w1@DEKc6fY$s29|z_P{H z(#@VAT)E(3a2_H$y8HgZ>sO;&df+~j>$D=5JR zoO$#!{|m-eCeoZ}yDYKn8|C@y*<`WV`Suhl=+&F5rYjk@?)-eL8+xzjTX-Z8#g@C1 zPX1~X1^am=TG090y`-goI&mRq%Drc;@@&;Vs^Egm`I%04b{{*6YU!y)PnR`50Y6;2 z5KR5&Chy%X==`=AB|a#%a*$T{a*g@wR?d$m27D)g`bebOuoUe**M~91g`l44Ui}E}x+ZKLuf(a- zHs86b7^K+vNcg$#RrKK=!;-tv9)Cr_cBd9=;mnQ|jp3VVVDsK-6_AW9)?sv!R~vtD z*mEu}j0tgcJhPBhrr`sd9Fjcst5hL+Nkw0_gXp4C5Tyy$t%4bNRd{oav8Lyp$YA&H~`Ptc~>A3sa84d65YDp;_EjKoM(kDTB>q}d% zq3}(y_pueC-aWpgG~8TnWfG}d(_9wg6lLDN+`Oh82 zCotPWQWr;Zl_UhI{HxlTgsM6;X(|E%I-2ic_oej^Ux8Ua(^Q#qtttDLv(90s?{*p` z82}HX*9!|)d0%~73;|aza1!{v#zt4yOg`CCruQ~mcEQvScG%P~74)2M8Ya6JjZ9Vr zD?s$`O$=;ZH0C=$)O(cuee0W)*+9@uLY`e&@-ORV0eVgjkVI!-Il>_#jqdz>vp+O$~Mm<=L z5xx(B*k#A9x^1yh$dZxVqIFd!DyQomN5O+51fWw>ayEx;<0|?hCwnG03}y|BM3i$(BGa0X+1XAM z499_2Bm2pkC4T?0lCY23{aU?xh@L}Pe!XiM0ty8R!4p*%Bj5mkXP4PePL8m;j8I7V z@%|L-M46xyabx6k3$<%xbWHlLQn-!de%do4fPtTnprez+>v7)9iuLFWSIYF|bheG_ z7lyuxvCMh#`?MbR;5z$=UR!WBlYrUCpN{fTia^iH{-s$)tA|y955`TwC7K-og6POJ+bakXtZ~mjjOHx zoF~DBO@ko;t*2qM;|l!c=u8EDX@s3Wadd|H^ZspZEnIQZJ{`B5G+&ND6j>br5DT5!C75yUzoI2HoUzt>vzGS%l(iwdh?=X5z*BYT+ z&ZD25xduNmZhx}QHf!yM3Tv6y^o`%kO|IVM!KEC};fV$lP3tGh>rzss99RN)bmHl^ zRjPCXOU2h7Q`mi0hAcLI&1KMSgEtB=3$Jq?$A`q}Dy0k^eh|xwL1SZ=ia4%&t=5Gs z8C4&M;0wq@x4ZzN6fnPbFQbjvivGS5_Zr;Cq-kJE-L`w*>LfGiMng4+ z%?LjmsadkYfp!IH8M?TwqCuN|&1RzOiS;9m1WFM*VaBA+nOa>Gf?Tm(m*oIR%O!XO z<60vNL3%1rl*>>T{N0q@?W|L6O(2i!6iHWCIMmNnCP|D1NIbyalfNfoop&}4#xEF& z$R8iY6*M7`X*~LZO3$B9f8nT{*>eiw9HHCtU&segJ#{_$!4EBp%)hj+H;O*P~%*5E}()k7rYaoqoeEd4FdVWo9YXoe#bP}3*At;0lP$=-LxMdRqbF! z)VHo#kC}bc^VLakkPofj{5YlAujLs-xOjs<$D4w+Ro@a@ks+|;uU_>SdvI!>AFMy2 zS$t9@CDTc*^L5=E@u68-WpDx9b-@slNxOjgflNKzJz@Ov|qb_Ue$`rBr~n}e)@ZB5J3I{bxj!G=J8$P+ca!A@%~AS5{b z<;sEY)?RB(#muR-1i88k2K6TniT7S1D+Cl6gQb&04EEjt03K~V2gw=wrkpuCg#U$G z;(<}LC`aFiLun}kJqWIV8zfcl7;SXHMhr8xT213jE)JiMm@!MQQe_Zq6f!e=@`*Jw zhFrby{lOOc#t7YH5FW$;l1;i@#^zjmoz+^Ka7&$<$ zR!y^BqxyZ@eD~kt=BJjheBF+_c3(hn!wr($d6^fYeHx!lnWZNWwBKuQ*6v-~fnEvT zQhd!=eevn`kg=@u@G29l@D|)UHbg|7_DP_}i(u;%uO}d{b7dU%ENkO0L4LPlCI5uf zvt3kb;R3_#k5z_GX&n&GM%IEdK0|q+vxU-&r-^jzmT3@Wz+dBJ3exO<;u*t!5XlW} zx)(wHoKGRxoB!$PJ0~`y>dOtXvv_)8d6e0{?p&iBGB7lf7}&d!bdHc;PueY8HK-1O zpcdC))nkNH15pa9S7+Mg5avgoow480V|yV>a4hASAzM|ITpw%y^sC?ko(Q&+&4Gf? z^ltNHe|$}~lE^yI;QojPwV}YhQxi1%Eg0P{#dsfvXWQ6FTE})gn~^b-f{)&&hVIjz zJbGir*ZX#-I{6blx_`~AztXkd91r3c#8JkZD`dP~1+Hn(;#(`nc>1R^)5y7NWcI24 z2cM+)yWY+-oEx3Rob!QyBK^HKymF(4y&&d%afD8cLYLvs{-?-R%t_+e3iR3{=f?MS ztUwE3c%N6RqK`A z8%>r--jPf9+8N$VZE({%2ooX`r8vcAzxfJT4@W2cq@TYp8`*Q4%_?G%?=ekO18FwP z8QTqZXnT{;dgc^leCjx6j9-)oGyi@m(_1UPx zyu7v5u3=Npcmu)6QU-xcZ)$q%8Izg&8L3_@^52v!FKt{R@dSSu>F=}Ts1xu@ydL-4N5+*U zy2XYODDxl+pS08Tt-aZ_k*e+>q|xXim@>`$vFF^-`BYhtHUGOCje^x|o-y<9aXz(2 zS@y7Qm>1X7rW}`yDeyyLZm5t3L2rhlb$MsJgxlc9n+Ow;Owi>90t*!qJJkl5BrtbXb z7l=cVa-AV8XFR%p>SPPmmo3%_o(=u8LrP#@qi+RY-eOx;N2W7#R>qp9 z)U{E2!+(X($fPkC?ZkgL5aqlI`MJv-LzV|){=~w@0FCt5-Bf{dM2jxH9IVlPHF+IN z_FDQ0kFEMwvze61m{iHMYm$2_Z`R+=_MLhLo2B~Fck$GFq-)2@H)WnA^LSI`9>Yst+M%#jZ zgU+*T$x}W)Ncf2xvwkt4ugBO-+m11x8+c9yt+*XSv)|yAJzhhp!%R>GtW0l^*Ot$1 zwPOxRn*(k9%wF_TeLDOG&m?D?bywE*#8{C?-B2Fhmv=mKbK^xzCV3!Z8Rwt|YmSrM zk}YtD=JgOz<$r>j}A8S}E(HcU;+POl~cCKy&v`(XV8kG=pnRw!wF#beA@j{N2Q z-gxom_ZKRGSqgLc>HTG)SPZ@}%Yx7M0f!j5NZ<%1n??6;9lw)rC;|BkBs{kB8!<*t zMtPXs=o^SkOm};}uaO$B=J45y`M1_*@z)qxi3l^5JG;JLrMU0%uHP-y!JugKU~^fBtD=EMrWq=rGpy3<~NwpZH$itXzz4a$VC z^#n)H*!uY+ZayPR*(PicMUt{&UOT6#|9Zx+7fm{4VCta(YNV1{M8IRYK#%ajxR0i3a&_FdyH;#A5TF|rQDD6GX z)J2b0Tsu@|Ul`=~J?=rXGS}|%J?lP$o-Y+`nEipvv`NcVjhMWwR_OQ$=Lpp#3)c0w zt8PnJ(Xf@*nLJ&&W`=kbPcN+Q&eD&XUaa%q+jxXg0lm+eRBGs&ySn0WXntbYRYjaljT`TqTCQ;8SY@H5bJu|HO1a$?gs>N{Syas~;s zeI)*zx6mQjQx?lHIp19>Jp(V9bf4Kf+F0k0!>hJ=ole=HXKk%-YW@h^CY)$u09s=J zE0VKK0(|}_5G9=e{JHuOX3YvdZXU(dJn5%gXmOKvXw6`vCO?@YAHemvT|h z&0v5C)tV}rA%y1(aqb%PCfh#B4DsFpC>qF5;ZFjvzP2mMR#eAK9t$G&GwnfLud83 z2C=Bn7#zT3-)$iaL{XXDQPTWl3W^T}$=$kmRN$as8&Nx#Tra(&O!OqErU0Hmj;7?0 zd(Tmnbwfqx?;F-5IG;;5h-IWs6db8-K;P{U=ICf=1yyqNx(+WPWpVDQjae0}ePx=53_( zRlhh2N^8ItTn+LkvRRh5#b#cRocbAJj~w^2Khtqa@u-!G7g!s#k3d`a7JIMfh_P6T z4cloeR4kOC4~Rxx^~?Lbm-mV|AxPMiD1Yp=Oxkuj7rTkS48;zqZZUyYGoC!4aTrum zr^D>Z3Kn9_&jC?#JzJD7un9@4JH0>EpYyI5$dg}`KMI`1$9>-jHmEmw8ufQ1BA#AE zsf49i8Qj59A(+cr$BHISI0I38Gtv5wHRsbZ&n^Yy=9)&+%z61ty90Z{qwycTOTBtP z0kjm=iVrb66H5&_+%Ek3E3lg$4Px-kT(clU_z`FQ{Z($yYX7s$<}b<@6;5`G3|{C< zbs_AyeV3DtKqQU1003%e%H|TH%cv~o?|Ahw1?5p#U*7z|zyaP|Sl|EDQS@dP@C>@O z?Kh&{|6NF@RVcq24dy4&a0bC9JjfRw{^2a zaPwZG%aAN6K|OOf`3M>~Sf0trHbyeDkBY|nixl>@6VTV(j{*S)^wM2Nmk;Wa4hl=C zvvF|wn4(DXL~;-)rE0KQr%S*xF^d0-vj9X`J@ehlf*vo~NuKPNvK}$P^Itzcg)ioe z41BWYx>(V$#`6rAxba;O4;}bV2Z*9xLct)KO9)2O!`H)n83Eh}g&78Q;_Hg~J{*+D zphp-tyZbemYZVwEIksu_KTx#W9H?FEqA2g^HPye)(+5xlm8$s7`>Et?HivD4JGJ_F zGAa&{F2=@oveo$%(DrF2u3N~$q-VL;1_f_^uKuSf`**vNhP$;+^>_|$x_o)3s z%fKDJCs0l3A8WN}A|jd7yq**^vJu`zB2eiAf$pYD-cKw`5Au-%^^=moSehvF*y&qp zok?ep^wE2CeU3eAUOoq-$Dzzvw%saR7uj=W+w#f`K<{86z#apl6B^_OFP}h*g0&^dwyS{M{xU@lTgdmW3psi7bevU ztb}O2t6!Dt6cnBmc2lOP?>srm{PWM$|2qqSOg1sS$$K3Js3nj3`+#*vlf9QGz0LlYL6HExqB#*z&>Eg<#|T{2-UCg%^lFiPr6 zHrhL(9GJD04xcVSil5)kAHf~F=Q&B3@h`}D%lXD}uGEZG=;r%`(Tfxg%#*f=C0csv z%<83P_7ix+iL8->1j8b6PoQ=F^{1F83u8f_31&C9$gGih!pgJ(g{xltr$JJd z0mi59*+yfWhtT?$wtw10B#8i~YOfN>mqNH7G8i0&VZJ~98!XETM49nI%gq$J#` z7@JCs0GcKQIluh>WHcTo5d)B%M>%8If2xIBV}sN)!@f%m1_kwR!CM7}#OeR=GT-$o=*^yTFCLKLYhM+AO@~Cjz1#mZ+~rF$)g3s{tzSB`>DUr*c12B6 zg<}CceGV0Q%A#7sFlO<(j>{IZNk?{i*9(H9_Te9T{K%EN&o_s4qRI8d+M8J4;j16U zZN4arT0J&DnsUeho9h4|EIMQLwW_IzapY;3h80Cu@`VxBW7K|rOsM_q5EV2e^2f2Z zJlPLZ=1K}~+WtG^%BBHg4@|F25fN{}C7TMayb5TF3Ml-h(3l^G}gfW7ps(tgZDgv#P!DOS1D3N^XtG z9_zI;EmY95$MBzGQs(M-)C)hm{IY$Y;od&@(}O+&#tF7Q?Zd7fbu@L! z;hIY_WCVl|$W~9b8boPtn0(wDt3Y>Dd@Ltt7kS25a#n+O@!Cr%)``ri?JG2FZ|zdu z&wjju$gffo;k_GR=}y(X=B0{6JO3Z|(nPH9FM!F6%zgt0Xm0!?Txp>pqQu8P9prLu zF>_jFZ2E+PH?e_~c{z7>OOTgm$}C@5G_1i%Mo{@8DCEMe^f#ORda*>p0haWHaJWK>AvXb;qe}KqJX9*JeR$WHZ-bz;h>Xy6)9FlAcCZl?%wTDr zxkaCy)@D+3&jJoqHz#K%e-)VXx@RzFeVJfgC?YMn#R^O3zMyE~lI)HbsCM0=ju0WsL-K z#EZgHV1C_a^2b%BJUI^m$J0eooJxZod+2no?6TqU?YnFo1l}tdDFfT0HU(hJ8owNB zOFslQ3|kvKi;wb#d)KQ@CO2KRNU5_|?Vn9V4wBgCt@W&3eKxWS3dE;|Bp(=AC#c0d z+6jW?JcQTbxcMzZsJeA0OMA}f^sGs~UmLFX9$E}RZ3jI$npKr69zKnOS(KN7#y{W> zmee#^0Z>$aAH-!wq(_%_&m6R!O3^}^1$2Gm^7K-AWVPgLzwYAx0)3Hs{RJ>4eYVZX zQXdTtkQIC|JWKwoJiK^#pL*cqK1H19IFYzkIWU_j(GBV8Uab&Czg2rFM_m@2q5d`b zxZORkM6F%Y$<^P(H5{3@YL7Ev1?~Ffy&W>I*pB z+DsNH50+iu&bCRj1h0_z0PS&<_;q^_sAcC*JFozq0eI$3yz8;0sdA9po93Y|GH3jy_lSc?uhkns)RoTS*czM z$VTyDJpPp$)@O3tb#*7jG=&->N>GlQH9s?R+cZasr}4c6&RlCL*A&EClov52wW?*C z!%x%hASz8hS-B!ZS@e?bLCsR&L9nPq2K^mB5YWImBFTHT_X^uFo|ss6)1dry87 z`nM*Z+}i(65Gur`*=sL+)EK>g5|wIKmhJW{rONBpbnUaHx;`rc(>9n7%oQ?Fsx4^@ z)xo(6%saJXByHwt&0xMOnQfJLwF>LnrNglRV@2opFYemr6$Qh}ehT&;^)s(e zb=e}KL~lNqf;U&X*CJ?uy4j7ULs|ho6|tDr))T|Wig}Fnnq-g}3GA(I(xvvhlW20> zdejIiuRTvORYV|2H!?`4=HKP44WngF549_VL|*Mmt4`$VqGzo(8Sb)b{p#YLKwY1> zu%J!(VL-J0WMeHhoIfQNzla4ofDi`zq9E^qTK(wKN5`UGnI!dJgo%-*?;G#x=KSK$ zU%NIn9QPOO(vq4zSKg|`%?&;D)*=5~Rv}gp4)X@n*}!^G_#k7Z2t<}BV{O?g<22Eda7Owdgp$)5M8F{xWtpCvvY&7^X^s^cjukrl>`ijd zTx)w`2=7Vn&f<~0LL<{Gvsp9b0q28qvtV?v(LU*K0fl-miz}>J^0De=H({3W>N+~w zb2*t#bm{uccO>mkcE+vv7>R-jWdrF|#4?~;4Fh{*K793l)!}X*rL@?B-={;yXRNU9 z>tEDR)Tuw}cWyybF?<-QX$rFYH6!Tpnr&zOrGYx+3)+1^V0jpG>#dI33F*B!>Ow+X zi`4!BK;k|@Kbdy+9#=fJwNTsc@)IdsNic`r%0W7qxFRD%H~RO?AG0t4;}u(>^2x~Z;0%;CLN|t6z1|8fibka~XL!Y$*6Nx6YwXEnGAWs4 zP#^C8i?gC2EkIBQsm|hs-^-URJ-_(NURh*uR-k?Unx~rn(+Gq}JxL2MbmkFGq%iwD z`PCyw(d6W_QRa>pEN=bm2l_3D9@)>B6d7kfx%x^AAdL(p7(BCFaJL5o9yQ813*|9K z#md4K5GNYT=##EO&rOG>j>bYyn|S(+E1nJ3)To7yh|f09^8IFVxKb86)F9iSU{Cbl z1!WERC^l&KVUp-aaFjywxZ&A1q;8GOi-mgG!Gen)9!qk?Q&3Yf?pecByfr( zGyg1jwafMvHY1{10$7iBDP1{GvX>aElNoW82H&x7vYlYkZ6F1G{qiStA3Rqp{GiZi zmsiLyU9ZEh7f#}H+f@R6nRTAx>RKTNo_4h)Edd_t;Jt@VElXKSI%o%<)Bc6)^D94WeaG1ZEb;P{w( zejc@7MMHr0QR;=;ZvGG{_CFlT{xV9Q5zNV(u%h)@QqSjxEx_R;%tjp?n03l8lWp%7 z-}xkh*?ffn!~YFfzN}_v{{q@dZk_JTQ5~xW=LPg)??sbsTnJOv@$}ZZcYF16#5j)h z5YRAY+clc9WU^%4Gt_~!4xe!-IEOCX>nhF=0mD=l!^6(e8h&cHj6)9TQxW_ zYsuc{c0O4=B3e`s_zX1kc4g1?yE`Z6KZjfr z^##L0e;vw0oE@d**y1t8DxiecoOq#sbRL7b1;j$JNVV3&sOM;L5m6=$vvzt1?iNz({yW#XZxy)y zD%w{4cPnu=&!0G}7!ichE(f~q&t0XG^r=9qK1bIo_vxqm#`Z4ib9Fn#dfW0?i?26v ztM>12(}nXEAG9S!7XuJnT8$jQ(1NhdRlU;kbyzw6|1tI5@l^KV|EGvjcuFFof$Z!p zDv>RFk3z<=WzVQ2D?2-zV-=1)D`amcyX=u<=WzI5N8j)7^?SXZ|DN3L`*VM;>w2%x zEzDZK^=RSRQ+l#32?CDO$Ra$ivdMDo)8sNR&6HF69lV+n4dJ6TrIG@c)rx%=Nm|^_ zsW#L-^)Xk)z1bA+WZFW#W-6IS{rYEOK(s8Z#UxF7A_4chv7H~wB5$Z?{D>eCCp zK#Y@!uvgVohBjp|h<=E-)w15%so1C0UfaC0S~DnKb50hfdE)1@@ z>A!yIt6x8|(sg};O0yWss`!Etbk(9Y7lH{GethhczeE~pep(kr_|a$Avg;4mJl{)d zdum_fV%b)Aw_5FBV9h@63aAZ}33ax0(nhKI2$McKQoyBs?|c==p9u zs*mCh*V3aE9hEfh?Ct=i5#J#E_PdV+ndpI`!z$a_a8PYFITJ#QBhG-C{kmJssO0((K2+t$&qS%o@AD-e=?paD{skg^NsE zniPwdSQ_-sDB4&JtK|j#X3h;fm+IE&)nE4CIy)6T64Bu{%`Z%o6S=y z{dv@yhw`Nj5XWH3;v4dExx9OZs!9Gs{GijibwWiRTYxH&Y{%ZcsxQd1J+@LgV$SmC zc^i;+M!KmTqToVBfn+-&w$4jcteaw@Q+N5c_Npa3(IeTG+T(4*=P}P*OR5FEt4Hcf zrh^x;dcooOl+W!wobUf1qZROCi0t>YP*3^br(N>Y94-c`7XM71_BiuUl2pqZsKol^ zuvP=o^l3G&x}7F-Jw?jv+hsXE=gIs?`_v@$v>)E6CqlNW)o$!mYm`<~;5`JP>j$-d zrDJ%}m=j_7jqT+LcUfWUhVD$Ksd@LhyJ)K>mtJP?{>@J){_dydZvTDneETr-d1VE8 zx4-_U5ec67ORIs)6|dV}ZQGiuA2A;+lGT&f#SV}BEsgRX_>TPQMgS7bt;T{T(&?pR z>mp4fbG8mxX4ZD*jt;Z*hU!zwI`n}~C17<= zfxt8rM6nfVw~Fd#joF<$GJeOZ#wS?gx;6U3L;?s%M&5?fv6w%i!Q1V`e%cY!2aam} zr#_rInMNO63yb*W_xP1olccYC*zSB2O=}s|R;5hL3e;cGPKCWaV5r2$+tcrzR9{9? zXSuLWVZmUlQat3zA>M|;_sGi}K8s=@!#!}0+P2g&l%cH(w2{uf;?Lhyh4fU%*X!5a zAY`2NMnH}@aUq&0Jfc58b}OhMa&E2of^nfwf^(a&=*V4X7N)hNILC-*8+ead@|~UK z@^dpbeXI92$e3Tln^S&2|1w|?GmoQ}U$ZB3Ao8|slK;IAMlSU=rAp0MX--K<=B>P1 zbzWAe>Dn#pZ|ivL_Ip&}$F-29t`#Pc)|CANGxH4O7j0`v`WIBznakpP>*#fJ37iYm}lL*%QD@FhdXc$I z0c8W0KFDh$+N?U6c6vvja3DfYPZ8R^UZ>4AI5dmb_oXH)_X(-aB8FV((E!Ah}L+|3*e5&H! z>~db;SahuGEbUmTfmT%8g|k;i>UjH--nn31Ur^;=FJ>nZU8jA?Qt$v829#GuN7ctZ zOYol`t`}Z&uHrPE@h)83m`}X!mDhbVnm|ZjRA?7!q^_q}{f#?*-BEJv!z1iP$gx1< zMkiB2iXGQ#>X%hUOJ3S}8e#pf)WY9^7h#o5#;D(GWd&L?Md3GU&y?%0N;~!rFjpwR zpZ{VOf4}+jSTBDHP9{Z8v`^W)DALqy4n)kk+H{Bpf}mbIK@I6v{s^Qlin59}9W$~q zl^}JV5xV_qrnZLGPH~q*p|f7m2eC2?n5h_AwJ?H^7x%ELCE`V(-o0+Guvr^PmANi@ zY=s*mXf!own>9OikM4KZzlFODJM=$H<>ksnB5;E@Ei?ov*NgI;T9)t0ZH$C&eZDf5 zWKe>W^82A&SSbYZ$g?8}vFL(CGby5LmhbwIR+-N7ZnQeAC<9>X`7EpT{a;-SQuL~Y zw@NV}ZLo6Y*#+XIZT5a1)rtelIdfD>yMVblZX}&{QXN~weAECbzUVMA#G`+D*uov~ zOJ>)`cC*%A`nUv8_OE&$vFZP|NM2l@ibh8b3y&7~Olx&nUPOkT7K*}ip}5mro|)yJ z&{_~=X1@evz+<#M0N5v=4>@~5r&n)5DJ2y}A9wLa`G`n*oS%j6D7^E@zt#+b z=`LS1i_j|2i~vEvgBchIUxFpJKKO2c>iwa;CL;1%@Z zGJ63K&*tX^7n3-#MWjU!Y56>C*$V&-SsT|}Z7SMIzL!qL`bQ6zKa_{9KrNONB0<^G z6W^x4Q9zTe-#42#q8C>|QV%_m&Hz*3rrw8w@ z?tPLyoYH~;hP0t%UMcFb1w~l&-PbP7 z?#dtF;K}B-<6>j>p4PG8k|BvVcA=S|iUc`z*;fip0u1{3`SfG2T&&0uD~Vz7V=(eh%(gC$g3G56Qyovh1V(VPkHn3I*joa+ zH}6AH6k67o$^G#z#yrdxrd?_M>k>9tFV()u?F3nau_c?gLGmGA+@w;LCY>$GBgXG* zucoMn{SCK?AeZdBzGxb-h)Co!?mJuvs~o#R_#QKS-Gaf8t|4pkZw;G*%S(@qYsPI} zdzNS)gVrJDpW|cl)4o61vD(wv#$Pw4u$W|4#C!4+$jVItA|Ll?@x0V^%*JrU$o!y% zLwI4%{#;+c!g@|=_Urc40r6+wlIkAE_U8|u=Np|Ur8OQDrW^9T0QNc59dA&d)~%j4 z*f~Xpx@rh_p(LEM^D42@IkzECWwUELpEY+fTm-$kRE^BX=#5p4a?n_V7gYGR`+{C{ zho5Qklu7H~;WLowx>-iUs$V3 z&(5_{`QD9A1!o!9w116KGe@=5%PfHwRDHLa?9!|ftLVRWAArRG0Lo#9=tLzvi24Y6SRp?L_jM?U8<0*b5*K z$Vlh(l;$bg=Rv)LO9dv6^SQ^mbpCx^1AHK;m~B(cDaE+Er*edi9Um$Il2iJ0_N@Pc zQ)s~;JC(&$kUP>_chI}bKhyCv*MRkE6`W5o>AH_MtLR9hC-u6oLtc&{zTrcJ5jG=V zecBbFL-?=r$L}gsv@PekrgiZYd-jWx$1S)peV2EX&`1uq);8iT;ayX zQ6YE-nW-H^DaX(T#;*tf>ME#u>DYoolHk%}#FcM)Uh0z$=Fduu#~x@Cag8`_F*_6h zW(RL|^IBT-`@pO>Xt8RoTAOVckbp~%YoEpKDI#l_`u(E+Kag<@9S!Jk0KvgxyN=E#SBkuA)%EO_ufMqXk8b#aWrL=1j(IKo0DYhXb?y*!tdxAIltvPn}FWIteXYApQ z7tC{D+B4D)*%gOhEe@Q^o7MxdmSVonf(r^W#h_@^t!wEEm1M4Q?Xmsi0XoF*-SB?^ ztny2TwZUN2+6!8+ccMLbh@swMevK4+|19!5GN|1U=%^U!Dm=}!&IlKxH+}|71V?_E zfrOE-?F`)N58FV7S*=1ib!0s0?O~7Y@o0xuq8R~*nS=xJZx;WqNQ0f^bM@lGlzEbi$y(n=-_>?Mld<~|(S!)MHz&~5PFHn-h7qQV=NY9i9%9`^@f+`P0`IJn z*90ouwv%Fs=c%@C&3+wuD7FMZ!TQcRB$dYS#(!8}#0Yvo)D=?_eMQ|i%-CL$d(_37 zZap<}oybkdufPfLImg#S(#ksy3`G?Ya9rs^Ybd05z^=AN^W;($tyHG|C5D)Dq~T?R z1MvwD_z}m5=bXxGysmed```dW@vL*yvu|Vav(HOhM=9NW3`!FjrIbo5(0=dHO6Z;e4WD3zL-)aLz(qci99OfRiQ{1_FVt%l@b`TdvC+em89X=YFXyD&Pg}B1|c`vS(V76Pn|21T7=(zDU6WiT(RE*D*oHwgT6aYnajW%1cHUCv3C^1leeFm4wRbeV*o(uAZu-ODvRHtKvyj>kX*j|VJ}Vc6 zLOr%pFbDCPM^#tAp1_35V9@&DucA?Ho)ZkZiTCt>gy+`7Hp&Mia+LclxSv7R3O`@0 zLa-CsfS-4xk3*D;QZ0zjASO`(tZ-dH%8-%R(2dXVJnnJw14}_*e$C|nP?W6 z&9Px+E&v${b@`g>vA#$t_WoAIX@(MB2=IS3AwXESQD>^-&O4C{5_Wy&9vv~M={Diw z!DjXvnJa*vGK@MCx9X1l8tA8fU3||fUgM4Fdz3Btqqwzs|W53^Fvt*e|0WUEET^qp%ISoWr z`xp>;7o{j@P4OTXAnj3f+tQDn1xufzQvRYKUQ}rK`gDPHA6@h4r7*kBVScm0<@NX;)VMFp);X_ZkVhbi9r^cB>Vr>5?P%cT3{p!QaKZ-2+4WuoAY;)nN+ZO))*BZAU>rL6EeX20%&- zsAKnc+sCU~hh;3(i#~iY=^BOXi)i4{+=w1Lul$VeO~i-r&|8gn=RldRti)Jnl}!&+`y)Uqc5_Ni-#3`L;gPvg7VUG0 zJoYwWr%*52Jk^Wu`jrHGwe$-j#g zEPTqBAO*d178JO+_vqN&obirLD|j92uU8(?#`LCk>q0^oTm_U(yTioTJJvrBY`2xI z?41nFH7gzYNG2Y?pn}MvNNa4YrtU(=vET0R!$%d{4eJMsHfU{a%@VhStE`bvg`F7L zFC_~*gc1fs-TM>3^63I0(KW^wVbu5EUEks&eDhQ~@|JwMnL?zLb-HY1l&l;B>nAg0 zRo8{eo6>5G7S#%F+HPwuyYubjJ{J7fdg=!&Yf;LYn1eHasCp(uEgE|BYZg5kdbe;3 z!5KsGqfHbTkH}>kS7YKei@C8Wv_t`SbrO6p(ttp1` zDq^-1mZA)MdG7qwc>TrRMDLRPP5nfC8`cJ zk+|QhMha_sDw>r?smFFtoX#v8Y*}W-IK?3RNuTIf{B~{hGRJrc?=Q$SP?+pk8%fo- z_M;0<|1?}!(9L_{m*H(ISf+zT9xsru+8<>%ntsZsYZ7F50cSlwng_U}boq@6yihfy zYeTNfE)9gLE9%8Qau6l^COOEa5?YQjbXg@XEiB-AHZPs?!mmz{i6JcxKDW1*h^i78 zjbmtYH8<0^T%9qb{jNuv{1Kulr)C`YJ2uwIkNZfJX~NMe&K#LLre0g&Rph4XUHkbm z&6!*z`saQvS);b(rq6L7T!!dulj~9aqtjqDn0Vcu9~5A{i*toHAJ#!eP;TN8X?-Ou zX6|CM&a-5t;>c;^_T?gFA7e4}mqOmKhl< z?q;K1+szQQc>bBZsEu@MYFFoZ7a7m<7#$%1du7& z-oC>{$lQj?KfNwC+V$9*1ZNRVH0RX)nB?UAq=P*t3}CCaZwqwK@;FCG_r#{4!pEl5 zYcP7zBG0lS8V-GN?HGw?Uxf+2-(tj@4|Z$}-5HU$oA0$2_CBnHBGI4Do!&o9c^kt| zV|gvrktB;Tjswjmwjw~D{c87R`4E!69TMNSN9uB{*;y#yo5^?fN$g3KGMm*(E;8~- z=c|1DY^NqVIf)St@Vc^KiCad%u72hG_nrq5C8}2K zW$X&|i_&rMxoiYRC?fZ5nFDKD3T3{`{j?jV=}{bN@XKfagbps9A40jheD{?JUn_3G zd1oF=`}}xyT}bLU=01_b>`6^lJt;lRnD&TjK{~6Xst?<3#fj&(A~>y+Lmo;UYm`~C zn39|>l{w1b^2Cj$4WnG+eH$q&ZzsJ4+rRK)=GHzzwnu573a4W6*1iVk9)PW!9_k-0?x9wAVxI^uyVHZwjU8lxn!pUVyIzAhWn%_27u*skR&9 zsS>bd2l89*e>~0KyfxS-**|*tqVyKijq^Xg7nA9^4d<^T=}^`ER8a3X@b&H>MY!?2 z2%Dl`I&6ClrMvt#YAxiV#S5aNPXT5!+=36wuz6fq$_4_>%7*f(#r5(#<~JcHX#8zc zBX>}!*e6k_=Rr!()D4gDSm?~mgBXDjnHU%2cPV4|-!_uvdD*A`)P&8*R*Cn!#(LP^ zg70}=-R^(vc&V`au7|R;7}_Vw>Ac!|gEGr3;qzx(Tf@`)`y{oD3>riq7K~|@I95d+ zB*{oAeRd`yLHG;Ii)cbm(kv;zQ{Q}tT}>%7{m`9FIXFX+veLDGptJbD&@*7i5*+KS zMpaGye?^_Lg|NGDQeL!$EKXqoeCGSun9X7@PBika+OJl>TlXF1Ik6YDPyS|A=NYJ#VfptuL+bgf9g%BbR(DSeUAMy z*S{ZhUby3*v{8BG9GoNc?~XBxy#2HK?Nh@`1#zM~cRg9TzP03UrO$9MQVUrV-R}PK z=};l&=XCy>#ENs3cjku@5AkQvtiFGwIzL_t<&e48_P`@@^35^tcPhPZ`B#7&;OU>Op1#}e|NCetvH%(JTzXK#)TihVHEZ6 z_UpWLG7w%w&zTil@kfnhP47#$@7@)uX#2tO)O)b{S-&g?V#+}EiuEP&{jY8XW6)X4 zqYgxpBp=6MFI<2>7iS2sFt_1&$R{;gd_-6{bFE+I<4~~+T76tU+IK36vzvl6(3DBH z?#kYlPwx>au5a(~y}3SFAc2YLMkNI_g4;X;95hxmWSH9?Mq7z1Bjj?aT=$;vh6Sj1 zXQa!fPTL4n7Eg{uZok}&)r~A(uxO!XgMr~cpc|K6KSH-d6QnJV(n>~=jrUrXxCCYW zB2u)m=$D0U>OpRZ=*?gHxRVU6MQ@^)>NgJQGuM(P;Mno~^ml_gIoPT;ZyvmQecu?m zP?DFmW^IJ$_YFOdyZ>5{R$|*rQw4WdY2vl<2u=kWi}WrMLA8hZT&5)WTyxAf7$p@n zMyq3KLx1!{Z6puA4y@~tdqkonYja_MN9|3~;z&z5)&#@pH9xX724GhMOO{|5lsuc; z3koIkSNbZ?49`#W4Fd0@!2=w65%IE}S}?g}%!8VBzkbJ3<@SU<>9;(%Z5HmT|7bJ0 zqY}qw8v2f(RndF{ttH#wj`^ zRm63Fe(+Iqg2dl;zp+}OL+9Hy2thJ?e1r*k2TVDf`n+_-mAzRJ${m`@IS~P(MZM4f zGgP`?MxpH4HZ7G7o1H{rn6UN-7F+Hx4BB*M*WpttkJ&KAd75i?emdWUcR=VaH7A_| z?U3})6guJ!%wrb-$db7oG+m>aLzp1rL&=?N;bJK#1ji4z+!ga5$VPFDT4UVM>Z?AT$JcYyKbwg#JCdlwV7>&k@EB#Z z^>`#oURj78U#|gm_XEswqL-1I(X7o?M3~*@-+f&(t-E7UJkrpC~`!eu9a zzIrBCLlV|28;(h#nV75GtZNB9zsyM&!y;|%e?jMet%%$TdF0bCw&PPIWxUhoRE^&(8U`nX|I!%|7A zk6a@*KaIQ0a{Ei@1%DTr6GRnu`8LMV7bQ%Jnen}YB^p>i^gPPIcqBTHQN`naSC83G z$E_N0?B3SRI{Gz*R`~(%uWXcHf8*s1_`J=)aQy<{D631qNMot0iH>;c`%9LK8kX$# zoe$t&;U}u!k=dr`&cvxg&XcB#yw#~rx|oZMHY=(o-JyB3E!Oz@vl&g0k7vrPJjdu5 zl0I+2Fh)SBbg6^cA(RKYB%O+}dh!`U_#%JdE6r{JHoFuP`>K$sr4sg@2LX=K#cRtS zcM21cj}3bwa*o-Nkx+CgVGYvjz^OS=BXi%F%`wwWD+~K~@z_4>uwkFP;owoB%hDhj z!oFVYKPW_hr!yOX8=%QO)vs{1_Db%ndR_O8KO{)L^o@zhlf>8xHdpD)ABs^ibCIu) zf;l>)07^SrQ=NYkmtFTQWFhixN zSBb4sl-Y6hU&co(GHWofnDZi(y7p_df*m1>R`jrcT6C*E9f!%hdcAejaU}&7CoUZ$ zu2wLhi*;Gpc$oPk#(&x*&fE&dziZK>_9OPhmoT*NFYomi{$6_I_a|-1Veu-_rF$NJ zCI7lrO2#+-&4Z4SAA~QN6xPOyt&O5zRG&5_>7M=c`MEOp$V;S`wH9#vPNbk zg9!m<6$y>ZyorR1G4}9YX$yX%%cpC~wpTA4&W8Ccei_WK5?B+oYjfDB1>XgjonmSM?^FPgU<^;7+zA1Zb zdi*elMZS>=lMmQFYH^7AeeSclHj1zS37^ZDmx~%`OkL5Kfak7Qa zoU1h+Rl&!c5he`NCAmKx^0m(nR)>CTwv7f_NR%s@CaLkn61`K7vNH@?TbvUa9%$Rx z=fb@;`ipJbKTV(`a?)TFt}yb2;rJ2+-fwD7Bzo>hiBxnpi9`ADlPJ$KC!6U|6&tEn z`ra6?Ia)qD?lgqFX1nC}MGerlvT1w}6sb+Jhq!7KS1yy%6B4B_r|wUw0n=FCCT-D` zmwIWdO2v%7!at z;<5TyeGYcD9g95yvfjLhf6ATjbdDHpRk)6rgl~7?R8|U$Y=yP6O>-S#(bLaphD zGh{lt-;qo$YQdEk)ISQoAJXiR=*a&nr~lrd8|+mrH~- z*U)zDrGJZ|op(ph=VQyf&U04MG4ge1Tx@>Dn5oOJ=I0qfN~zKwl?}YQzEyR3^;}7s zpY(NXmrBBo1o*T=TsR&UHw94U^nG7vR}MhG|LwX;jM5&v(4{WFr6Y9=y_6zUK;03E zs@ci*@%=LXYdr+%HD*5lt#*bKWn-#y z-*Yi2*XMl=?6q@mU74dXJIyLb>U^6AQZi?ncu#nm&WF!)H2W}8b$QyH(12$VOD zX}<3fAW@D@3Nsufw^+4%=P;qT3LSe+&e4f@?g_0{^_!^aMG%Kqa zp~MgOP&Ur+4ue zZw7x~ki^y?cC39-HiJNiWOZL#;u|>>Pm_STN5+%Bc-D^TW~04Cn6ER0-t^=muRboI zjHFV=js{97EYffP91RN|=b&MgT@b$q_duYFsu38j&$%`pZCV?32j!RTk-->p+5E+Y z^^K~TEr996%0AjxoR<{%cxi|&vh8&~ap+5K+aKS>4KE(u4kR{7H+TRW(BZBey%oRY zplHll8!b`jbr(9+m2rz3?eL5Vfen%B4nnND;M|#Q)+`!e@hE9&4@fl_9rau!RwS$Q-XM zJ>RX#7-FVYN!MJnF7Z%^)BG*Irat}mF0MJ=cbHnL*&Gqlyn?44lxNx%2TdLy!QW8= z$&aI+cr+(?pc;nr<}D`bzk)(eKU-u4O7iV|(oVsgL`>b}a!TpG4*E#&!(a0PY_@b2 zX_2O4*;8#mE7*Mw&o`QpMDwvAicgqi6J=LMnnHN zcf;p)w<*z?CbUj(vCYnR2OAH4d8svnmrr zRWp6$)URLQ6sWt>T$T0NAbCcmG;ipjY7(c6TOvQ&)9Oa@!gIbnX${#|!@j(jVPwH6 zL-Pj3mNDc>br$r*kf4~EafejA5QkjGTqX8l@_0Twn?RL7baZYS6eR!HvLV!O;14kW z)_EudNYstXEOk7eI?)1KcUd_oeeY*1|Ad4T5XeFMI@)fv@ep|&2vQ97WrRxDrQfhx z@g`>co4q5qJfIjDd()h@pY?+fdoSt_k?hvPmMHeKbJ8`G?as>?q+fkkU6*kr@X!^( zNNuQh8IVf7Zuc@l=X;GAP(2XYNqkhwu@F$e4=PU5WMjW=>J@x~2{bItwK&h~f%~6) zccLYqv^MXT@US6Lv6Z2C`XrvfBO`VEoYNI6#7>7sdfoEy6{&9L-&VPt;~d}^NZ4+Z zDe1R%`&C@#r5SIIEI#g?F^`B+LI{6 zd4@>=r-Hr3cx_2ra)(0FRl@6=ohL%&)ybk$0J7&6R*hrIo>r)>D7c;7c89a)G%1e4 z^UTye)s+J7eKiTO?%ARV2hfRuIXck+kU$_h)$kB-D%Ms6bCmpL*BLdK~1x?jIwI~rHt z*INK|t@bHM=4GR}Y}=K3tHfA3y7KiqzeXG1vXE}4_a2rTNgk8PJx9bJc*Wt@*}E1_ zIVN9c=ju>S9bXX$FE*I6KWOC~WD%aMxim4((sTMu`|~Gh7K-U|&$X~_4K{J+`QyTm zhi8Rg@R~%5?gYN7?E1u^K)hD#WD}1l@s_x>Vzya>*IVKrJJSMB)wdqc<5$@R%2wFm zx+wz7HAQiPdfXB;fFM&cr5U->sd#mdJK1`U8_GnzqB+AXzxC3@CfGyV=-abkJkha` zFMa=EWroo+_frXtCKufI6~4+cJoHiUa$6lCKkIbPDLP-WwnR?&L)HP4yRs(U4iT9z zas(ndy?-c)p;|v?B_R3{e6rkiN{P~-5o=zazukK`H^XRlX{>&uA~5p03O`w|{Pec} za8euSxarTc&#-55ryze^XYZCvZA>xGQqIhMhE{?hh$zn2UHhi@lzd4hTWPrT(qbkw z;TL|2a(fC6wkjY{EBF|Nv2Gu-H)c>7`r%}Hy?Lsv&Sk}LRANVHhuQdmI{H4GZ(Lx3pNfLn$0*E& zlUOlYRW0p!`glqzu`TiC;;+phMpiCfc-TYt43xfhOHn(zz7l)?O3vd%X8F>`j`S!N zce^_Upb?jHD1Euv4A*j(ub3~)GIPnS$yEJr;cN+?;~E}4Y^d3h0*-fjlMujvrH4Yr zyd`~vLTyB^_m)Stu8r@K*HLPV=mr1E6Q88QD26EUyCKosrO(fg@@$0u z6p=);D=tPTXPT<67)e!_BkPE0!{C<+%C!^~&56cwR9E0xj%RQv57-{^TT2(J#;sa9bleWlXW3lKYTfaHUA(kH3;9!p5T zA~%f88=PRqS6#eW5jX4KX9u?mgBU}ei+V}P(O(SR>qEK15YOg~UNRPev2=XMb#?6P zsdxA->(4pgrr!NoHGj{feL&gPSfHfczb*(q6!Qm;=fyA(c4#@TtO%09rwb5GOlIuj zFKmSG*1B%qQU~FF%l!fOlvL)xi0^7RM?qcrFh<#Eq)p;r>vhK|#L2~rD88z9XF9^g zbt`JZq=xH=2sY_IKQDIdv*0&S;vDZY1xGrYrwmbcZ+%Qp;&`qT(9#-OXe+g3X6AgMTunt1pIJ zXsXEiz6RwMq?04A9MdeSENFH?5_9LF9q!PPD%UH zWJb}CMgQ)bC!}?A)+&Dw#Cgwd29R)0c?1BgJ>w&N71yKPQpyqFpR47F8e<9uss7Ky z`@H&Cfzn%=cCuZ?wZf#!A-?S%txpa_s>8L%?$)^;nfv_BPUEqHvk^^KN>lyj1FtQ@ zy$Mb6Ut=|B&S;6oAK+cP7rw@OL*?Be<{RQBz$ee<@0Wq-O*`ERnpip@;r*_N^>#G- z$y=s3y+jr_a(+_f_V)19WBXlcE=6T-`Y#pN01cp41m$akf->z_Fp@aG_bjB;|C#9f zdg0yrs{Gr{k5%>THtS}fQm;D%?47%VO-W-@TqHD=ON_2{odE@Lur%hkDU&x(W%pVaiHJ~ z7ygQICC4rM;IJ0HV=vFapBYXHA2xtRo-LHxtOG?ueVj5o^g371X^`tF&t$K}WoxeH zFK})SRRO0=b0u_N56+nfT|SBQA>8izyM4u*d>R{xM^q4Ai9lGst1eH>+bD5ii zghOEx$(dc3Vk>f9v_Cwl=M%gtk&|Lle#OfFCqm$Ya>;W~@ooy>+YecnldNb+1-4c4 zIQ|e$Vw*Fi&d*b0bFqE7ejR@|Vww?lEO@oyN9+5S1k!TV=g)#4-A&To(#Dr4LjVqI zx(*vMU+#AeCdRb(<<xyq$V-j8}j^U5b`k$vX- z)IjuN)Z7@Q?v=rF;@xG-mW?hUR(Krn=W-}3|9i^d#~{* zAx%(~h8wQl+ySKT`w(juNFa{w(a~T>TK_ujR|r_>+P1mkeQvF+84Ycrgc~oybLYN; zAQ{Ald;-)1@m&LGf_@gWrKZ>za*EBi*io>)B%*FxBHHX?;He2jq5CO_SfmJtkNlY1 zr*GeC<4wdPt()mOJZ-N3F31k}$!krY5Wf`e@=rvy{d^-x>?8W~%F)yC{j8{^wpq;f zPb$Aw5OGWpwBeKh2R}MK`ziZGoNI^vPSoOdg{Pa$>>qIM87?z@C=?ZZfvg_0rN0Q1 z8u5db2{ie`)PY+!sn;`pqvh`gA2}SNBsK>Q6ceK#5({1}KmxAK?8zMxDPx-vfLC;bVb(rbAs6?u8vTuUd+MjfX2E={F^ zLPn$S5v9+s`7|L7djdgnJV5`r8(^kow>eP?^fzy1U^vts26JBP^^E2O50sMeM9rN@JX?|?EaOFSMuT3giE(^C=OKIy+!C1|DYL7!ed(HM2a9BC}|W|qGI`XzRiLHi*Pn(Hz53jw`$SwsGWFa7@j^q*n60Bi(nyODp!&A@_)?rLAv0taA_FaR@0Zb8sV&S931fcL%rDXe+N=;6;)k zvv`{1kT^Y*m_$iW%a)#&%64z#1~V38e83sC)I@WFG)H8B(GXgw$z>PHrb-$Ck;LY* z0l*Mk5sL+F$#9FYCNeElAdD!Z>zY=FY`46A_X-*Ofo||i?RO`PhJvR+GXQZW@}Y5; z!Vb$4@>=S}w{Grb$zHm+Z<0IR)0mKwqnpt-CQARF@n$Cl=^}36((^dj#0QMM)6y|L z*Oe*~Ft%{E3#MRqEX`LUKG8!o}+42pd_;Q zDnXh7B!;UCJJT5}Bhcp5^)BHe8nLQA8%JrpEKM{gNdkHNXO4W0&6zAYb-`EFbX>jA zD)x8dWhzGK9_Xk+K1!d<9^qEIy$%oZp_V|Bm6K`ois_6RvigNu*q|kalC}dWX@|+y z^%wv5xrV-dAKu~7UoYOCLGRbFTH8qh2`??=WuvFYD|(v7V*4C&@J6ffXecJ1g*WW5 zWyh-FEgiHN(cmO$25`b3Puw_>A;Bwps~(N?D&^3RV`RwWSBSZS7yEQ{D$^KUR%+S0 zM3R6%AVy@7!MR^Kb;t78t+@SUKTruWUl>ZOPltD2UGeB1eZH5{nt&bhziTCRycs;N zBsN|dtF;s*u1_4(W3M2P0Tj^fKJZt!b78Hi=68mF!zwU&9N9)!I=ahj9ul#?H8?RK z1wvjY%1A*FE2E3qBOIOD;7UX6ltPjYcc4QMXUG%?Xaz+oI!vL!1Hz%=_gKLPj13oB_*0onZ1;3ly5AE;P5+* z1hUK0mh+wl4oQ!~^6o}?J<>1LFr8{}sK%o{?c`U4v$ixw3I|0_QA@h1OAMfd^yKV; z7K`-9Us^}R^liwThtZTACS?p}qMP!E5JiS#jjPP>P8IS1)YPw>`>tpKK)9nEb?Jvj zn11-@InktP?Rz`b_Nse&Fx&D&89kFEumqRl3ztBo6E!XmK zVU4SD+hFS3doSc3TS%tz{&z1}#}=L+&oDo_t` z6|_MXpk@%x*y=Ie?(bb}gSA-a119(1y>lm2G4}l<(^tX9Gg-L>p0DQi$H4-2?29)$ zOXMFmFPiFkLQk~r7^_m2@#;6(CRy=&m$#?71PGx|!kXCz2kwW764=cHC8It&avgz_ zvAYb`BCnT_06U!kn;6{W!~7CC_tq@xrm)BOaZ7$OJ4jkat=YQF&z#%V+>8!>eROX1 zjQt=4i}Kgwoz{eijEC2~kEp_UG{^|Gf=_k>3MAh1ovWF{0L2Sksozwd8%)UQhXGrq zic(hk-}#^u$*-anHIYNUy#iW0XsDq#gO73mrny?z!p*McreW6WH&02Oxo_ff^8DFy zo3hakI%7K!dAyPU&!+AsjZ5YXt) zFsDrqLVm_+J~-p3 zhs0X{YN85cNfInufu#B{Qk<>8oQFD!F%x#h`<`=W{AqimvR&&7P}&UeEmDL+nk^0K zVRW|7E~7Z}HztKG9gD4l?F_#pkZfE^G&|78jU|V#j}EuD82eu^os5brY+Mk6YIvKG zJXgJ_F`T~^Jhy+wlqBp6pY+wcv&ES+Lqo+o<>TT5_g51tk1D}>#Uo%N^|%&3;=%*Gp-SWQvV`6aq_f3==O|4GhTO9Xiwo{Y&uk^A{4D>qQVAoqvO59h&aMsnnB zCGdc8Z%$tGd+=w8EjqU9VphGBr6;j302YYkv+)P_#F$-duF3$s2YL-&jj69NsAcOc zoyMk_dZ9cx*M@e%>@LtkE8L>fp@`!ERi%_O$OIOGV^mmpke}wxudUB!#k%~rJ|Y>p zcvHH6<1-&O|7q^1nc{{`&A41_cf7AML!+b_qd${R8$L9ub0JgCA%}y@!mVo69i$|4 z-MT4*P?>a92Na{J+h0xLiUPaY$7^G$Hl6iDX`jprR7d;mX5$Q!)JvN56x})XYl|2J zE8s2)n;6CYhI-aKiA`rYhMFXAe?aEpGwI~A8I86LpQh{7J8163V6lHfBnQt&T_rfh zF?(>LqCLIc@ns+i9lnOrmyBsC2+InY+wYUi9-5?tpolANrG}%mwl_T_q9Crt)=W6! zZwdk86Ic)1tIZS`XtDTp(t+xp1#+{8E-NmY?;n7kLZY5#vh-AGyI<`BEj`ePGe#A@MUtrdsN3`u@^WxwZQ%E z&HqosgcQ;hcC#mVxA0!nBhiG0Vh{0#=eNX#z<>Efv=zjgbK?!N=luhM<8*v{fxcS^ zP}}_X0^=mnl&L{`zu##I0qdk6ALH}FO=w8g+!yDZAB!_D7#IFvu*-m#g0US+GTg-< zjI8nv$D_?Sf9+4?uc?aC=ul`@=A?MW-@~u6KhG=hUE89y1KH@CzM*k~Em|XNJn5UU^44kG3Jp!@eik z9)Dg3aUwkAQ%dJJI@~tx@L+}nC~9Wp>RH2>I@&WQFtIpqMR1M-P3kXQN~Xge=Sp)| zbAkpB84y=yR4?64{f|Tr_53M$?m=<3*RdPcWnMH+K#3Z;z#k* zBu$?Tb+6cd^WRClpbzPCbA81SsA_*{@CPB3@Z#MN>N!0+R3GRn-ik?XZ5|B?J?$V+ zcp3hYLK5L(M`>0J5>p?SI7}A!A>?R_;donhTG${6sa>8JWqpO&@P$kAfBF@>WAoe0 zz=;rcC`b1kD}KiRq9A4~0DmT{mn6Dc;&A4`ms@e+!fA&yC;K}ayxJLlj`=ao%3{}| zwi~kYTe7o%6lFAK^j;3%M$7e`S`}ffKx4#j;brlr(-2fv`g{uR7D->6e0oy;GhrE9byO5{y75F?z#O1yxp zA+A``kn*4Ft&|{(<5&~0FIXAY_FW^+v|n;-{L}eb;M|YplK_%RjM)<_^U7pu2h!MoYWv9r=psN9nZo1Nc`{N_ZGr78O+i2N(!M zDr$FT-_5~!gRAXZOuCwCV*kU_m&a4Reea(}jdVo_QIsMykalIpL=iZv)_BKz1Fjy^{l-cuDw5Rj*@+z zUdh(Af*cW4t~f|ODYe5$(h<(2-sCpWl8e`msj2H7<$8yNBBv-d)07{c+2y}wN<4~e zn7gvg_xS;ENju$m)U%m-%U?@9;r70u(v=J5dXM1vnK87BA{Ov67bF+_-{M^H7?T_u zSsG0zHRuy@U!NeFT?+E}lKzj=JyrN~)k3uL?`>0^wHooD?abuIVwikB=PhttG(31N zR&^(O^z(7ST&0VKpwgMU^mm!fF9khKmd#W{pe<&v6ucUUT%)JPZjZk@o>0Ig4(nB2 zBxf+{H(C%EH4Wrv>9E{)YZ1YClO2-DLCK zMp|X#OF@yVJrD0Ghjkn>biY818vBSD#%7kzL(=%gh(By>9*c5)l?6?CFlMgz9rYkyvh0V?I9(+J2BGM=N4#AKtT1muCiR3 z=hJPVmj*Dwk8+$devO~$*p|uJZ8!7$rn6SJx z$`y4Z`oJ?907t4L3pG#Z42g8CPrq{-#~TEu84Y#nX3b3D5jxFCv7J@l^}1nn+&F4* zyG#Lgu-g*YfT&KZz8~74sLab+@VE|M|CP1OYWe!o`!@c0j+Xak?ult50&KW<^mSnv z%Qf-BS}I7h^f(3&Z)-wt5>(W~ECducjL??aQVUBrQ*RT~APb18AHxI#y$=QX(6Z5H z8lQ%ZzQkRl%iH@`6|XH9F0sPG>-DkSIyT{4y3>$hZcp#4DDqE*$?7%yP5uaw{au2J z2n_QC8c;kl#bPIaeE8{?Y{oi#Bx`NFhhRDP(S!#q@Vrq2;KS@$oweD-SL$G@CZLg* z{}RP>W+$dT9A=vuYWwY#I^K(Ew(&S72)Lgo8ZI8lr|^XEknaMC(ijjpe1z|de#tEFCaKtW zzObtllc}lu+laS365NKNdcl{hlPqMr@k;f^m^e!7>g&7XY8*Th9?U3z5vqo{B3WS|8)mJd)GD zKevOcBr@fW9n%a!@^|wjU`K)))wirv%2;vI3085x#{E0Qf$078hbpZy>^5bCt)38Q zqzT;9FvWfzfJU0S3fQC%UC;CL9({ts?Qr7Fr*AYFec->O7nZHgdCdG|+W;H??=@NB z6(nL70y_txqao`+yBO*>-K(edU%i!2z8^M)${p#&2D4iRiNhZBUjRckH*>B?2XXmM zl5X!4N)frDGB%=uyCTNaXrgO>EXU);kg%%{t6<^fDgsQtW4#92?Tpy{Ur%&x#$`Il zbPJT}bW}+|yOnyv>Ybz2b@`oUxu<~evsNxJ9Z2=4)GyQY%ov`b1@-=u?^SrEX#$0c z{Tw=|{MPyk?lAe}+QqQxZjC?dM^N3#e)aqeH=9%w%b^(!6hGeto`Q9>@w?FPY3*vn5ShA9U=%1UYM;BIo&^-fz9yrq zIUfRjKup1-6qbFE+LdPxWR)zZcj3XsN(l(X>OYaOO&A1bdHyD>C(-<9fe}r6v$tw- zrv~ims6a#dEUaFW{#f5zyVlaO)mb%`s+^(yU)=Qi{2zlk_Xhs~Ut(E#b7i8w<9YVL z_k{*~c1;HN6luF`y8t#9ly!$nd}fS_!bR9k^Gp@XouNhEF!U|Cq_u5#X!gS1U5@jI zdIxrPHh5^-Qz>z;ZYk_x{pNlJ;^&Kd@`Uj`m%$%H2FA{@*ioPZU7_OAmsXgMR-b>< z@I1dabV?N-FN0>6N}M`cR_5)|eN$37<=xx-3e3u^j`Bd=EWF~myrDRaQxub;WYi={ zefVC!IurEkeuR=Kg1siOg@OmrM0NA5(}g2?XJ3Xvf1p6&BSh)3yhW63x^@aK%K)9{ zZezfLd~y+5o0lV=S{FY}tlZ+}`I7kQaXkU)2dfS;4wur+VJNxzVN0Yr~&%oginR3n}|TQdg85xT72DK z0++exjH{WErkjh4QrK<6_TIx=)HGf1t^*Mzv>Jym`~=j{{vwS>kGUE22qP?k^|4x$ zugn_mN4p9UD*>PEUi0oOCH3o`z|2> z{jA|KEh%&)@9MD+eRt0B45xocyzFsj&*1B!&xgZfhiA`s=dyBOFXtI3{fTy$miaSo zSUTQ3@-L%WmKN{6HTkX854OZXBTPwG_KpXta%Iu)vRQWi8+^vy51|EOK8IF! zNiMVzlQpASGPw}!O23_Xi>FD!Ww-7=YPiO-ScEY+p)JJ@pc={{+I>&UP;(YuBgj6r zekrSIJIG$Y|3r#$ru5rsyXHKu^`fDDF_k-EsQ8LRP-w;HHX5-r{+V;h;iomF+HhrjnJt>=5bL4oRauRLAA_ZN8=iED6S*(d1_%T!gT=6or+2d$c~iFM?C+1T76o|I|{ z(BvZ2?~GF%bg#;t(?)Q(7x{1Wi@Db`ZUfa6yz=DuuEtUVq4am~cVRM=uo44ng+~tv zYlk0{JPnCZiN#RqPSUW?ejc{$xh+C)Im<2He^OG?H)Sd=M9pns%Xep2O{~HDt{pkz zy)8@9FxcXE58yfk*9Jzq4&2ydGkWcB^`?CvkUysk#bEqo^|6tew5?%!jb)fW7jrg<5SPMVQE(j!v18Fk?_clug{u z?hZ#{JMM~G2&D{_nYI`Y?F%4_DI7L%Y1W9*R-3EGtQJolZ55eZv6|Z6sf;PS&J4dRGo6o$ydf()Sle?NMD%Jb~c7Qm}?{1$H?t`%J;=U-DeEv zPBVL14tmimlBGCDRj^{)JQCXQIs&p%%SnKc<)QDrBmdYt5G0zV4*k`&T?| zKbO_F2l26t&UIab!@-hw^WPt_3%7EU_w_}H#No)_jS*M420LeV5(u?>$HMb477=P@ zu!p5$p4w~kFI4RqOQkUu_IWPpMFYp!8j?P={WlW>hhk@*;xMrA{q%7SP$^rVRgwR$ zB5Lbtykg7)^Ggc;j&b3tNK;AjcbK@JFMHOD3j?Cr6!NWq2-RZ=h z!+E|#dX17X)fB|MHPf(|H*@+8EB&@CDHY?vfiqJpSJA4hwqn1SoS)*Ra2u&Vd{`e) zwEcS6!P65Fw-hxAi>R0l6dj2rU(_;)Um7!bP)bRK}$H`2{3*%)sIUr&6Kq^Uman z8*rNIs}6k|8>uxZisp7Dz}a*cc#+>-8U~9~32Y-zYm9rFzT^ay&^)p}K7?1wOK%PN%NiKUYtERv>snB7q_C~LJ2}!l6Qk=oG3K&1pU&E5!$BI9 z(ZG;bb^s?rQT{A^>uc0q9NfS+5!dW!XsO|WO>U;~s@y}H7Vwp`=zsA=v=Zi;N=AaZ zVxLqFS4+Y5Q|U|6gws$S4S&P=4c>y7)++3Y#gnd zyOQlP#=;>``g2}imKU`@L9e^(T}E0`Yi6`+OnQ1nnI+Ci@UqBKR@qn9)QRVbTL(YKJNwt9-7>1%gy*!|3$Slyq`8N92IclHv(3Gbf0%n@U54tv%h#1j zWC4HfZLR|LHx*@z;HmGp5oThR;;NE!_V>nwKh`K9MTBZ{lipd$$pnfNzcB00GAB(i zDaNMuD|@~Dtc4hVL6&ZmixI=Zt#f}rzSuf6;buSNj4{=Mfc6@Fj=O5kl|wI9 zK?hRvy2F1dB~wOxcZkC6ui(*?TYPVvlxsV!k zHF@!mlBtGJH|}Ze@_bWgmX*b8X7FTT3AU<=YpSBms!o_CJ`bQ7$;>hE$~f@<|UFm7K~xEnvRbu}kEXRLl)ug|59FoLKg5ewTEzemcVF z4c|~VNhnEly|VK|vQ1~{*FZtUrER)h%Z|QojpntEpq5+Qb8-QDal=Qe_3NB^=?7-U zI;mmcVacojXYPPIY2GZg@5+*6357fV5f7g}iT^Q^d!K}ly;W;{GJ6;LZE;%_ajt{C zlKJzG|ETl^r(t+;55gbA0hsK$^$-@}RC%>CFM81W=5ZG zCh3u^B)p)V)m0n!Eo+Wm)(p!vDX2Zrk;m^TT+>)Nb=Z+Uy;|GvvXIpz)2ocAd|ZMufEY7 zAP_~jLkfzR!dD$5l5}q%vmaK~xQ+e89hW`h`i@)**RVc6P?D?=omwGFrl>zor)$yo z+n?noNW<*!c4fXq5+>ABm(Rk9mS0~NV}lgBh%a8yJ#-HN6xBApt)p#gMlYgOnOgci z4zo>GwadLRw4Z&r($tOMWs7vd+O3>i7kY8m$ zJuskEq}qBx22eu$a{Vs7bb@S%;FST!!?aOralIiH&ZropeV0L)mT%3JL13BNX3^kQ z`JnLbEs9Upwz~`1ab_ zbWE^^O);Znc*2uj(}eiOP4gSnsIQ4`7mYowA2Y+aF6OWq2|aG5^~9r$L5Vcn=9NRv zJI_><*Joqc8s2p{Pn(oUYU95!4(-Vykss1-=xWG(9D=g$5l* zP%bQIE?yt`D2~kb*|5*$$9=M)2n8h&q&H3vhgzT(h8q)Jn`d#vDd5eGa23PC1UY zIP{G!_A0$5m}~B7x*Qr=Ji>ltmp_0-Xj=Y1JPWRBY z`gAkFkUwt2TTy5o?KK{O7gWWr-$&gx#C3OO4x7Bi_>m4=FHVzQ8MAuowmRB{^E(99 zIi(2Cp*ITfwDQ9+<8J^8;4{RzX}5KT4O01;-yN_T&5^OkCQAw7rj6IjJ61x)oOaPJ z3*)}5Z6n7?4BRi%$b?2g(RTUKN87=J!h_UJds_CgXW3AwZ&DO_o4$^$HGwkV;udX* z!wt5dGpewO|616Wx)iCMfUD4Ft@q+Oc*WM*6K`=wne&?;4hc&@kebA-M$e|bEuCa$ z5Quig%C^7#o~4mLfD|Z7a3gZ?r%?S2a~Uo*_rHaU%5)IGDJvCZsrT63>Q?Bs`z$FT0ZJk#OzQLQ~ePmc1UxZr#2dO;6- z`E+@jy>j&{3)x)mqzvWe{0>;igW>F>zKnNuowttdj}I0-P1Z*vAtHAq*6)eOjjsE@ z(?yVzWWQu72{`{+n` zS73E> zKRD+(I;a8qzke*vPb$MPMyP$yblAm7x0^o6&8hBHHJWoZ?Jec&5|$^AA9O9Z-^{B# zBm@|={-7qE8ZQWcin#uRkMxHt{c8sWZ{(YN^7+ufY#fEo>O5|o^)3ax%OS7YH1CGy z=(I@9yme}Wn%4r8zj3VRii$K7d18c-o6(&6_C!@%?ao%*zz6v;^2B0E3_}2 zWEHh`VXiPy2{%*u(`^o49%?1LaNc+VP-Uyi+}j`c=)Sf2&&;~YO6e8;iAyH{1x}e) zl-}C-9(6EYyRaHLt8*_k*Lm#Rr04xHs@W@b(DuYgM9| z`7vAAyEVi)YmRQ%7JWHMQ{n%LG8l5aIEF9#UTvtV|AaU4E$&5b!_E69Cp9KI~icA;!&+{uXWtuh) zYAxx!l}cg~(p)kVJNy@t<{N&&K_e}&CF!}pTUdQ<1ECs0zL=qCi&LwWGi-*JA5aMg zN4(Mj*!`Wbz2=O;MzBVo@Wzm-1v(sGmKp0`ZRv%@K=A9_{Dv>SD+!_QH!YvZZ^DC} zPKpC34sF`ju3e}>TXUMsRNC+Ii-NPP@VjgA3o@Fl))3xcBI(3SLG<%&DoU7*R?$gy z$#v@3w#^ZBdq6W&GpJ=X?4zJNETSpW+NHbA3J2;GlErB~>L6%L+=l@DupQl#illUj zKY9|ZFH-tGy8aU`410K6|J=JHD7SA9g;j^i5d8i%1$=1DL?2{(IWK~AI!hGPh<14{ zQs*PeHA5q1F#*n2*^ZP;9H*sP2)l&2#14Rvz_%*jPuw3#%nyarMTrjB_|Z`~T8F@Y zZop=KE89CqETRTk*7eW-`Z21ve3dgdD>LndQo2l2gjO&J7y7wi)y!*!Wx)kxuLY(* z6UK%kV ztYljyz>`M4f|(Pja$WemoqygTQGps=M_|QSz3fn}i18KmPXiEUG`ByzkTV<9!K5r6 zXhuqJCYHDhezzDhUa1HamJh2r=ojwR0MJ~q`I3sYm#KgMlfy#PMW-L+k`PND9X*3` z!U@zE3krulesvOlWLN_T#oId_|IF%b1=g&5R!wxC!2#}}Ki`FPM0m++MA)jxkJvTO z4CnIM;xHDxvWoxpFG?Nd?gY>JAhdHi5cr8~VH{g7T;FAyz+O3e+a4v(bmg2A zbKBlTT)vUFoEq z_4*6j94oxMgt93<#aWT*`s&5=zg7qb2X|s94IsS#>+wxV_y)i>s6hTkVmOqt|Wq-i{Ua`Y#nxx(q*3=e%Ep z=|k3Va7q4On(Y$ENYML@Q%r(R&dJ=|#=wDZc~v_Bc0rs7%9@@_o6@)+M|8NMI+EZHj0twJx3HnNghEEYQlFZqUR=QJ=7+?0d%q3`#ZG z;iYZA4=ewQ!+F%CWnJ%V6oRC}VD?CBtjiqK0 zMauX5A}*_gH(%RhN~V@BI)%p4Q*9roM=d8wwmnW5DFXk4Dn9)r-V_4ysKyv`8<10l zgJR{ICP4p$KdzS|d3myz-=@N9_(UFwL25qt#N1PWs1>I&)a3X%>&}Ol$508&U#qub zS{p&0f-1hdHZNxwP8`)p_kL9Sv${b;zHjw z{3s?ewcEf~(16&OmZ&qZUjCPb#7Fd>0@i5Xqu^~M2H=|T=eKd( zypSuRTZ9?#TBNRRcTXUq+QbW7->xlz314Jc-hhVPKW{ zbrtbaD9hhZT>1kn-$R)IPI(Z&+wR_!%gjPf_``ZKWK+Dp5ehX7EYT^xZ7-6Th}SC7 zue0-RnCvPw3U(9@xivRZ-v>G=YBL1WfBq+Fz`qgEbw!wMU0;6m*k8awxR}i zsYgc|{W^{CpoOqjAwK4mL-PX@BVnSbAuoa<6lyNM*NaeQfryMis>blegnp(-}abA4qwwfEt)IUaPB$=1mUO7K-=o#siiXXlIAU8)fgaG)y&B zrsrORPda@MHQYF=_qAK`?@4)dY1?O2aAD8>vObVblGzFI&@#_b;daQ^%!lcV+Ais6JmAWRc|P}gje1vJvkmULu(ptt3?F&OGcSx^2kFTds~0YpvixPxwIXfHBr9(La87G zWcAq6&7q7wiK&@m5%LP+M^N-54IOsY{~MYIr|^O>;5j&!7g2^)P26wOy%S6svg8g3 zHK|+!oz>5HPXyP+zgS;0sd85t|Jgk6JYzqs?VFy4h|IRw!ggVyw(Xr)+>@{2we2(e zFTMZft+`4GE1&;mkq~;hZ;(IGlL2XG;eQ7SoP3v&IWazL$I`8oEP|E>wmZswkU6KmziMf2I zKlppUni>NfX=wBFWM6XTtq1pLAoQhAM2m<2d^b4*oda-`v0FM{_kU`iPiaU!I8^#H zedj5UJUJ3SlUrPvYG4s5g9V4KnqCM7sI73uv9Yu@%41b6=Ku}YZ=D-zzcRE&c@!m{ z_}L!Tzic;&H=#}%4265mw%6wvs+Gm&Q>zTpe%VF2^CKaCZ!9EdTNe&ITBfeEOiOG!W?-Ayz{oT5f}n&wf3^?6jzY z(3*I`5x|YkOVWuE9CJGS-urCek~;Q<`jk0D0+&Dc0gsa0Z1u$xI(C@oDJJ(T8+QCI z*~AqKish#g>ssTOUT(zro<+Et879tQq?|6rNzvwxbsnLNCTCV^ z$;11c3l8Bk>|!GD_kvSb*rc%QA#f~~lIMf6o9 z)W%=%8cr>VZJu62@b)Q$1HCS|mt*^2~k)@ke1H-g!s7aLk9>=94j%Ja{xUvx=?!5GCnO z7G&$=p{7TeXdej4T;UDgbiL3eGk2u`Zk0f(@qLs$Q~pUDh8*sC{4Vx=I6p0=xwK(C zWcGFkf<#}(3Dxl!Gx}#ByO*yxI=|RU%hJ9J8Jz&Kx zRD!TP87a6_ySDh%Ip&d5PahhGCfZkee=`zZpjZ^tXr8qd+c}FZd}X^aI(P{R@r~#g zP~+3&M%+fmnvFK~S_|$O%uUgg>+<@r7GN;1fs3ES53ido0C*_u^SM)FJUU-854=_7 z?qt;W+p|ZIQ^R$I0{f>W9`gn%X9_XlOF2JBo-Kr2Y3pfFH9DcQhvLnVK9>e$XMttIi2@?70CSgWVqp?ua zB=vHNhjn&0HV6hXx>Y@K$WCq%bPq~mGNCNHWz9Y(!F@2VZ(b0L9R zwr{lFob~X*XMQPhRffh7I|<6*aaWSm*9)NY^X2D!)IU9#&}-{S9q&_O?*LgrEwDd5 z;Muz0L(keK6o|}BIda>lW8+4{yTF-EoC%%^$ z$2rg{cWsN4+ltAkf!9bXP2GUPMDe0ugE8g(vk1Bj$*k$%bP$k3r^XasZDnel1y8u0 zIRn0!PHG01Q@(fDw(&KEbPld^UNXJxNH#A0mKkT~nr4Kjk}2AI1Lt5`bm%Ove>$ml z7J~^Xx);#bD9E(nL=V}%VcUXbcqNUor&E1{2GFj2I{N^t84<;Y^&HkB#eRgVjeaNe zrZ&9o4HYFL2h_Np{9#{{g8)EzED%@R*_e2a3jr=&5s-jr(T)|vjm zFr?z!S-bOv?h9kh00(NA04cJ9CIu|LbXvzDZyeJo0R%IwgkQWN zb@!5e+7k~+J?>`sVrKN>B?WpqM&OMCjEVtXH;L`f9otkXLYzszsbM@vPJeSW-DCX| zI{h5PV`=-`n}OpCB@+m0lHBWG9=P@ltw zGtF* zyS?=LXw^(jB%XLJeB1+=LuzqjZZJ(qoEaJAoWSq2AFAl23+hy2);5l*hfQ^bRAL2} zoH!hDKmt0(WkCjwAPhN`=TgU!;prjSV^0*)#1SRHFLd0s=rv7QO9vYg{e3g~CR*9= zK`i{srx(0FP;+S{f-?uEo? z4g0PJaq9Kx-;PD2>aVlJF?5zkKG=@OMJIgxne^6l-A`J@uuvm6ACIh8Joul>0#JEUV`zO3ZqzJ3 zXEZcl9aaujC=ST8E(#&{Ae%d_cVhU}=jw?E4=E;vhfyS6Ofbq)I^;AyMF^xSgV@_2ar`re*Li$FC%2elg{2vfo2Z<6! zRzHq{i%g<1rTq(za*ySirWR;9Z0?Yl7zi7gU*`I(+}^RTlVl~ss!*gU{Xz<5wLf#g zAyD7PiU8f>`ff9fMZ3si*VoTGY1H~R4eS>O?T4|TLjXtMAQKQLZSWh%Q}1vhUNCpC$l56LlC8=;6H+Liu{=hbmAiLxyPf`F{(Jexe zMaU{5UY5{ISA!(zj)DoK+|Z>g@nqZMK93rnRWdgl(1RVtZC$qCJBP4q{Z~|hrOqmi zB@WOtG!XX|q}mHzR*abRWd{mfUr4}57^*0<=iD=z1J4>wekfd6sHCysx~RZlE>^;L z;}qZ3S5e46~7$Ar3NMM;DGzMmlZKZ4HTKOpEj2=Nt_O2YQ4r`ASs2db0; z>zcNnGGP8?>^aX>>_^NiOx}LADB7%)eiEOVm9MjOYe-2Yb_BSIYV3bM#Y35pwoy1L zFR?L3>G2h!EGZD;AYcOR5w+y$dv1sGXNNXud#vN5K^BLJD9f;(ryEt4&XbXTbz-mj8|^d z*%UE#W;c~+^yU6w7-2q#s@|ye0HgYQaHZmMnCx6`j~^^&2ljXvwF7Wjs@R}rx3VMw zsQ8XvRVXEyS3Km8T=IzzGU@6 zQ<7w8>`}C*(?&-l`m;QC46#`U1&u%P*9EVqD5axukm}OK&i<%VdR;cXo#hNnjw+Nk zB`=Z!+5E>R9%-c*L05MOGi+l_YwUPX{y@wYLR_+5IuGo8b{vK;m>AbwTX`Dr3UYw! zGt;b>-+oY$Mrh6S2Z418y*DB3qPC0)8jw=Jfl@p(C9UhMe~)9cOfJQ!31aat>#AdI5kR^klTg4;(xHe2}@ylG%Y3b z>y?qdIFX>-~R%1}Eat4P_WlHJIZv00rA^_4<+%#4AFRGrt8Dky9d7Fw9!IG^H| z%xsezhL6%cFniFMJs1FaYBdln5>F;9E!S`MhIitlV^jn70_8W3qeowl;kRK$RkKfV z0cN@6`{StC-$u6Jm(*(*>WumKh}F)O+aCDq2yI`5O+lJ}_1JFPI>Oz_%t}<}1nHZA z?F1nj{vs5{p4eo|V};?@07X_GMdhB5Ot|q!0%4jQKH_hS&CKHwL7nlu0ft8RKvP|o z+`Wqgd5V)2RPa9e8+9ul$d-Jd>yH|6RLOL9*y)UzHjFm`VUoW$OtBC0>Cro+46Qo& zW~Q7ve!wy?B(t?>=;u19WyLBEEw0>Nq(#<|zNoQZC6VG+=+T@qPI0wb^D#_=kbA3V z+@9q;o?{bZfo11{M9fK+3W~>ojXwO(96upi<`mquW89tm4jrfGfNzk%Sad)83#23s zCuFcG4r9Dgw+d$;-@_MW0X8( z1X5p=Wh@xF2*Emrq49p7VronKMFie|l`!%p0UqV7EY;fn9+w(p_My2&#i7hEKfY&| z(O~eU2q)tI{+)za*pknxh$2A-^H-$#tjFL*9HM&<3_Q8=_(}XfAX!H7iv^v8X5X!Q z_e^_tY?LnGK1SC1NUkIt<(}`yf=E*OZHBW$=>`_z**9>BhsREM{O+?ty7^UyLWdP2 zeU59!Bjl*!Xzz$D!o;tUhenEZPVqsLnQ{5wk&68}{WplBQ?E?2?FDXWKO6WUAZ0{z zgNp7-O7pjaVp4=}pAB>)VLQ>ge=;NTf`cR11JglFZh)!s+)!pW^Z({BY6~g`uh?9! zOQxE9R)#w|Um*@Wh(y0$%nD-yr6Z_sSX$8( zqx7olLu;;dW$G^1`r0BlQ=BqeLkcSuRI}GBiYk9;c`=HED;NgQ)&6$Iyn?H0zo%Ox zy5=Q#r1|$pkjgE}{L9j|i{(rs4O(&UGH?fz zbBQWCck^b`5Xi${#C0hzI1~3s3<^O)^i75yZ9RNKU`nEL;6dw1w?5?3)zLBr=|BNg zalZ%xbbQ-c#w%>wBa<`VHTs$rqJeMZr!{wvPi|a~=d1jEY=|DI1r5J2qsN@>!cPsn zAVfC`>MO&&RyvpeMPdmmm(H~zKIn#23Uk}Ru)@-)F&<&|qX3)t7UB<=T4MUdb5N}#HjpEdDZxhqh+PjD85+!I z?;4$xWMqtKA9n>h5I?H zoE3D)k|^+4q-1FTpru+M8O{we%?FS{d41z&XrRqR*33F6-{8(?qrzBr*U#chL(cXI z@tF(L_y1IkFtR1i$XdZ2*6Zus+i9i9lV-#Ju_rdDY&*lAYP#zbrM~s$z6SG|FRk9X zEcF60_~F@WWVj0=V?ydiUEo(?-y6)%RscJ@t1^wTSSRLq%<8Vm+lL_Dcn(q>oif>- z;4w?JMC9*w!HP}&gLvKSI^t2bIo7s}s1@x?G@yY9@7grJw6lqB^>kZ`71R~Ub<-Re zU8F?i+N0gB3hekP{sL{}ya7XipfXTxH{lbB%Obkr&hlEY-k?lFKi<#&Q3$AU zZX62-M~TOVvL>fuRk|`O7}-Q~YWx(v5WA8)Xd9M~+wgbq>>=ezy8txZfKEt<4f=Ib z-Yr3Ai;kb6fI+Bta7-R*Y)y{eB^*Br$)c(qD4vp|abzPe6J{p+;?%LvJ5)*2pctXk zW%;|(Con0SqpI#L?i!$EQP2TV*X1XDRb)R1vrIbsJi0HVh*6LP<#KYNaY2C}+6rQ& zh>nJ_5 zIMqa{v6dD}jVQ3Eg>Nu~WxkZkG)ZLZC3|bOUz(m+%IV59=j_nlAmaw|~clzy$cFM6qd8Q2?z%vxXu@sDk#w1{_ay6_<)>0xsq)-uJ zr%uz@sKSxI*0zOAb&BAV=8xiOrp_q=kxVF9?e4+4Ogbsy*@0Bg>Kl)8tH=+izl_ z?)V0zXPv@AyJFq)eIzur{V1tvcFp>le(APtA^RKi8}M~z%N61&AJwN}X2o%aca=K` z(57R8h4lRRmfG#Sd!X%ZYg5JU=2Y3)7t0nl$)N3 zV916~&E`DqOQ}K{?_ZJ_#j+s6bpZN;C{8c#z8)|Qb+ip6-t4u#XS(jYv{!MdXS?bZ zp9OL!T(u}{ z5T-V}2`Yuh_*ABcJjTu*2FDlF;=;gzJV^b4&1)Z-Uket!(2TZcqxb}$m<^?m|4lAV z91$n;BouYd%D<+uF(>)kD2&bXjCyIm3j80!r(>)RBPw`aC4byo%h7Lyld%W4B7GuuX{xSPJ$}}$Hv9$9-i4m>RJU_d7*{O z%n()OUq;D-qb&pA8Gh`e8I&}m&OU%NwEE-;kBo;rs*}@(itrnDQg|xR$JmEn;aGmO zedF`eHssFOAP_yaY%AZK(=sWU*ZR?!qCQOq^pxgcVru1DUDSNlR{EiTiAwCjIWUG{ z6KtVMOqp{>Gi}Pl?zY)A)1UO2Ftp~KK;toRHeTthw&}jd)a&lvG~$AT+r^uwKR|m< z4685v1Gqv49AZeC0GP_oQEpb!m5K9TOF4ylI2-RL)U9k9B}Lh1FI8?|f0Jaja|N-A zgbxualQ&n)41idQ57gJ-GO!TC33wZ4J>-8rT&pN5$CnB$LY9W?s2rUAY^TXn^)e>N z2cYKT)EzP(@A(RDsOa*bzRPDtdKD6Q2yKixd8nA78*dvl2EB4%`paB;QUy$9@C}<%h>Q5A;t1r;9d)tBwKlkegK(F7# zUuKNL6d!QLwDMcv*St9B>I!T~@V@Pukg{1{i#BvqJ+1qBdwr>V<%x=XgJkao1dN+C zHAeIn0$;++;8?Iz{#ZRsb3UFW@({K`uAwrMy53|m5$mSs*PxStIfbr zP^HBO9pa_ke$zmiMN4B`>dgK}ao8M`P?`xnC>vX})j&9w*&4MiC+51Wo!QEVT=+Ds z(f)eT9FyH5s!N@(vkLZTz?X)6txB777s-VgML4knS+(kRC4=TzT8Tkhc{DDPyRw6S(u*2t(U;Mv{Gh;_@ zVZO@rBQlj~2%sC2JxvC!iAednI>NBFy+?yfqp$zp0;7B!S=-|}^QG>8oI;O^jW>bY zLyCM~*b!K&$Ho~DeOc;+78mp(O84bR91_eDjV_8H9Zg;w-(w^UwfWUK$Unr#xepf2 z+Da>EWRYFTmxG&sCZ3FlCye|66Gr8lWm#ZvmumP_?oWNQO4xrQ>IYH|(gHFC8itD&NyuV?nZFVY|f zK5w>ARjK5AT=4tHNlcDh9}L40PIMvmj`fjB%8I({1v{5-OW&$Bi0^d=C2bqw^KPWwkEUc__fKOHmTR=^2Y(B25 zwmh+3ONu_-lD#qmzNt|{tdsfj|Cm-x0H~^5pGbq1&zZfD|GreUuQEteNg5@@z%W8E zQofyIJBJlBm?<;}Xq48JS<*YK8UOoEtHS zdtuEplH>LM-Xt`S5;E4d zv$|9o+D5_=P+U=5bPs1==Rhp8xHBXx{E_#Hp!&WtK|KeZqS=>W>W|3|6_`b$VyW4M zAbc&20K^-wKAdT^W6l|OD*1`%g>=)Zjh(6Zpq}|N$jY!aqOmb`@@E%c@kHJo*fo(s zKUpFwsao9sX;HLCg!k^49s{C(c$T!tYM`7mH6QqHwi-C~?acmb_bj*>{+hQ(1nvXk zhgt1k_QW0^K#S(YXc=m+doWR}>>TDhTHTE_B)SiInhoIu>}PiSjsBzayLQVz?5PYP zB8>KAGOLu#w`V0y4)6av8bQhILdwk@iy`=Zs=F`(p=2GXbHeNF% zzfLajsKq*4WS~P^TN|%kWZOrK@V)$;HQn`1>D;`EQO?>v>Uipv7WIlltV4VS(ev=| zu!YNdYT_}*f?pkuksmExP~Lfx>D|7LPT5Xyl)^?P*gSu+Sqo1aYEEAvrR6Gks!WFK zo@6UeUtMsfsgbt%Eg5EDXwOqsv5d3m)d&xxh4WPVvF1h6h!UJ{RBm9}L{bc)h<*K8 z%C}t_=b}yB4#H2v%+{d=oDZzc;v3b#cnq)H9(wb8O!->R(7wM^l6pz8!zQ6nFm9=_ zH*y%9md`1~9UU&B)O0^|1%{#bx^bPYbY^wq}3uo_*cX)9NX>0>}1V zcp}p_PnGgKl?%VsTa7!rdqVR5$C{hMtG>LlKznaz&lX+-E7$wcmGv)Z!EqflvlsJ341|cOB^adC<-n7~n4N?>?|} z65S``njiau2w$)8@52nGJV|C4&+EE@1=_}Dc)-TuT#LZRc)VKnLF~|Lnu_~~i4}Q- zHk#@v#AQ*R@Eg6e!S$gdY<8{aK1Y3TRXuAl7L!(+w3*&#ngM;_C`Os^s}GSWMs^nP zKQp7x3BGmuerns-HT#6JO=pO2H(baNEMaw*n=KC$ThgA=Sv9*>%Be!Ieb6ohvZmeq z*xuN|)-2)Y1D|(VGS>+Y*yl>>wi7Jni+DIG47IV6FgOPXIpy^$mE*|HZGFXJZd_4K z$}t*HOQOMRahtn`l4UJ1UxU!M#KTdiKsfFFyFt@FTRz~?gEorwj}_ zdsPXA$6*)Z{2%6}y>0o8TrQ|b7qsS%Nj0LT^)&SX`%(IoVOfT`;5pftIQ=MOznHZ~ zvh*y{u052NF=^V{+n`aOkK5k~v+R{QfuABaH)dMU|83Mk#bxreP@U{V$mn4^x)YcG zGwY>|>*^UfW_2(XCfP6qS4~nNh=i ze;Aq-H{d(D22f-t*)Wp#BxT>g&3tSoEWGFY~0abQ0J_~lT zhM2(9W<04s%5Qu|=f#J{+snEE@IgVfmu&9)rpfq}tU%cRunw zgc)=BHrb}nYWxG9ox884RlWV zic#WdR$0^bW7sc5h~vy_lc#$5d_^oE;l3Hvadn%K|;R z*s~|zeBPL!w9tTxIA23^;zYoJS;~Zy2HT6RPNnfRY9#WAj6HZ#2Snr%NDh%}cI0Rq!V_1_aRl!zY@ zAd;tvz8>h&EUR(HuVpx*%eBc$Cpk6`a;V*ife^|b6oATV%m67lbQTxto`YN)AZ)6&ttvj;(L zS&a_Lf>yFw%*1X4g2k?KC%{nm845>!H0lV6RG$D6nr%$kU%e)D`dfwxr|k5pTH=Cd zTS&evC%g8DP*B*qAif7`?wIqFdS z&iAGC*s_%4w%W*;n^W}>SEzXzVRc4zT6{OOD9qG&>zSw^(Gs5HZBc^>yvArCAiw$j z^S#7@IrUOn=`mbU1rnfYl5w9j)=O__Hj|mW`=l>uf#Cg?-B+)u-QbFAlXQOe*-rJ2 z|G$MUv%;S4^8m~NiOyt#Ptz=3HWZ{Co@^yeR&b=l?c6`U(L}(N0$xHRRA|96NpR`I zBy>8<%Bw=%2JaRk>WdLaLvqK1bgB~pv5s012U-LmBcB1QkuOibv8?q|I{fHw)sWo> zsP?}k7YN~^oplV7q51K-q2fBA9xGcij$cBA=S`JQ@V-|(7oBM zMI_X3{J=OqxT$}F(uHg+tr{v;=({g6)WC%o(XoD2CX453-kj8V}GoRt+&AD|ocQI@`kR%xdn8c6Mf_ zrE!$V0ovaW#__4AJNl0qFqGjR5~nz-nB2;NQu47OOcHrBr1KP-2`tkp_+maUxuR&a)xUlQuo^O0(aT~?gRQ9}CJezqHH?Qr@c&in8 zXrMqKwGgMA#Eh(71O5*9kl;G*rzFCGCdBMBQQDi(ho&O4KA}>RuNlLvAJ6%s+uUkp z-~g3&n02?=XJ?vQsHValihz{_XZKBTTEE1(f6!C>i_5BNxJDSro24k~pLnSM@)AQp zvxVLBM+3#1p59%#Jj11%s&pI^h`t$B<)JS7NYd|ii>a0t7Pp6Qt*N2Cr_zEc28sTe zXYSG9HAaC3i$umXO)9yIBQuLxXwIi-P2K!m6NJgB$rYv_aP15^fg8|&@GK^t6r}J) z+z<831xldlv~HqtXeI8R53F{>Tt&s2v7pkFy4M9yu(&bf?hI_znwvy0A;HGYfK}z_ zq5*6a#;ptO*kz+w3-9Wjog5}cmq=GNIZX?lORhZo0a5%*nb{$FxT=k=cp6Cf7EUC* z?{7Fz{A-{u1k?mTGP|~zCpf+f#e}64NESUV;EBB>1>c}{n?4x-esqc8jftaX(dzVS z!>bqAOkIuatn)1zzh8%Yt$DW8v*#4k)1V+lt*)w6So0M> zjNyWRSU0!6s*q}vU6;T%=V-{{Bwkc%S`x|2D{3pox)WkdFaU#%INzMXW&o857Zz$@qRf2W}kqEhV2rJd}_4o=V5<$zrC z`l-=@6rlvR`&2y?N29*aa2M;+O9r)(uA2(?V0y@pR(2oEH9K7|mi;7$^8Yi6Qu{Q* zc0>gZ8J&*I87*exf6uElSl>u->Yh@|#!TnuL)wok=H6(dJS;%bC5 zkl_Rs2KII#3y+xE+y1Cjy#Z4>bB!Ry_NW&F=Y-VA$#7NlTGw7N;HGVoaAKwl(FCEm z0&!~g0GuQE<9tYiK-6q*E=X!{bTHMYw-#Zuj$Io)?j5HQ%mO8SU|-?Kl?+p9%^yYD&iQE ziw~BGAx>F?`HF>Z1F1=Eser2(5Bj;7O!b3Z@$m-i`CCbCl$qRV;n{aHk|viG8J!wC zI_U6|)i_LaJG#z~xZaK1S6QuG-$>=?uUmBJw?WgOWRMB`-!#UShf@vM*onWA0M&gp zIi|CR0fgjIbj8c@@A0u!9u|wum=o(UaS|_!+9T9h)WD19ZmZO2Uxq3UiP_o4U?aE?IBAc%-&%8v+RCagw6O z_y2j3##Y5GYRq zeHThZ_&vb;ou5Y}`vaQ-iMhedZfWA+p#itmT&Y#l95y7BaK*XQwm16wI!UdZ|Mq>i zaV8v5jkya*#>B8HKNY*PWYj;vl#2@17f6R#H4+W^v<+e?HnzYmb3*^oLL1q`#0>*% ziNw5BH-lf4P+g7KPOfL*eFj#rU}(rG`_Cm&|2THolWKtBFQqZ(`E2fJs%2?a2SRRpeBMgThi@$C^toqcE2fBS2^s$P1M|I0ki8n|-(+%3zN z-%DhmC7`%+#gnQKkQl^Cpx7)=ey@5f$wJN?wi`9&=K2GuQ(GLR&-9V+{=ph-7MgPX z<=ovIAk$p-*Cw%}^5$x)?rGUS689-S-O=ETW4rNi{Bz>4dir+C-eYZIytsxFtcAen zDg4e}(Cd{8L$QhMe0sFTd~EGG(0*q>Wv$OttHK(RdDC4_hKI_u5KhuXDX(G54!BCQ z=4&Hql;AAHLvzH~14Y1LAjy=Nzm1O&)|Szx9)|C=$oj@CA0mbl58Bo@Nj2%bchw)uZKqwOt8wxje^f>Z7JhOLff?N16)W zefffrL*iCOKTkajyiaOxMq}0T%PW#TKHc*JH91kpHEHGV$-Xn&krL(rr*h?e1pY*G z*Hle2eQ&1H2H4pwkl3WMQ?Se(Cb`g2YGx7d*gWcP5(7Hr^apev(YL(V9zg^4zX)MA zD@@M?59lEzW)5KU2{5(-UfHVW+0}EB;l^IvU%SWVY=tyzP?xiH#3PHJ(vw*`F&4kb zud0fuB!khVtt;2WpPF!+%>y#se~C(~m=J#F;(YrCO#O z{5DwBLi93=hG>c#seE6OwMf?q&yF+_Z+p#b|Cvy6AG3nb5JB?x1%@(Yxgs&w6*0hE z#ycHe%>VU>Ahhw2p1f501esxOMJ0gsXD|39Bs4gWH`?;E-e2;EA9?N!RL84vG}%+l z7nQ8fWLj;5)s9pH8Ml-{t;A^XFYTGRPQ&%jK)K<8RNji+JD|5T(c!KKtvnwPC?diG zWH2Q}F6KSPZobYRg z9%9<`u8b$G4(_ z_W>wo^XF3bU?ll)*HU9(Z@nMB=RF4 zAlrZoPWxVF$iI}`P&$mPV5q_V|YHkq&Xx@$J$?Dyin(mvQh9Ji) zqI6mWJg&|N+M59BhQyO7fugPDuA*;y8=Dcq#>efZnB!ni^WoKtYoeAv%i;@a&Ka z%}m`;-*XKOmPncq_JF)u#AnbLb7ptSjf53C^5^0JVwF#LxGIBivd~;TNgDu^=JOS_ z@spamW8ySkj#`K8zM~qgG2+VMPgfP@5WsY_tR5a7Xm$qMV6nP2vN3?b7`L{WE)B#f z^I^WHyV8)6RsVe;=~~}~`o~d%Nw$V?5}OYH*chY_=6ZW&vnrGmfe>`HTDM>%TjeBx zZZkg?(-~OJek1$%lg_}0pT!KR024rxVZy@gd?pPUaSk?u{8Q5=Hiqks@Yk7YxK-5$ z%j!VO;dP~#Db>X5%)6E){yRE_+PA1C(ZV~kAaDRcUVywHCCnWB2klQ5zc0pu$LTBt zwgq34D;Spkvvb;O5sb$`v=nBpKWJeG=ph-M-vc@eqjF*Fy#gUH6A=n(QO4(QD8U%R z$(+PaI?se`hY7B}%L$sV>@KgLT$U2LU8H3|>wwe{aC{F5qgGI6%f?5Cj|5UBoYRLd|@c0p>6`QiE=i?YYW=hj^l@v1^w`&*cSFtETiYo8jPWH<7J9w$Cv&}>a!18}Yu9&*Q;|Z(3Rr4c{vOQspX+f7#@6u~yj>3bBa#hso+9QrJ7X zFT#~(hbgB!&{LW8S%9jLkmi&4iCr~nCd@l3(_HMCA%K$W<(^lSnmg21iMc#`Fg-jE z1JbvGAt^AWk!I+jYh1~YHp;sMRQ1})SLromH5EmvA8D2~YT~&K#8v=O)B(t5GGL+h z`kl=TC<#ONp-JGkSlm@8?x|o%8#n?o9f<%G-7SrnLWIMdXl6IOxeg-qlhsVPLmHb3 zB+ekL2R|;jvpv+`P3qG)5?{oMurSVmx)r3XmTXp9NZM_5d?zj5`u5sGM^2S7Q?OW@;OV=}!6v;Kx+t5FHR{@LDWpoR2p* zWt2OE$A{geR|72BF52ZvZ~$?yUjFRf30~j-X#cR%z+?L3l(@Np8R?y;u-vomwkj|Ttw~= z06?Nup#JlETCS?PF5lY*U>37uButqCiIKY0rh*~ zo^=3$GY`h8HPSRvlsKHa0quO;TCtw2H)!;?>6F zr(BKJrz?f8G8C9?NPmq2{*!07U(?AYuln@U;*LsNg^BE;^RazfVg=)lPpKvVcdosH z(=L3sqZsW}u;x}9D1hTUUVB|uQV0cCe7v(M?}%FjTit>mKI61{F0M1SNz9E^ zW|~GJP*=hRc?L4M1#>Fzy7jYLKY4Y8-zjGo#)niiN)nWSuzbkRB@pfuyKhDZ_uiD~ z%)0JQ7)3w{uIf~EIjHq4z$H`Ryy)9aqz{jmK%vpkyTili0O;al@;?B_hrDMy8Ms5} zSbzwt5mjFWZ-RFoBK801D>5govWYTVT#%HIkmekwByZ9P0Rb~UfXtWJ4-|)NRWEk{ zTHN5B0j+g`w9vCE&d%VK7IL;qD%;j;p4}w2`P-CR&wwA$J4z-D@?^eI_wglGRUbKK zv#Oe2xp)^@&UIK4<&KB|aKy~sRusHs-gx+vXo+Y=nSBm=6>sg5T(K9JD-1t??M{7V z(XFj^gsuE1m&+r|ubl&ck@B;{cS1{$Gb__Jadkv}Y!djP9C>10LL$}w0SGLqJQX`T zw#ge&T$jXlou~k?h~7jJH+&mR7>T(BxJ~gC;=2C1h_{#h4MQxe*@xOLG zH%@PIuzerSE1!}1$HZ7RYv2c((Ma$ZK!mgMmxD1OeVT>+ezu_CivFsP^bgKUmQB(- z+!hr$4YSqha0A&a7mW3gxu^<;M%UOq>sJ*=Hq{oottO8K86pa`pv}u!v2b}q2}bfE zPg3*xZl?gm(UUJhN3%1jmMruBUJxat_VGC%1rQn^uKv(eLTGymNlg;Wy=XNBk*#($ zVW5`wP-#_&P44{EjeELlStLY8SfLdRqyoLoR<@{|}Dqe*m zYz2TrD-xTt4oaO^Hc`n;!L7y}$h9V5`9lo-y%CX22ix#>$t`P`(V2C3+{9diq7@8G z5Hb1vO6@SyxtML9?=gK0cdSRAkO7Ze;|3K1K@SNN5$vD+<%Hc&k`$%moa!Y&o9?FT z(RTx)&;jR{I2;?tLp0rJy_g~KCO~w2N9VufOkkxB4_S2nbL8BzteIePo1Nu`N@rGc zEG*M^Ct?dqf-sMnPjfxpl|gZYjkDNR7N5Ywf`3QABZ~o!0%;i6byX55B>Q1-uSCD{ zopU6wG@(*qHG9qCj!Tb;!}3A(Ro#?36;{J7Z-6NE*ufkzc+SDT)b!yLq~1j+&e_og zBPs?P^5nyw?~rQiraa^W+{hyykfX8Cy)%35GMu2MY;*SJ@u$JHaZL?*5peL$+AJv| z8sO$5XawvyIUtD}c?g_g8!{jFI#1Djl&_E-fp?RbJO$#5bxXCl&amIqL*U#lLA=Nh zJcM-IF&FE{O2~2;GwI=w2!IX1bP?Q*WIiYIwcNH|@~ZrIEos+$D~-FB^;9mL{cglPY_ z+(QKW&&1rc{o+eBASfMYAN9|u!WZ528D{c9-zp#FWk%l&H}>ZvGpYa!ZlWuxk}qm6 zrd>w`5`Lqc*%%9Q1ctGJzjpRV$k1^u>kiav;f(ExA_`dzrS@#_k_L4fb##Q5;1`QS z|M8me+E)r7sN``rhWckt$3fi#A5xlMQK8}NZVPHS1|FK1fEJS)^(OzG3$47)P{J}Y zG>cdPnHl zst5Z1M&)?GV}}Nc3ybppKx^I?!7=`~;UU=>u6Cpv0>B9`hBW<&x`v6gtj${H)WzGK z=pqIJ9R}+O0M*+f1Cn&1!hHU}18z)nvm-U*j04cqO4pB7WfoK@7b6np_;Pwz(xpS6 zYNZ$Ms=F}QKnQ{_ZIQDxs)(^vbE|d?U_LE{fH-l=KYU$K@l`-d_3pJDfEZ@%e(Q|A zs&BsSD;&Jzu(B3+>wdK^VP=O3cfy@ZCfvy$?7vNQB}F`-1D@q zKu9bVCf$YhL~H<#N!T47IN#FaK4U*jl!S(aRaLDt?~wVElReb0BP=OiuK5s=S71`l zmCh6UVB+Go^ZDUeuUUV!0{=JcFYDP{{8Dx@WW+hlWOSq)D*zi-`N|w6#&$ zaA%wKjrz4!F@M7ymB?uR7f7}}FvKantYJ8|oJ2{e`CwNx@Lw8JOm*b&k`(Qd=vlwC zQNFX#)Tjat-eJm*nE{95KS3+crhK>agIOGGK&!KAX&;{&@rgTLEU0=(D%|)^(=)2q?Zh{~ z3PUl7@qIG(jj-I-F7CwDlA;E0iZ}UJosRyA2apjfqT6)s0(2J=Qa?5PD#?GB*4N9; zei*peKF=LB3#%gh;{{T@b;j<@b{Br7e{L}|Zw`+vXYdrK>?D`$WV~2%I2#jT2!L#V z-9+hywyXGJ@N2ICd&{>N*=R&ts{gvKvbCtd>+4jfL*NWs96pPHlLa&DT;P+ zMMXsGkZWu7A+GaH#HgInh@!8ng&c^IgHY9bQghV^qf+gVo}c}NlcNI}w+<1&y?5=# zKLHs6=bDSR8&S8_+bd8vd{PFof&54mg_#;Pj#rPs3-(_cj`&c54|-Vu29dGffQhj6CE^TdjZ!g>L-Va}Pbu~-kBG4pe|a=OcKR^Bl=XCpWWpx~ zHvrf}+TZ>3?2bmik-kD-ugXY|t?PbSsQ-AgZKgdXL2@Q@{uhu3i1rHnq44mLCkBJG8)?;4SXE7Nu72Dv70?! z*PBv_-oTgPz^;M;zZVN!V_^B&(Uj%FLPVTz&U(DhGK5zAH1EJ3G*vSY-O_0qM z0ZG%w2Ov=1N#qMYwiap(4VoL+dOgdG?|hz4CbZ<(7;3!RX;<4Lt<`j$jc{j{H$&Rc zSuG%Q#kpj=Z7g|kw{5m8XUIQA0fn?OgtUv1OkV_etcmsYO?6C523ww8()U;%LSP4BeIOG7HroS+=x5vY|KW%e5@s zS$7S&?N_Q`0a*qB|8^XX6yS_*qulargiKu;5{NdzKu?*L*dTyFOE>6#$o+$Y*dZ+c zo)t!iZSiLsz!`4O9peHy?5DrC<=3<9K}9+llXjKOnc>^3Xdg=@5mr~4qp&Yo#{Hd$==PpY$m!pLx9=-1TuC&)(~s>9R`rb20TSagw{-t z!vfXScaY85HQ6K_$-LW)eBJfKD4$~mg0S*+eF9Q1)vC{}%cN2l8X>4}MvniZ?t89;M-%Jd(bj8r z%s96zTVRDF^*f_&y`F;Y<`)*M>yQT7{KH;?in`>G1);zKVEuQbP#F(MA1zCvKqJrR zIQqENX-5qfhO`&y_-Y8u4g3}oHN%HHrAk_uI@9}~awb=`>bz=`Fcvrd23-2%SSRF$ zIOLi$Y5-ft-T*m3bPnMHB<M~`GXbvLiO5JGKC2g;ptqtl`uelQdU4|Sv6F5q%atS}tEATGPJ z34F+Pd8FR`m3G89f(#yzQrVV73B5AJm22~Rr`tr_VCaOrjD=>r49DQhWLkbbuL`tb7eoLAl_ zRvHcG)Xt#hB|(zuNqgg#H}ZTBy&daECyU!E;a#^!{|2#(eJua_AN{x#hu^6{+g$6n z5nUU~5o3l5sq?Wg(Ru;o3iGJ`50$SMbmqqj7Mc@s69C;0{6{Fe^bcPXy}9GL z(21SRh=8AOgp+n&HfA?rd${ORf)IG5jhJc4k{iFpZi*e+p)S$nr|AMXiHTIow*0KO2welAGPnTJfYr}Np7GYFPng`dEuyRJ{L!9k*U}CKwNMSbW~r#Jht9@1I>IsF0?H-)t#Z5 z3W@pfLspFAz9l0j?*Gmife560r}q#d6oBpq@ooU2FD9LD0V)`G+KbbT3?T0aIUW+u zpIx4?{9BZBcz6u`T@+bFp_cf`Y!x*rh)bRRjvo|Cg$_k=z83i9Fio77kSIrLIcL3u zuzGcTU<{OBQ@E$_)M8Hyc{EiHK|f{+;;WP{fn?XOTuXrkvb7 zjj2Uo9+RNY!?NrN(U2aD&ZtKCfBMEIp+9`9yIB0IdHyM|CGbY#RUt=!*g3DV1Cx29wtrEwQ<5(5a1w#4NQ#|BMtZ zgnH~Y%m@E%R=0kUQvHSTx)1;DUof>-b-+$DdSl@Z24mj3dG-!ZtY|hx5Bv*BaOn+N zeY6seQ94Ec>2G|*_}4vMm$py9(E3LCHRoczobG5y?F+KtL#lK#ol%ej3$m?j&X)bz zd;uYFw95b_;M_O>@;YqaNpn&{n4*ZnxK&}^i9VLwPGxS_g1K$MFjE*yY-|d-bD%Om;Zs2B2nkVUpXU5sfV$lv;64& zWulC67?CLgPkE<$Gl+<6S24LyXe7VSWg(lz;V<)+RLc;&ztu*x6&2dIEQL_>1%!bV z=^yrLYkPh02Mq_7_uAU7B-o)H9?}eARXg!808AZ0g}Ccpn>ks*}<+KO{~0v}yLy_9g0?_%EF=3@N2;5Re$zWJnjq^VkbUezDv|0oOv7WQF1j2;iw9KC z`|jFh`^0J&2mX`e$AJ_>Bg(*i#)g>v9b@WTFQ1S~R)e{-H*sL^dy`DTx373cz#DBB zU;l%Qn%<3H+pLg7It69~tEoz5NzJ;oBJ?a3;s&bO>CEp9X&(E7q+)2Hp!w4?QSdO5 zTW-nG0!6#sgEQ+)i@ivAp@wZF)O<2cab6{+DjklCcJSK7A~T_gp=l1?jC14sbt`>{ zX#?O79|UetdHC-DSujc#iwC{L1KBK|zttqFW3Og->#K*UA3?tQL1GdhDE(#e#;wUA z)cUYpB>l&Bkmaz{{8Xn0@+jTDS{DbXsb&VQFdkdKsK3Lmc{UT1ic#H`-XPDDtNnGS z$85-CAR2Xo8oaGFW*4rQ$XBX5(BsA4MfPBE-y$HpE}&s zISBNBwJ5dXT?H?+^>YsuVUNk*`;Oc?%3selMxxvDl}UGgVgkNbRQUdS;PztWEjrr| zjdxZSutBEyqxX*UeXmrl#`@kfgrkTnWwB!6(|l>8SHweS_kCuIL;zCJ-%air1pBLW zJ=K_LH1AmsHKPu#H6YBWAoDbv5CCw>M71TbYxKEWfu1j{F!X#IlxmaBwf{ z6)l_A|FWrH#@=!^-EK#0{w>42Hrl*4!?ZTq>*$suUAoHEE_dL;n}8>e{O&b|Eh@|1 ztW~bxG^ZJagM2cwnw*+?*YiNn< z?tJsvPJ(U~@tmTh&5g-<-6Ym7(;dsafw1oP+2Nb~s-CS;pwa#qC-n#_*%|%x#t9p>(#!>Mg8_7c z3OCC`8Do|e8>f`2l1C`g^UeB}hR1(aO8xZ8ZIUJI3h0FQgWFjy+O`E%$L%#{Gh9?X zwI|$mE=p^zE+Wz`Qhr7@3ae$FF9bPkMps^q0`0XSrB|SXQvdNAho=k;gS@V0jd_Zy z+&ua1qGK^TJYInb|LAS!_jf|YaSAf=M=3rhzeY-vGp_&QdtML03zHBNokP{C?%ca@ ze_7=kORTK*zWQ4hTe7G^t}XV=kE{))6@&3&blzqlOVuZSKj`)^$4q_wSbqybk99am zxbZw$!@p|W40-+x(*dN76o&-TdEkEgY2o?%_Ij)4dfG#1iepINxy7{&`dL`Z4K8H% zBk|#(Mu%ev!BPO@gY8a?XzXqnTj}vdVU`;m@sTB^R=u2N9B^}R}*K+oZ zRuQj*Y#pNn1i33&1s{VIsJDy2Rl?p;9d;r!1 zDnPPRo;=k1F=-n8kf+LiRZ;icKhfbNK-=$Ls|DynC|AZwe0<3hLl4t-N~QZnraJ|V zf3SWR4df=pa`%1+`psIA?0KuIs!CqG0N!oZiW4WD^GesF`DVEXB7lyVbHYoSSa%{l!A7 zMK^J6g7`y?vK;@AYWU99!iP#P+6M_SgF7ar!gjN)L3ZJ{30mDkmTMP&3 zzGBdo#GHel#lX*f5w4#FfH18chVa|1#i%i<_A1$R@?XQTi)X>P*hN;F+Fo8Vy64AJ z$IFQc`4foCio%PokMsXY*XczR_yAZOBxOZcsp$RLZ56-n|@Z zLA;6DME5>yA`VYLOjH)VRI|p8heD)QpzJD6Vz2p;_t<1g9QvLct$xUt4ujO8siNYlm*LB#8^0hy~!qg@7ANNaTZ@;j012;Hq6jnk% zm*u)%nLr$DyRAj(Dko2&qMu_8-Slo>Ufo_ha>2F@n?{mlO(>q49kof35C+Zg2oLMM zP>TB$IwqxrLm&Qn!YHTSJo2`|l9RWGOR$Fv{5jq4xP2Mjnc%_&x!Db{+n)A>pDnGd zpKyC4;M-xDuA;YFnO-`MJJo)_56*-Nbgv_#w*PiIdbvUzd*m63ffrc5l}Gio6-VlR zw9sYm-XyyB7RL4T?L{N7UHC-rQNzSbg>TZ~8;6zL{MsffZHiGXw|?Xvhv#1tB97Vv zJe`C}zq_9(@~dH=bUmnXSg8`MdIbM=ua-hJ9@vo_Tsz-^ChDGY@Y?x->j_}d2TlNqX^4)v)81*vhW?NS!P3&^&c3C04gqPF{K>{ZvpBbkUI+?=DzrHxp=Aubm z*W|2L9dlTW*&E3xANIZwzH-0p_CCYrA;nq{fI2`A?5DV>deVXi zKV&Ye7H_1lM})6r=Igr}O-eFI;5*ycz#mp*)4YHG*s080O&xap z2V1Cs&U;_f(^7TwD~t7}MP@_M>{R#F(!IutEy73!Ama*}MhZ4qkP|8o&5yze7)qAC34c4_ZwqmX8( zMVT-BtVLnjqt7A^^Nd*kEu^| z3y3VQ-!12Meh7tsn{;A+uUalhMFTSZDR zPGEdhc^hPxs-5v!a5S6i-1L)N9`AW*ST)ot`K*)cOmz#L+!H)pIFj-FyLT~+;x-gS zKD=t;d|Y?C?Mk*g`0Fv!!X?>wLPZ5Tdy`-weXvn@bk%%>XFA9q340zfituKGYRcwv ziWuT^%e^jIF3ql2S`g^zr5sNm8vZ;W{EdD)qs-@Am}&E6@&*1z{!WfC!b|YXMt5X& zy@ud4*$!t@&1=O3ACEFq8(cm7Nfj8&Q&CCs1e_YcfE=P3T;LW!N3O%UT(wt4} zZxWBs5iA>q=Rj?>^h2+awWa4z zZ+vJtW8Cr@iW-VC7kSTS7@^FZ^W1$K(DD&HLQ-FlJpme%fVen?p(1CYWaZ^*gbjwIjANZwJ4Xb7uOt`Sc^SoEno8ck`# zee;>pIPa)z6j`~G5tjhpREvLy-^k7msA(+{j4RhU!_4` zpm(w#;ZTECzLrlxNvwoRF7p_T*~5c$O+eA2*!mGzj#Va6lqNYTN_6@odX!pxiqOjY zKWN-9Pu5Anl$$aCxQ6PVnIB7q<8K_8ZZ>}&&NM+z4=%_olf~?>_yjs5 zxzvB4&aGM^qQJy;5-(jhP0VHXpT9JioFU3o@!GTOV`>H}+Bv>(-*5jZRO9wHbCO*c z3Ej`Z9)()nlPeP4;gFE!p0O~nx=?am;!S%iB1}SR)G_wDbd-3e826r zhwIC^%guv_9r&ju!U1E56kKviyFT?xLS-cZ9y)^$+__~>ov(Vunvo={sW7Q98;u^v zsB>lP^aW{_+n|4zVm7=Oq73O`p_`bTW!+zi(UqghJ}=y=vT<^))Rtb1>|VLBwcMs< zL=;WtkN5W~{B2|7pK7PX0UI_QZ21}?M)kH#GzXIiHLS4cMMqB|{zFust}(J-^_Qk? zD^eA~GzxOBm0tW9K^*}15Y)(!AdHRuX!v}%e# zxv7-R!hLe`Mz*OsduCPo2K!nPe?ubNYDPM3XLjseHj7bhqetsoh@|L3b6NirBT!aP z9-veo4tJM$9mkHa^WY?HPi^`2Ku)e?>Gfk&X19;Xj1q>{+<|rv0zdpLVe~5RIYbTK z=e2)Cq)CA9LZ(q2o=V0zJRp@-bN1@d`4GFWyx2H)k&_j&wrb3+(;Rrcs+VXaczG;+ zc_$%Q_mk^rYTFk^RK>kaCv5)_{26f6|1*9V9{D2yCOI-sEaj5c8x~~nt2QQFIB!ir zJpR8-1hF#DJ`3RspVKVTg?=%C*^ zBjav}C9E8Ov;PE(NK2@*nj?{ST51if#E4R6%SKX87~j`xTYYtUL{Ylpwqa=YGN_!z zw-yinqUE^~7pB}4^EHtn?xe^bX7$N5jE5ZB(@yCz1Eg@>G0g-c9#X9S?l27SZYrwm zW9PbMNfXN8lH(?Pv>r@duk!4}E#%t9$Z*75ecsCaBAHLd(q>oePDeIf7l*z_z~~p( zr!{*2p@=4Y4eq+34fQj9`_&%POXCTwJ>dp5hW-ubxsi_c)W{Q4^5N+q@Rn?zPT3-% z+gi_t0JD0``DL3{XcwOtAuBz8HhcJmD!K`S3O6_*jr$%wNk*bwru$-p$*@-;-EX@! zFY|{2(*h=*qjCJ^zmol!WpIqzYkq&$st!*l!~BIe9qu@cKG~K2?ALh66LZi| zZRXDr+8cDFdxAx+YWl2~%Gxz60e$FEc&ae<^F(yY;xW0+4{ol#RETnb*)u0A`7W@2 zTM5)v1pA^OMGJ$?0-Qe2?%Mut#>Jn-%D&2q84?6B*?ZZe%sS9CVZ`>0ROY$cVdPC}`~93A0McRjYc+iOYB%@kcS7h4$}5PHcMl zJCdCZZDg z{Wzm4%FAXNo>^EFraRh}7Q-8gDskPJMHBtzP~xENpI0nkO25LDVbMO+a>VG?!K!R8$p33xi1VHY9_3r*8&2*uQK4)u-%XM~>F{Qy=z_L^mi>uft~cymS1&w2Il$ zOi=>-c3o7{ZL6RCM+9deZR3QRDs)8*&j4{+YtVTz6oQ0Q7tkDa>kXt%FACxQ(OyZ{ zif+q;1!h7NAfw{eIOQad0-UsXro+SNQtj4EQD%r6ED^3`F}R|T!lK7;Ap6zw+p=v% zCA>#cK04^$j@nPmr>GTpurR73@vPu3ytGd__cn$tNm>fbuT2g@|F8=9F6`Q%DlWo> z+fn(RC~47sv?r&oo44-oX6y>G7DMt$^J5Oq{!b3EBJS2c{Yn6F%;>R3;{1$%v7ZvK z-Pe2CVI>+dU6kjksH6-wpQiiB^k~Fs z0=Yx=aaL2|gB`hfqImiia_P@5S&yG2XMGU6m*DN#)$P4ESh2{iD_Nr}dzJAXhs^`9 zagtZJoXF-kfi&UKKE2OI=g$RuKCaUVMiYl0Q{g@O#pk3%;@}@|l>d9y^oaY^yXAa> znyQ``#!csI9ff&JkNq?odllbE8;ZINPLc~8mBP+{P9_DwY$lvz<^J`cmideq z?MpdU+lUp1<@oFLNc=gY{EnHdOIp#)YaT&N?4)}=EJtj<@~RE*MKx_ufSD$W7h^9! z>42a9g$Ev0zmT#<-p_w`)|CUDel1OqA7uLS-+U( zIaOykAIxb5@+Y56P}xe!_j@60ytCy!%O-L2IQhh9Gw^Y>34bs1_jbIP7`I&=P2GWw zZJpN3;Or{z1V?Ry2hQ0{b0y;LS=^s=J!~EglA7s#gJfZFaL=Lqp3U1n9;5c@Qmd?w zp#l$)HDZKEBb>aSAEV+Frj%uWJwt9Q*X!zdk@vpAKLzPtsG&!9boD|PH{xiK0~iJl zmAId!G4$P4GC5r39lkbZr^4;XF~kr0_gVrC+y~k80MCOjLyh{fmA}4gpSqRO*hB9d zab%&yDfgiAX{b?-`seN1qr9Kx$9s@fxUMBZGjxx#=)E@vyELbe@4~`t%c}UVw)Ef) z+Mn;oX0b3F%Dm+rs;-vLt^r2w-J%<{^$j3sI7liy-X7!~};<{z%MliGL#7(?Q=5Va|IUQS{$s`~KOd;V~i+ zO#?!{l!hk}hJ*dZST*4l`mQ}J{eCP{Xw%O`<8EBYYY%UgDW-CbgqS@_Zi>9~x;VvN zvmHBq8g?o~A24ZnTs6(NX`P93v8vJY(zZQy#jk(ehdS)DaQT9(K25BHSnsG?Gs!dW z<)Oe(^z~!cCWV6x?m8y{4Na9tIfV#jfk;ckx2<@5F5S(d+#=XB;0Y`4Mafv#lLP-21#7$EH8nQ)}S; z+mIu2k3Z;n!(jI6hoIFc!64-y1=9XX+T8!OcdJq{P0ir5Tqh3!8ALkoi6!Zj{-e;J z5b$&Gee?5QeYLlmKyHr?hlJ&)*li8dr19C_1IISond@NslJ+HWDK8Y=3z$@DQOVtPLW2sIh2%ygh+R%bVwgMr9%z~64D@&($Xy;4bt5WcaGov?z%r+ ztR=(D6Z_eF&%8poRYB2Vz1OaKVPbPn{Og}qef{~xgE|tOP8AvHae5gI!4WM1rg&KL zXrR28a(CwdVf2r7Lhfy%k zi)wB4e1{7AxDp{up!L`Hw!NoIopri(4e8uYBtSosahzVKGZV#z z(_Hfz<7{av3r(mBQ`aHd7ygV9!h+ZEt4z32)G9MYhWe<&nfR z*R&EiaakKuvI_34_j7+1?IoCS+5*=Si6LB>>{CHP6lvi_PkRj4{9A)g7}M3ND5aly zYTt)kF;TutQaz&7&TQ|3XLyxO)@5+T7 z{o-$af!toH#4qOarOAg&D!ar@0ci@wCOdu^MJE5;I-^mwwAlml5BKcby>W=lWf z-Y0)@*YVs{j{+q~($u7(Ols1&phki(g7{{E+%~MB1m_8Ji|FYi&Bs3gr8r44A^|rLxeck9wAdOpnKoaYDp**npqakhAhkgHH zJS3IFf=>T-`1|q!78jH0zWZA|RlYou58TC0K8v1BXYFo0nO3y}_rn0I_nz>)QC0u= zBZnmNyMHtLxAT6GDyLs)Rk?qV(M-uQyk@x{!awG3zNCh z^qN2AyTa_*m$QvdJBaH1T23UQb$I^gdpDe48-m+DClwV@1{dBRiN5qbCam|}dT6)r zf+9p^b=2)`8J|YI)|f2l>BoT48E+?$2w!8La5!5(N!)-~eLXW)8S8d#(7BL>Oi*&N z8gJbJcwA+9>)>xuaTZWtF6OC?i>sHTW8*cO>)Tjd-1Z<=rOniyXs%to@C5VkoPe&01(`}BG>h$gK2ZcySsp(2lvr#=afz@Y z^m=K~jZwHezz$4kGKVA2pG!C$9!C#w`kkY?)}9&PMN$%R-RFDeab9GHW1M0R{IC(O zauLzK5<4icW!>vrrUwe&LGbwC2Yi&iWwpcUspO^B2S56EY#)dkzJcO`@r~7D#KCot zWE=+#5CHqK-Y$F%vv-xmv(;Lt~~b#S99)rnL@Q;ni`bBI9;Lt5Jr}3<({j zXTx^!;!|8D2zv3Ej?5IlV2D*;#c?2;4aiDbxZ}pqVk5{T4vg31CKhkCUuh_4Y4v7k zRqNqakm9POOVKpmYJGtlcSD5-3`;KZr#Ih9&$M+tlC9Hl> z$OTXL!F=SX{ciF;Z0=#?5Be$neFxr(xNXm!epnz*y<*eSY$0{`Gr~h>sTOz)Asn4d zXmW`dc-{jDJn2chIB`BB%!_-C&nIu$Kw~j{;^nVA2FB+7BoU^=WZGeMgzpJYq@5*k z*m9JgNCeZpaO|_c%>41|B#8$6ENpirg*oUd;Z4AOJhvkee0ONVnyM5$C}ZxuueqZ35Nr5%1b%E8BcxC z&ATJm=VRC(4viGE#RO?z=&^)7Up;wnWRG}pJhpnHnidV8&42NH`g9<3wDf+wPD?*4 z7Ntcrve$(s^2j|jfmgG*fgKlD=dEJ2L|CoA5MZ0!;Y3psS^ChckySDI*+v9C9LV2# z^!Qb_-6^k^5q}|NjYKnLfi>Da$pahx^7bra3Wd6d1LGwl^UD^Fz4<27(VqPYQ`XIX zGOqw{%~f&d+GE14s$s7WvpI?&W?~vNT1vFNhE@LRz2L;LdciUr`9))5~lK<1uJoSKO)Cq+ACR;4UOgO2X0JzN`obuDiVLKWBt>ZkROUvxT+7E zZ)#J&hy@Vae1;?8q)ZnqjXKBJ3>|jT{6H$`uM>ifA`D^NZM7zF5Pt8!DW`SpxiI(+ z0DS0izV&v~a?AWGS$fGg3 z-ZLSF@@Q#fM-D%9a4EJnv(c8Ri=IR`zIpdHZtU)Gy~0Gj;be_OOjdorv~4b|TQcc# zg)@Eve4ig!n|)_UH>_@T+)iu@Mq&v(w#op9mF*D|s3|sL!v3JyQm=k$w7wg&G7cQ zz|p0_Vt5m*G-*$Yr~P$DO{-XpRcMUZ+7OANo?3@EzK#Z6++103JPw>~x@*xNFP*2> z#50332=g6a)O3IPwWKG6J9?(fZ@ah$BR3l4*PDr%@q0l zOFS_*g5@|)H&y$?%E;&%hH0SnHgek*uf5*1Q?(4oF9sqFy#XriU*E8m`=Kcm0B1Ws z^jI!o@Ajhl%N;D$2L$Ixu;FP7ULqSP=&*O zo$vM5rzJ^C89=@kQFc;ge0Ju?-_)duKu8v=eFU?7_>g9^jbgOyd@}p{d*%UH(a=SA zLrgb!uL1AZ;yTpv1LEFC_%k017~FMQhs@#2xK^07dlO*aF1VT1R(H9N=qZPbdp1~_ zUomcV_u5qH`(k=XAXWPp4@^>n`1f&;G=mxpBh%VVS=cg>-%YuV(`Jx520w8@J&nUp z|KAy_p`~&d#&}5@#3wX?{j1_mW)z6Ffq?NrZZkfPtma%!RwX{t*_eCvI7s$Wdc$D$ z`R82Lc4kgYMj;$DhKqat!e2qsDpAAG=ju?dfC6iI^76c*6bT?i6j}~kPmP<`I35^3 zb*F)Krk-^t&+gUvmD*0X>&26C6kp%cD$fkEzYBG2AaH4k~K(Vj7OkT{4kLD#Q@2Lvpj^bZm4Lm=UtJEkVFuSWK? z0?MG>?#OY{D6N*n?VE)AT4Owz<+r;cws#$mk+Tz&laT#xem2=2^J88s_4b`{)H67F z>tO^w0W?ppkEN=N&+kkGO7oZ8-)qma&k9FpaaRJ8&DqEudXBO%<;h*nk4Jw2YQG-B zOVit{#C=+WGCEIaY6y-}e)3zEc^s>M-hf-h-B3m@Uua+kc( z&vN*~3nqMB&B{pSck??|f(tjk&IWyCJPkOt)yVL*(DwBL#osaiQvf)ugIBfId&-pw zhTSL5)(hnPK(<~1zJ{V(uZ7+#TD?q7eu4aL;rr0D&?mi zFi`FJi8m96+wYetH1H2~b-iqvD5gSG*b>Mp7;63A}YzySh@C?ZPPzH_M zi(oQ*C-1yj60e3AlW%FdIP!B<_%vIJJg5@&mH&$cHmt9)?lCkmIeK z3oZ}#MZMZJ+d4k$@09L{GcaH9n#3#=bTrmN#I7TlZ93)vJ!O99)6`#+f!!C#$-fH7 z?1=+Y=Qfytpip?u`)?6A!+#d6{3(qkEv6Sn1{?JOM+cceseTcp6Oi5Fik~p+7EP|l z@|Jb3yvDyD?RH;w2Pd-XsBx3|GT94H}fZzkf^OUPM!-y28$)aP- zSEK$`Ou3^(Q|52y8|}Re!WVm!fS+}C1{v^GWrdM~Sl~U^x~PW5pL5zaqPlW{?r@Iy zR_Nz@7oS2Ng(BqQN@1>D=8xR!A#L=OZAf^=^}!3~D3ZIX`v3W4Mrl zKqh+RLxDs-s-(jr{0wGIRBZm9^g=v81C(@{-YxguN!J$gVLzK^4S&1@BmhrHkdk}W zMI@R#QL{(>$49hia>UgnLtik;lHc>9eSDuJV9uBwQ6Y&M{Y4cfF(2XEN|!x(XjV1Rm0RT>;^ZfOq_J>ot{AJmg>roxL1wT_$$?l_k_x ztDi7>yXv1v5o^OR)U?y=rl(Q(ayH(9U+84~#LLG^DvM7M0Uq$DXy)Ji%b5CJ?Vul< zX6n;eqvn1U`_Yq91{0d-27Lqf%jsR2d+o(aTA^0%5m~GPPynuJA0yB$|AG7fyj9vo z440kQ5V>{N4=L769NL3`x8$UL{v1}T@>S4~*hwjZnT9N;L=TtD!}SJ?>#VVgqbDtI zq#*>@C1TZJY=s%#R!?MSlm-of`ZUU@!qFhWs=Qlq|HFFxY`9!7K6Exwo3$*DW!ieS zu5@Vlzg~d43)i5htvvo7n+I4cKO9;k0QthN1CEEU(xb~2K8w%YAG!`SxRC8;(hz)U zVHvcsc$;hMk!^KT>A7ZPnLZRWOy#{O5PeQp*O%jLfqDGe7vrTW@(aGhIY)DU?il;l zUsU+G^TFg13#Pq5v2>X8O@`BrHp(3DAB6SI-|iG9lM>>D+CDlm= zaNJ*bmoHC9h{jRT&@{w@Nt6h%A?EV}Sl`8jMfMDm6oP%SkQ%5SooG!+Bn*0%PArT> zN@Wb6EOCEqWr>0ZaLrLfu`|a}oe@}2=!s*(g@dFyXa{UYo{drJF1Ew=u9L^QcXjP$*{t*)q}j?Vi#5c!|&J==D7YG}Cg z(@UJva0muD8ge+7vQWpPf$gZ(eq9rjGEk7GC)KS77Jx(@c_Z%{;7>!*ZeCWdxgy1l zIA2*0cO_SS6+C3fEf~aVJDa_X)Fz0HYr^@g_2&DB>6?2eWin7xL-C(%g+BO=?RCUE zsFmy+svrsd&R)u=8x6^`<8P@3A+4nUK05IDC2}%Tkl9*eyn*?y=kjhFp{;>SR2u_L z1J+yhT-{qG+AW3)>fWDl@Nt)48RGq?>qRr0IBXNE7Oqh=1bsu5@xpm41^r4L4D&;x znFl|}1wRl(7=(6Czl(aLipP3B|GViecUZZr*lHq(L>XOuq+MZJBf!z5Qygn|E_xj_1xFQsui3ApWwzg@qt_XJ*HwMLYz z8(I6g=;W(hOEq3~BZmH>^OPanH5#@eGARnbc8nsKSrl`h)WeG+KSF zI8$E0oyFg>ipOaAXo%8E=}$u&jLEPq?+k}tRG$u35}mI9xXWtDrMzGMbZ^u4ocj<0 zp{xRRoC#V$H|lL)fhAb(2u3a8%G4zB?_S`sPD?o4GXHTu?a@R1z7oV_ zswp_J4Lrzk5b)iSDFc!ScuixWWJ=Z0DPoE^Gx>62^ekS7K)OAGeKDqTJhafq0PBE1 zdhynf6=u`|0sS%w&6N2*xSnq+gcy{5vsSM^LDS9m1dLxP&zpQn$mQR$@RxUnSt-bH ze1*8`0i#9;6?%c)@Zz)fKL7cM?(r-PQPof!q{li#!bxiw62J)PJQs4vBwBBZ{k{st zJ`8)N%!lB*Pc>VJ>_@h-`tRS$C~k6}Gk^o59~f_d{o*CV+t0Jq0cBMPS-<8YW#l{W z{Oca!+Y_f9`wDn2A>3cwlv5Kx9gRUi*(&2*YpIHwyFKd*T}!KSXuNmC3)tsiWX1v@ z)eZPcm`%_E0c-hNLhVvJ9$LoEvKWn<*snBi&X3atgwP&%PkTg@;UdSDWQr+MR>)Kq zBaeV&D&V4;@CqjVF(7q!AOaV6tc;KiJJ{;Lcd*|@@jMO;jrsZM&rNK;HATI zy!!)8TD0X87Pr3PWhN(m9hqNu=2weJcv~Eqz)2!hERA~ois<}_r#c5HXT-vu+YLh? z>NYQu|BR_+f(J*Bo9WRQhs9Q$OsV_RY-T-lnuAK|(&EIi;Wd%qdG(7Ca;&g3 zlP=IenE5vLkF<|3ZBZqy7nAoax7xT>$|Zp{L>V1e59c=#yE)2xdK6Pmv-08WMo z?wbq&E#K+tdmh1buSBlj)4IrcO@cWR^e{gs&u0To?izh{t$ULu)2Cxbo27`Mf%=OS&&ojYI}1ro0vWK;8@L8 z#dv{>`3n@zH-bDL-e!9*ibkKc&h?)l#fJzysEIAGe5wFP-G7_+>ckx`UoqV z=aC+({%#V{$7zJ|&&&i)Fi2 z8sE6VPPxsuIRI^M%oH65zk8E0xzRAWL#<^KP$t1KfyVnTZ$m3AwF91mjDZ;zs=a$OJ`p=&)+FksQQ&wi&oQ zix%vCy&&4^1%a$*zbVZYMge*D+CoY;WtDre{;9o#HTx|P$DfMn#bzB)tmgx4p_56+ zu&`{tFZIE(KI$N;`e43#(;Ch6)Mw9chet5uPM+B2KEg|VJsQdtMD^&quQ?eQ>zU_c z0kZJe3ez1Kr}&$Xqw`V9ZN`|?T;1*vV=?N*sdzBFwUCYhHv|Q{-jwH0oG+Dq%UvGF z4k_&^iGRZIfXX~sO?r4wdh;)c2^r{(?j#&y@W)d~78|tGUo|~Auebc1n#FA&s_iXm zrxeMZy#t^3Yv}2}s6?)S=3?r*} zQK!BXhQRc^xq89@##z^6`l`kRiL1+9z5@wmzwIf=F9yedeGYLQ?j>P-&SGbnkJ%_yAZ>-j#s zFw_r7;*al8uV}T;4rZP19a+F1A2f61`Kh?46mS}^pSo)T#5IqjTYs_fw;xF`{_S6a z%flZI3qdP=w@C;6$+b6dv(N}nG(KqRrqqE+O(091oXWz~?s*2Vyt=&}ZG}C$<6*@?%UkSm} z;+b&*B_$l_H{q?Oq|G#J%c=cI({0=5Q^_rv$vNcDwq*nk`2L`5s3P}98z+%KEzR0Z zZV}0Ekwz3iu59Gvd`U|seC0!``pUR}x+uN2WQ4vg82uESKbQ^kfkAuESQ^s1di3M4 zb8#-DjkaH{g&n0XtJA@+o^Vty>fdHOHcn7*rqk(cdS-%IetP5{Qh76|UYC=TeNN?8 zCX@!w0$5z%>x0%ubApOcaaif4p`v)#TwJ!}vGM&T@jT--q!%|j1anb_pQOfYmnYIa z;TaOVq`wyUgm&uT!1&UCg}zTi`8#5)|DN*11orEz7P3Wmiho28mJB~9sOekEa`&rw z$J;&=L#13kQx{Di(C>%hz7<-GM}{S+Ss8#3=;%&4_F#;Vq9f|KHrT(rEMIrg!+0%O zNc{M_2&X73_*RAC&UW53Aj53y>A2$Qi@XL7(T^No8V*nEG$H8mPzO*%ouBJ)0Fx6a zsIH)RX#)P!S17)g8H<>urHQCEF|5GH0GP3U4!1TUw=Hlm!eMfBcpztNtoch)G?lvX z5#LxD4O>0mi8>dT{WnjQ_v~JKkfms~Wuo*P8Zt(xzbsz)RToM5FgZ?W_y$j83W`7| zDvAXpt1n&myE^_P4i8?GJkdI?FxRD6S5jU#rft42N+zxM1NmIFV;B%@6GuB5ZN zSo_xCIgzINfOxHSfU$0m1 z*xdf1;;}e7B6mvei(Jk(ncWMFdcq6INtzp5utE{2-i|VZa7_QsPoU2oKVpQ{a0AsQ z^+8Sq!g;21s?algbR1hpJx#jYiSy-+L;cfbZ9w#N)i8mq5?o^yse;1c?ifjUIRRh0=SD;*E|63m$hF`BT4$^=wCn3yq*A31(#4R?k zdQ7HrF#Je>fX?3=Y)5^Gx;F+9Cn}I z|7s-M)?v41+2b-Yl8UA1ZcxpHB>SP)Yj?pT>EAY52MWZ-F|e1(*jG-C*n#;4i(Kiq zP3!D9Mdwg0BR)l;wJWa7Fj1#TI@4pNQc>mbTQ$mLe=}rf3{jO%S`tn8HLaz;_H1G@ zNw5DUYiL2XOZ~(Lba2L_>ki_Lt+>yHRA7}i7=P}fQKj!5onl1Ap5?4PEvRwaAOY#? z%T-eR2a9#?73dRvsI-h2-6jUj_*fR!{@;0&I#|TKv2`CDKmJ>*BurkRNiG6<#j+jM z{4nm?^l*&ZKiJm&9E*lfsnk0im=_(M=zhni-)5BKV66u_7Es)U6Y~TYdis-c%bc2sZ9}AVK~)n^2aa2~%xfLLcady@d`{vF{hhoz z4&WnOEcaTg(0_%of6JkPg~!JA*&M41jZdR*ff0yx((3~7a+wQLnTPz<`=lYIK781g zFr*s3a~O~b5#uHIU!o-YS_$@*oT#_3rx&bdfpwsYv|aWwnDu z=uc2n957rzpg7GB zZy994)?ImG^s2z9;{*kq9@|dYV~tD+3CVSu`RA)rDi(k!a%^d zcb@@b>R)WWw{Zo|#Nr9?C@s1sD$}@WBkGy#kiw$W-@W}f?B~K@6I(G`@-(Ap-aH0%f-ikG*jqb>s7d zKbOBfCxY@&v*7LD-qJ;-s>M^ZPB~kM zw;b}1r^JZ6<%XefsRr^kI}%wZg*>tP;D`4F+(vWyn!`uOdcHAovr)O9ruW7~SQ*MB zqV3D-!Tn-H3J&A8Gmu>pIDVIlPvz3O;A;j_i4)MF8qOe_6*R$K`kWu^OQ?G4_g$Dh zB;#L#*?^jGtqKnSnf3P=h33`Yh53;q#_Mmsin|j57KY2Nqh+hB1I^$dNoym|ejkUT z0}{Rv)Z|WBc)`bsv*i(i_fY^FxO~Vbtt!RfNF4YZAAx3=Cd^~Ea3BU=XQqUwS}AitnkB}XiMG{x zjNZHkZoE++77>`C#tw%t-mlwhL^A=Dktynf)_S>5f^=u3i~-I(K+W@uF1T~~I`l6I zuiwjJ3#J1rJwy4aNsvekvPwW zEz^fT7_DJ{&tFq`2T2krZ9#6JOIJ@Kig@Dw{tC|Bs&a*dCHgp?kQ;HeP{UKSzK{$;j)!iVU^pj=sL z@AiXJwGgFf3l{N>M_f)y6@_02)K-Kn&0zwk9Xgt zOgHQ%yz~RqyIvI(ndWco$~S}kuGv=ba?m6Z&Qp?iT7h5uIiF!^ZAa>O2}x>S0Z1S^ z)djX>yr{U=0D&8P{D%WBq38_`Bvq`jjMz0HH%!GNrmQIRp@~fCAxRt43z{M8Bp;4E z=mZ1HH5glfZQ5f!M>F?ZJT^UOwNZc@@}>6ZBYzcIN9!Ow8L%ousz$M)#7-vjGJQI7 z0ZfbmO=^_b?_horp>KEnP6+%?Vh>g}>mo6rU7xfzOlDFFJN6XrxYOgrBEU25n9pNt zLD&7IAy|5+Xh1OV0|iaCT;DzjK=D2gbNUc))5`~FmiU`kGf&F78PlqxZQZF$M3_vV zu#Yz_9F)L!uYA?w`hU_P_KJy}j=}L}A$t_!hKi z_+WkXH0<~zeS|t4LC2P_lE(($&CyA#PAsU>fpKDeKc^u^X1x11eHf-|ji7(P?;+p! zpzt@BhXdeWP;Y`Ad@TmJ+aJe?qGfcyK*%m*SK}oE zMgtvg<8^ExpT50bNZOBV6@qTG+`XW<*%9IZK3!08AW3xQZ~f=w;O^XP)$k5oHX<%R zcbi8y3j#rC$aRse6#tuR8MN}9Krf??@JUNSUSBG3;rH*nXV2SC4>fc^0^kO?QgU}P zy@mQ&&yB%Nbi>Km?|FHgwf%lYoS5JXX3VODD~=rwl_kaxkmcJvHMg8 zBXG_rT1GLuYtps~N20J`YOii7^JMRvQvGsk`oT7Lp~Zc;tx{L$!|JRweso@b0VhfjE}GqiHNE;qnD~D9OZ)t-OsTlqOoq53z*QfFSUL9%9{c1&00xvy2w#! z8@WE!-nW^`C_Ef;xTyA~#wG5K>8~Q8F4mu)hl?+L9_jMWzt(9Z&YM5mjL2VdMkJ#B zzPywY#r6B|Ri4v>`ct(QiGQOoP{(RFlVY?}P=-@Cm9}T?6r?$0mTy{Y7_}6CcQJ{3>xt zr*@YCacEXiElWxq99)(&zVn9{XeULClF>IiXTq^%?WG#(V&utozS50y4o!x%R*6B_M!s&Az(yq>8I-W;0+K zV13KsBLV{j2C4r^pHyhw-6DvdO?ylXALY*eZ?n?j7ra0xzQ=s_ai(9E^qNc4AI{^y zeQ3d?_D=$-FQ2b_|D(lFQZNI4J6F{6nqzCuC>GtdE`elZo(R|Ia{4vg5WjSs@23-l zS-#(G(>mArY8?vYQslww7!|P~M$OQhMGyQ4z&eKRO=ENJb3jw^neOh_QMomHa^-S@ zrxh@y-wFuxhC}f<0I{|%%EQYVVF2u0D{GYY0j=Df;nc8vP}MlN{c=o5SU_5dIvoQI z%eGsPoBKv(xUw-=2ubVv=56K`lC zc?Z%4?P;`4TchtFCq%L`Keq8GBOXgu%! zH^Ksi6G%f$@}+n07o(J`;A6h)U{ObAVi?GHx02rzimBcVTj6Dzy!DTA?$*^zH#|Xg z_U0q+IPkwEH0CVhl!chOrxzRb|MA6u1P1fyBUtq%kO1ungIH*qv<*l!34-gO;PY3v z3IzPX;dmGol;fh@15HC=FrVwCr!_k8qBY8$*Pv7$qv(z}w`72NAHImKb4y%n*hjLd z_1!VWa~emrz~qS+vH`y(=Pa24L{m*8J`7w>5rKP{KIhTbVk7=n(A4zVa${1~C&9;^ znVe;DhpnkGBF0yo>SfDm(gEvS7>I(wxTA|^7i1F)EqZ!+%E)3M+$;xo!#)3iBR-1D z&u1y?lh6zhq7?=7eF&JR_kTa)iS8E~H3L+&_H~LnmDgBwx_fXJVKi4vJ;;xS9?5Gm0U%1JoSaI2EE_#)^n`gIPYuI`rT(u z9$p*@DS&4oFfDYCb|_n%t^#n)f_3_Bmd}Y#BPuH-$i#J~BhcGYwNC&|Qh@gysCds=^y9zHFWT zQYX&nqe_H3^Lk{s(I~a{9tFGvtIjcIGyNpE%m(^Zu^V?D%GVEo_JJO-G$$PUTJg_@ zTIA@5?7N8pi1`co+*T0&?`V)sZE|=lO?bP%r|1r$98TaFseo_-q5U`Fybov1vu}Qj7#Qqlvic#Kz3}Q*r4t`d2#(TPu-(mmQ?p zH_R-LbKR1Wg}Fh{_o}Fr#vQ?(DY$J;`)cuk3NMC>N!~0PG6rQ{`&#^}phIaqO_^+w zvx4xb@ZuHx+UsKnp~j!lTMDv=9CTa(bdjlU7pQZ0&()WYozn^x*9Si=!xy|YGSS+U z53HV_!Ya=uO@M;Ez37_&h60Aan&8rZaS`Q%$douEEXqRDZyfjxgz6j~ zy_>W(x(c3SV+x=D(}kIoMf+&E)c^5cU#zt6Rwv3sl!{hQnc!JkaccH@ zR_?-_YVW!Wn4Z;Jx5+msbacT?T#|S=MkW=Nk6WEqrX6ex< z3xN)FZHDNRXnq!RThAWu4~0&9SdM!T`M;(=(NZtWr<0*YgsM0zokCtjjQ)Zzv{f6D zHpedgn3Upuc!@(=$xS7lvApngY=+bAopLsCa2S(fnw(idRDW)JZ%Dq$brvO7G3h~? z`s}+?DJ56PAV%eRCgNh`qV4e*Z>!YG=$ScNi4m1^Gb;hod;ejwm-SZGnq(>)@zla6 z2M5%I>y1n_4Fq|uvr`3>2}ud=Sz8K+Fltz)13!YSyyrN(+^_F8eGu^4?C2uIhxsEL@ax+^X<9T z{K(h7x>(zTZaJ01gogsVhev`~N`B3A$u>K0rr!7A%O}y7HTx2x)QqIrEdQCcFzc$s zsEJ)JqHKw*5E}xAv6?_f@*UGnhJ($b4AI~K*ep5I&v4Vi-%_9snt5tl@uIL=CyVfN97beZe`1jE;3|2 z0By`Ru1Fiq}mx0!s2?Z_#XcRru_V zxI(hxqSCdl|KPhKl`2z}@sb>~* zj|8iOab#_-2|~vlZOIjEXiE?<%^^b@gxV>C6A??3j8VOkDTm1Nd$8yFrhNQP^o0Uf zw&x|Qt+$hy3K&(^UP|!lw|1Lpv5dyZ=Z912;=5bbv}x{IxM)9!Jv( zq*%HQ5VN3iVhZ+_ZCN=6S!OL4wu2gbwS%!HF+hXU~aI@!4X~F1;3OZ@D_PZ@Mh8 z3qAM?T(CQiu^S7oCDQ0AR<(9?&wp=GPCQp-^*SI$?$`Rg$%%$ zX{}$Bo<>A-8h;GbEIAt>y z^B|odhy!$!3mbE`ZxX)M$Le`K?ee=m%XA6hJawW@9%HjW%}-fc{*ol9RAC;VBl-#-J*7^?ESJQM9N7oc5Nv}vojBE?rwvx z!+c})74DYBLamZ^wUw2!@45$1z-ZeeWN(>KxBTZ>JvZ^M3p_c${;A}=m(~7oVS9Ui zupq;gyIbpQsDNUeixk+hneVsw$)Tq&c6j}jjOn8?VWA-L=WDD^t6=p(8}}}Zp^bqHG(=0A533iy_B(I$qsBAqMK+Koi)>QF?g z@LQ>;t}y0BgB~;M^0c=|;{Dy*oYZf9Bzus2;mBzG#<4Fynz81N)=G|UW8GopZx_)I zJ$S+g#TG&uTk6Qle9pCceaJMQqtBaF=(4r?1kSGn&#whf=|qVSguU_M;rorkcWseW zi#{$ZWUQb95m4&y9=!x80k*5IuO5uj+Ke0W<(e_-Pwe@!#o~A?O*WJ%<%clV%SJ1# z`&$%ule4TyKo;%gdbG;$%C?%{V@5NToR|+D!PC_0X|YEDF7cZFFLz;{$8IUlNLy+M z)Xh^sZ z1-6tp#MHHGO<(wO50k0r<;b13*KFwFp?i*mAG`xJgMpLoFWW8dy~uZ!HBYOs zKTxX@R9r83;z-|##f_V+PV7M-41Nqh&Q2mZeVuCed&b9iD&a&Sda)P%wUmAIP8-DH(V=^G z+vb~(pajWTM6*s_o*K-fmkhQWGX1XP9F}D+IWY#i(m>hs!F@1vc4w(p8e*yjlm4lc z(~Rz-r40TPEZZ&d+55-Y1YAwRved`BeFSRS(xk6Xtf|ly)7AyyLSWVpYA$35#C-Xv z!SR9bcb?n3ko_JmL4r=HMNdL0kvw6*jtORP<>A>JG$#kVf$9`EK8*b9Js)=J9=biB zrB>#rKPSYEpP5y8tbG+equo>|e_`TzmvV!P$>5>(t72CZ1%1AcE!q%&kI6NKyI#i| z+YA))_&Pp)N6gdkRZB#gK0mft#GJcpJImBB4@lZD#RA$_;5=;@BrMl(j-dxuN`%T)=UsStT zEc-Wq1P4+L%5N&cn=QAe2RTwbM}BZTll5(;CRl<)1N*hm*`QhKj~bbiyRzvC1eM1I zV|c_>!^>4G1;ayPCk9tZ2*@D}w)fk}r9T`*kyKkX&pP)-mDiNJn;za}Ilh}DtnZ~ShSjV6%g`^@D849Evkj;^jmvKfYSRlO~K}{@}U-oFvE{I z{hJDzj3E7yXY=8z3c9;Shs|Lwd3W&j@%dmdpA}h@Ze@kZDfD(~nIAM0nxb-P^LO%wk{f&Q?xO z10$Q_FGnD~w`I1)Dq*|9j)GJ^M7w39auIyjn}1x1fW-feV!TVKUsAGmD2mq!?i<+* z5;At9%%+o%dWNft3{mae%h3%n;~NA~JjRR`d$%}Maj|FeFB+{Z%Zzb0VeqyxU#d2F zQGGjP?tv}<#wXmzljD(2Jt-+|pDd=#9vbxp!#(JF@g$;-+9!gpK3Q;9R^XG|Nt zp8xQBZM$uyS)Hl(5y@*Gnqt6xfT;hW=_;VATG}>^beELUB`w|Eh;(;KcSuQxAl;30 zNK1osgGjeXcT4|sy#IF=>z?CXXU^Vx=AAd6=bhR+nB}vX@;J@H$MrG^rV!uirZcK9 zufso;3^|KwOj)$d6Sc$WPgcNWefYQGu522MWvM2q>3v>u#k{AaC}=&~ly{mQNHLcz zHoqu1F&g3Ufw+2{8d}NK3wK_JVnhi|y&=uzMJ2U%5M6ZRxbF@aH1AX-$)0vusw7n; zv$yY8Z!Gw+r*M^F%mpiFX$atPij?5Vo+V_Q?+OkA40$Prt&yYK(s%EhhT2Nxi>C#o z6V+4N=+37CqmNpyG>un$8EYxL&KGg`&%)N)MnL(x5GXmLI$a=RCUmLKUXyz&#ruQ$ z>mq(NQINKgpBbwMBltVK--5s*{g@j|apSa+IU8!&o4r_O?3Fv}YEplMLw5vS6a5#F z8m(WgNk!NT9CrzyD|lu0MrYwM-&D*~Z0fpC+b*9C%vtsYeDN@t53}^Y!#9h#(W0QJ zoF`Gpwfx0m^a42NYMaA#o7XIcpZ96(QnGlzea$*~3Rh5I^+alMSN-mS2CT|7w*fu`812&cZ$!W^f110j;*MF5c=i@7Pd<7@!Ok(h zo^AX<2m?Tl>I+#K%hw^)C7C*f3qr7zDy}3XCa3fxLo|y~vvCPt4>+&>WrEYOS^R0z zKc*?b2xc%$-jv zX94_=_Zefx-FL#epMbwLcI|c8PKJtY0^(KDwcc`tvP5NON#H}5!S^QP5 zH(C^?5y&#E@EU96FuiuM*@7-j+0^B&&^>ok_mpn{aN-(*Lon~*PVwmvfUHLXh|PA3i`Mdd?ftVI4XFVf9X zfcIvK7evA((#x1bHHnju>O65>0*!;dE?RGSu4fRR4hjpzelc8&dGB!m<#*>X@tecn zm-rl6Pk>iK0{MQ#{~34OFOUjUlU&V7VXskukktR7cQrW*LOHwaJPBYbT|#d{z#hfACa3o(F?8#u#@|~!YM`@N$iF_uj4(J}|^!H6G5o&LgXadPTo2kN6 z7&4_p1K=2erix}XVrK&ue4kef`G`mg28RC)(O+pXh#Er1k;KZU%e;-Ps>cA$wLr0X zwyc=I^p>Up^n%!|LWDA-OA0akYRRx7)(s zg$$ED5loSIX`%wdphef$L^b?%muE}LNX9mBNtsS^jK?Dm!o3?1sbQ(it{+GMN%bA= z%)?UMPAqSU1*&;9K?otE_C{MRnf0ZLWhu0*QV%&1yZJH*iK}LE5#s(uEyrzCpvbGq0zG$)%+joMpk#`~x^fV5MJSa@6 zfh^GtRNnR4&Xep{jV^&Mjpof->@fan3pZ5kX3@d-uZL)|0s{+-XlLh)A*y_oR3&cA z^L$3TfD22W!Y!>~CZbA$wWlFV(4cBSq4{<7vg3js9z;jgYm{E8^;mSq+BZb(AVbm{Z;ZetKa^e>I8G0$ z597<;cwU}wT+7{q^H6cJZ8*uy6ym8~lMEhuGtgbLeo5q5_EK7YzPhqXRsVWl(@Wtb zaQ-;P<5m&|3|jc;WC*xozHE9^3-s59{WttvzxKisErOI6STHY)S?80c(szekVS>7u zC9p+R2O>|dX{B(P$RCA!L?xOPO5aYl_?s%AG)C#6l^y-)q6Gs_7Up_1U^W)V|*DaX+c3$Hrs~#cAXGs zw=lY9W0X&rs}+wqHYl(#)3i?4=|poj-+V}IC6+o#ZnNq68u4(+{S)M|!HZ6MrYSKG zq>dVnFkGYA2F(a-nJ7taOj+kcg#3x6e)zuVm%>sE#gv!CRF=e&4aJoEyAg$&JFyn} zB!c+Esna4tyB!IE9KKu~zI_l<))cbU$14+Id0!2)B8)wpZjl^qSGbn5;5NRuL?5PI zK_T?i_cpcDB=mTILYCd{dCkFS=OD}r&}wADgm8;psX_FP5H-hD+GuC!5DrZvE+;Ds z<~&+5jFn`+@7*_Ium;h=-fS<)M0k!TtcE>$*#F`lF9IomM8=5R66P;s^3Olc#IQ}5 z8TXm=jr5&B4Q9`iyF( z5on;JKKSbWI1U^%AOXNvwH}p$?3$)u6Ku86H5p{7i7F5!fxb+;Q4ZYxu&=hj1r&@$ zt05tP8yCQ}yr_Z{8Snbb-|08JLIz>TyDgfW=weJHsVfIFuV%tkM+b`o%_I$c)^G{( z?|V2S-z=u%D|nKdvX`R9V(rJ$+*-jLo+#li5d%rKW%E@@Vu+~u!h$SolA&xyCYAF; zzku$ayW2yY@4UoN&fIdTC=^7Xm{z4XA6pSHBw zB0UBe!%pv?y>R|ZEX1`u!UU4SLaqWb1^M3IHNTgVsEDRSGNEbmZTCW~Y6;>X$>6Od z_3{)wl4IiO#|xy)A1jjxJ|DmQOrxHaD^C-lXbfS@+RrLkJXHm z`}%J&1R)+7mU&W|e7X99{rm*L=xd*@JV|W+OMqA-cRZBZ(6wJyvD0mk?UQn7enaiI zcl%*?^W%U?G_I8dzUsYaR-fa-xe=CK=wDP={WSHQ9fFpPH|M9>qMR`yD)`#FYU?AZ{{Ad`|Jx8uLW< z$l7@)>~pEEq(_YTWrt3}+5in%e{0YM_lNaGmxa?0W49jb5%G>XLh5V=T%WS_dUFrw z=KDVZWpmsrRT*`*FOP4}-+WtSY9W_D=(ZO)kIle}qVj}n0;kL$psqpJ>WQ3nn%si7 zW>4`Wa6!W2bpYNjP9zQl%S^C}$+7k3T(4G)JHgFbb%^sLEc8mog5E~G*SmO4w5Aml zmG2+fM;R8^-I8;ARpHC2`5Axm$J|(6-p{0@83%?35NXbALKM;=dN~MGz1qly@9!E%gM-|_ff_e`C%QL9KV;?!+j*4YsXza|R6&OjSB9^O#23P5`lD!xrhz9{ zcyvHA_4d6^k*i4ps4n2N^={HNx0y{Mze^z2IGs@4> zI-Aj$W5sVw_YMfVd)B-Vpbb+Y8Y!{offoxai<(Gzv*yI1@1(=yuJe=-@oZo8Gn}_T zI_O!V@M9NmR|oC)7Q{sYxj6HgiISGTib6f~^Y{E=UO&{5cj!Qy(k}~dsl(|Ule+Ak zEXH>B`FLMw$8-EufY~qBJcW6kp4(D{y!`z6>eN#o7+J?PLJ)3poT4H8TZ7l5V7;fL z7ctOHN*Dt+j;EisN%0XR5%JJMA{{ug;t91V5?BG1wL%*n!dd<2s1;8ZH>w#v$XE|?)(mr@mlNB!YSvicwTu{HRcN0r=wP^7&LWjFhVoAg5hG0Dfl$9YXl)V7 zp+@?~l!z4hH^lCC+ml1|g}H*o6X^Xh{7G+@Qr^quRwcl)8ar{(-{fshO8cUQEg^^+>0KJtL^%48M&=k&up?^ZDo3k7SYZM?B$0Ta7*?x`1JB7B zn`1j#4@1f$h}V=|!w#B7j3ieNiwCdlf9fea!5uFU7e&2_F|YsJziqS15KdEC!bFbY z=R-MMc18XU2dVG0?nE%`y1*CWa&cP6XIfx}15o3nk27wB*8XTtrF#+arP{lqLsTmm zX>rW7P@}&{)b+Pe8YqyZAs0eo=ol*EZP)#E~*LLmcMTGCx zkN511J5uYU+E1#Ars+-{fsXG$oxhI67bov5)P`?)EJ!`8eVwmY)Ni_85rVcEGpS~h z6(hrnZrEwq>6ykFXx}iep7o>O-l@#>tKw)kX(iF)Q=+yyfqrs8IL&z0PVb|kVeB1I z63`JKiCF!7g+a6i`jgY$41Ren4@nzMS!YR)E9UA}C~)r%?bR^02u}R~M$lYYSONtS z(esJF6KL9da~-W~Ueb$u4CT1*SRFIPz7}+`2iFwIh9faQZsXa2`!Xtb)6=33pOeLQ z-R_gjYo#XAoZCe{h1Nj@YMuC>*ksRBALbvo|G9#|Q)w}Y6Ct4#>~`_%4=(SpMK@as*|GCg{ zbG_G`E*rS3RN=U0E?Q0}h82Akl4vdDQA|Ug+Z#>aj`|+|O?gctEMT;N0MA?LFZ18D zL<{~r+H(B`xRw=j({~XH;_DmA0gCm4PaDq=mTDnLe5$g59-N6Hv7wv02ZU3lS?mBs zUUnirY56$~bSD_yy<8Es11(OqP=#5eyQB>q2K1H zzQS)u=~H7wNxK87I1~1S6)}p&==xZojClEpR&c|OUIN~Lv!gVl-xUkX>ogfinPiGL z6$lpv6yf1z?3&Ubdww1qY#|OcB5OE4bk#?YB^vEQJ-IYT^QN#=UPFfvWD3i^a*qpL zJj*FZ5T8p=I<*-t_0I->_*(e2aDU8)9dG*BSS-Hg=<6eqT@TpChvw6N;c0%(H;`w9 zpHOP`(!xXlcr@U(U{}KEIMs=Amyh~I3EOE4L3AhO`)kKNd2iB7;hNNXBaXgUGKB5DTiJZ3qKZRzdIMC|%xI!@iN=LQ!e&%%7%Y)hLAmMY6Dh zuqN`Vz>Y~r5Je*+gd359l*wQvTG8JbwwFmJzlr>)XQJaqAU}@>BI#NE*uYuCg{kX#yJQ{kxzjDZ1HIKq$L5x%6}V1`Q;~p0|pEw7J@9T zY0&GwMs>ZUi2$(q7U6?+-WBkiebj*Xg4Rm`T9x<}THNHd6-o{LB25_JabETfXGW3t z#Tyq=Z57j@_esA^GeF=Ggn4M7bvJLzN+KpWFA_Gdd8xh{rf{2P z1gUW?m|#w{sPvhuQU zCX|t!{WWGU^8S`s3}lB>pPiq$o3&-BH`U!`(h;jG(s#qIEkikA&E~&^7(h@Y08dQ!xU%_n-gwhSGc!;sEYU(b2b=DWZ@tU(MF8 z+|Q~M#?z|{2T5{z)L|)0vf&ebiXP$`X8(aC48xXFH1OE3+KTmZ#P)MdQL`JUD6BsQ z(8(vw;BIs~IEbjErF4Vr+=>judJTJtu5DQ^5nJv2RMt5VEY*80Jr<3PkT@s-N9ZFG z4QGe_@A-EH@}&;Emku?Uw0QL!j-)?19j>{HZS~JSj7d?D_Q;f+L>3z=7j2$~++)Su zzZG1~%T}>{?g<=P%qHxF!fR}#oV~U|=Yo}de;7_fI zGb|v8D{7-a5W}a69p}nmRv18okMLcA#m`+fEoJC^c<^O*&!<)Hf2DtA(2R_#e$P>i zpFWiiV|_^^!bC*YM4_KdU04W55drmaLnvAb;Y)n{qL2Uxfc>mcAh}B{&C1lkVec4b z8um{Q6c}WGw-MA7*;YL?EM_RM;x>ky(?wrV3_G2jZms#clDzzV(HUy8LG{sBRI<^$ zLHix==b*-jQHgpjG~b6kSGj~oM`X~u?1C6mF#)K9Dr*&ogZO9>t+pxyx4X73+bhB%t^=~F)MbvFr zm_*^UYZ3RIHLH#z%re-9xge&_DpdLBmR4*1IX@Ah2;{k2;`!XF;T;o82M)M%*Fds$ zM+uTMqsZ@8?Pjm{%K1~@JP}0d_vt*E(muw3Q zseTW=Fe&|Jk91M@`SSh@B*i&a$IH-A)ysF(0UXTc^IH%tvi}uzmAdewdp%zpF^x!T;}=`(*=$ zrTYrXKmQ>=WG7*8utEdn`04yjs?SyZ;mIk)B$|1~LjrRZX8Er)PYNgGs0Giz=z@xB z6MXJy1WGoXZ7h?zj@&@g8yd>UeuHGuSvtTxfk^NJf<&wC#sW)b(P)`iW#qve`pqpM zl$(ae5a{pjX2(n)EL&Pl?Q56hs0;z|&nLzCwx>L2G<7wm?VAZvK!bnZL!M=elh@aL z;_|-9mDP%Yzp+5N#%@ecrw0uAlDmQ3Xfp}+%+S9);Rg}8BSv61fc;v(Ea9%b7(C{3 z%~Eq8W>VUy%E?<*5jfkNV_#KrY9{z_(wq!tTve9_dH(OZoL-%~{VZ9*ztzqg`U0I+ z7XhqCEh8!JQcDwgoJeM%s;vJK-JMQS>AA;uyHsyHlsDm{GPu$DfmI7HcmMgsDNxXu z^+X|VPgeBBOM!Szsk`co=(Uf`jCPnBpX4IJbD-?i+d{V?+ptol0byT z$Xjo8Dq4JvHOO?+RRJ4@D4ogW6)s5k6M+K$DZf5yu$+5`8>R~T!Bx#X6la#yip-G^o$QT3;x?P;u68r2cjR0+phS^UdpjqE~w31r-jp=^(XMh zMHk~PWYfK9GjiQ^d^#332y|fma@MG^((%HLftgf9;!gKM`X4ySa>WUfQW~81BmfTj zP(1?T>i)aKEL>l+lId80GKsX62lU(Ks%%ke@!Ci(-0yNKafY#rKWNnOICEQavD8~38*_R3P3rZv^QFP5`_aGG!`LR_XlrK(W9s*GNv~yEO44}<51C1EnM__ z>|kB>wx9Tp^uXv}Vlka(J*Sp@_Q$_Hrm?4;$f4X=15d+F4>T;<3cx$qHhp}Yv*r@9 zrGPY?Fth#Cg$o3Ayb1ny(hUpH7uW5pqV#s_iU}0*H<_ z;U?}>Qwc~KS(sQvM8%*XLjKV8o9i}b}W}X9|n>BV5avN#}a{x-M|;thhsbbXxwpQYyv5 zmY?XMMk%)IUPSond-@-op_=vqNL96PGYY=}Z|a#llLl7K!$rpWSqq_BJ_(2at?S3% z^4#4fGtaAUzET&yw}xP>f*>{9@L*Y*$%Hdp_jb>CXf+Y`>>tT%WNNlPXf7ivItS)P zL#*I7htF+{@$3>-Lr1nnwGBv_@@Fp3*n32v-^_}5%rj?(1ZiUe|NB#}x;Vw=`vXb5 z6rY-pM<3N=*FpJ=r=`{IXD17LU{%Ci_HqPw&T8dg`vHrWXaeXR~#~R1-S}f@s`p?a`nEA5>Q&=n?Qte6TU>vg?i-!2`)2 zq25=NO^q;8v{Wt-o#d{teDmTcB`c}f;M zU=nF}v5&^FLGQ@^-Heb-I*(xq@71db1Ea<7)=jRJd{krqA5=)b#C}uknDbZhIlaM! z)!YKKiK)9e+MGWd?nWX{2~qvd$V?Kw5qBaS+ta!TZtQzfw zB1%EgnZ_gX7}hRVS#-b+|jt6*A>CMBJ>apdmZkQ}j&itQD zQ%;zOegEh*&Np50klig6Z`_1{IX3r261tc>Sa;U8!5oh>)T@oyG$+vE3Iwib&X1zh!Qw(n+i?ooFqd`B8C9m;rU zod_*l6BF)`nD4VPW&^PsZ)qt8JT#*a7X}Bug38r=e`wJB!>)#F7Rh&oo{?=zoCq!K z&gT>E(?9;5fr(-fQ|YDVe-Gyzg7{+(G|dYyIQC@rLEag%1y%Sf#WGIBg6C?n zKL+i$&6AO$I)66(9F2yE)}WscP*kW4yQ-F~aun$j@pUy{VEAHD%jcxfLb0v~2+dmP z!gQN$`o&LNO-{;-34b|}EKLP*`@3Mu31!bUC7P`6OChA74%n#p7bzBK+8AEC5cwf{ z{LAAwh{K>UPJWf`@Wh5=YIWkO(qR*y*_7{B^}HxunZLTjDxBWreg|4K32CoX!v3A& z&gr_>jb&^cE3Nwgi5ywIviqdMVA39+Ug1y_lBEm;PWU}?agLU~_ce^eH0)={>r~B3 z*{+cxp?vg1(1P_Pr7W}cFzbqyh(3sjmwQZgJ*JwJMieWv)l3eo31nrV8UA~BiBaZ5 z>Q=D_B8$-i?^%EOr*Sl z(dk713hQ;7{jRk-_}sr${cqXkNi`67qtM)9Ew1kcg%IM@%ssT6*5+o7`yag3Z*V~J zBfAt}#;+-TLA3Sfi5USXZfWlsukpk`BjZ#OoO}1NnZ!XHzvHMrw!fZfTwCHBj*4J+ zcJ*nxyPu!x5t){3zDUa1{(D4=zWF$}=q%J&i+bq!#AD!VMouN$Y?j?e^E(-lplh$@ z42GuVFZZNt&LBl3rqQ7ahmPXLajoI^dUy_Ck)R~daCQ@_M1)6vEi`<0n#ShI+!y+L zCoSf=L2zQUHoLi_R)AtFR3!syOAp{w#ZUmOL6m)OznBI<`cbj|ds14JnOLuu8c4^O zRDKqj)JE@$Yh=LReB8nx)#g%c@Rj(am&2<%aN|aSrO6V};D0ZRplu%EiEPI}Jqf)& z$lURfW^3q+i$z3ORY6ag-|V=!cw4U3&-X+1G($(LI6~2A&W&&{4GrS>>ekvX?kVT* z<&KqhvX$gKw=8{X4kXQU2H>H;pLN`pSke9IlMnqWiPu1s<0jet_ar)L@T)@bQk=i> z!`~T70ND>MQBiMr{{c}$iN3G$=8d$9c1H^aMtt<+lJf87ur)D6QKih+}_RIadCAA*mS(cuV- zUy)fld`+3h6O@^E#B!|ZcE3^?w9B(PAyRI&#y=1zS9XKFep6@xnz+G_zo<3?0@S@D!Q3RQ$@I+qQ6>eXCS*a37&j<@7v)9%FO|} zD(|)4JFdjE!Eg(?pERUb=n`K_pF%KN{U;vAwsgHH+t@$42@M`*xE==91VRgZ_!8|# zFEalgt}Ta3&T(^0BGl|unnv1Mo-7RzDf}!7qK*Xpfy%!f7V!?Rk;7JqO!N*t>ZemR z8Ty;h235|w{d<%8}hYE%$Nbgv*-U z=o4-W%_p`)uLI!rE#5A^2H0J%gE;IS#zv_$&xDt67!o~3+J*0|Y8u>F47NwF*z-F{DGeu68mY7{`HU*XRlZ*@=& zd0xyRI$PW#qhi_HihF_f>UQQr+A*C~K~I z+7S3X3p?aL%B=Ao^f8KJ!U2FIl*iMM4X6|PX`}twTI-0a!hwynWlT~K4+sDXo>1%O z=&yyh-xe!*++@-kxV|O^t2eQj8PRT}g*BG6*k9i&ksga){Fd@O&qD1G6MLfbyQpTN zOnc{#++xLp9;GhPQ(=RUo#^$H@-v)ftewnxDG<8t<^f6Zn)<6boK${lM>wphwTaU4 zUQg?Ujc}{XJb4%sbOVRHC-yE$l4{iYHQ%I{)w>4YLTqb@O#?e;zsN-EXj@C^U9uU; z;$lb&*$y+sU@}K6y-xhjLGe5M+*ohBVLSX*L*K)v zmkOX07UlCd?X!~{rV%T_U!MWoGhn7a7~i_s!iG0;D!Lonalex_=nSKeeq3vc7 zbOjo*YdPe98Ch^8!oAIufsm1Plc~^l-by-u-`JQG6i>AY3R|OwLP0^V=q@_m^96b` z*o;pkE0|uydYHPtD!lFB$H2$~3C8nCy|zKC2fscmRORQT*6@%^{o%b|Pta^7*^sm=TT`j@ z&ey;bkhSJ}VNltN>oY?KsvcgYCng57utRlylu!0NG?-`V_kE6+8Nie+=?u0tH-@!P zGl0asQ+-rl3oR(e4mD_P@mpKa3g<$KyxgDyfXS20Iq>JL`X3em8ctNi(2$&J?+4Z? z@L5Tv?@y0M5S?5d5CLK29C_JshsdQqgJ}emEqi^c$$p^3b_9XwI|!%>nciO+$7{NW z-2eaB0S3eJLl?}iF@`O8nIc+bisADQ#!Yn7)kIO~uL+6Y+O;vX zqm>bymC|F91_~EF)!m)%H5o6|gPt>f|A1kLzlZ4hDa6wcM?k7ByFlh*Vt$D$SihP) zbS>p}urde{yj=z#bT~!W}R`mL*l?<#?58>{4UX$Gm!Q#-# zX|CYO)awI&r5w#LUEy{T>LmCF_Lsjv{m}1`g{7ODaMCshsG#pVj7_=UGe20yH$uov zDbEkT3oSOkpH)$7H5{IiTZO&NYwdPv5JLBGxA2dc}J`3=&I~9 zOQLFQEhKcim7(%Bghg|2__JiF;@gncYVL4hfi{AUr^DtsFu8LpGoN_pW-dOBI7zP^ z_3L)>7lOl=YAvcu2HY}qjB1>um|DI7mgAdF!EoT4rynE=qfS6H}US!G4nyEySi!T>90jNT*sQ~1xempl~-hA`5hsQzYz z-G56;P1SWdNnxV---KwulGtbZa&OPP`Io{sdA~;`CFTp}9AMuLn5MQ|hR^q|STSsc z17#e~r&+UkoM3{2f+9&%UcETQ_j#wG8XS+z($?oiM>j4m8i9EHkwA|O5&SXDMz{q~ z$%-+>iwLAkP!Wnda-w#!1b0~xn)bX#y@@d?6b_IOC_Qk~1mgMD<4xZsM$r=OW*imL zNykS_2+H!k?2)1K0;xtt`;87$iy!_RFQwvD=F1-}ey1=Ld5q=JIv>S3xRyKPp#>)1 zoe&!MJhWGo@pTe_zzZv5vKGeZh5`b9WY8oGiT-xqA z{&r?I75o|QJ}y2X^qJ(n&W{f6#hT6A&2$=`s@_E2Ev?%#e5Hh}AfSd;{T^5suQ3RJ zd!h_4ga5x!;y}MSQ~=0fM-bi}1#A8zN1PQF#15-ZN4{hG3I%UY<|)7Yt(C^`J60?e z)~O^GNsQ4q#(?f?HdNQ>(w6J{z89b)KR1`j!W|K2b3O6aD^bjM>kN|F1*efXz4Y`* zhTqrJg2V?tV@dqNe@oPgfqgt}hIWaD@Fs_z-CP!vSCzBb6=~Q3CLIn_g>T3 zO2-Gi0Dlx7TqO{vyDa7*m{_c~GxNui0!UlaIpr zSy<{#n#0`)Rg|CnQR;{$5B#!~=@?D&wTL`}N9hWUw~4>|Ta)vx?cd<+6{Z0fitKh^ zOq0`PAo}*|?=M5Y=K{O>77b%nA<`X{$e)W5J1?m`$4hWs0e&HtB&oS{RRwT|*^i>I5g94+z@+ zyzu7s%bsf=)s)`+hL_@<03g%@rqJpA=EK%+^_gO?_QiQNCiXr0JBq zTCtQ%pAY?RW(+0Wj42@~H3Di_*g&PRm)|Hn%-vlreqsak(B|p5>WdhqHqrfqgPu~= zV$T)5JdjL+c#ES3?0BCBsfsu3acZN%)S{o*_H=l}n7Y!Xx(#UQsE6N<6c+%sK3s;9 zra(IbDPUfl2n%n}4~%?>I_o=GNvb=O>s8$q;tklm=W}xPXe}BMDv`vM53kjd1aQvr%~Kb>KH*yXfHir~vD zW4cMxwoRoG!d*h~gH^xG<(05+!WQ2rUg^Lor0K`(MU?^mJVJ38hjgmsJh}8ujWNmE zLeOymauX_qFIC%4PY9Pk*5b0?%$;^uuDCa`g*M|a+#|8Cv)WF zYz)Rxwpr1LV8SWn7={5IAPth!u<;?QzPIb&lXCVh_po-lYhHYW^}BRKrNoyd+;(Zc z$Skp;lT^_bI!9Ug#&5St)4;@q0!rW;AtDLeilDWoYTqcAx~`3+2LseQwrgM@c4$f>gR*p0*Qv&FMNnQylR4>XO9SejtTUw(*}kM z*e_HUA%FnJKKrw`Wfb~p*N$9DAf0ulnUVP>-&v0SyiEy2c|3wKDC4@a=Fk>;eQ(oU zBB+Yl3a1qGac0)IA_K2omNL~SK7+q?E8M4U*3j0V(2DK$MBtZ3GF+^`aqHP@>^8xf zNNbt1Az<_2?ac8$zY1a2y3v-Cm!bX{9*X?}RhpF#Q_6_PWS?iAL?dzbPeJY|nsOXU zBL>6AiS6M!F=;|7QRb)=N-q6lW6ibm-NB86?FjCn16K^<(!1vht4 z-*|Uo@<2GGzVPXg+_3E|2d?$%NB1AlW}Dv(*$Ix`!FElSD^1!liUJ=b0b z9wcFTnmbX7(XU>S5&N5cTe07PcWFjU6640#loVsD zCu@6nkD}BBl7_=FqOxRU58DV259C~?r|d!BOyc0cCO=fVP@C%e;RK=dw_yqatp`az z9M5*kKebYG*PEHc2&POFJFq$~Xk+OlD}a9vm~G;wWQB#*T!i58dNB7R^biuG8eX2P z7$D;8@*N1jjRYLb?K%;GkPg*?U4FN-1#P(x?FzX&uQFCX0Wu+^|3NkmB~N_=%F#U{ z(6mIEPHJ5Eztfj2^IA$%yIMpXhVpt&Fdgwl=z+r?zv3@VNYn_`!L@+3`mL5Rd1yGHZ$|!8)Jr}iiGyQ!o-UEZBvwzYvGL) zf5xe^b~12w<2~uJOd|i%j3k4ITFZ5W6XL#wh)E=v;C;?|64dvq^7C1D|6?c++7=xi zE=19lztBO&3ooC%7AhGPPR~&eDlT~OXH)>|k^5*xezzEAT(@9UqxD6}CGndT7Mq(p z+BT?OjYYj&cXe@u1}t}Au5bK(1^sR(Sn?mO)+C?xLGu|HOhPn{7ZM1sh7`=eU3%N1 zqgcEt@2)U^8y=DXqTKKpagfv#2$tYX;&1}cdn+9^2%$2!ts{T{0n5DCgzdV*Uq@jD z9m@W`4`=ru%#tZ@p6-W}-K-IVV-mph6SFG~Ng#uXoIhXGAX#+7_A{j0sY+@S<4&)( z>CL3(U~Tw{ZyyV265?VnOI^k4qyVK|cPV1+VQB;i=!fObwbL#+fWsRFy~hBp2d_`d zEHw}yeiUV`i5~3=c7^z&k2fg^t;mBCxeOT!^7Ak`lGRMNYhUu{abJuT-O=(9?sd=5#eZDY}2eLM#NRcIpTVElJ+srnikBKj}dZRpSf4;|_G9#Rlz+~sq_#EyW z1%9U8P&;bC|MDgJiGW-8T1HJ4AWbsLf&!9+M4{jmMQ%$e;f!LR7V;}1<-C($ba%Sa zQ)^tIPmOF;oBfOin3B){HgA=x8CV3L1Q-oowGlE-8URC9Ol6n6;I{A&)dxpz(jcFk zkteo5p#xgBsA%X1*X~n58Ua9c31f@xocUcgVia_`z&&PBzCg?mz10ICWjr@(fSwn1 zGo=TZ{`|sPC^Bj2<;IUMKJf(3&jV6M@KSax=vs1yJLg56zid(y`S&lu+h$VLFj5V!hnmU;#r znyze6TwRsT#v(D-UIQtUj9-}!t5gbC;sqw^BT+qr(4`}UhZ<4yVsQnZ8mBG1HhgoS zJeLXs0(@NlkUsvv#EIVxz=A!#e$nU%>%^y~X0HNjHeSUe9r1Gm_<1fBc^{rZqME0Q zo=U*)VHlwI^GZg2Ry!MVyR>04`gkJ{@llZVzJ}9?zv%z?5Rau|@3ts`!o(?2Hu`crV5J}sPJP9$oGTHq`C_f8xH0RU_%YDD!3 zWRBtsk5L1;!r(*9$#6Bc^D1VeKtq(_eU$af2jS~uGx_*Uxr+x{2EO#P2{!1!2q+|~ zZmPP36@~1Cg;u_X1M{ra>kpTVmvxtnzxXd(xZLE7Ck^ueU(NRJPd8xg*ufJb6WX)yMz*8Jv9ja69HPihTJ+1pbax zXSI5qKqrgka$(KyRP(uUslLUpd1kEqbCscOee)tPWzJ?{GR=Z;Nxe+v_Rd}zR ztxU~RgMLEu* z?DtLo-j1_RPD-lUW6d!iny2bRIosXZQ|z<)SnW^$d;cfxc(u1`dQe?@Jkp>|6Kl>1 zS+VA4rRtZy4^7HXzt`GOTD+R4@3xdO=J@RlJQnqu_8Ws16|3#JOB@iJyBR%wZl5>$3UDo z64X=BXFgcpr#iT??BD7v3^P%3v1`IP%R&ZBj;>GB{^V=##n)(}9d>O3MeI5&Fyu?)m#%qw@L?SwenVPg)qI~U zo^D%Rh^aso%7+PW*CU>KvA>;OlhS^rq<(Z61`4XH9v<8g7_!|v-sbqh(KpdeY5#)! z!2^32>huw#KOAot5btqpH?z|e-2Z5^nH#v&g6A^&nMwQt42NQ+oGqOE;ro#heBY6q zjj_zMfs*LzlfXqXtHWCH+1gJ=yS^a=D5wBgVgYbc92m*WoV{DBBEz4%-v>J9QnRk@1aklQ_uc0Zts5BxV*>Gx3`U{qHP z{kik(^K*A!y1yUIql|~vzhS&ClPA3vRFfyeh_nQ$^xzr+*2v%{z&jD!aCFucS?|lr zJJX8^YRK2GF$Tg3(sAQU+_x)@S1snMrG2fwA8=e-;!?A<^4{Hg7eW zR@Hr<3WUYM#6@Wn1))EMOdB%mVgnd5HNLuHmX4-7i? zs)+!Fb1HB|x}}irmi|Y_meuz8doBle7TzH+kkwi(_uqzyTe{WEo&G@zMZ;_%>Gs0! zm-Z6dyaeMUbtx%Vi}g*C1crCRW6&o7S|TutW&}}rfC_v7>$Uo01(9WE)3uvVSKw|c z=QUVQ2V=8D^D`le0V^CbeiM4yDejeqswD^N5FZChxN3ro>R+ufIH+KFLO)6lcj?VFd!Y+>f7Ssg16O6?zbYrr^G zuhoSA_7_!-7mT3B#zHM$^UMeWZx}9ATh`kU@4nkfbZ4Yq#z%R%6F%S8r>x<&seA#I zxFq)Nm6LqlwGS)pvsH#JVvtS8wY~Nl1^g%v8CM-Ud);c77G>c$L9D}-3q%C<$-iJR zmLbl`Uy!N9e}uowIS8$6F=jN4KCeg|b0#vcO-mX>zGbzVb12~f`=pDhSp*)kZ&(T3 zw=X8Xo;XusEn7S_OF&kB$c$@IhZz+~uQ2u`9$W0)i4OJ__^OOt+iehdhaxK*99}pP zuOJ!na&arqF-72mY7@8Lqz-OZ&JFSD=hp_`5kbBSB#?keW%HmNGTyFJ|6cY=Z3sWu z&z@na9PMkPH!pf2kJBXz4}bMP*+AAEd*YX=N50errxSpMIx$jYPV+oyPSeaIPEZMf zY~E2lMVOgM zkiRbxVc~&nvH&9aJPv&DVY`|+Sx0@REGlG_-=Z?q{2lkQCq^(*Tz1hH`&1F9lEm%Z=3V7Jzn^h+5_9a=`0YYD zcc4r}xK$h`iwJCbN`ix5%-99}Bk}(UAJgmuV`TJw_;qH1;GOE$7R^q?{00o`vc0OJd;va$SJ7*D*LxeLHT!GM zk7^0(2x%s9OXDWEZBUuvfb_f4m;o z-tjPwo+_BRm%hmKJ~mZHTw|2DLw8;kxa)MduY<^O7;_`L%1$orRDTxlE*RP+5F!HE ziI{0qk{4=e3x>n&92x{;34bV4i0*SEAL$7x)N?Fpre$5uI@j(1(+T_pL;`S9xvgY# zA)2G^k0dxR6$jY1GE(c53s_b(^uviwD*vl`=jw{6-@YN(B z^T&m-l$P1b_l~ z9(qGgodM{`UTZgDl30(`9Ns3B_*U=%`4h9f@ZoLVzm`L7J$kjcnfc)8^5LN{u-VGN z`BVksfnkL#`%m|wkOu1sONzVP18InukoRk1bn-=)N9Ig84$ntMW#cspzlYF>Q@`AO z7oYPj(T`$KnVuEVRCu!r^4^$g^K+yEG-P?}XeD2y1P0)=69n@6PB8`npcK8-wv>y01GDA8?%tX+HY9u$j;3nmY}-<(fJwbH@JM z+nnH&FyH+<5;za=@Qi}|+dlhyHLs}?US^@-X!y&3Dj#fbGZds8TBVEf?)ZtGdYMbb z*APO6zOwB5v;R62tDjLY)J@hI#Fnvc`J7CP-@D(>silHF5;Wt3&D=EUHE|AjE}L0= zWy=@|r>7Sw2%-4y$T|N1}}5oc20+L0inpEvF1&76qgOUymS z&~8yLNjb)b6Twq*iGe2%dKdX3^;y~Yo=%)hpU#S24<&VpW}o52j;Aaz4|@(-AZo?R zo#bFh0C#U@%t%Us2gT{?=36_4ZhS>?5qdFN71N=u9den-9<<%Y(q>6B1fU&o;@vdc zf}P+ORKT`DPuE-VSEQK{`Gc|8Gx=;UIJsiTbgPDZF$3}hMnb9(BGCqgj>D%+_jUc#CI0GO690b5fN*NjMGD+k zIssYB-Wiu1>~Lnp>Wjf6Gk`GgVqYD`lMW9Tr|0ROYY&s?`MV!;%0)k9vA@s2gYjR| zFVyDb*N0rGvpG}JJ;UVx7V%zyYAls7{*4&ee`cooII^k!gl>EXBRD8S zQv2CdV-(___sbEg?C`cZU;nWe{Oqua1BWO~- z(}H?!E7kM@=ESZDe@q`q(@4F4vHKqPa>16U79-bbV|cunsC~&=TJ~Ka4$p)Bp4t2T zI9W^tGmmCUS!XjIw)P~<8A5=RtP;aFzC$f1ZEl!fnZRwj&=^OmjBtEXd$KgEX12wt zi($CmUz(_bJ#K)sk%R6-_4|7Uaq3ooIS}5fY)-&ahfsXf8xjq6Q*(LuJe&c(|bpW>bC z0X+0i866DdLVjGwt(LU*HAniUQ_%2(548|n3}KX(frQ$TshD}#7`=ZUyRdmetkA}uz!PKb3Vo)zw$-80&JxW^~A zt%(dM_84`F1!uWSj1?A#2wL3hTQn-VS$O`~n7XSO<#kbt($L4{_fRKzXSeRvTN@QnSAVnvM5q!JW^w-=cXI)tefApx;D0*N zborgIPc+C9Z9jO>N4JdwZZt@cx);=Y+>UNleps0}R3JrRQNQ<= zb|Ly>+*PnV8%>C}!4?{`c9y;{9c(bE$hbmdG&rD`eqE$*jC>X`;%5}=%PDe&G`?uF zo@366@J$WNUku6++aq$^-Grg^YcaDc^^9Y&+xziALeJpON^+FxU#8Qko=1F5P(~}r zS&;HcT`n%TtPFv@h)v<|zA|^gQ_!u4t4GM1J&~#8k&Xv+LQCg_LHw`fVX)n@D!h7q zy6d)rLpUMav=8-@OQ5*4GJ5`ij=hVFf@p#Uu-I5v;u7ALli!ugW}f#SmC~_PO}ElX z6S2Z)n|DllZ!a4piT&tVK~+q&mkY%G+C6n?BZ5PoXSAY~^xtA~mi`8M<+7Q$LDD;o zR{>9)NWFjXuJw|}E*jC^XxQ<$Yd5nA8?x%qBj#}cOSxz{UaG2shn7b_D! zRvdyeg_crD{jfo2t{dF&Uwov{c-1?BFS|OwH2nU9J7B?X&Z_6xrZ36E3jZ5nC7?4e z?f|m;w{2Y{GD;|fcg;J|nwCpPQUfre*fdx=7A{nngO*vJG9+h-fo#< z-jgQ!m%CwQrZgb8%FXNa5PJ)AMApVQd^FJ|9MJ@xwI@s>vdJ%IK%_kB3e0363&JkpE3 zbGlONN=39zj1#pd1KHnX=pSG~{(j9%MEHFC!k1#Rh?L`iZ4LlXHkP*#cKC3t4qQ1s z!BvM7(mInSw`+Ru>HH&moW?*V*YbVm!d4%+Y0%S$D@TQWz9XH4fHVF*qDjzcFL3BP z$F(9dP9;m?S^G_UVy@6TL428cu@_H_zH#gNMR3c{B`hC+8dByT>O;vgq1LF{Bf;D& zM2!8Bo$(I@Is~OdgYM!L-ir$LUx@4w9asd+y>e$|FN@mbet+bh454X-;Wt2og~WPz z*N)#yC&(7qPnf!0lFf<~1s^8q4xfiG92cRqbX{7YxNZ@_;lZrDc~b@@3I3cv6#pY_ zQu(CDs8Sh;OXYoRs`-}C{d-~@6lY0i5qSLCawXoI0SXeA==1iR!!FQe_bRaL;w)$(AEmK*N}17p~;rkEL_Sy*4W}65t{af)o?I z$9X8_@k~#s2((=@7+&46LN-|&Ie&v#B!1o?608gwlVx~$2YYeK(vl8quLpVUb1r(a zOorV2A;fN-jDmy+THfrVNS_*YY#up~V#{|g(Vy=?Jpw$^Xn24GT;%tWYvS|W;mkt|ip9~Qf7kQcdB~{EbV4vgf@%n=}dN0Iy06SgY4d^H$ zYjEFJfC{WfEP=0LnvkyXKhbA87(|82ZoM!N>AINIof=y2Bryt~6sPkBK7(Dy3%j-< zPNzw`iRO6<-mwC#Gk#2e<)6c;*JdoEbJVSk z#%^>HmX8IfPXU_7q{Mns3Yh?+AAQ}s{Egnk!%pIoDsmVc2^RQA5N|FmG@rRn{nim5 zqqk_yroj)=SpbtXB=??wxc9u@tk>15ENPgpy{mt=Ut9S-kJMg zJ-C8fFx_V{$d3i(vu;UCgE{}}wkN=^3K$7BP3(C<@ihI1AFG`cLSmihM+7imV4=L& zlt9QSedl3{Q)Nokxn+9kr=CW2`mxCCYoDr|0zGy&9(LULZh6jFC9F}6?}v9G%R{U6 z;}x=g2yrNiyi9@)9F&hXP=WBc6HHpEOhdgyWBiaatpC-k|3>m@J;vQihe-6k-Op#a z3BYpi>L&_V)T2|p_wiL9SHmPzX1z#ycG1&;)g2$NVA+3PKzXQ%o~EThF=v8ho8IZ< z&%Cb*#Cq27wEVp^Lx+=0gJzNObn=NnKO7$n@uU^MD8&2*jb3`ox1 zI=U-!83IesOe*xF6R{YVjqS9~ts3TXwP>wA_c|{Y(->b3=qST|1=AtIjA-qI9!*}ZYF!p9M5 z&n7Rw+{9JUoG$9LVM-Zt@obh-%VtXEe__Fpq70QOV-o_R*h^{%a2ngZdQ~5QkCR*g z4%_IgH7D#^``>Tq_gX-eYZcW5N;LrRhOeJRWiZi!#iwWL2@aMY7b!RAnWh} z9k(9MN%iLoz!0OgUOH@_9OP#AzJ~I0+{qGe%BIlN#mNJ4@qk_DZu|TmZ4&%mbkk%L znJxQJIEwh0tXRk1ZY0CB%pdhsQsgF`e&qr9k-RBtVNe=AXG~yzv*v?9(>!ow^RLGeC?#A&Cj!=LH4Gd=AB{W$9s4izf20a7KH6PYOqC+sC9Og3OH{| zS=i9Jm6JjFqarZ3%1PfSm)Ri>cw)igao?SV$nDH5NKxOe75m=<3!}7(Djk!4ZeeTk z$x;IE@Vz`)4!q+Fp*ybjP+JT03ZZcy1cn^z-@Gwe^!7F%SaMgYR}cy_K?7F<-jj-a z6Ij;F%zbn^A;Y>UXwL!g`O!$(2&4iG%=fgia=8f+KhKg%p=Vv}96Ad2F-gvxY9cYbYftfBF(_1aVL=yc3-a%V zTkbf_Zk5_n(;+YX-YU}bSYTWiYvgy^9Q+~`j4O8Z-HgVaSP(YB$JhD@OL_m^iUE(w ziAsAumAq@>;v)@|o$VYc1czVWRSy$VW=QEk$SZ=e&&8Z+s>2Q|aPeTQgPljSj{Rv6 zSQzB%)RZJ~7xHj#zY&(Y@7cuLmm)`cL^EnLGacS;op)3(ajWyOF)6}l!=@VZJ>t#L zzt7aZHd+HE3B{Ot5bss7yQxdNt;<8Z7&B9g3JGt`o&eFQO2_D}^VqF$mHSei8IU?Y zqDUk-M9e1syF#`lK4jdx+=>B}YhT-iI}-hL4kvJtS~JsO@_e9{BTdKNpLbUw^P+-N z_>$C{l0ZJfspwUsE>w|#!8WSCYZ^@-VS?k9M7;bfPWH^g+NSO&9H3UeMpoYh>tvrf zbmzJDr_s$Hap7uj#YvS;LHt{ESPwpr2c1rLSHBK{ZKaNdQHg@0OPJKaq3UKiAvLCV z2dG|Bw86&%R2}=y{`>_p+i8IF#9ou`CLcayQ7&jab9rkmbuaG%k~5k0FLI}wPb>j; zamCzR5f|rMWj$WoIi;@pbnv{OQ4?}N42#p~<1H1l3a>}qPZQ1=>zfnmuk7YVSea{91@=Oh+0D;5+iD+u4JM^0_-h#j&XA)72dQfrZ&I;Q&fN z!F84fQ$ zS>0{w=It&Jre?J1ILjzD?;I9+vW&l-7A5(IT?XQ%$lMVDRuuQDRB!1~JoQj_0;LI# zT^`a7i;&(AG9O=qeTiIJ^XYWYLjhk=JDq~1+y%fgiMJocPG?UAeguH18uXU(XEMA+ zTs#;X`*yOd%9+#dOAh}Sj+b(Q`2q0IVkZBM;H!tmvugPw*a;5Zzf)tgF})uOPq(h& zA4X;oF@~(| zl|=4>U@P$F$Q5d$JcCOsXOfmAkxJeuE8Ln(4mL#1Rc}JhAND?~b?^3(7_>HssBkl+ zoIoI$q)Cg;{&oC7Xi^4U5j#8ZI&TlMY>QmexHI1-KjBe^JYm5NP-q%m9 zo;AIl-OChBL_Lm~#&ttK268~T)RIY!Gd5I{exK1K$!EdM!YkJ?}3; z-hehX{~jPk9U zxdiDg*Nbp8a8Dn{ss22#mB5=$u|V z4+*$M@uK%?J(#Ef|1lMD{LT}h^DCt{3HUoF6X?3`&jdtC_P?v3j7CAff?hXO+)7y)?W0!VeGYWvZLsAEE}&%OUsu)IJ-z*C%VSiUwTGvDX->vS zakfbnVqGT=IZ&M0`fq|UeNe>ej_Pv3>pt{5({BBE3c(%t8~s7xEf}=~{?&{vxBITX=?F&2!{!nXzuqj!y3|ZS? z&pn{9AU#ibYWQ7w+9ANRZx|zvSxYdqYl;P5;Q?-RXU+?d2(m5=58*bA8%@HiBbWt_ z>&#$9J^wH0JIj`I1BW4XJ0F~t<6XE{iWl{ss7pY9_%AHTfk5U@tam)~>)C!u6{_Bs z9Yv_j4X8b<{Eo-)FtTCtZpPj3gt;E0QhL}2WFO9KIyl?_jL)uqRaNf^BvG&^0>*1m zf{Q+#thNbV%@`j+>CmU@T)gOwUHG1QYcCjMJ%V?QQ&p;{GxcT7&e6VK_5f6-wGEVb zOm_x`C@ti1P&z@v#-kZh{Hfa1F4P_XtYsyIkEzvy1*VJ96pN47ffexI;oyG-t_)m! zidMPK?fjACRV2Xa1vVy=7Ab&rWX~y_3;S_d@Jk{wqwM@a;G^#w4Vvj8KhyI1i<FZ~%DRId92Cj^nSM~Bb z01mJzmYR@qwlx_S0Py7sTneDB)BBQE&#)(f5ct?JP2l|0OR+qmN$+)w=hQR5z%MIx_kJi;y@~yEm0cV<=~7ptn)*jj zeP^-D>d}^S6u4K%Wrduy8Yzuq=^anX5t1_zef6q(QefOCL*xJ{*AC5RDL#WgRwY}a zCvgAH@$S|vukT+yKaUP;2%M!gZRUQQ z5E01dtSsu?twz*qql@YUy0nMeK&f4Br8_6`=^*%Fvj@$Gb_Mv;s2@J(O$ifn=#J&h zB6M+Y#iyE|vpo^X(CAZ;KX;d#E>?8RU(MiQP1`^lPr~kM?5AqSKYBdzj+yNCQ6eE5SkFiD9TL94)`q4{fLk+4PeXPK(ixAujLg>Ul z$Y%26iKB(TRF?-iqGTh%VfK)3y|zf*M;9ST+63^J!u~Yu!ngO&RwA+@1p$G@QiY}a zTHrxZ(tu6IXihu**X=l|ca~Rzu|rqwD|d3Hnhc<_`Cwzr(-tLZ`7H|dI%lw5i}Ply zZ8sh&t|YuuqrKD6fX;e+aTSRT2WOoz3~P)MVP*QlyY{8n{0csw=g&TE!9`^_?a^5sg$>B^CfSfRF&C}$X^gL!5Ij@sdAZzNkG~?^eVTjx9OFvPwV3Kl# z>KxzD7Z`)o>s#f_NuYjQl;rd03iw%rh!(<9hk2b5kijPAiN;I0vt&OTuc;}`F4-G= z5XhH03YFi93d8p8C)7XGXphR*vGczN6u2%2W1cXcv_Q@+v|`m!dh(>+Kf&)x9Av|` z(}s%)B8`V*c2h!lBepQpSakpMB#rLzK1T|b8Z_1vRQGTGfa`-&VDYtY50Q8L=$_E~ zUNSc<{Azv!_q#zE5uQZ2-#OO|PydxWikH9y{i0V5Gr;$d$A%f-@i=~S7aYL&aWKYx zu!+VTSz2uro%k^$%-s7!Pm>)(^U8WfZu1AYAEc^3hP)Z19Ou$|gByCZ6RH=*0D5Zq ze6=Y~A&-n4P{~=|S<}1PG>zc}!*RR@<3p%+D)t-lnH@*IV{9W2q5~ckwnA{kc$0Eb zaY%f;Hg`}YmIW#{DH@=B?i+rE#7eZ@x(t%~IMM)>y|Q9MyMPS8i4P`UdOpW!uIf`) z`QtjY0<*bGEhrcuBfbEidS=A)g=L$?(dLu zF`)*0%fH!I>t1QY$RS!b7R6gp(Ct?L(IGU|%GSIcovRXPzIYRVsBlt@;+II|EF%G3 zRb=-l+NA9z2+%=tXura{hc)1mcT1++507_5F~}Y4!jV&h@5qZ>XrN%aBrw;Bpwm@O zH$In}TW&=`(C$gC2nb|ChX-sjMT>qyV7Tc|o7&_xPmTiz?ZU6{8yl>4_ibJ3xSY6v zAsDNHy1`v=^qIR7mq4|la@nY`Hwg?ebycYGcAH1U>?w0x0kZ-i&2|62wq#PtrV=-n zq~uBKj^Lca?Xb;3vwRkxSZ-VA<{1nj1G3}L67ut__4WETlTEY6khsmZOVqLWTrwz_ znBEFG4gOq<5LE{``n`(GYdhFf%McOY&2AkgN~^ylFxP_?UQM{6jKLr%fMxL!WBfYL zc(E+piEok>Xb$Oc<9_cV^=i=5p(+_i66W(C+nXSxx%Ko%(R&Zs+_cZXHqEAPM!3Yi z*P+;L@KBlnSPZse@Vvj;fD8{sVk177B=k(#;c8crqJg`jeY~1b9sXASrgXb3y#@8& zi^6M-5poPK$H^6R&6>Waau7y>!M1OShXI%X6d~yCR8Qx*3Z4hu&2_P0%;WuZF|Y%S z><_uZ7+a)<&C2eox#99)M|1M39W+wQ#n5iXvMFuQ9O?Pus$m*3zl&1t$JZ`O?!^#^ zo|=DyVTjk;^_QTzgg6M5KB!C0aAy_JrmaeiM6)%dU1&-r}r%&vYB%}|1#@O zH;1gbapf33mnncLLCjQh2p?+?4=4gNUP{+RV-N~?kqwV@xH&JZNTyTN>o?|_dwb+^ zT=3RhHraUA{n`Qs$o(KEuUsx60_q0Tm$$-Z)0`Vuu?+Z+4c`k8#&|)QXG(rpNW?qj z!RJ*z;?2$!$C+r^4x)p@6m09 zrS{EH)rJOa+Z#iDmb5|;H+Vp47?`0avN-PDix;e8Mf~WwpB8nX{NS5nZ$zn3T;Qs# z4AG;qng)&wrZ`-njPge1_!eKArTBTgvV6*zI;PVBb%T2PUyRq5l7;`>(9EJXbsH9% zQrJ9-zPYe0Zny$gCTC}6vY+K!as`{x-VztJZO3b-Z-F!^Ijh|2Q%&?O{UnV9MRm3~ z>sHr2f6-eD|E*Uxy^r}z0NT#`*N^+VE~?yxE<^+{%kP7=zXAXej{7Lp{RSni)MH5# z=suFt(r{&p!(bDu^57rq>p1($BIS;{e6BjQNhc98&=CfKnzQyWfq8pD@1S>k-SV7A z)8Xt`X+p|DZU)qFQ5*`jnwxTD3^6nDiok;dp4o*!oizMJ>iUIOjn$JM#K1wo_)d{> z+_L^{@=9v}is>Ntm{agcHL>lOYOhtweviBQn`SxDukyOc-@|iL+Ej)2N}q}YNl#@_ z>w1qv-UJp0WEMM?rBB4(L(vyJESq%#Hv5TSiO;D_4Acj0Q0RL)dR6Zg#689Sasj2B<66ez24MTAi-=Px`Mg0^NFVB? zu8Y1D*;fw6{!c5z=Gs_$^+Gi?!M%^}Q~r0xJ+i8zUckN9J%CsKJPABGaWwqwCIs=W zF7Ul&-DYD%lY?MUT-aQhMiyyF_*eP&5RYK%l!sYyXXW^oAApDVSmVYr+;O^Ujo3SV zoFap5o1NzJ9R%PcvSO}5EZg=qQ9jqBu1m%}4^Q#5X3!9RD(a*LJgQ$`&c7UtptxY` z^X>x=+%6}Wo^biM)`0EH&n+E4C$vGT;+4ow4T|vj=!9YWC%fUKWylNQCW@0Yf;*B5 zcL6to-gQPO6$n}j+AZ9RGWuno)t7PN?~(w@Q|Io!3-`-mDa|fzShb!L;iD(8ugAnV z{!U(n#DNkp$NaJ~b&Oet7Ubh5(eKvT6cXKJvNa)l)_8%Ge<1WRC z=$&h?JS25VFHpA0ZFa~%@Bg*7@KCSm-O(Ec5M%zhJ3OHL@2p;_n7|2YA1TLU9wVFA zFHdRi)PZXE#Ah!%NI=i1ZVeG8|3$1i#D!WI;$sjv$MhhHpwvnx2O}h}Z{#gI=EcN$8n7z0>Ny*D#H*-ZZwA zAg@8tfvDR-G4T8&KOvu2Q_~QHHRCx4bkcE>l#i?HY6_(lr8^cnTIbj0_;X`UVPwPS+NKAKNw9 zUTkBx!hta}nzk|tpoD72K{a1o^lY3A=EHZD)WB2<1aOsY zV)tsV&jLn`JJURI`CYbf?_#)n$y5~>zzH6IA-O*FEm^S}{W#yhe%VmU0Nc4t2<9t~ z9_rHde*=o{@z&!p$VD4?^Z7Zt$Viv~@rkbLg_jU7?|g#_9uIQqn$8Ly+w=1m4A|1U zsD1}d*F02?wGp0b;P4rF34GkHRTO3KU6`!40mpr^(~z$~CK@yfdaN}U388c%pnt}f zB>+6E!=$ykp(zaPq>UduZFvLzRBjvF@kK<%CJ;>as*Zq~vr*G2>7q8EcLyeWC@G>2 z^2HH*NLEAD<(*nI&jFcz@t!nbCN0Oe42q|a3Vb~*j@woT>5P)v_lMflz7)rf{m*U! zPFApR%JA%`7LYB@k_@y9#H21qsb4n0jD-i6q z&#G7)n5yv|77{odi05UEE&OQcS^BBOv(B#oMVWC2E_eYl1R>Y=yK_LX0sCb+y3|P? zfV(xqeK_BPXac@q<4}I=(j+HOoHab1&eJXx*B6)Z3H;a`&K?A}oxh^XqwVJ)-^tLg z5t=fMSapmI%l>s%ty|afnU)EB3y@OKW9>Ms`PxAPTVX{utx`&gBFLM7GK?W2>J8H= z4mS@Re+|$I`aM3^@@V2AFnKcn$?EB!p_Sv>XSpdB^M+O&oT~*lT0D3Pk0k_`$~gy*u9|X4^#PqmvjgJ({|yJ@gG<9Z119lC8GtqFIT?$-C!? zGflKIP&!$_EQVh){vulch%Lw_c8 zhkA_bGgXNquzyHW^KAd8z`}eG7LCz~4=s>rR%VL@gHf|VKu!47#6?ZRCZA(R4dzLINa09Q2344TamNX%v-v{<4c(8V`(t(Zc_J9r=32wdX+6DY*LqyWgoJ%o^?SXa?Atee zE)V%DlhD9^%z&X{H0k^JXXgm+eVgq0zJeRbh@Gu72qXz~7+FKa{+A{SVFF3=P+CwT z(TcTeHd$6N`!66F+>z=-0W(~K8Gqd>`#pVwMrMxx$hzffmxd6!tjI$gN+5lzX)mpH zdL3$4NrG<+ttA%px>0Ivvmo3sutZhksqzG{ZKj(7E`1FU{7RN;QdKjMWUg+dS44PS zW@ICtV9?M&UDH8hXw`$%Ypd5cY`GrlUcIp)R~FFmL~b@F@-qI7xC#uE>zZa{^+Bu{ zpkvI3Qa~>G;fy&0aelaWhr`#m<=X%1w{I6Z`6Q&(G%aTDu0l1W3kwZdLVQ;KljR&5 zPozpsmT#BFg+cDMvb5;(5PRQUpx$E5_FhRn?nsr}HfTsJe=>Aq^O|&mV8?hY|QG>+D7{+^m@6#Hil=KFHhN5_RCj&{XL-PzA2MTbYyPdSEA zXH+fB$Z@M$S$aC=3pt(}G#7g$0E%1UiA6!wa_jSVI?+Z|UA<yHz(hd{>>GVfWf{e5-`|rnPV^x-3hVA_wq$a zt1cR{O<&a*zDNyTcgeIw=L}b8`a>n$-R^N|)Uv;QcS6oUj0ESw@7;N+1Z#CRLAiR5 zJ}VlI!=00Eo3t8a(FO>Ta!<|C)O4UY$Tn=_AURVs_WR|6MS7&jg&h@r;_db#0c4vjfJP_W)ZiMc@FXv?l}b_J+YEJ4 zh4uM*1~oSF3GPhOkv%Yul=S925aLW+jXHE;Q`w^gL%k z6o%j)tzCt^v=sDZ(GyVAe;6 zA6*=veB~^dD$~S%$fv@+88=mRZ9Kubwx3aVsX&FdJ}TGSPwyoqRBHL^sq?E*4iKDq zujr%g-_)agzY6KTHAxvDoAsLv;;x?qxw4=RA1j}*ou>miAsO!w<(@W|dU8( z6(b6tmFRhaJJcBUCTbhLkevj@z>nRL#Q%L|W)C=m@g?g1RgO>ESN z1ENsJWm6E)bMCkWTbH@foLaD4soy=GgXf?+ZvrDjwzXd)E;wT2vBeaq&5m9u%t^|A z1FjjA%Cad#jHtD27AzV*@Wx=8_BK_9n ze>m2d>u61u4+PwD^-kDx>VbTNldF<1Kh7^9E#JEC>x!433@2@~^{A=}G@G&hVAgn| z6TDd^p^_(|H6Oi4W4ie30xW2AJ6M+m1X5V8uAC1Ccj8 zjUFDy&wjv2+SvrwCZBw1)0Il7sn2;7)wJc)r%&oFHZ}q+wS;Lw6`+MIQlLPB z6EizVopc;a5GUR%QByn_XyFDTd2M<-udXEu-#_EO=Ks+@T`Q&B1$kJHZ9Zh>{e%Bj z1h0G3R~H&yJg{{Phht|C@`(KfYW@F=)_>XGOaW1CL|r4r28qw$W?-)ki+(Ejcp4b= zh&0!DDeWzmx)AQ%!s~86Bb8^lgW;Eg)bgYaH(JN%zIjE+lm&IZCv@6CC(Jc9k$zHl z!EJ&r40NEk3Ye?dhk=OGv;sTd)Du8r;iS08@YI2iXxhGP<;AmKGZtARN?+>pcLVo} z6hyuPJM8LWQ6Fxy7nyD{`M>inurQ7^bMQ#8u z8|!7Sw-UYZ;RQU)a>xQ}tD)2y3fzzWSdMo7Dzy$o>P~6&5XxQMdQf~|6jyqr_U0fd z3Ch~mHJrI(=iIgu%Ghx_AtJ<iYR`w;?4Y{if}c>cEO(2GJPIe(;!Nq8ECwAJc1-QHS3eSt_IHX);%y5E1v02-e1Vv^&BYZ(|G3S9OV z`pUAsnqJV4$x+iS*$0_M5GylA&AJXBpj*yST_ihD?r*^8E?5?8W#Ak#9UJ!<3i?We zAbTEb;)=+Am`UuPB*7Uo^8Ncv+~ewD7E$DeQI#&rvOLlI3*fiSrVs1dLlAOcp_v&& zsL0}~-d;VUy-CWz&p+wPic9d6QX*gFf8Hfa0P_8mLD4Rm8j_wsZ7yh6)70CS;tz^) zB}ESM_B;Y(_pHYG zg%1ehgjLarW6LDEC87mFSWGYwT_&w}LRA0*i0!bJ4DNQT8|*+IBke1Ddv%TWehE(h zek^im=`lP6`3(Io{$!8xlB(lk3A)g&*41pdM-JShpwR8Hm0j`HHW>syW^`{Ot>TRS zDR{)Ibj=J)OB>yEe-1fdB*5NpLolQE4`nFzPtOYp?gNM8)^)!he(c&_IpCbWN>LjV zCP4kiP`cGJ`V*mE4-Mx=p*Y zRu8sKs}aQ=EzFgoceka!_tpzQR-ix~!jtBRt!Y}bgfC?q4|gZuMA|v|c8)KqKB?~| zuZCK=~ays^x3!tey zCkp8(XY(ruodzkdFF$QxopdkWavf44972_z$lt(i)#pLdk(-aM^yq_+tkRtqNe!8* z2@b;ShaJU~yL|r-&>M~W#G#>i@g*c^^;@RoKJRV>7PW*gJUu~mZng8WVDzo)0F=#l zkU#GsS2AZk@g41IYyY64@4}A28r2+B5(^XMAi|gVKeCoq zmk*b>0@Ay8*@y8j_^2chko}q^^NuEQ^Que!b)H#Ztt>et_GbHV{tUz{Q(=e45-e^G8$Yj=+41RC|U!&g*Bp=8eYIq#QE z7kso{e-`EJ=DYbRO2#uIS?Kg0L$)*jNZMF8G2-=IkqgsT<6j@MzAqoPNvJY%yLrHX zz|E58i%btSUw92T>Mkues#IQ6R#je*!6(}BH{!uX!06q(zqHPdKi}{&J+-N*KfHAN z)H9+`FAnvD#FAi(d2xe1~dT&z{R*!eL{0S`VyM#wc+pXeQh1~ zib0auTbNYN<9PIz!HKUJsG*JsUuzflgfICch;9>_)Aa~#t%YU zIV?fA59lqwf_Po`1S}O4+oS=1smwNHF%6h-1_}}}5jVcQ8`wo9Gq{wOG*ZpcE12Sc z0uTtCCSpuq5CSh>$+DeT1_xXnQcaj`Qw`e_pgwsJCLgWF4B{}TwYLh1#EhAG{$fnN zF|VTZsjYW>o-P(dV0SLugzE)=OoGvK|0Y7n3>yw49z~fnr?R-&Qrm}_bIzd~vLrZP z1Owjdt7_Wgo8@=9Jtd`1tV(SePqYmpn}rxl=4@_o_;}l6j`ZEtfrH>PJle>egO}Mt z6a{&RbI}D?gxR%h>?XTwqrM*)X1oINd#MnF+DS0 zv4Vm~frw+^ZXIg>aJP`oY0ygrV!X%Rm)Y7dfr7;9;a!iL^%$?7GH|xqS49)@<^O0^ zy%$Sj9JwO+a_wo8w6MK-b61}L^UC4jWzl~Aj~nz;WRN>c`|zxySK!8%HhFjT9qy8B zWBfZxE%*N@eNhs}zvz_SdF~v#imr zhB%?7A>=d-=IGOoJMA1=u=M!gbVj`j#ZLwZ^fv1Y6iFm~R&pK~{CPX_Jbtn+ovBn! z2k-yZ$>9bo{*k~-rTV?!Pp{6&1ySk?uqzFN&xskqaCH-p$sj{uzCXoJvQXQ=7=xI$&vus;88bhiiByMP*4|Pjgcsjd&O^u9_xK;Ku>i_h_>0GK* zr1RKJoht?r)5Hz;R%Pu3e-`X}a(C?zLwooWDWH^3vp$jN%>!plQ_wy`<9w6icauf-fB--hCj*bVhfAIxT({fDAY+ zCUE2T@kx4TvCKNs-mU&v+ana-=AD4PCrE4jIC61gb;@v(+jq3H-f`+6$3ZDQ_pnh4 zvO{0KsGKx-R+4+kCwX7ryR?L^yi}l4)d&=O$IB03a@Z~_`*6Li=b!_^j#yaxw4vT= zYqH9jAHF1oXsm9OPe@1|9(mF}K-#=H;wP(zcRZdYb9CV4Tv5`Ztc-_C%UPgOIivGs zLHbPZ^}n+S4vy`{*}^}e;03tCKw2|PMh4)=YHLP3)2FazlQmd>8JRgjjf@(op`S@H zXT_1?(1a??S=njKlB<1x-SQ!*Uny1~1*MLqxS{5J z2ef#*L;A;Uj|nr~61avA1|!fuL~3~29r!nHOys}Z^Dy1N0wuG?pC21}dhB0CXt(nV zM--RxCtK+`%>h1KQPFfTD?TeBy=#=;`P+6{I!b3v-b;SaNfxeEkn@Mh$VUAmDX%k! zTsoN*c`tBx!@6nq4f!I0du?QJ3y0B&I?pBk;QPnsxAR_Rm+Lbgs>U0KT7%T?JJPSSor`4D-&&> zw#Q(lH))Kb=DNnE-PQApwiL7q_J;1&4IxQV3Xc@s39%KxKd#v}2c%67K`+4t4fD6d zZ$tpknpV`@%SnE~0AS7wx5)NZ#qO2(NDrwPQMDV20IVb(l`>_V1dUtiv4H~4%L}kf zN$cp2ZLs+a*W+qcL;D1>u#T|fvU`Dl2KSnVSpZ;Lb!^-wQ~GppY8L7q4Blu7(!Bpy z1HEGX42j`I#fZkD2<*xgz-eVVX_d0)avA$XP04Ylw9#w3)lUFI=4-WFgC_iD{6fD=1s_JldXoKJC4#^@esBdL1svquDE>Q1$V$-7%;aI@{jQ25uwHKw?H( zm$9$YHJG){m%Qh3xML?(lm@8Su(R{wk;c+L57Mvx zR+A5tyZ9&lFWPUO^mcB4{93~v=kd!^@>d+@L9-<1u#m{|>cfGJJnD{D&(YDm+3aynmX&PiEnn`fm} zX7!7Q>&>0Xyb)2LTl|l$%a|Z?Vxfzj>`#RegehA2zspmnX`pOZD&X6H7%I<^bFeho zZ|^>m0QY~O6WBZviCn%w%-8x@Al7su?c~=_I%bk~0!lD@$%>KC16%^+EP?lS58nvi z7Jq>lkp5d#vD01u+|b^adYUnL+vB9)y)LS>KYVYqrsLeu zDbOHyvrUwh3ru^c z`-_lR9#&8WD()P-gv!fH?^SQOX@wZ_!Ap^WhYoq8xnK4vFB(>kG}9eqPxS_#hh-{& zye$GBJu_JWi2gnIDGj`+Jw%>XbbMTTlpr)>8Ap3*ZS6cvU?-0l(Gc9qB$`yE1-QzhPkH_9Nt+_=#1? z#=7J1rw%4ez1=Zf_(%vt?`zP0Zww6fQkRN7H$e)T6T=gpIJQbJZ{Lc7@~&PH-BX}V zIn4SdukGSPuTVK&Z&PzGb7s`!OFR^9n3tAIXnCVAVlp7W1VE>sui*a)DJvb1BqC6I z?h=QxC&gvl$&#MC4(8WZCuCYgFTsSazNdSzRcqs&7(*B1>Yo=U)~YV1~eKq7BEU{l%H3;&G|wf#9h$sCY2TxQT7 z&(Y&h+mQMTB)8XbpxtBSK&Qp9cuf!cq~O8E-q(%gBE-G} z@BwpqeftGPm6}75gZiTwZ7Vh#|NV_h7x#1a5!Al{i^;0=D*1qN9jjTpAHmT4VaSSyv8AK9`&TLWyGIE6g@c0= zt?`_-%ry`$8}@krL`BV(bdym*f><+6(2&Zlx=B;;0IoNk@jdx;6M-@8ezxX?TeyGY zOI?)De_(pchx-D?xN8$B_^DLxdW*wtn{Z9P(awp8Xy`x-fFOC`WqRHnT%CyR>tunxSo|~Fg(_C{P zLZm5bZcEXEo$VLt+vkU$7`}0i5MN(soIzL4jwrP|M2a^mBoj%4Y~m8-k(xHq<;E8# z#vuq2w2$DrU1m99Vq(_el5oQ%pcBZxV=xYLy06QRPy@jR{lcm`)esb% zM2E-p(ChyTJ>T0am%g8mPAzL@DmSdZu`{E`9{p*29X0&~w>^FQg^?y{$}`hqJ3ycU zo4H@{CyYCcbzT~^if!Jk1a(e|3`A=rITH-V+1UlRW>f(fmAXzoh&197S7gM!dpT<9 zxsRi@e{_kkJr@-qvk1nj6%}oZVV`L;%z7AZHf+f-gm1p`^&2|pWWEdwpmX{!WLdEr z9+w$Um#3-D4wmROwRWa9;+j@BQ92K@o(_E^g0T^-1AlXPtMULqYwTy2 z%o3d=10e;V3$)GDZ6yj#;5LA96B-|LM4|+*^8&XPT)rbtY?0XT!PRfzH1HV6a^>cr zLbML!9RK3Kex7`iFW@A+6f_hoV`TaBoIc7=6Yz6mur2!%lTyw;<6JxW7ki|VJBZqYLQ7bDGk|tma8eXWO zy7Ho>z+bP0qK-spB)QtxS!IEX-aajNZVZ)ur(d|Re?qG&$kaJuzcDW%=&R;1drBs# z?eGMGaC8>*oWtgqYORSgmNI7vFE9Aff(lW?o%y^} z3Kwq9SzJH9cfa{35?R&BCE~3Ep2;I`I1r_gIX`2CnnkOepq%wWRN`Y|UWseb0{yAk z75m?xfW3)+i_srZ#x+9`yrpG)xaH>G*vVZe*PH;8m$Aksu<4z>u6uyu*olhgZk3ug4owE4?S?*Yh7bHTF)F;q%bk@5$tD? zn3$Yp`pmmwCf*8V+N1i0wp|RimvNM|?xm#PuEW?zap;*($9ptpe2|5G%Ad;4E;VW& zcNI&T0Z~)5;<47+J9>y~)o_zh@k|)o54eh5v44@0d}q%T-r_fl>dw=MHys~8nf1rw z%nHE>g_NoiP(tp*9DMC`b;@j*g{gc1UT@L@tjdOC?S*a$j>gAN@Sy zibQ$&!nL$4CRIiB{gb8!K@Br>Z8r%w+5qc7+R%Pp4d65fVqJ1&M;|G50PNSxr!5#H zQG>McCt~mEFcl|fyb?rv?GQFGzEjrKt1;>cBJ*F&1v7Gc%vK!gV{|MdpEdf~uPWk8 zI{7ZAO$#gWgj&^1uNoAJY&vo^G}R&fIslKjt~kbZ;`!h#U+OnY`W`SfO?B%ZiEfB< zP#hV^tDJegyMGl*k#IpcWulh6zKy7OO2vILR$`A}a4X*6>GZH(#=nzD5z9*C4rS@T z_-;ne08c_=U6Zm7&Fw%6DqL8zFviyMy6N{xh(gljLSqkoWekm?ab`BplI8`=X|H z1%~x-oW*5LhKu?%!X=I-Hs4%3-fh6GmI7pV;@U)v`e+b93#U4t{RU54gh2eNshi5S zx5m|}6b8M*(|rNc<`HG^2^B{oeP9KXg2%EPhNMZ00Hx>Qoc53!qBonN>D38glOaVr z&isfAL6H8^4lyZU_m)NwgFI$9LD$bgzfgn!^oO=DMBd`x%a5%*sJ@r(hjK+{4jvyo zwLlXZWivG%>qnx*p%+K`yIcOUY#{2ZD&Sh47y7ZZP(7q<2@J|KzO3{CY+!(-X~9U# z{^wFr``Y?muIN~FbG!S^UJ6j}E84X^{hD9nt@HP^=psgLKAPO657MpnUM;c_*MR{0 z-#2|Qx5JWy>6ayvuZMu?DZJ7}O;F-Gg##qjhJQSE(-Aiq{?R*h8W^63x*XTaR^Qlb zSF*A9`s>2BArNVk0gu9gt9pKTILi>m^X-o$vb2~}mbh+b9|6#}jF0TF{*i=(j$#9# zXr?e&o9#Spj1?`IG z@Ah1Pul|<5a}fONe+Cje8F&!yPyh+tfW?+{_3kX!58_be4GFvL zHubfP6tuC7Al7WLMU7j&^&Wc!g7XBw?t!c>p_G<0_^zO4b>j)(A3;Wgr!>zX8KtNE z;o~t6ZyUR*!LQUY<o;GF zjmcT8wu4NN?7Q!;4g`A_e5k(B@16SX!JgR?K*qmoa>^g~@)Mn&Kg%BA6B$@tmeJJ} zm~~CJ3I&Z>C6d*{7k%7GZ?`#@Mzp&3?rvTImk@(_7MeN1VHgBzYT0K$YW&!`B7Cg+ z2(c2ri}BTTbvH{(GOjChPr6_nLE*Q6r&ycryjTbdT|G#MqO^&@w=BSjFz)udH+5K2 z&epi>oRJ3^r2%Ls9Ht}%bV7B(ir8_Q_h5XOUgYIlK;YOcD$C{t)c^~$Xxjjx@o707n6t15%Y=vg2jbT&Y{YdPZ$UNL8tAAD^X}Vz8@<>8$P7Zl4TDSn>DqFkbIzDV& zUaxR?EKUjysq%v3fM=aUF3WfJt-py_aim9_oC`>)>qg^?6N|X6BUt&b6%QBp>p?fW*Hd#d zYkwR@S*C8*pMPWN!%;(%mxMhsmYfZov4NILK&IFAVU`I=S8`%-;Nq+TaG5@o=n2fk zB0pd2;Xhdf(X+6S=<-O+MK;D&f_3ei@#*xPw>=BnkTnt9-;$Ok?AHSo-Nra5IZET zcRNLETp@*GTGQN%zd`g_lSNIrk1IMq4|Mq=F&(Ep)ae~c2h|@U#Wle7oljZ7AZc(< zT}#o0g7mj>Ep2FlT~u8cl~}M@{oe0jr16x^ZMql=J=Zq7`-I}?n}*`z|E3KIOW4K9 z$$RJJZbbFN4;iyMlnA)lNMOa1T3Tq^oHl{W07k(qo41d`#M(EpuCcDE<_E_dWf#iuQqSuqM&L;&|G$?w*-<~ZRz9a z*nf8*%1v(f+&*2HG0ut^(LE{2$7~p_b}lB(cpqDU90#r2QF$TgMSFWH)On#5_{Dur zPVoZ~-KJ9cK8q6__)dqEB)@g}i~nf}m->hR{wDpaha9io;lbGYnuYr9_s({6Kc*vx z2usV7NJFIK=rMW>)D`XczkCmGIleQYY=}yk=?&+a!G|@CylDkW>c3|R`a+#t#2Q4u zsQH*1uB0uDVjwy)9p9+9FU*45!YQHeozAt&ta>&TFPJQ^7EImWwc>E^O-p2iF(m!G z0y8kO<0#m1#^tYa6UJny&jMQM_<@=3q{{*AUz2>M`t9VZ*I#kj;97h{W~Um8D?LA? z8SEW{ag=5AjCUyLgF7)dB@%$my)$Mg)yuGG@tA=?;)>2rWq_3R`U`*q1$dDf18$_mKn? z9hOh=%@ZqesD=)<)nl!1GqMe}9>7efd$OGSP9^>N^zw!<@}&%JilDI3=!4ocwwRp{ zf!yRyCwEw;(_z6BI&&C&+Ow?>Z^4o>v4fg%>+FRtRe?sTHs}2{Fw5ka5fM@Cq5fLv z_RUzOt`Dde6FL=8t%Ame%4s40gHiucmbeR5H^v5tX2Ft)u{x>AF6SbkU9CcKvCz~0 z$#+kcK6XwmZ|vzPc+EbE;|i=wIG7!D$l{cMVv7jLK^DDhfAx0RN7yAd_xS5P_CC${ zZWC%8%#}b>-6G7~mTp@;M?3uxk7npdSQ|@zLXIW>xg@7q)=2nZ$6EAzJs1--F4a8$ zXN<)bNegHX7kN2sA4&i*uzFzTD%>{cjE_A=Ud&&_bR_=^6x*)M zWlDN22m&BJW3f4}fSc}>63JI9?vgeSIFVg>ughP6%8Xd`{iGA0pZ4qpA=8plCB1z> z*vBJ)iD~Mh6hlaH5&giHyGHzbH43ySh%~P3@lXAz&v&RlbGDcMs{~p695?HLblB>- z%{$O0SF(S{bb}Tmr+*H|$%9P`Fl4CItR+3GYSv;{KvtS7)V}Lk2iu6WcfA02vnf1nSfts7rzHrj{G4zm+m=S$w zObi``=FBR(BToaADHp>DZX7C$%Kx_>JK$vCB_*L!6nK#0m_=&<@BE(H@oi~$D0C5ZaX$pZ_blNX`-m06~j3{TM^=fEU=N~nb zz^W2Jm8C9&8JwkCi?a{tHM!rssC41pMjd1X+z?Ry@*YO(MC0pC;OUmR9`X+80kcJU0IoIN!IfJ?dB^W+D zU&3Zus#!F+NuWwNd+}WqD5wBHjgx^A-8?n zKv#_CRNcpM7o5_nSo@Grg8R8henrjL4ARlwkxu!SneeCMX8?C7>Dg+G+=qOsYW|Yo zpxQh6w}hH^-qWpmxM<%Ds;w=ho~C)hjOC>Ol5dZ z_o1v%b@@r3;UdJ&klr6EhPEs`FM!rFrfP20VwT@b?B|S$EcTj)q;po*%OnOCCSg^ z-!UXBUxhXCk0|G8df_Omy;!<}PdSnz(nR`u(a}so=q4tEw<%RbNm6ZE1HfvKD3TlY zlNFK{^q`>V7}T=K2fC4`{o$0FnRX?#DvR)RPaoC}8bKsBj*@p$K{}B-hSdb#x7FnX=L2(ssmkM<#Cv!0(E>tz4+=_Y@o%%qy*c z*g|eiytXN%v{L{yegEqP;0t#kUG}1z1h*SxbSE+OoeSFx*f}^9p%Nj5r-}bXx|uLx zze*h*7vdQ0}l-duR;JKpRg)wFw__Dyn^VRH}j^99C();{4L z-%H$_aXWEH30lzg`lEIj6J$z z)XgqQHwSKx zKo8iT*7ld(p%-nPehZ?P2`y8%Q_Y251!6!}B;Rud)D0<`|vb8i93d#UyU)AQ0GkiiUmd7?OkO9RLNObowma^j@X z_?C&o;I0B6@mpIE0~(sTtP_D0N!kofY9&?RttwJ^3$y@0k382`d zQdk>&LVJkNOxxH|VqdRUYjm&Y?;^JcxAjk!l_XPLMGtUYXQF({VT&T?PwQee{n$l| zt;-Z#ST?2plEb8s*82V;6HwX47eMS%Z~7(Vm>l{nR7&y_w_JOA?K=jCC&;NWIZ&R+ zMQ=x>QlYtKulp&(J38Fs0?#A`Ihb(<1 z?obpQBfR$~^41T0;a<)%bzRcDX{~kD+k|EZfA7)@F=)T1;@APa<*O-RMTTGVRCW4Qu3vm<$II ztL?M(i!v49p<~GR6GK7<^w0A0YCQK?cElR)lUhm~u^40*a!}ysHA!BUlLPG>^r4Hw z&9@E`HFc@Fy8b*Zhv5fEJP(2A*5Ka%#oyDVIN=ki*_j0&&;i$%B&wuCE#|4UD_OZZ zxHnP?PhECPa%Y|05e#%Bh~C8tpzZ(T8Rxn7i!&11K-)HH(!k!Ixl zk7j`@M)^Q7PzEtTC7VsM{PiN#t{4FtNP#BVj2~c=9T!X!0}KM+Jwl>?bvd0wGu$C0 zMnV^xx0cELpFhOZwUZXe*K)s&uRgsk{wPxa(mXrLGaJL*zx~=Xi3f$@9x0i?Pg-G{gL$8!2S(HL9q^RSdfve^Kcp_~u?3f3z8J z_)VE4Cw^5G7sT=69iu+LgX95YvK9mx8uV@?I!H!P@V-7Yr|$)AJvLZ4VEf!jTH`V^ z954?#Ev@eL1jV~x)joyx)}RdM8X>N+(n?&fQ|M$K%F&r?4Sn;pTqm!DAMf;Llp}W) z;W^+0Eq^`_;!VXLck27+O%73IOUUOzPS^hdK(^0j8J>rn;k5c_X5SQ0bCHSre4ed& zBn=9vq}7Bak_~PH0o4A3@rkr9_~VOwi2ELnC? zmP}BD*E4;?7q0q-Y_X@3Gq$S>GdJ@Y2*oR{wjGIWE~&m}-@l%fnEF2pz?&nY)lGC@ zd>&c{b`R#(#a$RCf4KPlb;iKpJLfyobZ|fHhNSiFS62hW&PnfF22*Z#^?|z^yL877&H zE{uDX6(3Iax1#Q1jnE4-Dmryt+5)%nrQp8A7J2%miLNzf2BW1o?l2&~3AdTc7^$z^ zKNZiX)h>~|+Yj5iB5P=wnD~#cw1-8aZdVPY6dn&u?UIj-)$AhQd7q^D?X}JRCnleT z8~42@aGq`O8#RR&G_Kqt`A~n!G)syscRd)ThlLxR=$&Z$|liu zls_TqaXkz4oVyIGd23$rqHlQYlL@rHqS2RcDW|758y+HbtkezYf7=|-*)$G8?UvJD z3LzSpa_>Er_3d7=<^B8}bxmdAs>8c`H{l;>=yr&wDD~&O$#~ItQjw+a^6C`-(QBFa zRI3lFd8?ppwU`Vi2ixk2iuXbtZw|2EB!r$-gfReT^_+RlLQqhlmr4GHP-j4Z*do~S zmYMwu)yZT z=b(!AhVhS#WHi3lNHoYwfj%7I(jLcd-RZtep8aNj*E$Cb-@+~(uqTn|Z7#5IDJG$| zk%DcKKzauEn&ao@iA+u2b1QaK1d#$P^z94r3 zi_R)G{upPR&=UqhJ3x46NB$zDUKAa^SkotVkuY}eooasTU*=%juFQuKf0^|eyX$qd zLL!wrOGgmx=NlG`~qbu}-) z{PFeq;-mQ+4QOcv+Z2_(+H{;o6sGHnq+u3q$PN271Vcmm7pVp4B1*h)Xt{G9|?WDp>eJhd;``T-t zS}vkEZYn>6W*j&cn`d7&^nLrbW}yPlL|3&Vc*)**K*Q=rUiRRKpImiHKMXO*oXE~` zlNcmA*dv7%#0a^2<<6`hLgO8ZA>?k(g&^rP+^&3|HM*5a!=Q*u>4EqyT{B$c^zm+H3=D`%(%jppC3F$ zmRC$+`dNh>i35kwy#O6r$hJJx7= zus5-j__C0p1MxU4r7-uEYzzm-&DP^k3j*ahUbFHczt=W`&k{JpqQB|hYunlB)!v&l zJ|~+O79hX_-b#ujBJApZAuD6=!4)==^kAn(ziO*=nB>rfQBp&f!XQEQ2FjU2vk)O! zHOO31$iZ2fvpoKt`O=t(rskTDlQaYbXxn%YB7Vl0lp?kRJA{EJ@oi6r{qns0=Pry) zU@pP#Ed&GaKXBT03Vv0k8=@?_wYQJ0ycnf!NgW$Y+4O@pql|`zRIk>0P0xVF}B zePJfVQa01Z*Z6ab?(P>BA@|@5X7`WJXlgrcC^U0{Aw?aF|8j=u=j3uEwz64E8%u+E zoCTvUi>x%{+gI%_Zk_Rlh61~MiZUnF|IB0Bf>B-#sa{DPkb4q|COzad%?LL_ z6mx1LO3Xtu^eWbBUa2k7kC2(=JjeYiqG8h6I%|YZ+yJM8Ob7dcXDF@ItQLPc2uAp<+ywIP|GbMqQrR ztlP6{bjdgBM(0v(*e^BKgKg^F%+umcs_^l=;H*E)@R#=pgF{iIiNK0b{Mf!9L=kfk zMFtU!!f4S5k>&uBTi}M0xu$iUPzDt1NO`^-P}Fpb+E0`iV9JgD6YBL{0#{w1qD?r$ zdy6z}`217qUeIwT`IYC9Hy+$-aSd0Ng@%{oUi(yAO1roGPVW(4AFFSca)Qc{#bsiA9~LR7EC-7#D4Wh%$PR$kd({y!$r6c1fr3d?m3l zYV)JP_c2_kvYlXZFw3FBC}awgfDmHeqkN+LrLK^xFR79Gk z-rsU^x&%wg%~yn*;Zgu!h+6rlZ-0F&WploDfE{-?inbHhKi=d5VQber7yY(8h_?J- zA2WNR^vijQ7>eDt#r9ABv#W*EsA_7{va#i&t!=2LrXA&zZaIRnC9<>%2F|`fegXyg z($M;yzHPfCsz)Jq;}dqXF?I-Ev7$9awlZ(_ywV#B+w{hg9)}h?^vNN&Dlb8ig06>J z32RxTCY}_`gA@{u&~QNKLndCK3%W45OwrGm4Z&!~; zd%tqWp3EoFcQ0MK0?XDa@D$N9V!@66K7pU0Bv$s(u2^VE`D6o!Lx|wxgg;E|ohY!| zg2oqxGeSO}x9BU0XNN)G)1~*5uevt{tY3uK=CUN)?SCj_!Z_G5J%%+Ey^&4x6iLMy zx4Hhoghcn3gl*5<%AuNjvqY7MOGA4GyS7u`w;_+g_S!JLQ(G9l3t;11tR>_zWt7*W z-u%q}Qh6u`UNrrDS~^;Foe;*Sq)R1YobU=H#sGr6(P8n#&^-nid zcd7vl08>q=C?an>xi$22G_tAmItV|GHq`sS497`D1$+_`avzixBvMSid~4xLh(g=yo1a}lPZUkX_!O4F9(%11R!%{c>aH9xMM<6 z$p~3oCa@LS*xWE7)3YeBW8oRzOh2Awln-44%)r=Hh4 zep7OZvX6sv&Jt17*ag|^H{9yt5eUNDvRSJcwLE`0IaLa9iHUlKY`!caPW|KCaUH80 z>WfO&;uD??f7p&VcYd@fI2-ls|2xo~#Q#Wc>o=!@pP4dhqu$r|rv0;#R~Wzh#6L12 zmr3s(#>58I=KfgPWk$;3arYQ_-uhDI{%~(1sqMuS!wJy~O8+Va^WpW}zUOfn0RkbJ}8quaZ-qvoR}il%e8`u@xVo z0ZhRqKmYkdP?l;U$Xk2G2141I^+QQ6&YSS%!&V~MZ?|gfK-Se~NJ5!s4jqe+DAf>r zn~yK@*LI+9knspBkcMo6^6J#{f|SW(S*wiQR+*7pgpa-68iCB?!B391+JP%CQzs(| z%S{BWJ;h1W@P$cj;$bHHr92~II4Te@>FA)%BJxAr|4T!Mg~$)=JFC}cc2qTXnDR|q zR}UktAjr^d`*XC6XZ)2w=j-Q>s4)>-fc*+VOgXqdbpjWoz*@mX{Pik3*Q)~#33kFD z0Q6A=)FO(Wy4>j_dhnq5epk)#@!5AM-cB)FmGlqn8Q(_j+MG3(Fsk^c4vmZd+EpT+ zignYQWJp5hG6kVL&D!`ZG3b^e>~Crrzc7ppIe@r`=Ufv5ma%yc;+=VuzWr$H{0d@X zc`Hdqzg=FT&7VQu$HR0cfT*m@{b;W+j>Uz{-UTyrS!t*D`Wv=>&u)Oe`%ii` z7lPEsPP=y?6tI+YKanep&b5viOeB`;50H8ut^Qr)nSA$#PQJo=#5E)Td1Zt6wCk}- z_Vj_&xp}+bXw?3N=6_x7hS7gBDFSq z75$OYMamKH^;sXAa+W>}0_%bmHalMeDSF~$%O^IBUqKE|JLE8pwX45aW4fMXhQ0wp zc5%j~%8U*NqvxUua*81yXjB&FkZ1tegfOT^G!FKCnIa1!u#c$g;?qsQ5!N|cv38`< zX<;ZrG#$QM-AhSM$Ddgnh};7ypX8Ub6wV+kRQjVfKf&pcEW&DP8DO4NcSYvW?8hp_ zJAGAGSH*)#r!JWrTJwqiqpSP|A5mOgJO^6HyPP*cs!+$$g5*?$qAoI8eMO*zo%l?UP=KN?x3Ltr0jJdhpU`UlG z8r_>tIfVhr0c3ML{!tpMK2ANeEMT;`I2+Sn&N;bC466P)rPjSQzj$ILPEJ%SfGrGQ znid-AqQU#gAx=;V@t-e48fxwZzDQz+b7jG~K z<}iZ@2pL~b0+f1Tyq!I_PHo}ju<_W!%M-=%5!GY?rQm>&D{zKvDh!{!0k~ls88-a0 z;iX{4(*E$lsX@My2GT$qmc-fnHh5R~-??|VOS>g00Rjl%Z~k*y{=&nCz0LiD?pbCr z!lVF@8(}`|j{3o8Dhq8f%OrmWomrp1T685E_-1M7;1@N7AODc^rg9Vx!-M&4e#F`3 zm(<>6atc(L9DG%&)?w|iaYyEt^W4LuWnj&XnYXUpt}1AQ+e9eytG2)R@3Hir%9E7q zO2viueuM<4C%;$TRiJ~s9;Pkhan2F7sIi-`+gIi`@%iLj)ZgaXF1{E=Bbe8mbIriZ z!UcYN0CMtW1P#r$*DwCch?w zB(-o7^mrgEXldbR2@ zWcphgh^1UQj3Q}WMG{L^S9i3+U@P4WNaNB5>YCr;SS>hCR$Ujo67f80WJT2C+93`A zuH|>2Ewg;(<=&K8-LNUIDta4)8r-ZqH^eTJFgJ%etNiPm1vd#YP`31%U>F0Q1t& zf)1N*cl~#dw8NGZYdj*Vp}a0_Rq|4A>*hix$o8KKfTb8FNV$V(?>e2obp~&7->WV1 z@3@SoEd??s>~RUTbzVL(=b!>K3rLTmEhm#O{OG_t`*oQ75Nwx*;JPwO`H6zceJiWG zUb;(h7O2~lV=&J|bOP0XU>&i5?4QgRGB=+7HW}fGrSFc}{21vLO<4lymV!i<2i;H1 z;c)l?QQ!R~^USE@_T05xkR?(tavM%ZlBv1nn?}yV)WEhm(j=y^&63gy7V5>cZF7rf zGHpXd_ggEPUpZ2pvlQj8_A{S2vArGcjizrxX}V?%?BUqUBZY4Cs>-Hj#o0^-GPB{9 zb&;=j%ZgOnb$#0ph*>r<|DW0K82bEjO4;f73;oKonCeb>7shNw@Ct()p8>m%4D}i) zG|2L7&oKTNL#{b;f9PijA$h%}YFqJ_`Y>Iua|5KhQ6`9c+ph?V@c>R+&G*DYg(88S zocAR_PkGuh`aa+0(;PrH*j^>%`9%pM0J3=V)jvZlUbPPy3vAFxtyJu%A70G+e=>?E zp{yb}2WT>&o9mPLumh$O5r@2z36=)Jp#7Mdu3&2!{(g%uqlkgJ7K=#Yw043jm#;EN zjGk_VyH~iJVe!C>_2~IGzQ_V2P<}(B@xz5m0 zY8&iCTEPxt5y%TRu}Kd*slsO-6iH&ax^iCv=B}*b%5$N!>mq&OX>!Z25$@99kqtrw z1mv7~LDcdc?(-2A6YWoNm9nDCWZKA~+IV%<@FR+?EIj6$Rhb_hJY%yFTUj%7anm6M z)`!EebGf$NAX*^D&$6AwlXiKAScHAcQ_KoX^a}~*-}h1rzX5UTUoQX^x#lSSL%(`i zryN;<7(QRyr`9VDWVYFQgKwNN{|6{;7@5DR6>E@6Ei|K8bRi9%CKDnP;Dvg#tQswU0?(J)w7l z0}xTVNkBxCX8exzL1wv$%Pcm#2~yK0V;v;`h79e6Dx554_Knd#BYbBUZVDUIvtqP~ z4SfiZeN&{%S+k;d%&E$Q$-j4|<4r>|rtj|~8admxzy2j?`0sQyWOf*g4uor+o4BB5 zAYD`tJ8QQ%_SZI3*Q=LB9I;h@b-*_m&jkI(JlDwwSFMZv%@eCPvzAQw%;>Y}ENrs0 zV%AgP{Vlan`L@oSOod$mo4T6A%m%E!|5FdLN4pO6_rW^*@)-6gstV-7>>`x;RLAX( zIOKpPE@Na%2Q6sn6ZRl+#E^ zx#5kyyMZ73b@ltRnDndX6KK#ZH0xl5fb;kmc#Rztw^F`9|#MXSe*O>I$X8&p z`As!pp@o`i>-<}ZFTF8T=K5r<9&cj9ToIN=91q&x?FYJF95gJu_HZ`p$a^vY$q1ajI^gsp5KK{(N;wC{=G$EE(e~kf(aOCZc8ubUa?oI#nh&S z9~AY_(I(NE?afb04uB#CFfih6JD@kj`Uk!iUXwqKjPvA@RPMc9b_>fJvV^E#MB5h^608)=wz#5&)ssVT^zK8$26aI?`%$^#SQ!?y(4q6@7B? z^^0qLMEqaK-q6IG+Tz{2*FDxFXb)%6w~R}GDh&brI_;hIvN2`&zOpsVVJGv9K^{HK z$*V5sQp!GI`^(b~D*!9u|8h1_h7@*~R;RnPYYPLQDfik1!&HH&y(oG&YQK{-ZOQ67 z!-|++b}9RV4o=`;O+-t=sbXyZ9akJ$*~Dn3mVb$%EZO17Fd2IQLB#T`1nFylS=sjoA+@S_J}0%ni0HFq;-KfGUdL zr6RR}2Ib}xb@WX42CrIp38#F^%FCn$^-%q$4xqUM>^Ys8SUa^>^fbyb+(i! zxCiw0vQ&BT;^j>xXeUNffggKeYLFXDQ9(Hy7dPk2~CGI3MKuk zXGZsmDq3bZPMT{ac@eFz2b*dj$`eqr`U(GvJuj7Yg*80tR#?G#XvlTntP(ECxp9NZ zp<>qu?FWTeUcif9yF8#=w@fQ#cNOsl_75m)zopa9+{fQ4C5mi-3ZbDv(`q;|(U0^) z-S;HR3Kf6MX`0v*N0YRP*j$m{yhztI54j9e#kuTgM>ZX%%WmaEL7`1zJ2}xn@)HLJ z|7?{fKX1poq&tkS1$mc-Vy7g#5e7gVsP4P*8^2p?Bt`{A&Y}s$hdBZY09?lFGd`|V zSTpoVXeIPLSdYAH{}UwpcaOfAim`HVWORUj+41~L$r-1Tbi1>{(jitMfDd$Hos&9$NNdO3-8u(@M$RyWX=J0wd> z`|fa*d2V`V;{#60+b&q_j0(AU`Ws{9u&V%Usaf`(AEXzKYM z)_*vqqVGmP6X`uEAlnN%-!g>&1ss!S0MI1m`ZNEVePrcBX>4v$6?Ge3eh9N;P)nC> z`{N_1MEi#+H=hKeV*ULUdqNTeft?pw3Ot~C(pOcP$FI~b5_c8?g8RMGmkzAyPtj8K z|9wlAL`RuT##1MWBEb9kht3#R<3%`O3Q=PegsG=)ck9rWAG!*7XO~(VQSWp*ZD1W8 z?$>$>{s1ML&!@C(?d)9%ANW=YW_PswYGfuiA9kW{)1k2{S_QkuCB$y`!DRT1)Ruq@ z0REJ6YA0d*Pj0$teq+ylp&OyV%xBDM7W!!*mAOcUTVi1-2}N4BY}-A`)VOQx;=V;Y zFWi599k&m)&!2>y6!_|D@j56;2=|I}-C{*MlZeR&Zb=nl)I$ljryci0t}b-gS&|S; zsZ>O?Z9k{U`T&+-n^4+Cf#Lt>SVQ@!6J_LhClW{-T2 zDw4s&+JZ!V?^%7bpzowO4$*fHq`3BvH5j-WqJ5n5%fI}{XM40qFwk&`EsdIVzF;+o zAX&J5qQUa#swsFA=p_mGg>yvolOsd(i63tFEwllT6J-1Or5XZD=6~$S=6vs`w=CDj|&QIju6;VwRZd>5f&apx&GlyEFCec zElAz8moNT6Un}1>>8{(6z%aaPV+%?^LW+V^Nsw9Rh+*o&!H?arOJ;&MS z=#0<9Bs}&f#ia0PgCV}n2a14e^0n6Q2t>nFql@WwzFw*X_Ag&mb0zfk!}lXOnjF29 zacvfmtAmzF`{MM((Zq`SXY~q{;~H6IAbZjz`C~F>hc@fC<-w!_xBn1>5>mUn^!~3z zxp1e*1olpg7o)eT+)_aD4*Rq1B$%4NQ{khc9^TY~`H$i&>-IlQNCLe71BPys6s&#- zY7axl`2750{<|9vX!4meRk?&2LOxoE(OUilxpeaAy6>|u=jHJ)7A&tbc9h`wNAcU|yQk1i_f_AZ%QQY-wy?XLq!z*=i}zfDo$~IBA$fcZE2Ei*fAc z$J+};5E!B)9~s^{o29Qy->k<9esZ&HdOIfw9yj9CTXY{LI)O7T@QDw7xQ=}oI&m(Q zX!88wC=16z1$|Lc!Ncr!O2b;&+_GbCoI)aT5AgcUx<`{j(9q*m1M|mdLgS&s*<*c~ zMlTcREjwV8v|Kyj^=yED<7;)4`ZSO+dVb+V-XHqyx)JoCp5SFsq+^KGdovVc6jH(N zmSUDu%HzA&1b)y&RJSb>^>GIy6}!}du+`Mesp|o+FbtZe))YP*rd+#>4+<(YUT-A_ zzs=5C9Ssg0-USKY4ADQv$HWA<~oi26hPy<=9y!x}mEWu(ywLQ3RcU>yb(Rg9; zjh$~z4wyfq_%j{94#Sa*zdmviCAU3eZ1}vTKyGXrHUk0m%(nWRDo8JQwI>F#719*C& zNeh5*!s%~Cu3kUPAMLjoVi21h{i?@WAZ!B251>`6M=&TX480j1+pd3dRO+{PE=esw z4vP19Bb@(^4=GlQkG=2&)4}SyW5_Iz!<>y;sKonwaQnzlE%ruJw-;v=+s|| zRhIU;irmMYDDZSl`9_D{V6W3G(Fm#AgLTe<9j^&=-GlBiV8gjkOGnc3IIxwe&4z4A z&&^FZ>Ut7>d>9mcZ1e4fH0EUCjAmNDMZ&`Y_-J-#CQXwO7VvG>$^euJ(8J!$nzfv( znucp)60BnFuYfuowHhIm2;z|~UHiL=Nx#>tT;%Q_@Z^0Rgv>%g>ja<;G+vXE=7xH! zZw^W#eP+hEH(ZT}lYO^lmFMJMP&<;~2qt`##GP(dNt8=HkuwZj+qiXbJ8a4NI^))y zMTWKV5HG+{(TinVJ~` zo?$o$u87J8fak-tNcdrulN4lQ`Gi1@q=|KY`}ahQ*o7@%9V~y|fGJeA={_1=^(K2! ziWRB{>^2}}%;47=DFak(P*>>2I|*EuPk06qU*0`88Cum!*PFXOd1(OAWGs}Bjst#S zcu-COR4(rrCAAw2*ecmnmfk!*y+DUJAFu=KKXI(bCch_8BPo-TA)ZI*a z_<>OZgMII>{$j#uc8=?UX~<+H5em$kp+7XI0U`o5J>NxQRrDKH>w+*%fYx%2`761cKzo%Sskf)E2;Z3W*JhFNdrTRWdk*{elbw4jEy1_KRK`9iEMmQ7uqUhI1h5;flQ;cO*2Q4|ZI|80O z_^-PnCCpE^)o0&Y_f8z-FJ4j#QVRP7SG{r+w!9lUrtn$$X1H?Uwd70<8@IJ`Y{a}` zQl*8mRDz9&>0JPwW=oPkf+vAOK>|!f=`*NW-Ifyj$s^;k^+%sk-=%o7jO^b;0PFC} z3>!ql6s(FB*+$Reh50z<)qMmJq75BP9kA<9j6xmVZkW9MqxHVrs4tx-CFUy2ft0vR zyAS&7XJi_9^l*F0vo1tL&fI*7`@w=6P0YiX^xa%PF7{0Qd5dMMf5@qo5aM*Z~NNiI2^8wCOlyyX@M~m_%MJi^|p* zb}*A{Tiun7Zifc!$zP3Q-Tmag{UHh0-wtD9c7;v>e!Oq(xfF-Ehli5N*bZJBIz~4m zCg7a*6KIW^Y4PY#X2O9GQ?g!6??Sb7+kPm@@I_=q=L2xFjbW?fZ z8f8U5^b4{TPD?wi9d86n5}CY@a;yugcKl)IzI%>h=`6;_AI z|5mhnLTB`>7JwO)@J78pn)ZJ7;^1bYd2vj3|7kknT;xN8PwU-> zHp2sDoB&9~3~ZaG=Uu?^J{Seb#kg7)aINL1|LO0K%qR^x&dDP$W$fKE7_zkt``%>~~!(wkEb6!@lk$?6q{~kI6PJhDL zNv|IQ0Yi6dc%V=LWNx0FBZCxmC?>ZRSs&*QX2a)0>+bxQL48oG6aGbhPfd#EYv2Gn ziXqpKE$)C>U>Rhfn}+ewH^4>t%{ALi`Qr=ICNp8oqV{X7_UIFK0y?6lf<<6v26ac^ z%7MkSTXWLS0g}J`oL5Rhl2R^+i^zw2AX*Lu4E4hXi9H%{|3NRCz_8^yh!fi1^sv2; zDvuoe4i8IEz|FB7&29w=jUlIAiTtb2DtTCVjG(!^D|W9|aiiK)qv|Ud@C!l8>P*1E zWK&{50*{{u&$MJ^hAl2()8RH%TX1p|nKa0YX^PrL7iC`4;@u6}rdKwtM=ns=OAG#eINl2-+^n>;8Y z%_w8UTC6h0S!FGmHaiNH#+C*^iSZU1SSNq+uN}X*Y&rxdOZ$EDZJQ5!z{nT9pI3%? zx2zX7`7Hv#L(SX@&fCyV+OV?!?p?)p<>19(0)U=K*QLUEpng@#0PmiTXiv2lf+IL2 zRQ`7}5WI*E&IsvZ^3zUf!BW@CM7Aa7Im?AARBKied=YH4iz5kfKI5 z;OL5^iu$DZJ25-AoW{kRv==VY3ZNWF!B`7i49G2knwN>7nSe}el;cB6KoHg)mH{(8 zVqN}1GvD(w3<~{DnNsIl-dyb&y?vot)9h3d#T_LZUMWBT;8z=Tzj;f}UQ`WeHZmrJ z0GT%|AISCW*j9qTrSP?uX&j%2SyYZ~>*e~@mV9gf4+K}np&QVAF3@|Nc|dJcB;NExP`nf)>-L7TLv#u14spAiCqlT*?vK-Yp#E;Scq3JwfLtL8bh}}61^xFcpR0DC@dRg|F9)P;IU|U$s}O-4 z--saCB(5|PIc~ff+H1hgBebnFtNyrH5E2B~?>!C^)0b`(SBs=M8kHA|XeUXARcU2AY1jB9&cA5KZ9e0OhL> zn&Dbk6+Zbn=9o**u(HxL?*+OdkXcdFj^c;o;0qp!zz3bRLq3%@hJO;0BR8EYJ=}U3GxpsnJ zn#fWPV1Og^APFk=`-`>y-eI*VbMyl4s8!t_I>6p>G{U(yB_=_sA z<*a{%=?m>j-fmvspXdl}+_RZswm}{iMlV@~Z8i=+PwvQVMOOC#1Uuc5T{2;k*y4f+ zA?W+cbjju?Uu^+YE9Mqw>r-%ZyT0~KfMkwe;=s1E#ZWkq(!j*5-?c26j&yI2MxcX5 z*OjK*!FNpZ)6U3df?Zb`K|*ZQlS`>YaHlzoqWK<+)=pP})Yo!Z4|W2bX-it@15$Go zp{4#5$e-$1i5pg!aeTOhg|MRAZ9(h$ieD&%W*nD}BEu5L^>ZPk(N| z02hpZ z8xDG?xn$~6ASy9cQd#Z;sSCtbqQ{v)Nk4H0gz+6ZmQHdOR1UO$)=T#io-k$G)A~U1&~quSsx=%uKUJZ~XcijE+j^?R1G{LX0^UNC9K9I(%bVox1Io zoI1a-u-P{JztC* zuWvbA6Z_@TdZpR@$Azrz^MN(scn7J#*#&Za|7@ggVFfElXV`fJbl?HvAFv^wIh9L@ zp`xAyP-6J;Otp`UzGm|Cyx~)>+6)c|6Gd{Bx+s1|xZbcd5!F?tc{02GOma0%59pUN3G(62#qa1tTG3pf; z(8jor7aG=_H17ncdg^S)AvreezksDsUn46}V)6ezYjvY#-9LK1ct_HFe6;^16;Vqt z{Q|%nX8S{4D~b&(?;>Nwg#rd|)cTHtLj z)vOBnk@jB?RO^G#1z>7bT2?>YtDFp#QFE9Or%?^B19ty z%(h4t$ziSkO zZ!OxWMC+;7%N%DHyQ+XITk-ZdPhDuI1IK=&Hqu&P_Wiu}Tt8v0qJbH#39DFl7eM9> zZRh`=1qf-@1lF&&4fQN1oYtuq;=_6%WgAq{36WW*NAel}FojrvJQk34+txxm3qfqg zpf2zEH_OkFj&Y*D+Zi?naV56yt-!Eh=An}BX^n!s;MX0-sEItc4) zD2$)Nm2EK82OD6vh4yCwuo5Y?(E2Oz>d7vZp3i-JJ^?#(BRI+~=z21z2hPa+J61!s z3A9`L$@^pz47Nnz#jdeW^i*?+}N5e=DE!l2{;y3o6f@%DAEGZ zSUbKuVmu11?C++OTT&sxPLY#luk~BqXrLec@yyl!n43Jmq4ks|8^oA5YJ7HGxPB7I z9d)hW=`NzJ31x!N?)ba!+JZQFG@w`H()a13t)6 zve;ZntIeSu2t%sJ+iRhCc$=_ZM2QV3Jo6h&|B*SId68LAIq%^UsS8 zMKdD#W;Q4F`VN+nw0nbXuJHdEq44lh4Gr>X!XuNlc1YQMZ1&1j9zEul?$nqBhH&L zul&}VwELCy>-mI4M>xLlJlIn6b%3K%y)~qRUk2l0}B z9H7GCGaEb@_;_{7y}r+1HvSQB$%ctYhqBPh>Jf9 zXe6EPRohK#=cq&9I&)8j621%FYMyqw_?St=IIF=CADoB62LJEWnng7_LW?iaGWXxv z+}(p(`Y(igjzo8(h61RB75n#JbAJ8+5@@f5b>v*{!g{xR9S|O_rvSjoAVt2xQF8u+ z92>!_o&r3I827YF8F%uO$$h=g-YY{MSBHFeLIA~g8=(NI)`k&r6Kyp&;c8#hun-v{ zY1b%2EJgalse!=cJeYXRUo1#|KtcN0KWGFl@=dLD6 ze5_nc;4)srv=%txHWvi2;RI7C#54_8oKo^YLd1SFVaTp-wA!}MS{alm@UV`pnmE{0 zjjHu}64WhYhct-oLTuzot$>MTGb>$sq;qSh+^xsuiTFl!3!A_83u@-?LrSAYed zn1vI}3ZS`48O(LrI0E>qD7%9i%q);hYZAxgUC>=a3bLG8eGFhJ>r?>M06EEN!B6If z#e*Qf@xF=wZ@QVLS!O(q`OA3Hw`zUr4YAga;wau`R)TN8E=2_Q}hSImsqf&1h8 zJHw`Mpcjl#0@wPF+KU?qG;`s#Qz0cWR)$913CamlG)t1Rwa3U;VMsq*{{VWa+)eg>|Qt@$F zKAg0C%_aPe^$F62|1zrfORkL!($5y7QoZr0LCbc&l>R?B9TN6tKZ@0pwQjT!a3$7T zYC<&4ZhdgjseO~6LtpYZmH#@VufN|mK#Cz753$8sCM2)&P;Z1LKXr?vWXe`T!a7eg z4zdQMHf zg{)SKJxwzmjB+dq6|06ytC;up6S~DYHt@Vr$)Sfn^YtyzW#`6i$jyFFQU!fEuR&7! zOLE(EgSLA?nm@RwU90o>9Lg^`?>JAUOx6-<+j6&v)^F$fG|ZxQUz-$xnwmOm{@Ju7 zVTZW-z(~%yfplD!ja+1tq=kR~VC6_)bfyQYwt480z%w08xfljG=_E=)FSJSU@hQWh zjxZ@HDWO45(!ildC7Y7J*Sm|+Fw%+IxDbC?2SL0eiz0WHrl(adE+rM87e(h)_ME5Dm3BKa8SUp&EN$VhO zcU(ksgp?U1b2Ae1T)!wOR^>{&eA{QwJ{yM>DeT?3%njrlpW%Z)wHf0fGueNtH}13Q zGc}^1aNBP;nBnjuy+GMv`p2) z{J36PD7QtfoEFV!x}D2`a}6Q3&6kTSVLu&GA4&aUC`yn3i~B1`I>pTBsa~+@blUY? zC1;0lbC>UaGJkGJm?wINnkShS|?ztVhH9H@2qwRyVfv+XyUNW1*p;&f%Hb z{OBd=0%JvnWerv~UQ!TPpUhCcg~Rg(e1od=BEnWd)=Ec5NXjlRgX`k7bZ>Uo=d1?M4(wN!tyVuj8CFFma+mf_}Tl^um zFO)7YG}Iyl+&f`%eeCdAG5lxac8tWT*m|BL%&C?`lW)>pq=lZy;!a!pcN@6h+BFUC zqL=h2vsg|3-9msclW#$amYJ-@oDGtq8c}A^3z-F~M?#N_gG{cpvDBmNy^y8|mbNDKOb^N80Yh$uy*v%=lrHA}_;?=-s z63|Xa`itiIBrH(V&`8H+)dxna@K8)qSjK63Wp^xDw@C|c2x-pkihUxA9G-vd`T{mo zm87s`S*)VN{#vqV;7G=^LW0qmbTQ5kY)v>RI!2Je%bb2BS^P zcWL%*o8@&HzOf#4G20>Xk>}YoqYP+?UVJ-Eg`hbe4aW_}7=FBVRF`87Kw#I=Nm0CQRJtWk_ zi~ZbpX)FA0DAp*ILxKEBb0tJ=oTMAc_IxV^?=97%q-I`yb@I#V%V--A1Z@~olB*W~ zQlVr| zdD{pJb7l6>t`^%i>G27zggJt!YiSHEdgSjX^xLH6dKvF6G5g|kOas4b877@bmAv^y z1d2nL%k{gDCTvoeKn6yet*reMEW=PdT|gUz5Y!{Hray`Q`s9p%f? z)A-HZ9#%Xr1Mq9&DB3F}9-X0r2Rh`oY@9XM3q>u6`}!(QODr=$d~h;gt zxcd(2M;Ag;q|o`freYWvS(#kH*P*7+O?WK0hM;nnSkrmF)$fa!9Ay&=bB|5+PfDhm zGn33Q(_0g)>-bSimiXQB9IRPEpAySwG0W6Y{0X0j^&(K*2%rHhN-aFA-3rcb>c;8` z46f}C*tM1ayLR~qQ75NWQH3@Nq72<_a(_cxTatL$)Pl}D%eFyxFEkXsi|5t5K=l%S zY)j5Fwh|UF4NP1K`PGV!OfcDTaCTGOBVO)SFz_#(z?a||t&(>by(q&Fo@WGo_Lbme zQyc7JKL1@zVD5nZP!as)FU9jKihjXP<~YIzwZxcd#H2mt)U!}$1&a6e5-K^QUwg`a zdaqwImW9}%+tPd^6I^L5riRU=*Yg_rb01dP<+ay~B&7CDENEr+sPrNxT6vFCnAnGH zY@EIfG?Xn0;Oy=;$4ZKW-dBYZAv*u#N9*(qrC}j4bH&_2F_otWT-nzpoy}x0NP{eq zLn8#XYW`#hpZvDskhiUf+0a1?9C{b{vbN>b1fAaaLjmO>DW_U3tnnsOnum(bI!sY{_$@T^5H8sqx7~^IAdOwTRoHFpI^*y;rc` zZ7%EqCs|w87d;BCaq#g0U4M*(kRNxNgz>Ox@e%2Gh;`koemt(PqJgT9wee^eE{@!A z^~M)FmW*sX#AehDw)6E=pe%udgZE&EZ_4=@E~y7{^25niicK#KZBEuc_yv-fQwA7 z^45@f(m};bNE*c~C&H`N2NumNlJEncWq_(MGN7Dl&OOO;6B8vRG(Uhlvz-UAP6Kt$ zE_2?TY#NHMO$3Ve7n#m+NjQ-nla_~cycX*xO*FdE^f4Yp$#b|Co+Trz9RUaxqgiM; zQiE1bI9Q)h@34QpOC-k<8Pi{0>n~StMM`^HmYfRT3*@0^!8V8AZT|SZ*1@XtmORDr7YECUIR8R=OkSO@_}Y1ewHgA7 zB4Z(=-f?%&#I;|8&7F#xLjHk@ck=jD^F6U4C3u8EnJJ(@bo?T>{SxO3io1hXn3Qhc zF)IqZYP62*o>cAr@c@?%Ys3+ehBO~9)`+`Xp4ZTn{-Ce->z7mv@mhFX^b>}A=WqmPJmNn@!d^O9)8{}ofm%;Nh1Nx3t&0jK;E>07?DJZ=DaQzNak~%4zdW8uO z77Pd8H&ZtenE+>vs~5Vl%8~kYuraKSypsnp{TBw#g{jYpLdlzxlN_r;9k5nuTS>D{ za@7`)#<-&_7On7ap%zn{$t^WYmt^3IfAoJaH;+bn(C7fV@oM#HCYPV(6;{!t>Mx?t z#-KTQTq*E5=H|uKTYLenRk#^xVw7iXy482leZ_Ezy{T_W0B1C0UQOT;s6{Eq5L(B` zrYEmfAHOf(TIHb6!6rCvoY~mkB6&@5#GaW%Sch_;6IQ2{@uAOvG{Ij&n?TFV!T054 zaUpSioK*od`<5Xr^z7#}0L0+#(($NSXr4h_7zY5x&^}`2&g|sX(+WmZv;d4tqx#L8 ze3G!NzBjeMZ)#(P?K8P#EMll0tz@SBs^Y<)*;TTA<97CE`{GLWKFKHufjZ0MA50Au zr7R0!=h^-qqdPYUXRJ+-n8SxXYr>y9{H5V%$m7B8>BdLe=3pfhQRd)&XJ&lr!&4atR(zj zq6{^yI|Z5zo;t+Emq5{)=72gRt37=30-AUv)mRuLM za>2Z_8#FRmjD>I{*KGc9-AInkbbuQs7q(OL%S=Mup+fKom6_053Rhy|Om#ECchf47+Sw&P0|I$(8BB4gRA-q2%I} zj@|4|Br2lYR=z3)`29DiB}9~EvI8=UCq+2Ko|cb#P(Ojt$1Vpn;}bs4ShOk4`8=nX zzqKm~gETY)d#zF_GQhen@gjkosZxOenRY??(HSum@nW<5h%I&B_?w%I)A3>iE%GcQ$PBy;jAA zGN2dqHDpcu$%M4{JCrnZxqAc32?jU>rzTq*)3{Ue~9eQDhNg6w5x0!+gQt zvw&QCuAKTr1NO-OJ~WgZ5pe@nu_q&rgKf|20>7`km`|wz&kD7e6sbN2bhBK~@sBH_2daZ4iMztQ#D)Sk9+whxR>|V)AY}PkW!)YA zGISCIV(!dL2n$pu>x0}X#6PRAjgmu!zx|M}ad;N)2SJC)HfiTmL zxzc;8^wgGK6dmd;o~5)z03$=mQ)tl@ZsbEFr1Rt(nCJl}tN4jCYjP&DNDM!P4yf4? zqqI)RNX?*3OMwc{-=D=wnhMkA6+`wWr!fF``EhyY!&Ul&$_1pJj8T`B-7$~kHrP+- zr4H+;)kdw1R0@#LKxbMc{ara2K+8QH{AA7!XIhfmIx0VrCYH(Z#V3PWJS(dftOkFH z<{(8+DXfP3YmHDpp`gqp(r6YSd%3wkF&*EIRHFRwO*od68E>-n^$X(--qzIj%?Pe zvplVrpw?t}+P1+%1g&X7T!b?ohy>3&kXMoSN`@DI>*^O?9a-xBNjnYfx?k@nl! z+1rtUX7tIaWxPl4D?{**QwO6f7j{>&fm8LQ%nzqB^X{)R}!?K%kunq?vNJJXw2NmP%; zAp~0QdWByQ8t4`H~qC) zQ6vS&?#xyMV|ujQq>nW1!8;I?3{gN2E_Zor0$#uJf^U9gaymC6)1mBakxPVBdp$9a zPiVB`k2iBCO#EUsXgI>J1W=pz7QOEQCuT(n|K7&TT{e}vB(3NZ@?y50qhyR$;S~iE zw$%yCcpxUYWex;Thg~B6D^6-}V<|GNNid0>#&}CvBsRoR5M(7R2nBH;4P9hjiIOCv zb#g9iQw0w_E7wau^Dw7OYv|$CyEwC>`DSea>dqhH-Bj80EuJZG$dx7~qECjJ(Jb{{ z)ulOgXZwiAhU;kprdUq_e&G%3Yb>au91OVlej<2IFaDrN}%LQceh*8FX_RX6&7O`;*02#i_Qc{D8pjp{& zHag$)m9Dh4UQOS=bsB8z74fya0);c`D7+mh+B>2^*nlL-m6>KE{vPrqN9U_z`S^=I zif)U5nJGA+*#kMCcy=(#T#fSLNtGzFpR!VTdE4U zcG|EYfy7eOwqYBA;f;Q#_4_nH0{{CNU?^Zo)|U50tGL{lIu{a4b)ieAQC0Pvv%FA9 z$+Sd9@IW6Xri_>W9eIai3b8ZFKiyCTWCI$XRNg2f~CBrKp2!N*`x7^m~FU5+}Z%SA3 zA#6{2^3nvn&L_NK{*+JYny|!z{t9np#QbXfwDQ&8@`@tM2W+!9U&`m1v$X`5HeaFm#|?*u z4s9ky{6n^L)RfqQCZM1of^(PXy(lW6U;+I_2dl$-HHq9JZD0!E>C$r=uMKj5;6lR% zp4J;vrU@40a|IIG-gq1_&TrbI?&4T=>R+!DCx^wlz*nx$D-B@*PMXG{$K{znunl-( zJk!_eof}(8Tw=^l%d(T6IirsicZqGhB7hG%8{>A-dLvTgGPSp4$$^SzkXZ&-Q2L6Z#Pl5M3I|htCJ#l878xKjBZq+BbR4c>%2>XkaZO zQj9cGUH0achXAs@b;ZbaK3fdR5dx0IGX34L9X&lzX>U4ZIq5o+p5}+w`=ymBc#`NX%XAFWqJr zz04Fk{|Su+S`b{S6*s%%lZKH|C_Y5ZIYSZ2DZ$tB5>RukhQVcibC^$Pn`sqpFi_0* z?FK>gzN5CQhW8ElMYsPP1F7}CN|Z81f_|&uhRwSt!@0!(UvzSK0m7D9iL|-!^#h7~P_Z$YEz=25;RtfXKI<1`@FJ%TKs+u_8>3RJ~C*ATq0qZsQ z9c_5zA;%Sn@kYDO1nB0W5vF;ihAzN4Xg%B#>lzO!<=jpMCe43@VqIxUK_Ni?fp1Gu z$Yl^f9&4MTrXId8RcgU(RRNPwK&YV|zOyYV2F7C9EO`YHYEPAjAACxYcos28#Az}=|WU#YKMDy^X zNavcb9ude__y%*;P}4@-ERV&cC0#1yy=McD7cku7jFaZ~2D_8i*_y*jTaFgP6|GXA zQ)h9B#@PKU8cjf`;+LuWqCIey)L~2@eCvBr;}C&u47)SS5$8+o{%WtkD2SU+&ZKkF zepO*ae{r$H2ARkw*Hw|&YmH>=HI5|4@+ncy%_k)$?x?$?vtpXKg0y!pNV{4$+LDc` zMlUl!XXcrnVLqvdw;LR6ttljq7oxi=+YYeB!kMN)9&(YI_zHw3s_i0z9+MQ-mYXq` zgR!{Uy8h%)3)%m6hfvoTi$Z7rtMC_`MCX4Jp)$tt3%=$-U>g3wzfN`@6|5NAUss}> zuMMGxq63dln}F1HwtQPAOm}qHj?e&9r}$1=!EiD=uz~**HJqb;#Wy0~l3;4gH1T;AR&vUynCA6#b9zx^=)8k8R2$X! zg0XC;NKr8ableKgmlW;{s$xLyE8bW~-2t2KS;Lt{%z=|GAWbqVIcfX`))6TY+}199 zoC3Kh{dT-ppt8f~4)>6iSUP9r4*IyQP^p|b>A%ErAn$<c_GP(P<0X<$NG-})!WLWIc&NC3ne zeeMK^F=n-?AJn71YrRZUqtKYM17;DjQdpH9RqXnCEq3!afa_oNUpRKk!u6c&6Ip4z z$tQMG?aB@piYu-j6dh<3W`&t$cU$7PS3@>$xOmK&7HKGrEdDAXn}fk;KueN_RWYR9 zk#?4BT?cTE9@DhW6CC%t^&G&=t7J($OCr-dfUeMdFMNc#_=kuVhc@;6zpm@UGdEm_kQQJy9$L9!HwXncz(R2am;xNg*N3Ir=#k%B94GP0v}8 zhFsgj#(^{pYS}LI5+bYTG#W48CQ%&zk-NTdgr7PBE}pqfti*OgJWl+KitXfLu-TvS z;g+b{^ed>56C!n1w0`G5S?(87^kzG^mDCJJpU&Bz#ILGA5P#>TIywKlGQl+p8jFXM zOy>DA*Ah8+?-`KMK$nO>0!{%|tn=EIUjvIuzqe_$h9VMDbsTe>K8s~3nwMqVYxKe2 zGxZtP9KJ366R6PKABI}9JeG&*7##t{NG!z+Bwo6AJHU1 zETpsA%S_`15S9!aE!Zq1Bwc40^`<_Sc;2pW5maWr>=u`i&vh}0osO3gV>+)pK^<3^#`^8%?W0GY zmT-&28TqPGT?iFJ7beejD%E93urk&mec?~1ZQ{?fp%Z6&I8Ke*DOu0(XRO8H{CM;_ zZLNZ&y-B>#)N$q`K=IV!x|*R{hw!2AG9d-2mjG?pV?65#9BuoHrrXi|AlRK^bEE@{eWzB#Hv>dtL*m znqhjUxx@&gZvBPNDp-w3Qr^GM{3=n^nN<6Dt}&i_aLvzg>Llx40%pS;0DzPka@;?~KGXuLH$+#Y{=rRnfU>gbCtNADbh#KJSU!4{T$9kn&`$iPO7V{JF zEb2qA%Gr*GRc!nWj(SWwS|IktM)mMg1lwH+d%b-?Xui)j`oDd88U*$!21;SNX*Qy` zMz_m#c4Kag{(j5?6!|?)rpoEUfEYY{&%qKQ`G zaXaGlTeIQy1)>c!_xGg$w%|&X=?VduAF78bsyu5$-(<5CjgrpxdJCWR-#G6>xfKKY z!7}>z_Gib~Aw0!7efCda_{EfLX0&4{ZjjsDFJn80Y|kNUKucI>LO9E6CRjzE@-#_d zWWDGZMbx6aNoX{ws|x|hZ(D}$I|C9(LEpi^92Rs?6htFc&af`~^GcQ3#hWy_1_*(` z<>ft^0qFprcT!{>oV)X?wRoVEoIU-Nt(B`dEMvZ6Z2}95=9X_s<>V-kw9pe{xtUq`e4Vq&Jp#?uOi@K2{1J43R#Yjq15KprGbaYB3-a3xJWvh;pB|9bAgQ z*d0#!kJyU>qNKtR^5L53pjr{Dg(omGiR6|%M`sl_eFX|8#U}-T0;us~K#ne}@&Dwg zM*OqC#MfbLykn)95R)sw38uR{&HpYIT;UMNpOinvP+^-ab*UH}gp**OYhv1}=>QEz zqq|v%x8Lp{X}u+kuhd7yPU$JmF5Q2_f4>SUvMJ4SJr^;{;T1SUh_KDdl@HeIDG|p= zV`hC)9LLzlunM#I=Q!}mBVea4&js`_n*0WC)TQ;Rsn4Dp;0M(RAfyQ75HyV94=Ow_ z0^xu$GV!WcaNm)^C#%J9PM8U>e4{O1l<#WW6UZR2W%|tl&UNbm&+{~H{|?ZmdzN8d zoYA}gtd6tzqK{Htq}Ib%C-eF{PvD0i1F51YTg<&s2N?Mkto&{@9AeuD+Vv7T-M@>( zrGP5}LV`UfnIckHlMThd+cK`-x#d~?PFK8TN*R)G%iQT_i?OV|EUDjb;X+UUol`=c z^t@zP01l+-hDGk@TzA8UIoDD{L;u?^x4G&OQYD{$*46gpzZdkhM4dW0boFx#YABBF z#cNyoG0DM{7P=rW-d2r}2y~#zjQ3abmeS*s7RisL)5@p#aCV1uzRvtjONir$zXZn6 z*4m+n&!(d1+S)$q)MM9*u+WXw`iOtijyj|0MV4o9Zn1|)WUx&~ZEzXJe&#({4)HsP zKa^GAwKH7h7^wGrrbO$70Q2v;ZNSqhx|+eHag@2}>tE~|ConLb1WtYPe;TdSJ_sv> z>75CV&T^6hf)s2vzy1w872x+FF41(Byw!ju9e1;YX8yK__;*F>9_;xorDJhC~|Cmjv_s>d{hrnM5xb$`-5QVl=K&}S-2PMt-t5EEG>_B4{Db&@aCdH&JuXlHz!_hro{ zK$G5xwg&h9*j5Z6VuGc!{@-sDO*L_U$n#Nuxn(xY&-esGpd31InLO_%-^ik&@&}{z z9E?DbwUa2BXlz13z$>eI6qwU^9hJz;qYGcm{eTTR0J_m_vFPbd4&UKg!hP65(I+~P zWF&w5Z-PjjQw2US8dlJbatqv@JOP`Hm;ai#w)Mpr{wRX@b%0(hz+vU-Pso!KX`R{h}cD2>d^Y+SwUl&jMA;O`prb>WQ?H4mGuV)#!R18@6&kE@nhT&*NP1+T#OUj@`H|oBO*Nf z7;v;KPrnLQ!Siz_dXS(4?pD?LkW5MPQ()y8*SC!b_yT?VlI(op=|0T_b?MXZoc={z zn3yozAI|Y@>IqM62(W4q$7molAWQbn^!N5klc*QDq1$ESL+`8vQOKAGq6d3VD(1C@ z09e2pnp*^t2}%YHdgw{)y*|h%Ip&pNB+#|M2oWP-Q-q4T_Tpd_&LU~K>oi;&RDPL; zouxC7gwcPaD*bFgk!Oo_aJk+Zl~C&Kj|;q|p~!zVnZ@-6m@cn<-tSmwyMWY;pk!6> z*m2GB+f}lTG3)dja_H>SB%sOR1Z;xcCPEvC+;aj%Ot8CYxiRI2fq75|j3OMkiIsZh zgHg+G?F0rFWN(Qum4$I|ir4Qt9zr!Zqcc3#QFV!u8eP%1`WRp+Lv!`E*WEQjGdAy#fQJW*NkpGKv7c_AiPY zslq@B0lkLnkyWRkIR@AU0e?j%p`c@nGZQppr3&T742zyCHl6)g55$g4!^IiB`cG4+ zlBwD3;7oI1+AjGjG4V_b&!Nh?anrVuZ@jX0VCJk6i7~^(0D|c zLLA_-|0Tl>bqc*sbi4jb+a*BrC*Ibyfo<{$V$p0Z7mZ-Pq^T-A6r+?C(mN9ju0%+KHIqOTT#BS8?itvYIHS4vb9tuoz&A*> zWu^J9!ZSuKVUV-#-XxdyXbddkMkLO(Nj%yv2h}6J@es=^!UFK6Luw|IXHMNlpv8_l z?{x#m6J!S}C2D@n;9leNW2NDUUjME=IGRvqw*yzw?fh>e1PkP1&7FMeLru~vE{1w! z!*(2ihtT1-g*rt?L|LBEBSbWg-6MfRVIwBVBjC9J{H_XSO=kYnRY*rVY!_OdrXTHS z!08{H?zLDzKqM0z7Rj8yUg2@XwH3X5;klJm`b4c(03Pg(My8V1o8@oxn1~NQ)Vaz! zK#Nu7Ys@#6l8iSzPYXSxYY^LCQvrPqJ5$6=U{oYK~TQXau(9ULP3W%PzHyng5JT<-HigwBtvLg)b~lLR^H!j-FOde35P$&Xa9 zBM3SF9_0c`q!l}+$@Gc4G|bVLGa09ryiuYo2LZ}YCdWFd&I!|etZJP9VojYM2c=9` zzR0{Bx(L~u!t=oG<10Z0HtY9NQP`X9D5hH{U%UrAcv!G~!=`}ZbfNy904R$!#ULe; z0+!5q;N?H5v6K@vW2;WkKK%-NUura$I0CN`uFT$^P2@=(|ZQk^9Ga zS3mtCk&TxILfe^ENPv>QJjY}!7qH)<^Y+V9FV#Xgli46^D@Kt(R!Y%-yUh32_Z;--h#kMF9=oR&~Z z1~#hr7J^R%R;TJ~Q|~y#l9?h-vz+dH<-@C*?sQ02x06Q!wAov=9rZw9mU;eRq;H)} zYzuT;>~^mv346#QNX6q19v@T_aNbQkl|>wEDPb`b>vvvT?vO82ylNQ#{>zw@^y_&2 zJ!XhxYxk#Q7x!Z4Dd3Hj^DyER%g;MY)e`mvW^-sSMx?=t`tZf(&DFK6+CV|^b#|kd z*EvYxXWJGlyxxB}Mrw)N;h{rPkX`C{&zP`&&3bfK86IbArS8vqII*`t*k2<+lo*eC zrOOYbqrPWnA{L9gCW3yTpAqI5SRBnv3%V{?WttFaQcenQ;PBY*`sCh(BiTNg1bgOY zr&|i^SMG9c3voah-M^{exe<*?(vyGF)i_MwG-6uuBy{I;t@$S(p4>V}-nY(=MR$ zYDeCBY@?H~oqPuEkF&Xc_MA%A%JYAGk8@TAW5KJfQA)tQLkY|2`|h$u9=m(uJ3-0& z;&p%G(F;*7Z-_4~+y(`@!tgkYIxEuevwh3;H=plpE5hBEb={>Yn1>=ps8~^$W2W6* zEhDFevWtXHm3xg?76N4E|4anc1#zcmRTvVV_FrcIG6=HyK`~yZdskay^{=;w$1hkP z9(Jxd3lf}on^pda4)ag6S-PcYlySzB7PmqjL~GSp%2_Cc$`^b!8AD>fZIyXaMea)8 z^M^ZBjylVjHKqNaKR@k9zi-4~UyK2X7Y_MXWp6?{A6|U&gX*Ax-8=JUC9FAhyWMrf z9OmB612!VO`HD@*Te-bd&`ExgJ;(@8&^tg}M8C5ac`&~AaV99ndpgZsQs(OZ;19G+ z>lbjt18^ko2xBV_kEc?E(D;FN*WG&YPch9)dx7@){pQjghw95bKu>uvcimr!07fxf!coL#pTM{hiIhzf5dS18|6Gv)?&-FXLh9hm3o-s0xUc24@3m6MxQS zTCEy*A9`ok?W`qB4eVe;Yk+$vQt0;13Vf2u1SE%Y3~(+XD)OA*a5W9f=df|j5aj~) z^k;NNGaxVQQ0=DrO9eE4opqUCZ(iG>uf6HvUQZ$M@ZmxA;C~ihpAC1G*Wlo<)z7QC zwRPD~_YDZEb$NvK_XJd-ysAzBydls~A{lLv(Y&2U;%J%;<*Yws11ha5oaIX_C1<~M z8AFH-YJOvWU_<}jnm=`Ovil zYPl;et7LR9N>ZZ{Y)P*4$}*BzLC{&ZY9)HxHPqK0D*6dk;r}X7iETcj)4+r5jlZSH z&YgLrm!>Cy!`rv6dZ*moGYLwXiEm3VWTI>*dWHn7PHW9_DwXuLX_NBoDLyQ*BM{R9aT_ySE*3gTcK^ zcGSSZ`YNu34k!&AFNP|%;O*T*F&hpdXK#=x^wcKTu)7Re_8;qDD^{RaQZ)DuvQK>S zw~N^t%|DYpKg7Fx-%`Lxjs=g$UXicqX)4Wq)Zfy(v`MPYrnQ~;c8=uOF`mh5B_jgN zzpBXe_>p|2=j6D6K|Z{7WznOs`6|>{PVpNw=MCP!S#kgR)iktT(2=t0Vw)oVs|5ht zXY&Np1d~G`AVWWrcmMa2%tE2pX5E<6<_224oCl{kJWzVKRXXWdpALoA^h@;y-z#Zi z?1nF|13NWA@;RQ2Hj?_HD_}M5_4Cd$2W(6o)}1@p{gxOAr6e!|vmDJ+{nw_fbavg; zWfBiEyHzvPu>P9T+j-;ULZSD**|SgOuEko|#Ex!v_na6<-{KF-xzqJ$Rd920ByBb; z+qVQALecPU1WX~m2BF)c01{9 z73B7R_CDB=?HV!VauP1zT9J+2rhAdNeBko2Dnu{|GQDek*5tzuLJ~khu>Sw*s6i?P z3;ju@u|{s|cb@Kl59gi`-*3eC}xH-xnc6g|DY;Y5*22C?mIbi=_qB$2MUTx)asg$$J51l^;;L*kN zu}Su~Z8QXYqWuieBRpv(^E zPE@=N=R7a(Jh^9Ebn)2Z$VHNVoDdG@nbdo_!$9{>Emn7*wT_t$3FqSJ9JSc3|DkgX z@3+BiUl`k!HMu|hwm0;C#U5^(TE6ai>7MbNT=V6{r2I=BH#J<9iP58k4hC<{VuaD0|!(bSyLA57NW9d>S}?6p>gW=cs$?N5jew|pYL z-7d(Ko^hP+vkbQIRI6D|yI@-Ag>cw0mphqva(oO$sN;jeiW3 zh^;4;njSY4`;mtFlGG$;_4%5@!$DyiwLwPBg2^^}j~&$I(bLlxm1>z<87_@|TWxuj zDXpfSUXq6l;cSue5bP@J_0Gld9{!5_rb0=Is3$pUombyw$D(APKD+sCiMz?3QQqxi z4^Mvg=4k-TMhPNl}$0Y z-94kq;Zr|ZCY7&J-K}b`i^By}IqfQK&P*v6ameuL(8Vs3#Rt+osgV&;LR-G=2C zc2n*c{ucZ<+s?J=%39s#$yDxy`mJl283T@0yIt(>x-LBLe#_K8>2VUo7#}p5B=)Od zxKJ=@>9y~t|IxFt>w{y>=d#^b%i_z{;sg$>K~1ifh;lfSswU7#%eC$y{gp0vTnile zDIaNCbSHkrxkQ{C%XscRiCdPSqVvzHH8-DrEKvWj4{@wJnj*K4CxEDuy{z9 z!KnH;=>;#+`(WQ7zP?Qby84Aa&sRE9WeK_pq4xK@3ll%vni!hmw*~Vuwy2lANA}I~ zHBhRdh2z4ta^u;Qf4>!N&bnM*QZxB1ukSkAMhrv!n?L?W+tZADo^z8A2kb15g};~* z4rQMFal1n*aRM`l?iAaHXr{iY>fN-sI2Co3p3KVzsb_=dpg%`n-t1wk%PmVuxDf}l zqhmUJBqFE-g#}&JvpDoG-(`JZ@XG52;mfAwO>1xI+qGWRnO$u;43X;OZ3(`on6Gl@ zU0Zf45$AGHDp4CSZGCkk?@4SsIp*Y;SYJzvqaLG>U2q9&Ky@g$v2LaNc*SJvhXzi^ zF}_i=e034??CzZ0L;)A-gNrncT4=BF!mWvY>flAbMrf8|{9ECu@rt1`t$*pEqL>0CaJ!DvdY zo~H5@knek&YaxBN(fo~{2S%}(LGPoRthN3=Q~P_pF$!4(Vt4u?9JL1+={3cmsmiud zC5jLHvGa)1j?^JM`hE>wWUr&AP#ven*`i9mEnRbx&5nxgKJ=(s0~B%^QD)1{Wf5ez zZQEI_2+aa0pKotuwM-L`v4OMblaFBjfq^ZHfqlB!*0&{m@azfNsbc_y$p!n}hlc}& zrFsS#mzB!PQ~`gf+FJM6g^@6JYDw|JwVTkNxokq1Z75lCZhC9tTm0_f#CruU<_tib zO|Bz|-F#i&X72S$h2E^dc82wuV>8+4(4VeDuX*pq!!tCc#*rIAFzh#Cqu(!! z{92D>b@D3;Ox4umGH9o{A~$HTegQ1bUbf=HP<@+g7uAoKF5pK$_8(~U1J!659M5j05oV_|y%fGCGiQ_I;?eq%DoZR6YfGNgUTaX18y z?5O1BO1C{^pZkPYwroDZ%SG{JQ79f_A&xO3~S&iy?3lF`YZ;>_6s@|*Lm ztjnOjLH8{1oGtt|Az6~YSk^ANxf0Eetd7#}hxyqBZdqx_W#usI9Keo@Uhg!`jzfG9 zTV)aZ4fxU6(HSB2sXtD#C&s!+m~rqjb&m&1Ia$wUdg8Y0<~Pa+Pn(41JHOj|PNT!u zC1ibL()r;y;VX1zAG8zW;%pPU_8HrwJLjhPy))_Qu>x>l4!~o;+nuc~g1IcQC60~Z zDL6494Vfp;KYFnC*lT}K?U+nHHfPR%dh{2_*`RFl#Bo>Y_X)8pbT6og;`M)JE$Yr) zz@^AWxlRS*XpF@zl-YY{?5sVyAOqsy>YY<_v^G6*+j4kQJ&FDVWT|Eg`Pm!I7cJJacVdl`X9;Az_4?d;|0Ps~?ZKESY- zr3pvDt=6j=JrOxUGKrzKp)n686)cogetL!BgR$<>IIzPMd!C9EZd>}S9cLBKQEy!ne^&E6HX4AK{j zh@F>8YD_xpIOYlM%tV3yz(tGZ(^_#UaMsiiXwjt&fgq5Gqx&M8yS15YV#T9=xEIBb z&yl(Is+xDjoqW$wQ6)|zY&B>PC2%(VEOldrFAIi z-3_(nBv)j%%#SX&{)Lb6PVXGVYMQDB+SQ8=YGSiQKvA37wMInRY6>{5ZpR(?{HLk7 zpLIqaf}cQ;WAA=S@O+z-gJwkxuN>mZl)er|9UiV*tC?G=gf;w%yeK17my)2VcAvBL z?xja0hy;e%cG?W9vdI8#6+PJts*z;U<o3vUF0Y zk;_pY_(}5evaQ&YAp+#f+iG&0dc-;tZf!J35HEDmnE7(yr9%dE>D%OJUK3FH1Q%-% zFCMgw+6cs`;Pmwiq@0BjhO45;wwhF@9_8>J6R&4;_u!6C{>{8i6Z+4^C#%|h{U2%W zc5mn!2oHL}R-~=6v`sT-G?~Vw}vb{aSkz0CCh#s(KjH{xl_odO( z*~|iqVuwHt`4lB9!GR^9_6rT}b*g1$3KF5dv%8cS4?#8jhFypZG*mR;Dv5=MTdqju zwBnKzrk!Rezy$RUZejpMOCn-IKJtJ*gub0`P7(g`{HC?1sit!s*!Fc*(0>;f5{~re zD76h^dIz_OQQKj5fC7ul^H5`}R@cwsD z=y!B3LJIGB=DuwPU7w}rBIg@R`;-q`t(+A@K(D;QoE4mZA7jkCvuBn)3v`%KDvJq0 zm3J+uUsgLX`~utWy?Pg0X-y?`0X@Cf>PGkeR3{O$k&32a(izr$E5E1nnZNNfYf*lN&cLT~@>{XV4}a=rwMlY7`j4zW@!=`?g zFpW6EkWcG3%J5QopKKMp+r)^!@Lp$b+R=8#1YNOuoejY8fj5SR;d@C_FHbi*F&^;b zX8&;YJ}oFwSbkS9aP*ERH7T#x9-*$0aQxJE{O$E+pUJa${mg9s_F#O8XtQQVC--KR z-a{&sqyRR!gYNKthok(dC2(Cnrywcb3Cj`Vqo|X={6BBBYE^ z<7&Jmh8eA77hc+p2fd+A%(zr>C4H1+jB7dM`TO2bP%m)l!@C7~JEiR=s|5D+^pf1w zPfgNO5I6mUI~=L@zI@=}GMA&f_Lrun|BEo3OdT3JtnM5fCA>j>tbHhP;;%&)^vb4k zYO?0o%oM|$ZwO@f=g;9h8j+=VvJFlven%5Tq@6JsX?Jv&NB7&wC-6i*{2b=zEb{xc zx4xnQD5~2POrqBmVV|#7gcf%1|J9+WKhW1g>ihx0oSVO!i!R!g0X->jy0}RD;>*_= z%9;nh@e!j;dzV~prt;7|t%oy`|XWgPZNK;8jp z9YC{6KDzxSCm7h}3u;Z>k5Rg(CWC@`j&(VMcf6hMUe^oL^v$VpwfGE+s6w|KtR;9PN;Nb6f+@S5bw>r_!MPp7@g7{Y19(X?qm)6D}{w&ap z0Z)aT$Zu(cLbE`2@GO>IOEtHT(Ia*f%3`C-#Y>X&4`NUS=t8$FNtFOIOWw{kI+xTM z>>qqr+q;ejh&Rp(MRH4%vr)b3_jyQpvrhH$F1-GYGq>Hnm014YbfPfY^qmHG z`Lg!(8DrR7KCoMb#8Y?HPC>wjzp2aojG@hPh%3QiOYA5J!H4=Y-GV3GQM_zjXDk*r z`-k>qz3P0Z!Fz~9ObEDT=ZQvt7i%;M-w4u9@RB7iRAmkjA5ma|2*nkzIK;vHGs`j! zw3Q}z8-Nj~ETi~w;M-%>zD+?T!grnYd)ZE1n?VFT&8XZ7if0=1w)xbk)4+$Jvh&eu`!MS@eQ^>R1kW{ z<~mB(!zp+*uQ1&dkoSa#@i%cOutxMQSg&0sQZM267Kj+)Pn z^D(7-wJ*Q>6AiclmW3uoyiVJ4R>s-+KB3>#^HE}jEOGAA;~#?UBvc>@keFCw5jai` z{781WA}3>O(aC(W3T5Sy{ZUqzV_R-sILSjW%u;j*tBmV8 zb8oU}${n0%?W8`z+LX3B7`+Fj&+vIyK< z;r&nhCRPNqpPc;$y`W$L;f47@)ylF*hgOw~MHAAFMYCXSS|~}kIbwKG*Etx10}OJp z3UpZzbldh@ub>a*kh6M1FO)R&(nv0-mv0|LD)QCp3FTPN@Q8RG%-&)c$H}pNJ-m^=B|qYdcf;I3v!)r&>ruE=$ikx z)*Ed-C}<2u-V&kzAZh3?8|KbNd(ZawH!W;u;%>Ej;Bjp`@4P?VfC=Axp=PfT^NMXR z)khn{7Rwb7q|G`9x;3UlUX#$eBikO5)ctBF;da0+d&C^Q&K6yDDIAC?SYT;_zd%hr zsf=v;U_1grYNq1qJ55p5c-Q7AHW}t|KkJ6p8}p7q&SYy2;tbxK_7uH>Hk{I@R3U8k z2-uf{(?Z}Hb26B11oIuz&HHLrR%rCo%7XzXEA{s5>unJ~h<@E)Dz_QBzqce@XyT21 z`YiJrf)4?nK-+NgZ+KuLpLH7f#EFb{{Hu4-!ug5dOJ^3PP-dYH|4}9gs<9a}3y-|{ zD=Ml?i+8mn(&XJ86Y6iPJnBQI8mn13L^u7}W*KDcjb|NgcjJiBvlr%}S^i+yL)`{j zq*Ll(r<5(QRBKeG%UQ~sM}xrY-$CSEdV?m8MJ!L%cs9fsIZBN<5nC2JV2NMq**&SP zT~|=Hpm%$K;oueTW5I>jICQ8}vn^hngOSmAilKa%j?j{Ph0OgBf0{e*$$+*=Yvl6r z@p~BRre*A`d!c!>TvMxdn;6|@z9e7ThU#ed#Lv5l$-AKxHO#Oe2xP|rE6J|m@7%&6 zxUuDXf_;Gtq7{#ggbzHSI1gsFl^D;31%kA&2vA=iKV54L+_H$AO z$O%3WwA0fZ$fd84msaF;z&TYj<=`z|xFzC3ZCO3A4ut)N(T(oAeg1X$x!FXHeQ&o_ z^!AdYx%e&jZ?o6LY+qHX<2tpg3HUqlKZce*;6@MPmOa~72wu3(CA4X*=c<_Rm9gqn zMJ3&y-#^-jAyNpdi5daH?o!$lB>nZbiB)F%ZlGQvk7aooI2&hKm=Hd`dXWt22?l-7 zyyW=P%>=~mSU=3VfJk^2H8aDDvi97K5Yb6q*dE|m_ZNyGEe z)^#YXC|A)%wm2i|1zjo{_WR~4 zlVdg_#M5s;M3ope0tSRGw^zkGM>Fi4LHv<+XZwwZvkj#cX+FXr31^SZ{gK;JRA=h1 zcFHQsgxDFUxhmk=I8cwL$lSGw&p=_~Je1Qw*1k0CEcqI>U+_+-1+YsiI3BKXlIK7% z0aCOXbe4|K1i?C7y$OywGL^h*tuO6UddQGowYq_NJ~~~aFYJ3AD}rI@k>72~ni}A_ zahGBTU+sKe(@p*c5i~_pRja<0VUhS`2eK`QP)%NcP2zK9nn4E0#U5rFo_}oI5HxIw z?LjlYe9z zuQ+D$T6;HvE5lb087pul0M9IpUa4ep+=_)12Lp^OdX7Z+%o9_7tNxYY{HEO8e3rZE zx#5ww-y*sSNqGmw>O{qJaz5<*L%!kU(WF;dMYZT+cT0~9JDt3!;h}7-s;Me0v1c=c z|I#bF?c=gsH6FrJnaVztP-7CwQca(|t9Rduc~TiX281XkFdV`12FT?Q>hrI}rT~j& zy}QD68CsG?h_Sa~SiSq|+J?*!BI~?Tk^^*mPjA7pUt}n^@SSl1V(cYdL$PU_*5^$83rn(Hf3m@6wxx?YqdS-0%tU0U2DHMkW0%jRO#ae}H_xgxDqpmO z@K)bj3jl-eN)m(;gixx0hI(WXrE1`J70eFkcK2-$YKR8s14#x^#%3^GU9UoEN^vDft$M~gfSJa zBlx6!nR3qtJIEBp%z&RY%>x0m6OW0DrMa}QVwJCsdx>__jNhVTUvfqy7={w(=y!0H z;GgD?V-m}P9LZ!QNh9pc`VXK$^^k_c%w%E;3UjbI+5I^=bb^t;BPS3HwWmsLXJON9>Jo{*nVF#5BApRKjWU|p*A!=s66lAJ&^GJJ~@{5Va6y#L~#7;vk z3yBB~G1d*C&#E?ZHCKd%1|7mq`JX=$QvH<&&F?<^RvQk1{x4R!0|g+YyhH&Z%d_== z8|dQw*ZnB}e$300yyGe)zQ+Gf5e~k!>H2)5J>TF}-^#&@yrRtIT?@RXamc(b_;xT2 zUDjQ#gx-hd8bG<>&OlQjn8^j9b3_0siLheLPt)9TXDrqVRjY2foJ|LYl?l{Kb{|hN zW_c^E=>zUW>&_!O|8sQTih=xJ3l5GZ z&_LtY<_gh@=3r5NOz1jOxtn*mgev*MoP2%cUA~1XI5l;1)Gru9y1JP4v)R22WW8+F zvx_pmF{%J14AJTxN-wVoZg1Fmt(&QVep015ji_amnUq^_gDzO$%6dl5Q=cc}keQpk z8=c(qVm|V^yglYijX<*ys0|<^%wpCRYynQwkhY~xh@v`p3!KZGvc|S06bxOb(r&Mh zC>!B|*CD)>z9Sn#m-%FDniHN2c@9DX%S~~Zg%ICSE7Z&KvJ5$UDV3ho2lWVG;>6!x z$P@sDdT6Ub0|Jend2)pz^#ROTZwzyi4TQ!69e;(^Qz=nhcFO0t`AAmA{DSJ`F(j`W z`okhTUkC3ER$8bzaK}s~D1igMs^Fd|XW6C^7Z3DP-Q3E&9ItI<^=|J}{o0S6*9T!9 zrF><0zAw@n5p_JX99|Z|QXgP8pDZ|0j5!rTUrTDNLe_~kEXewxxNFTspH`B$u*N{` z3Qt}nQ~xug)<3An#4FB7#AU!rd#RXP9lYRG8$C5t&SVLR$g3os^J`%qINkN4{YYYd zo0RxtauVS#jvgMc9|wwP^qw1AZVruLf2dAiZ|H=;)9}Xg1oXrwgVJC&@S-aqFp$;d zeKtJCev&?DeD3`0ChV9^ddvKUL<}o%HE(rGtGpqs3L2f~=I%_G1mdcU>=qSN2HzQM z-BZ*n!la#-Zu;wa=eEjsg5VA)69sw>${9{P4sWuM?k~4o#*}NTNF`R}XpAT)IMAAX zm=FjrV|@M1P(SR&6f%jFHxd^B@uYPL71z3K59Oyq;zpA13*evwpVX1X7eyDBTX`1} zTa_**=H6&m=og_A+qG3VkLTla0T=$ZVD9s08eGi%FO3$~Al1kd#Fjyn@gXFZNFR##Ta!c<^Pdk@ za6vsNf>BnZ!Udj|p^aHr^sOCoEgWuGNIL?bf*S$Hy2$1lh~Ww+@RBIJeCtW}kf~-? z+*LuLx}$4-NF;sEBq+;X%u6h#WxqKbxM1<1^Rn2v zrB+P3K=jU+sny>9;KKqYOtkPR(t?y%2D}M`&5Ir$6>d~Eji41|f*`LB=v#uftxj|5 zQR6`64q(}Bn%V5UBpSbb?&9T~rBhpQ41MDzk@02Ewu6^*^He5n=u9s8$0gXtol+~-sgg&RabVa6FvQ+~&Z%(T)C zr^wP#$jMEVrX$~h&V&oNovj^Ki*0{lmB}M-V&Y!-)U?`YU)HPhPSb}E^R;m(`U(iu zUmjrqy`B~>5N(c`4OrJR@WMS=>pDH{URg(7Ga`OR5`Y5X;subR*yAP=sAIi#-DB(9 z)>mOnA(xy8peKYg757@tvzvcHF0S~eeY$DMUp6`OF5h@;dWu0N)gX+m_rTM7C~DT} zz27=UnV9-mMca4~%2K-82zNH^#7`giF;A|(r-6Cu`5|-4X9IFgke2IxmZN|T?VZa< z$EyrzH=19|Q?uRuHRoy>^~vCpO!AAhnBA!oeU1Y`TNcfxiuWEJx1|f#YB$ZqaBXHD zCsCZn!5)HO6kH1+HZmoI?GzLI6KkydT3<1M=>c)Bu9ADI9Om35ITN2Yg02O5hrf6S zF~y&m3)k}Xe;rP*;5%qY{~$vmhc@Ni6OkcF&kG^sz=~;LV0^8FOyhF-mc+hV<(|;h z*2a*IHWaaGFsKT}^LL((hN#S<%l;VP0%DWyD9H?e9eu8K_dtyPzmLz+;U@om3j>h* z?{CnNYya& literal 106269 zcmXV1Wl)=K*ThMY;O_43P~2UMyF-BD6n85Q#i6*nyHngHSaEkKPJw=T-uWVP&t&o^ zxzBZW&+eXRH5FM@Btj%8C@55UIVlY&C>UqR8yA28`DCTH{s3}8Fjth7g8KL0Gry-S z1@g%!XE{AL$W`utZ<%qB2ozKhl)RL<*0=SGcfAy5`83)Hf7vh7Q|VFdPe5l-e&~1Whp~K%A($S<}9w|#Ki|fC1w6? zlQoYr!NbFo(N0-hi;4gLV_;e=Jh+P#Z+70}SjlEUDnW?I`1&c}uh3iSqfp>SLqbvh z4u{JcK7;QctNW;Z@vSHS$5hWZ(Ed9o5F7h5L;L7f@Nuk}+K;$COxt7j@|Z1)Ii++5 zMOIoX`hR(~>q8+SA@7M-^=9~qH7kp7w*Es#Z`~LA`ycq)-g?e7=Yc5F1XAY>rWT*u z>%oxk(#4{&25nf!)mq^5*`%gVHHT9SMopmuq$s7}dcI+p4bO#!4`6x-0LWpEpn^|) zr|g6a&*e6-c6Yabc#1>K)uT9G*8{sOq0}aVv7!JY`1qXU6MZaDAv6kgdiWNm%BYeE zsTEOCVk1Ff&LbrrncWpNcM=AHiay^i%ZC}5jR&v>ANlKhyLi_ca+Q{C2dU7XSI_-P zVRi9EJO_#AM@gA6C_GK0PeKtDr7Y2K#fe8k0oEi;{O?iD>#7LkJI%A)gd|ak2#6u_ z19(YWU0_|PARb&xW)yc6_L0HFa30q8jr>NQSeJ5TR1k5cJ&a`7cScH1=T?MZ)9Rjs zoI^LCcW<_RG4X++-B9G6+DL~tMa1#X{Ejo_JYq@Qb4V~x=onIF80_8Gf1u*&QPQB2 z$_EJHf=J6qLHHQm3b4?4||Wb4QWcPHucDYFR8uKdE^2I5DgW;c|?LZtaTX!UYCL zjddygvf4e&J{?7|R-@Al{>4p#6}&LS6nHC_?!=AAXrz|!YnqF3kcR`w^ZLi01I+E!6*VzXU)d@GxcK| zZ21!@U?ozod^SCSz1+4_W$batTKMlFAy08O2C!|{0mqpvioi-gq((Sqb%}ex{J9tt z^c04_?EvPk1zH5B9W%OvT#!6q7*0&B;XANZ{rT74Cn3O6B zs$J?!L%48qT%21eLBtN`|KipxZ&@V?5Rs6a;?d%RiC|jIB1oXsd1D6v^D(8|7C)7X z5=bcPU^`fuF(Sp{Bi9;d^5F-~`t)%ozR-er&j%^nh}&zKs(X1oP{@;j?8Oy}6JN+V zCw7v=FKO#X3ZuDnHBb9h5O7w?JAyg1V;k(y(JW0(<6xu6>-Xzj0T)O}TZkiAd}8R~ z&^`Y!!b_eAV?7$Ae*~fXKF<|AOvdH)lTLo1P~6+TON@rEN`x(1CSsVw$)J4j@@Agj zT7o_^6m8}a(|%rw<0sOTL21End|0>gl|x8%A#W_NYONs`ok?d`?RzV8!?5+Z+Zik9T;knvVpk?~U}Z zNYpFf`fy1;qR%V% z+N|%v51~W&P@(I}^X&+!j_i5e!^6`zr*DY1l?*_eyaB?Pp`$3zxI^ab<3Aii-;j}h z3X0$(sKkWB`d#c;BuIOGA0v(VFw~?9$V9lf4N5y7vfVFmcH!SF>$|txJ^Gt*?g(bs zYVVN(+H$aVsu)o9Crf{s-f;-WKfZB7(ox*pZAGJ7+SaV=R)%XVP_BQvfaaCM^~l^mZL@wz57Gh0rTdiA4oi z%p1mI3RKU&+D7GQaiQ*Az_A@GPGw0hwTd4E>A3;q;K!8>Ai&l(QpE1Ov0*q{%bN|* zH7AVq4b$Q$mZ%FKyr7ucS=wDL^Z8M05=>L53~d0Y;9_1bi*3`Sf6q4K67C`Zz`hC# zrko$x*QC2}DUX6R(FwRq!STB1%_r+x>$1n(@)1SEr~0h} z_`4uHXjhZu``DGcOh@K6TI*VL?C<4D58f6Y43fsiKDqiJNV*1`#UW#vyh zBA*ReV|xtPSVn@zppQ4BnLDv^YPR;eg~iX)zjeI<{T&)~bI)`!aM5w4K1sUGeQybb%gG2mZRe6`Qj!r zqn+R=C0%QcG1WN(vmXaL_J$ssxCueeJ28&9KkDRN!5w#VAd>W_@3WR6@hX+8lpLXw zh)<1XjPdCkbE(kWzl~SB1iw;IUWZ>U-ZyQhyVl9zABswbRxPIbpc7(7gMvm>#+7_m z57r;FA)W(AS;RYk;#We-qAxk^yhAx128ov`FHCygaY)7 zKSxl~qVN|pLNZu;YqBaGaUI^+3dL<)&^zDyiGGh0ZUj4+mbGh69V8Ll&$durb{wvR_^+!}3S!52n1!QjXZRwIx{t@Y4>l(jks{66-;Yjz z8j3czE!G_#`77v7h9Btg_SF^G3Y2)#kLU916t$vSKR}mG7kt#BH8ZtlB`v#y^!HnF zRv%j@TW&Ji?PDuR? zKIcgY{5p!Yp_DOV++jzDY;wLWI8KhQUc-YkTvE`aarh0Yz$KYY-u-in;#b?w!~#1= z(F-z_seP(FHz*CkVvi2e2wFqk1Oo)=y&Ap(W@Ze_uAgP)O|O`%%SvI>+UJ8i8Ri>WtYE6k=U9?AqG-y3O)z^m9n1?K^zEk02{_NO z$q!4zJ<@3-W|8YL1~au3=!TBB|IvK&X8e|GuA*^7w7`^Ii%wL_(TLN0_|(`Wb+v~h zjy;`7E-=sTxqUsvPtHlm3Hb75h#+!AWnSxabUGQ=DtEP0lG`%`Z&7t#i>o^)P@Hv= zG*0zf13L3_zjGPu(qu?qi+LLmrLyG61>7r%r%%k*D@&w(^3M1PR62|5LB+W=U?d9a zZ2Z%@)8MlEx4k;p3X2(om`dW_)-IEj7gT`(BF?f8iO$|j39Vz7->d^`Igh28>VWoA z30^iE<^j&p{zlf5O^f|{ULjus<4t-$keUB-bqoBzlfpDT@Otqf{Vv|W6MS!ZdStoW+CY7RgTc(UGh-9j znvSURSm)*VhKZ3v2<0aBBkHI8VK%x3A+N)wX6vOvpvivI7{{P&1R4Z39gHT^L*zgt z*2@$}4-~yV4_w5`q65uY(d`WOD|64JedOvISN}q6;(b#v^v-E+=XP;uz87CL^h3v7 z!HKw^NFbB>gq0fyBYn9HQ0R1rPl)75=hQ_RxBXy+$@`v_zF8si2JOO7uW3M`_(Gw~ z@Jj>XGE$lpaY?^2^#YYpI`kG@c#E}H`aGRquZO^X<8_W=1OPpxi;USsG1D-A_qkXU z8YTR4f;w~e@0YYGy=4}+rT*f*7iMkc;#Rkf&A5(=ox-#|l9AHU?u(O=uMXW^7zd7h ztXYD_T;6b;VE|Z6gd}JzDhW`UTc!KXgX<|ASc$aWvl>InUuU7U(aS+UEpy)d6O41W zA1awIx99hDWe5X72Qf2h+VJKzE{4f(2Z|54jchS28BNyM{Lc9vuUdk}60h^M#iChA zTrRgTi%_{!oQPKq0_wy0;$@<*YP3sENH)a4(mJ&@1lg4I@=G|x#q@)VPL-odP^T#$ z@2sBn*h;L@L6B8jVyft<*J$5hq3Dp8O`pt4o%1>gm8;k`TWYH8u(W*EZEKKnBrIX` zdZPWDJ=Z+tH`np&wvg|$XI~X6G?SIt<~RM;+cz3_1jpFZm|ABqd@4L-XqrqxdA1KJ zsMW3E6p!+YDM|i?&<=(+8wSPmI(B3I`Yn(4l?03i`zbGIb6Q+)Cx^4_71qxOH%v;u ze;6{3uWoYWJ@zhX;e9syofvFD)Kw*8ptP>g6SlhFEgGI6yncluvRuoKc!`fg`bmoN zos~~|;IhGUPRtyJf0?8uYHz0#KIBr28p3p=$Y*%V9ZRN&zYxKn!~d1Mwo%UHkiBn6 z5orcAtYBA|>`cFWoqH|er3(i-_DLNiewJFkg<4%E=yYP7dvfy*4^U-nt@8Ye8?D|{ zDQepoZs3VEc23BLJ%;`l?>CP7Z36PE3k={?S%DoLz&KF{jesHvhY=hO9YdbQZfKE> zLQ=nZC|K;O7xCKl_F~GElZDh`lJChw$R`cC*$+xb575opDlK$1<12{_&;|od{qP7* z!uw-^W!ntj?g}KyeDxk|bRUjSBxBkpR{#yND{QUyvaKMR35U|1UGes@Ms$F331vau zQJ#f{vJ3wUc501!R~F-jA?Ktj=y|Mmtt!)7oXp3)AXVBc1oB$3+1~lv^mQ&AXDjC& zmjdCEoNnG}`VE~-Pqn6|0X(;(n5nh=uhF?!=9HJG5x=p1$V=K_X%;?n6V+vN)RF#K z&Lxu!1lA__Ks{K~mE&>$=?kd3aZP0BzdC6Ae*2~Q-J>?;=G+?Df|>RTDh_mWSe0jH zOtxEvC;F>dU~v8(htVMAJGzjAA$bXO;m)T$eS{gb2c{m&eJ?UW_ki%3x?~lDY(bet z2fK1vmTH-SzoYH#`hn-qM1^4M+E=$Qd~%7kSOfcw{)#{4G~!s7 z`e^S(+DMmv7QAV!k@n~*{ekxo*7yC8nD7OUWcwwhKRG<)Ak$$x@aXW;Vb-DQ-rR0C zxU|Tuu>W+>BAz^+yt<#*xzjI3#Qh9qex~6fe%FCz(I;M>M|QtdtLrr48YDiT2~+dn z>nbQ2%8I-ECScr)p&Z&hG=8Rxxps;0opoCbJy^`C4G$h3Ey^!0n|fFf-}StUQPZ5L zJI6l)uW8nq-83Wu@UNh-ANm%Wb3jnKjLjoZo;%@mS}%YWtmy-#8#)@sUL( zZM({0xV+r&_Y}8?iN#tpvwOCtHWr`a-ZYkO3iDFL>@Lh-j(~ip8#Wz+PqE71VwUFq z{_5RE2BVw`rbtRkh6c#=z2tP-#GjaFzt!rPvKKx#Pt*Wc;-W)?TUZ<1;V2|hhKEH( z-#B{tUpadHZZ-X0_MUzuDYi7_yt%EMzPn=Jd?rQ6qVNhu@&z=HX{FB0&cD#C=QEyA z*fpOrx3fFI&mz8vC+3P#FRw_&I}^{nG7yR?2h@6ED;+=!#k2*qkJ(j-@DTqx!JwYd z^Nnpa*|B^;D7!SsX}N~rdEGB^8ILf|0LHRcuIaRr`;vL6sZ6AiRSMy=+xvvcCarMJprmIFckqvDKr)L1z7yT1-pmg{bV9slyxb zt27v`dfmE|)F=G;(;h*p9zRkJ9@#-aj<3LyM%0|DWbEOI9F}v7&}#hNufEQ`;PeWA(4@T+%=nU^aSIbBS5LN*9Y0Skpf3p+ zFb!F2^$rNMs1s%Iu4DgMMem=k`{a0WK7#iwSN1A7&i}dNeut)v2k{n;Fd7|XS<0A{ z9gmUqm~i5UoEAmyUzw|jibKRx*NE%HD+ttT70-RkESshPo=N`Kd^tezH!t%Eycq&0 zQxE!gYX1Vvk6U*_L(V0UkD|WlQVtKdhvTdVL??F3Nn1+ZzKw6*K(MC&nNeGGGtp2I z2E}0b#B5BWU-t#RP>TQhM{d)X%8f!2kT2<&ha`+Riap3{D_8DVhu^42{MaC;ddqw{ z&+K$I#fgf_;O9JUO=@1`8VmP%&6iei|8)5`C#y;dIjX{Ggi!z9>MG-|JgmEXrT{Xt z?G}rBr|E5r{c#1HZrQoM_lL&207GSmN3Rc^ z|H(~NTVS-6SH#z?57gIH^TXMJ9g4$YOs~%uD`54G0m#gMNz0gHUpjs%wm93N;!pk3 zIZq*2?fSHG`!Ta-H3OghOCL|8Jl)^Pk+JXL47{BkV@RTR>58XJ=eFN7J(<8-Ymso* zx$^Rj44L8P%*FGRev3J&o074f`4OrVl5{LwYcbP%tA-0s& zZBZQgsK=&urVcb<{zRpfS~OK|$3j%ub(XD&)aTW`X8+KwHpmSfY7`pM`>P@}*#Qfc zp)NTPG?gPhy@k58vk4S8gjWQndW7ycZS@AcA^q(dtyV2k1oH?6p5ws}axc&uOctR` zPV&q% zQG4%Tt6wUrH3TCxL$KNnO9fbqI4vM`Vi#n4$?ae`3%U;O{w?=bNj>^BY7mAhzG zlP%~ljBnqOUP090SMuA2p{~2@!)E`OzK^TGp-PjTYJX&F8N3#i@@u@;#7R`d)4LZl zYt5Y_hqwvep=;(KBq;VZbdhE30Yg%)%M+>Yyszih^BXcaTVPqL<~8X;Yii1nk6e`X zP&mhUc{?G1B;16l`9|5pyVeY9awj*?=Fd}7asuyoTR)7YX~C2_aD6!>K8X6x+|1L- z7Voa*{7r=3(@0kycDV{c|BED2i9 z7NV%%E!R{Q)!S|QKe(1`HBDu+v3RKq(Q4MmkkN4D3+M~*94kHj*c>@6E!UL@bwnj! zsH3VQ-OZvZ0Ah6}Dk&-O)fqwaFUu=5 z!dMKSAiiKjg|e>k#`uy{d2aMyP0*@NEHD*KDInR$uC}l-100%b7p2Px9T-SJ!zEeL zYbfrB+P7zOm-B)~Z0Mus`-o5$EkG`XB~xM)^91_!%vwagv>}3ZU5~~g=$@|ygF_fI zVEf%;lU>=(*?}nucFP*Y8S?Y%g8Us3v45L+?|fu3OkczYsDe$^Itbxn?1UqmaDdtA z$u;RI(ikug35!&D+FebmoWV<)`65V? zjS*Yf>dw3CH#!tjvv44_OM&69Es7p;Y$d^cifE6!z64wvLJP$@Wc$0c&kjiie~1JuZs)GbMRpE)xhCjSWOG8-Xva1TSy3 zeZK}raDHP;JN4%8#G|td_QB-O&_E2E5JAXr%SE0kj)KPXSt%2S4wdhc^hwv9?mtGd zVq>!Zum+)pGd_n{q09YW3rLk8gL{DE5DxAi_sy_Bv3k{kexZw7v%-tZW- zOtKK%Hw&vwf8!T;^q?<&K^(aL4jgr8E_1}<{TZ*Y&{2)&V{_65YRm z#U@`OPIgLtOHz%d2u@n#d%j4^t{?caDrt(K`ITGTHp!d^?{)wCS*+$KqVexYWQdbEj{g{Rta7f znCjS!Esjs|7)3ny3x{&lTv+J>t25}(Fe1U;E%pd!moTNDtHy?@P zcYl_)gZHiH88yJ`k7D(kK-#;xbLyBcf4r0ut6duc0`5%Zs?&aHg|2}fo_sPTQGQI7 zUl5khZi0uGsfq!oxq`opcAo(oBf+j@x<-kBFkzc^mR{*Cv{Cr+hg#46M*_@N3+jID ztB7*0y?<^n2I`TiM+jx%G!%REJ7$y0=>#s5$C}4ef0^{57=k2g6aYm)>M<&c9BJsm zlA2H1as{!6*P=4Xz_ToWDNgunylE7>BM?fgy1~*BUiQoSf?MAR(iWZql#bdBxvnf9 zWvoau|8@Ls$1gpvSUrF8#WN@f=DL0mI4*fa8}?W)Q2uRN7CL=H+9v+TlD5eJF-&$X zA{Unp?$zNt5%-^xJ`b+b3>12}fGz&&0t? zhC(MRuM4Vant(m9#N~B1MOlT*I(kvodRe=?&|y3(JT+?0sv8(~7CR>#Y;0l&+C{E! z$@8lhOo<7F!ga+RiR#(?AhkH}CRCZUVYwI;^d&H2j5HWlMd#X?-#a>VT=C^%hsn?9 z28V#twz9&>1ubt6!Ubr8nCQQm0^p2vE5LginjDdfI{Z1^gAi#iW{qy>;&Y&cI3D|UGEUY(auer#(?Pu!c(HLPNoO@c@>Ln;o2IU+O~C+c6Euyn+ocZ*Sh)L=SzsYBgF zEi`Znc1=gK+@H_2kaLd37g~bEvrI(U*};3e(7WP`nKR35_TgnLfWh86he?mM^=Jc_ zAM!;@0ywyv*Ae`>yTU$Ef%qxc{g^I4MuJ5@{^Xf&5l3_3oY_mfeDvJ^#>LKubA}Lv zP{p$-&UA>|a`%YuZwoD!Q^QdMgn+@_I0Zqu7Ge;^gIA*;i~q%q(=EyR2l009J1)eP zi@A~_C>Ou8Z;D`Qaz?=YVl;#w!UgfILnAi!KU2G!c39iF;*r~^PHOr7SZk%n;P61i zwZ42YKiT>*t>4yf+-CremIE0s;8UZo4VdWE**PLjlmHTNPS62gkUpEm8u!hI4urlN zDyh)Ev@0)YLR4RG-;h4vJF z`oxdZpI@ILf)9;~&UO(hQ||IL{uI~#krF&;0d|)Efr*D_D&BL#$WxDR;HtAHhGB=9 zk=gVaKbhUhnxPq*0{PHd{CBSYS9mf&t@h3|!ku~{g7>M`Z z)hr<2UT2-HxY+Dt!$idgE$M6;`&v7qlDWSc+SLB|{8L`sj7$h3?Mj}f-&{Z-=k{<)Gx3ai7GA0+dy)WN5khdCrB>?6CzteQmf< zx!qVap{S2kL}MlR6b%CsW&{0j0sY2S<{?={0+=yp-UcM~dvtq`yt9j~%n*U0{jg;KulfOOO_OQFgnT;lc0%edW}rpE?d@NjaPcEBa7(Sh zVy5qy96rp>2-av5u5)@4^qV9it>4-lf)7D&R|h)$8QoG4`%2(1N=x=waaKViPw*6c zPq&85&27Pd=SmeoStt@KHD1O|NiD-@Fj@wGN4Fq36jaTLHpBnkus-f3&i`Dy{}y1G z4Xz8u@HQHyMR&ePK9H6BL%O=l{UGwW?WZ8?MDFhkH}`4KEKcUo%x30UJVwf+2|9T) zbo3Tls=T)8T9H>f8C-gDSDV>+t7uoPTu^X$qGyf%{?Z{uDgt{#i6bu`p}!|$`w(9zu^<|I)|?s6HiTU_7WnyC8P|@VwGtSWQ5XX z?WSBJTRcJAf21x?s!0t(Ke;C#Sww=_=3al%mvp|om$Uy_pK#EazEKUhHraFjgB19D z$X4E{oNOy@e?;#}MTUh03&E+V#OfrPkf_8nOPInNa)p?C_F4y25Hk!>)XmMRYiS4v zF3z&^0RmXLm1@|3^v-d6jHx;fAv%HjhY+uGLUrW>#iPAuc%uEz2*ZNo=k&-K5$=PZ zdE0;hH_cBLi%lRhgv^7^tR*;f( zp9y6!4{iuj3pcvDvZPhz@`{<`b_BH|C7>GQ)Ao!j()zqPu4q@uP?= z3_&9a4~Bpc*{XTp$hx(WErx{{o<2K&^qje+Z^=BiTO~l@QTRvq7p4uAG-)BnS|>rQ zZEJL12Ms=<=lcPyT|EN$qafB+bZa?#i5+SzS54kiI}5|G04}aSs8ntXth)|ys$eK| z=h#y-kw>$h32Vn@mDqDcw|{$CyB&6)|6L>jeJ`)8f(CwJf_QEQrF?gs6;-fp8#M%fIT|Fuzd)<3t+DaktLCbB6Zxj)n*6(*80}q z{`Vk0RnsdvL{osIX<8XKE6Mo(*Mp%}r!&wqzN}QE)o?lu0ys>K+QLZC0RH((za35* zxkju&@ZYX5XlRFCTfj>JXs0#rz3885_y1t)$~;yskC42U#{(o2IoJ^lIt|WKg%x8R zleGB31Idz?givC!(9E$Swe=e^c33YSAQTjVsWYmn%AsMX(+=SLP$Vwh<@tc{ex?6p zuvKkjPP>5hvbeb!K!kA!OTct3Z}YkRNzWtR+5vylZ zBl5hK-P3Tf6^+$^wNF2jFm*nDS>EG^y{sytwXqCV*M+j*7Ri3M(8yEFc7O8FtmwbU z|C*lYzX8#tp>d|H(sQ%V2&u2_XvqKLPiShnTrxkPRAN(Ky($U1%rEr>$1oCurCS~u zXt=P=nW(4tT5we?H*YJhuP}zK{#Z73pq(*HNJ_%Qh?%2}PZ>=8EZ(Tz)hwS#Wctu= z%384)?xt~jT(_%H&z%|#)dVv)R$kxjym&u+5O6v(;B#AwjXy0$^(T`PLKOIZI2G~H z<;{+;3GRAu0cT6rXc^FocjBg;Di!)08LrA=4QCIv2Z8Hpf2H*0vNAILoSx?9ky3k=d#A#uv}^t;NIA$f_jfBM-=kA4q{ zhrw!_CEaf{wzJl^Ht1hJX%2Y4ahLBLuf(nSCNz0fQ^ZWD+!VgJ50PPwwJ3#+kvi(e zXUrM>Ej^x2l-#$5BpH{h2^2xhIa=k@aIsPG`!hixhcu@#Q*E+B&lK#pI^=aGgQ7;B z&i0VgnaF(8ynStP+Zerf31P96fL*;OpvYqsChzwvzNhnB7naZrMjlC5p3{A#8#NXI zNbp6e<*@qKurhml?p#!K*FLmOs~}g#Kz!ZEIl)YSX0y>yI?63jnp}()|gnl#rBLXM{95HF*?5E{B~?O^G^PmYgnuF!THw_pz%u^RQ26xLo}6ZYgbP(ekw@%BIc z+vq&>C|9f61j8I#_EyVd+?VJo(25&~Qps!RlcoL><%AFtLYAWK!pZ)0U)Ze!ICM>U}muTy@aTG!0p5nDT(1|y!xQ8<;{jx4FjQBltt_@cji2q~JiJLbS7^HJYHl`}_BKh{5A}V!ByVQ;Vo5C-+ zT*7!I4; z2(h@LV@%}o>ltGl;c7Jj-;MyB(|Jlhnr(K--bA~lXz4&`%NGUJ@a!)aRuK`boL;U& zBPJLs$}A`{aO3>nQY=~0Is@$bc#xarBf`MU%7WII#UAGSgzZ3!M;%p1U@P+!>wB~C zN~WHeDT{$y_DFW*pnjG#fedYM^w}X%B@!jfkWU@b8V^Zn}R(e9Fz5)%F z6vB+I6xglJD4BxN_AgG*#fRE1WX80)6q?GW;*hc&v**$pXp(+;L_)d>NlGt>>W{=# zhtUpz;QgB%A%0{{ni9+RuMs}a%?BFdzVNl&x+i`yQ2?`4 zDoC0#ldn%sX?Dz32wpt#39H?)3vl3UeJpuPc(AlB?p8caLc|?@yrx(^tX5suV}kmuS*E9;^qs&12}$X&uH(kuU}(;%0E_lcj*Be)~+4- zu1{wZ&)1GCAhCwyi~Hm}^?1H}(&?A!lMN{K3GAYaVcE+Cygk zdLU?7SPWm})iRUTw@E|K1ie3hG98vb)%VsE$k-VB9$HjQVKWY;riZDFXC1g996CU) zuVlGf?YS*jmD`;IPCW-ueO)&c{)B?ETT0)Uc=BWgM=Qv#rwf8EpeF79c(Xb_taNKd zg)N%Op+IM%D%;kCqb^mse;CQ2qJ+uN1MRPS@f-ApI}bMtCQg94YmVE;;VO3Q{x@v- znyyS~LCGQc;ePlfu8hSYduMXEip^8kFf3m+R=5llXI1*1Ic%sM^FiU0lPe5Yib|R$ z%&>U`M&H_EY&^dOaeJV^riaN<`Gy5g|F$6G0?|{gMAalfjHn6;;fNFY(suG2K@ z1LhIK;^!>PG2kVVhyjB!(@M3Wfv4)h{DgD;d@^f%JYPhjCZPB9{>J&W&Y zcZ&1T34or3hIE{na+|RF{0Anu#}mmw3h??Co5EB!7Z0<*RF$M|W1`l!08JGq4<+C? z6P8z~f6u+|(0lziyp@6!6x#Rrv|VC238EQsOLmX5zV#(H#~^%;hb}^GR?(=MCs{0v zke`co9#9eatz1%%y04l&xUvEU@Ay>TM^~;t5b9=o2O#=SdR;)gP7~V!*~d*@3Xjvx zm(8D2xH4CEh5rqDZ9&D6sG*1$Mx?^-YzTy)+_JZuItO9wjRf3|*&bqHU90E$;Lp{1 zd}Ar5LXD3-^3XLr9!U0|P>KZyzrBB#8D*{rh;3bmxqE8c`oqJdNRpjI$=~?+s*j_2 zbfD+!HlmfT17^KWze0<%l&;+#t&tCRR_J(*vqH!&2d%Vy#6kgx$*D~TDv zYu%KKwpYb%qLB8b9n)&CB@DZR*&wDA23$`qeRQNHg3DSiB(FJAC>rU83J@dGE_Xx? zWfsN*>L@s$WZ93vSU+OiY zC`ITHpaXYKQZk%_02Uh4{j7zJErLwcI#E%H;H0NR{~S%>w28{C)$V{KshF1YBx&Tk z^mp$+(T2WZ#sTs87!%z3`|K4*ovdJT-&57ywV^WF2&TM(<_2?*l6(O0h1#*PhEKqz zE2FaNV=J&HU1#Dtj~5JLaZaEQvk1ujY|tW3*=Vf!c>%q{X;X^#bTKZqXl1OhgKyxm z#YY_W2DzKK35Z^a71#aBhLvBG{zzi#BamkAN65G>)+m_I*} zJev|XV};yDJ8{Dq9za}giIw^GS4^Jipc}E0F-oHhyei(|1_QlZyTZ~JDRcqI zVb35iGDMBtKv*!%a*QP8i);Y`vv>)W8n4MG8>YKUM0cbKFlL4QE@TS_sK|by(ngpX zqQbEA!QqZnimpVU3=8q8U(EUe#Ncd*v|0%=hwEbeUG10Gk#^X}0_Six;?dpTF{Q%5 zb%|I}FcBy-2(EWF$3GuzXG!qzS=MukEf2rH69~(Yr9~ag=`>kVq>+hNnzbPz-#2J7 zV6V5tU;0yP)k;{|D^rHF~05^CJ7?r<*I?f;iJFm*Wc$fqza9I^N5r=SIB&=>&B&?(feiAD1(x3S`dsG1^unQ^=?UW)`Kg` zYqLY-=qQI}R^TOKKi2(y=}%S*c%tF*u_7>(DGVy1QgU%^??^Yhh7htdPnf?@%0uO_ zD``^7;up1aT9+>?r!1DQCqQD0DfSUwG1us;`*)#i@aUDl4`wjwA<_T;)Y`GbujOMk z;3(;MfI30VC4v9P#u_04LTb&I(74JU>tmtYMhFAbnVw1u$iYJFRrgX(YO8Ex1U4?< z8Td2uf);;@BQXzAiwJF6*^Y&(9r6KKW{aKSZ$s*UsIrDSO z)KfQepVD}MGyISkiDPzG&T8(l{%jJo8Z>u!3$94>i3OsWMyW81B04^&Y4r1DSKqJO zJs|*>^+Uni$8lgD;|2;=wFdGN_kXStj00nTT}(`qet=qmf!UXlK5TB3vBfj7b6L+N z#7)nOjT1F&jZ0>@W@C6uffD$yBJG$_aB!ieStU;T1Ub~ydO?@fh>CT`AKL33zXQ}R zxlE6Mp6lV!HxXm!9;3}Y2`g*{&H^V9b=NYzWoaKuqCH%5TPvMGrekIrB;^t20c^`d zYn>9N33pd4&)VVEPl3Js4Tls`?z#gsfeU*602?D$s|LxHyOzDamv<&y4omTo{!An+ zI^8)2T{sl9`0fTyN~^Pg_p7>{>LESfkfKvMNYLUsnkPO=sl(tWIs^@FZq0Rw6Wtdw z9!5uvyZF6Ryf@er=_DM9@F|y^U@U%@FJAG=k8KLV3Pjxkq6PsCnwJ9O!5cnnyD5Aq zN3MywnFuQQE{$IulZ5eOt@uOtapllrGJ0cJ zS-<*;AgkYSyNGVKZlGf5z-K1-QW$#r5zfIkn?=*B3wtT2(1xsExaT?G1<$R1QgVc} zpBN@~S*rtHjm33l_bn1C=Y+oJqis|D*~3V-e^86BiKWX9MFR!wt_h)#GWBa-1v36o zN*5Sxl8qvDOt)fURXi=f(%dg)A1UOWCLgQXX#W-9h-Z&>uD<*ivi`PaQa{tdh7L#t z-Dea5@0o29$0R#xBzfu}){s(7cD{$aI%T)XfNa9gE|3s%6BV@dNTcY=Lo$36dZFsy zeqZ`R+Uk zsM15L*Lfhm{TOs9-%W$;?7NI-a3W0mBR<5f3H*4-D|Erq@(fW$>rGwE-QaG$a1p4=ZyBww$sV1N#+3yz9aq4gNHrI>!H zS4#)LhO`S5qMk_d0ouOTms(YJ5IUf>1n4X7U%;VUM(tlB&c!<(ex-DUN|C=pmlJee zYdrNY8$|j^D#NReIsfX$uUX*PM9om>;VDv{yQ>7KhL8GW@V(3Sq z|GOQDlhLfzj}Hlxn?;Gy>RrlX9!^t^NvvFHSTFIKaFuUD?DQ!0QemN{pl_ShG!uog zaBpSTA1K?o0daZ$w{)&c-NWK;QZryEC9_>pXowK={9G^0p&BR)JdwzbE$}rpRBgpc z!azaMQ`%IjsgKWP0tw->v>E+ zR%CWNE0;LgK#MVa+jrD{i(lEknpI5JkG?sW|Am`cVMcfMd#_cf`5J(_L&#rYstiwN z5SCy4u?kcnKY2sK1_a){=M^=gLSp03b49hKjF(igj!zMobHbnhJ9YC^L|wkXik}W% z`WiMzg28;5Q2HheEc_w3Nl+nl2)_tMjhj)fzwF9u*8p2BAHl49^sC0#XKbPoKE_&$> zAnd%CxM)COtR!mR)_>U~#Hk&$I_D%0Vzz$32!l7$E&K%|MRgpZ$U?PWa2Yp_ zBw`rb7s%r{XZ^FEM0?lo{%DM0FFA9Fmt#dsQ&-b$&pM;s&KR|Gt}vlyD`j)dH(TZt zWQX^rVUZJc5z_Gx7j-JmJ+trn^r5~vY__};*sxURLgr1=Pv6>nYA&pl{;)n5~IlL4f91qnluC2=J@mpq2jJ+_1X&7Hi zif>K0MD_FUj&`-x921l4jU$*>edRNq%8Fy4I%RdcSzr-db3iaYtS%|vi@}YV`Dlsv zG;)_{-KnO8MHwTklyzu{Yb&Y4O?lw+>6jBT2Sf!D_j$HL12G>xC|>HfC?~V5yS;aH znzac|qGhL>M(ZX@@BC-%yC|mnR3N;)P6vjy0Jrl3;k z$2R}d*ly-cO}F^TzkgVQL9s3N*r*XRwBEQ`S{81iz9}Dy)6g6~aKMV0qg_%g3nvZ( z8+AspBksxVGH|5arB4_Sj4KYg;H|ZkT8gf_Sc)oHv?jB zHp?wyvn}Fe{RkYgBvTv zpCXvY^ux5OumlR9Cf$t}Z=HxCix!I#p6_vXduDN*u0-7pxW9!bJ{;@!U;uVpzY2xH zI}_9Y+nR;4sWcFLGj{K)XU_m5np&fy6v9Db=Ww^6Pgp0qo{;6uCP)-}gYUVEFECp4 z^-tbTLcoox*;rx46wL;GJJWxE|A6Ge&SYVQrkP6h#ucAq1!!<)oW4sXZOI>-@ zw=KJ)G|_f}MM8l#0=91|xX@VK2bHheLr*Cbb+=BnX%P8nG>SS1!X>$@x zkC9P2J(c#1p>v(@j!atBvQJH#$u3DU{Exqe_-qkH{LAbg>H#y&USVlbmCxqEuRE+t z8`>5|OnE?G371z(!29atc)UqBwWYo8?V~7 z4h)zpo`-`&F=G56bNkX|6Gg{SB3FK4ap}cuqFXHo(bl892$H6tScnvj#L!!Rf=rG{ zJ*&}vpd)n^*$)KDWC)3;TgAJhpo|Eqt;zZs8f;a{=Df->PD5yRLgNawZ`m^A99G_!)l-=KT`D0@=6r0Ih!7%b&MR`C zFlxIhZT!X!op0As{mNChY;E^a3r>rtljt zWgXvzh@v_0EsMI=`fQyG`r0`&6GNk#XlK|ikEOBZt7q~KPqHl!f)X@~GZp%hRT&UI zGcBrPdL{k6j=snr(E4OIOikJx`K3Cshl>Oqf!~H3dNeoqW zN?~7;82c#6t&V!uTkkb!60=sT2CN`PL4a+t0!ysq^?R(Sus4{+-&K=$!`j-cVgbmj~BSG58XHMW)!4e9fwiKT@hk@dfK4ZPpjHPA&RDNMWl zHD3Q=#$&eiXMphs(D&_4HS~x8>JqC_%AI2(pxukY#yE~hW%7Nue^tAo9%rY!YJ9V3 z?c(*Ppl|BnznF@h1YPmp6;qmqnguhCt0B6eCt2?s^TOtpIV{ale4*(Vor>6T05Juf z`S?c9kfF@0NsfmO9${V`8HqVQ;PBDypeczogGsd`i`8gXoWsP3Q1xnI$t%w)-p zxGK@G0gj-O0x5Sfr>#QLf;{&j2KNB^{5+BfCMQD5#Jhk1V*Y!2Z>fKV%r_-+^EX3% zT_)fUwd{4u4V&V@Q6f%n;9VekbHXNPb->t)b=end>b1IgIm)U@VV)E z{KaM~^!#Z^CpjF_VumzgME)PO^0Vg#E5BdZAi7L~ zqJ!@kAB+L@9NYc2;)6tvh$3KQQ*B_2>7;m!@<%BW>K*- zY?p=(8mS68CsqS|A?t7Y1fHL+JDYv*3QLX&5jA2tc~PhGb4;ILbRI^A?3(k+~+SH#^KlK*mPjm$o zaCr<%S1D|%-=h!Y;sEa#fpjJ4%;KJ!!&(l{IqLCE7(;NlNt|9YDI~468nt?%)^#yj zmxSuJr`mFIXR6HNd07{#E}gP`%I6dJTRcV_kUTKc){XVRWQGIBcswuR7HSG~{G~P( zLI0j6OK1NVd#O$=*I4pg^yaj^cxQ)o2^~6HFyRDT9{q1y!YF${IfnU4z2h5cLe8T zX}Duzl;|=Hh|VwrToU-98)=JgVj}@-W6$HaHxqTF@xp$>I0-7X6qFH`d0df)B?(|K zsB7K|DyZ*!OuRC6T}LJ=%otI^ss3v|;cdeBbAc385P(_oR{>5GzdZ(damq@YJdq+utH`C42!!{@ZYl8 z{Zk+>+dx{cb@jzvm+39;bQcuv#y_4ORgT-z4tx0QZ#$8uor%?dsjK0Ye|Bxr%$6snsbfPO7W5&;dR zKAjAXUVB#m8(**XEvb0Uf0&QKJh!c+Co6Jp`T1Oz{85Yj*w;Kj79d%Gw1`#@+i%8X+V*|VfeUX@r@U>Z}6&c@( zOftz`smR9a^KZP%t6G{hpmRCo^-#ETL{RDD(1A#xmeXxH9n&kg=&D#`BAUhhOSYvK zsJz>a_5riIll4vHZ_*b@y~%+08!)D%2)?Wc?L>ck)P@D+^*C6B1|gsijzi^Tj+5dv z=k?J+i`#F5ci^t!R-54dQs?&Ng!)DBj$RW4FuO!-&FH@wmupCC?%ETbo3FjwvyZD zKnvW1{1n)kC=yn;ShJ_t?>2f9(0fO!kY3&s%<05+nse|y9eW?7v^1xTSQ_FClP(R6 zyFa1zPtua&s>!Qwk!3XE%aWLD)SU8dQIU6oP%G&yH7)k5)`*|I9e!qY}m!I8w{ID&n0oV8t(A=0$ z*RrM>GvxF&TF@xinXG{7mA_){sEd1R>|;A`SL8Zt+dW4=qDSJzqqQG5RCFP{HhqQq zt=f<}iI&fAGv#7o?pZ%)M2pKpSXnkQ=7e`oN`Pa*oogTU&eGm?fxd`}T+(_)^^F8b z4vVaeV0Mt6fJ}&DJ*X^g(KmSYBvlfb14W?4-K#k`Rk1IIn!E)2NX6G@XH(01;?O~+ zSX>uUsiwc|KhDzD3j@P7iF;M{U-LvR2KE%L{k@dd23n#|A}4CukAkE`+_xJ8fWcOSt~Sw z7WH~qs6r39Qq!x^d*HUHNA@bg>gB(3UP`;JB4~F< zI(X&(F|A+tPi8LV&EBS5@Ab}}pcR3=@sphz4d!m2#guI-E|L@sqxDJ>e;U5i0h&X7 ze>QZ+rrKfS!ohf5t-P?-f2pC7GfUFU3k%^SBqUhF8hy#- zWV0f1E@%SfuX$$aQq_&#|37HGgHRupcht!qW?d)X$I(~<(sSe&)FS8C;UXUiz9yLk zk0klv$_cy%-g8Xn$u1`<+VRAViK(52@6K|*9z5aIeSh#gqb!xY{8_R1B1&^VX2#4{ zO?_rqwmfx$qi*}fYuc_GN*GtD&_s}b?!U=!y#uVZQt|x0{C9L21t1%42hR!dR>xK_ zfhB>>#pZ^}2g^J60e#JJdB1;)pC8t352){dGTyD94NWI~s<4@D+aDIaS{d%TeXP^p zlQ(2l*jID@5$5%rfI+@06@5?c_Bdl##u0IqAx|1*nnSMgIMvm!r$+%6{-ZICNa(jF z#;5L@x%`yOU_OhggLW|2_i)VZxZ?23C@~miw#h*~j*m1-l>1`Y)Tr;+o?Evx@W~Uzl=L&(l9P zX>qdFw6kA$qhw-=cTTi_XkB?xllkmROU~3E_5UIedO-<`Tbcl36#mompijGAe~&Cr z1L1&!90%!}e{rgnL(_O5uaajkhGNdQQ%1ub?JA^&sZ#KCkU=e77Xm8PDd$a-q3x62 z4h4Y_D7wvs_U|`@ICLA9*qJ!RuT{u4<>FpEO!UH!Y5s*X-g|s~qUbu0(vEy1aklz6 zXg*_w?e{`6?#6`PELRNEbs26U)ujH~uN^D%h6}xlkt&u$UqOS(EDy>6cI0W&u)U+cq&mB0uq$_Z&7H>;bV6GcY~$kF(uHTf?vOfoDN+waTE|# zS+;lXA3xpIlHWy%i{Z@AX6rSU1hu?xr*Y4o75uyKk@zY7y0ng;U0(C^;eM##D9PUi zxdr0k1sQW7y`mD*9^4X16uVoZpWK&$5ycw$nLjWC=@9 zcECzO!7YBk;-$reV+K(@u5-^o%>RzB9?)DSdxzF-5zdgXLo<3ROm(Hwj1X{C6 z?;J_UBAf3Mu61feC=S>REH6I1@2GbkPy2}$p*vyHWwCta)r3%bU%hP3Za#B({?;db z?a6O3{@Aw(q%yMhF(#9`C~RD6KZ+1_l&Ro11ONQHZ{pU;^Y~nV#To{)-O*)tH6I0Dd|^XZ%f1*M0@e$b$b0xGApQ;mra;71~GP!koow0PY4Se8QKvFcxnH;!68tJJ+VwgAeV0KpLDw@gFVb= zsV9;j3$Qsg-tc9JEgUw&8oUbA?%A2YmOh(u9o+v-crz6SK{HCaFfl-l5IyW)^p3o} zB6{)gPjmZ9`u28PO=xUSGErmqRso0>48#?gv5_c#)BRv&j6XICUzd;j>6LYjtyerd ziO0JL712_v*KHIIa>mdFe5+cjkOaqTb0>OFi6Ag7;&`13fMNEo-gM>f!xm{ur_2)# z5+po11ZIjM$ZzLB#XKDs>@qht`2a2w+gnU~5qJj8 zfHdXfWza37;Iz3#%WA@s5@>~4s_m!%_wV0@n z8^<4P96OBQ8w>!OszHcLm-q*W8|`VYj8%bC(88bLyE=z?Vo*j6GC{u1lvX1fg%eJI zUl&Yo1xH0=c{%fCzZ{D()J4%?@5FEbf{9|{)hSlj)`We7m1_#wJs{-s)y%MtFVx8U ztfaZ8xPA8%4*&S&Nen5#gbSDU?91%>23T7F^A?O5K-OdcLY%V^v-`%RvA_3QV&B4^ zU=%2`?Wi+nUt8D$3%F`Sa61c>{rxJ_6se*+v5$Fx*V)pK2}R2IzGTC9%caiY7K}OxMxVk#J(J-F!N@ zwMv>Etg!wj?*!xs5<`O%!mJ|BRYR0)A@ZH-a_3{-s}_N+62vzWaTvD?2{!LRscYIN zZGL9A7b|={?}1F}CHC30B{g-JC|`RdFuz)WRSe8Pfk>||G?#8+ah}jzSD_b<^XOf# zDv;e5S!4%=KxJjdRsYtGR7>}Lr6Vf-`F$-pC-$S4`s+;U9HHp)w?N0mU??GFOVY_+ z!n7NhI}W_{3Hyℑ-wPKS1C?_NTn9qWBfY;|;g=v`NYoD_DDe>@)s2Nxb+DHQ9<7nv@43vFl&(g_m{8GoRwy|2dexBNrF34$_;j=TN5 zl0YrH@#UC|?UVggm#ijqrW$JnL>(D|${eY%wM4PCj2YP0JRTt=6jR+0^Cdz4iTh2D zCi;4F-UcwE0B->5;B5RK%!~|=!LY6C)|5`M6Z&`!gct#Ov&WznzT!5%`Ru0+r$&C>!ADkPc%zX1DlRvy z(dPY50OJnS7+)bteC!3 z75J=)NqRbpybHa`?j|Ws6YU{NaiQ`Fr7wxLqohkQeL$$bVgU;F3%V4`%KGhMd|a=`}lxeaHT}k$H9Kdrj2oVEyi4m<+rP9z!`saJOXy;*JPmF(XJ|5dzU~(gZ zWqjL<$RF(+p_HyvmES=p0MLN6|1^95o&I`kVaKiCX6Y})Ikiu}08WE=k-YErV+ka|w z@Pyuq0Dx^wkCTt^wv30dIti<8q+@D(e=V!MijKtjs0pWx)Kb<&Dai-(RPf0S}LSio6Ru1{lVIPhLVq3}W^Sj@orcXh|T)W0q^n zyc`%fXdZpv3V~W_<_2~PU$O6g^ee)xi1YF`BRqx(m<9dTA0>Mtr}ua>H=xEHs&Yl< zUPhI&VBotxp~&0ep?S`)==niiuxAynZhdH#^%9k;Uf8?Y5>33P))=QsNb_eRka(o5 z!xM|l2gl)P7zAUkm@t9;lfL+g7sC1!$(77!zYNoni8q8f|1*(1g?AQhh~qqe`EIsa zo{!73pi@Lq$cDHX`wIm2X-3wxU+s(j;1jNh%0jT& zd0msXh8)g&hCZ*~X8_HiUq9mtlZEDPC2)x!`$8+!od!baa05(#HZfl0h)(8nFQZNX zrZPYiddS#9h09CNs((-CHh8(pNNl4=A&KeArIMe->ZOu`L5)6r5O$%8x#c4n=|^AJsWWXHOrcWuPoKr!=YJmF*` zEG!Ab2WP?Eh%5whnvJ_2uxd}c2Zwo@`p$JF0}XbY&jTW7Kh;xS{p+0<){qyn=)E9I9l^ z0Fv`OuF~gk{z;)Yrq03aXW{HtbM>oak&z`g=UrN3?GJiYh#tOPI4Z-8fc}#Yb#XyWT446jSm7sks#SlP zCs(}^*AzR)fIGDpTXnEn`gLRQe^4OYw7sf3gHiWqf%)Zla`=6SY7q#3LNVLCfXcW2 z99D)KeNtdbgz!+AUl%tQi?Kz-lQ6wLr}&qd>DJMsF3O)M1HQJ2ZZ?-NAMhHwy~Tx` z74iG{m&8&wwey5Q*tQ7|{~!y87i!3>*0z9SjMT44pU}>RM7nkTW%x=`NEeWkICD~8 zUl#Xxx@ZlcaCF(DY47nBvZng@wLGjDdm>A2uF5OC9}SzgPy{$LMmV3vRclkl(B)5$ z@gSx{Y_f+~Bvbg3-J3LG9xwsp%|zeDbt2(^qHs5J#oNp1!L<s01IlrdQBF&nfMFpA>yRy^sC6$rjKQ0WNajFFSmgQ$#5+p~H1IEa+rdW}RjgD+MFMpL%rOO}AOPn&98Np~Cz zjNq_F3Itu{Soi`gg6QjOD|tM;qVK)@0F-D$EiyK*@=m|wnf;6{Ui15bA-uRjtM&aJ z<${W*!KFXC=D-@2+ocVP`dEj4uVivAm-JzzPNwgqmga5fLAxcX%fH^;Ly8 zz@!TA5)$8*_YD!oZ^Xxwwr9hE65zPy61Ntm>pPUtien7W9PAlnd5 z4h}8X8yR`&C~4_Vu6%>gcz3=z)&Y zBa~kWGLvamK$#(@n!WI!_5pX7pX&~&es5_$gaAz(%ffrwt?R34N=X`o!UeKgx+rU! zI;wioHRdG!^S2W={i@1ur-rU_kaWUQR!iM{0X#f%qcnC%rWY(L>U?GOS-(U_+o}1o zAslvHx)b66Lm!FQu$H?&doK`?`>O)^#mPzaxSJUp&_8hB>i*q}Ec|e;3ygZ56A>>J zr^#&|2h&kT7@&$CLJ&2y0bqyJ4WrY(`}>MtzkzL`s_=I`paJM^nmqB2Zq&fthI6}g z%{EE7I35n{$Mu!G|FAvOCA0T^%!{=V0wy& zh~?|&dyZEk{AtzOl6>3HJ%j|>03qf3QAZRCmXall)T^m4~e{=9Za8x|IL@h@hRy{Ra9`)nNO3)iaxHYWyb^Mlel zD}Yw!{CoyrxBXXO#K(sD5Vo6;;6NObtgb<*YtRCW1U+8dBOdFK(d2&x%C2jLwT&FO z0tUD{q;2XW;hh!*3dbl%*EPn092)Eke9=y7J=`zylp3xdY!bOqIGc5EWBUMZtCi(% zO+E<8zd4r&`XHZwPyfXgK79WTr24x8IZuH(SHcGo0pb^(G6=rmzS5JKWobf>WBGK^ zhA$nr6B!O4s_XfN?fgVgP<$53{Op`CV&d@g5dOM4`)mE4ldv$>Qxg6$1JkdylCy!# zTKlPkmvE)SO;$1X*UOO|cciGkJ|k?VX|=e~sf^qtr@6v`WcfoOdB;)H@&f;NzBm2` zXI9CE1dNi89j73vEo!d}8Ufp{2no&UJ$27<5HF(U^PyZ^UxBmb_^XT3TVh)+!)4sp zzc*?BYs;^P38Kk|&sTj!Fj-1-?>RD-uaIXt!~mBHPR4L{7<$K>NNH`no!n2NgQPxw6( zi$jmYE3VM^F&4?Jb7i2FK#(#^IF6=9Y4O%drgucnx+x|!G$ynj3kz%c0G@&Y!2x)t z#?K%Y!FM->pvU;cDa>HSe0GHD-;>2=ydm8BdsK;xk8GqUon!i1esbL5*F53GCdC|U zD+i$Gc|F_KevKwO?PE$tIBW%O&T5#2wGop&r~JTN)FPy2Idw}R-m8Kyb!kdTE=epI z^AQ&<1PQnRhB9tSnZJM%#F3wMm>2XVfeA24v7%$IB3&*&IGwe&C+L>(@tr`Z1MYu3 zk-PIY<^8-{@#w$uXL*Nr;eH^wiiv@NkDXWD;5=e8+Sg^>JA*xasM4VzGml}sT=AOI z*6=J8PA@wOLBA+sa(ssJhokNnc*cUVI+Z|RBEJ5Z6!X{&9>!;gR{_vGeEm`ltMHj` z)Cif>!*I~u`d3-TwR2>>A#f?p=!xjLz{jY9I zCHAHH+BX_ZbVlPx<+OSFPM0x6q%K4&%85F&u3puoym>4d2LMdV>V^M2Jg~uk|Dq{{ zS9xs#Z*ne%4x2O5yxpCNocsvPrao!Tt^Nrozv<_R(JL-ygKNQaw^&237(1_`&t{El z$F+J)q2UA7;xg$W8D_9|nhwknd-bW9_a_r(#rRLJ@BX`I3VQexETj4ItP^-w-&t1U z;k?qb^Dq$Bynd{E&nlsWs06Fa%l3~Zx@G4h(SGQ3ufmJ{q;C;?gS6Av(xB-2X@T;bD3f|UzLq3UO%BFp>R9&%=A1)}w zRY|s-sWNy~!h@6R{8IBJG2<=VWA8rkO(VlIcM#fM0HW}GtpTB;To_2V&?8;o1o-cV zEFk3F50^P%VZ<2r1~R~?D$PD|Y`$DgDwoy2dXx!KXrJRUVx^kv8Xh8C4PpjX8aUnsO2%Ob_H`*#W^)?nZ?yDpoVPNtT-TF=YPU@BF7p546|7tkmf0Z+e z_Kg57!eRiYc6e=BxL5EvE2A-n#|uA20QiIAS7g^Ukuf-v`sFK#RZ8apeVjg=r8>v} zY~fZ5J}Z2v6V@E|Ye*AOH~hh!tl0@aPe}3;Wo?lR{L9X}I-)4yHEcU|I^S7#Z;?Uh z)EB%=IxaFr(|M70{^sqn3Y)rlz<$r7#G5zm6N@u4e)_9*{Xq-?24;6?NYUZ4sRSe7 zb)eCra&Rna{J1n#|Jm7dfdvCWr5EnXK)&j9Gp^Z)exC&rh56ZBgXP2vHW2 zOIMg_I8r>_Z#?$Brl|@FrE1jt<^TK2v)dScVEJeoq)N6X4>%%Ol#B3LgG{y|^1gqm z4m$Gc@Lyfdd8t0G;%?OT(9s$ew%iBBkEQ{kpd!!b$;rvBwv*NRNFf%5xzKq|y=FLp z-RoYn6j8>g`y<&PY*vo~uUwqG_MFq3*rnNTs)vY`n zH3xeHdCjl>{b_ls0Y{O^(s*G4#F4lm%)3Dj?F`If&4JT9^~yP14(353Cs)@}z>Ola zcc6Emr2l;U9pFj7Q1y+~> zpe$6J?u}%^R?U^3TU=0gh|sAU?Jm_2GyVGcq_Y%mqa9~xqO&}yZ$?KQW{}+;=1}kE zgoFHIVe#bbY_I=d{<|%3nME8k&3Y&`+Zh#N?WMs~06~TLq-Dsb zboPB-?i4L5YRz$YEp0r9SD zmN*xV|7SQw|b;~tRfQYB+ccfksZ+&eq^>Y9M7l(p#5qoc zB3NMV#5TlkBxGVCn5lO^ei|=n)X+dnedokMYFn;3|Gi8`p#;F_*SMM9=_<_do%>rIbdBeo5#E>xu+KOVl`Eas0ymJBQ12$soyz}U$ z``|9bZ3vt>#S7D`?Z^&6G+4F51>MVyUzvhg+%RP;WObxGCjN!_#8pAew;$AUv9*r8`a<Z$&W5co#OWWgEWqm^6>+4O>PaH=AZ=7vlWc;vwPy2l+>rM0;aczX@=U zCP7otX>E>0kNfcg4G7gS7uhkVa#Wq3WG_&!Xx+Aq5N2+Df}c9Y3iG0r#kWs3jZYNb zA$|EI-93EYJ5~6Y5{H&6A}jF%u5>R2w%3T|B#e}CSjBicPuy2Oy#a6?x)6{mmh8@27d=o?OP9yg zj#ZOjVJo1qHrj|p*;qhTRQL!6$8tH=%{K9@nre*>E9_LcR0}OldVx>fUQiipUeDio z@tYSqSlBP$9LT`<+D0>CItRnRD)ucbyy~W6&3a97GkK!-Yd!u~>h?w_so`=%OSg zJ3w94iO>5{?ZuVIV~9fqdU|LbkBPh5m17PLg~i~t3W8Z_$(Jrd^Fir!K?IAaw@M0( zHi-t_yMU*G>5^1zN*osGn+`f*35)MqEe3Pn%2x))Rm7TiHQfw|0Vucj`Xm!)E}S!D z?Agj%n)SRqxp&P*AhSd|u%l%ZO^OE#;|SQZI1Wd+wtOqX!D7dBfe{=T9Mfbg;ndS+ z^sHFLYZ8*i?A=mi)3xC)g*>^wyA#*tF;uc3cp~{d{-q=KYFi(}NY$x!C)WD=dz$Dp z4}i59Z8dR=TcZNHB9UX2wemS$ZkylbMbkM!pCE2y?$$bk!+fl1t+#Mq+i&9GU1B=j zO_pTZ9xnz;J~`J{d&_PEHSf!&_5M_vZdZ$e9Wu?LTj)R%>fdxn+Fa1}s`r?*FA%(7 zgVk{Dl3ui$W8CbbbUZUT3v4br;_T3biA>(c0r(QMosy?M>!{@^9^26&<(w6{s7@Lx~FreqyKe>hr zxN^Yg6(Dm1)^TpaHDN;xz;d@4KXcPcD7Svs!qmU_NpXYW>!yOp5AWT{zb<15P0aHvFnlMAp<(1`|u6WUZ#cEwX1p*rq=iI$+- z$c*!Yi5Z6pq(IAIOCEL1%sKp-Fd?8t0cgV?O^*cqmGnR?7Y@+G`NX-9ERH|HHcMr@ zE&8wQ@ufgw4vx;On_b2@pq;(8r;|=f2yzCoj5VTbc7Zy(V5{RIqYia^25K-mH9k{l zsDPX7Y1wi|(MCHnel@MI%qJliehv?cg|g}ir|&_()+}RTOo-{Gn#j(*B$cesHoinD zq0Jf6GZrtnOH6-|03gi^d_6;|lTZmARnS!m41c9^*RD{fEYAlg1(OBkb1l~t~#h3Jn#QD?73-Wht2hSLx_0- z^sF%hvtZkh$Y0~9_fmkRVWaUQBRg(tbYuRr#~<*E0$aaC0dCVJ&&*J3b)c&8yQ9;8 z5Z!gM?#MA?Vh{JD^{$L`xxWsr# z%~%|YxjF?*{Kl0Zgu>_~3Xs*PI)Q+XCT%1HUy+tS;xVuuuyl;cEw$SfyxL^(iOu61(qoWa)XMS{k0I5* zE6HCDhK}%+fd-z{DQM|-XHvAo$}%n!ZNrcF_$sH6X>L-t=7yOu0InEu3G}T(akTPi zTHJ{RQl0`oNC-DQY4NMoRvw7m%}CB*A6tuqSx$4;YbzeLgnwst>Qbf22f#W@2F8Js zfeuy%nz#U<3Q6l5%`;9u_zoTdc zbp0*Me(?`pJo^~Rv`h*@`p~$$URoxs1?>@4IdUgArsRMFcV)AR!&DXb-fn1dl!!MwarkU(`JiGzv7f2GRF|xSN?}I#kAU>Wv}wcb;V6tSnQ})= zxd@1Vk*5+B`3l;W4orpklhgWU9yTHrgP_SKZ-HH+i`?kguHHI9wWpRMhFaEPxFFJP zttXsG_K@nVKD>FZ`@3Bha1x?JNZJ+Ej+XUR2|<0-t(R|?`=qI8L({Y@Z2qQO^n4Nb zK5oTwWAZzQ zjoZGk$b@sx<=``gK$?Os*e3`Odl409 zNwxwP*atb*>-VOa_GCo{E87%06u-#@j_pkpJMJLC^8TTULWAX)S(<0y z`O%sdD)9gLJ0Rvdr1Dvs639oL;XQN8@KpHTmY71DOT9uTW$5Xa@!t5Lk%nX{iY@5k zF1YX4wZp%y_1kAZGaUA$|Nm5>%;V16={v#=kXj*jDfh1l%lRI;x)EuJGr8H~FMAJ% zf$R6LfW1KBLcBtkt`*<7Iog;A7FpSTBGqehn4t#lAruRH$av-dit=nMJPn-$bVguo z&-vzuXr6UkpE7w>eJ!$X_GA5#lJMRpd9qU%V)%CoXeg{&^1KErQ5gshiFO`7D~R+N zIXQ${Pg!_y$#@|U2f}85oCn5PcM(MKX2k#EQV<$l1sO+@?W6 zw{Vthkw6DvAHP3tg;;;s3{C_r1f@z|4Oe}G8C=W6HKIL{XLlw*O)0kfh$Q_oLRCkOq++w3RT0S8*>osG}o(1d9~k}ZyC zO^r^0scBZ9nUW{JRGB z7XphrQ}C2kab9U9!>3N1+10PhoqVRbfkcNY{^@(=rpps{+c0HUk7aQa)2T9=lO=t= zDL>b-iYb%Bbbi6Ir6e*DMLvez?t`>%H%>Uvvhh~iBZ8S7NQOhy?!HNfHgFO{nGAdE z-s~xq?NHj52XtP)e0F%#x?rY<;E*W1wNmRTfLM!y%A1nO(g!GA}w~p3Xxl)$%OCi2_!0?R3{p1g%3S6vV zK!j3Y4mbj7f8KP|9U|RqSEP~%)So&&

4vLj<&7=@5RRC>$jrdkGA&jy;jJSzX{1 zc5+R#=W+>I$$)y^-ikc_xLMDFiT~^jbw3w)yqpGJ7L-v(#hdEI26VdpDjOYE`>9dn zR)7I|n{5g#DIW=A(s61dZK5DQA`HyLwIJZzL;Kj0#7*+`V3y@dPqVa=P}w+L9A<>xos1q>w2VZ5~v6hI0f z%iTX@rNXGuCR|}B!=9~;V$v~bBae`UJ~^fZ-0P-=!59t&UAaLD&3Pcc*2(7?z>5iQ zuyj+Udb836nww$r-&%O_IZ_1Lwrt|NNXgb1+U`G6r@Wl%pJxIv+9|Q#V39((*nl>a z2%Km(xk=%VL)N_aI?3UZThGmO%blqoc`Af}o1nhIb$PX%d^$=$?}7dENmnmhi2qIz z;WQjAE;?y7YG^bUF31P8^Jt(AD!+bUt(JXJ%~)h{4elhj_ltQs+XoG|6VgQGg~s&d z(r}Xm9!*8h)g|2?y#ZoT1e;1&$s8Z5Lz6QUXu*>#d(bmrzid5|>2@{-!y*{I#=ql& zaXO|35(;o=O6TmvPk^UZ+w0zirUNZD+Je$*Ko5_R^aB!jJ>&0g$ zH_ui2g}^HN`FZ$PHqG`0pH`01wv~p$3o6VLj9tRP+zJec@T*>Hoq6_gsvov1(q4< zXabwp!qo)Yqbr4NrK(HyG&9!+qKOJzu;))&fG!s{ItiRS1K-3r#8LKe;%JY)U;O~5 z>2_CIJQyeH{7tJmF5!a-IX@?dZxjqb(ploXC#ZFHddk7!3&`q;=yb2Dx|dSSX)9=7 zB4y^o`ao5z{Ae(=n<-Q_8}scs_or`zXZ7Z3^u`j;221WT%{1{HR&(+=y7O!>5F)q< zi0<8DGeMiyb$^5n4$eGMZbOf?vYfeJf!+*`@)&5N%_zE$hJ>G510g{6@OmFJR zEhD=b)erDYn>+*PpBtPEr|oA1fS|zmuV5iM9>#J&x}M7BG}Kv%DU-5O)qC?UNF5G|B3BtD~G=YMQu;RXdPwuBdri`0ne) z&(aGoDdtXn}W}>h~ygUv5q5jzle{CcNNR<*;vd{>X?#{{zJnH7* z@Rf!DTJ%jdOjXKBJ*4eT+Q*ynHTLZBD@Ut2)*3ZJ%j1N3^U;226ce^XekQ{g@hRh` zCk8o~l&hRKQ~itfd#_iifFNUI;FGOWsTvoP!?etw_eATPQPu26r=%x?qXqb&4a?8{ z-B$)lnc>eFw}z`0KWv%?=mUGwVetU}hD20^&O|5HLRDcP2%qoc8 ztg~5^M;3!^YkMn>_H&%3;Wh5a+=3{ic|xQ)hDK0T$8H3$xPv+V6PBwiLJz&oe^8+= zqI0I<)uu4!MEmXY21dJHrAd<>9ovr_IbPd3bwq10Kt!>-3x7^ycwcXTqx3=xA+pTTe+NN8(scruG{!Eg| zOeQn$yt&VP?mg!`=la06c8pTM-qnY4Vh_dnwby!C(Uwm4{pLX4{>8$CEz^58V06XHwYcgd`4vh zX$sv4&(^w4lTAKkm51&7k$LB}6Y`h;1|)BifCq<$$DhBxF+r?zN}dQRZj?2+xF^yS z0#w(t*F#nW7!S^CjUM2)4!~0!u%NMlR^LUB#PB~rvGc*n3ztN8@O`RD2w^4YL* zb2mc)R{wIo04(Eu>wcLhvuy76zes0FWDhVw=Yqa|Z@d|baaZ68A@)^!0x4hYQQ;85mcrMa zoFQoB?BdW7Q?h#wvtL-a(eWy8@M{_}X388>2W^E_i8K5jWk)(}e~#9Bxn;S$5t%^tZxuA$yV5*862%cOiV8nPN&SMMC&EaNPF6&GQ!v^fC_%b9bdy;K`-uFkJe1z>piMY`hX1Z8viI2#SW0Vvuf#KuJ+JzPQfF8ka2A_qu zLt^4?>L0t6X?mfwtZ79R`LtVLVnd*OiQw)1Hn_Hxw|Bo>=J|e)?@Q?P;tdfOgXd{- zcq0&za!gd4Ey=|>!ogJ3k63cR-Im>(v%S3po&UKHSKA{r)ISODf3>Jxmm1h8MSH0m z@k}BV3oTHA;x{zzA^$eq`@!8|8NZ|(KKA7`BTZkq*kiY48LeVl5MP|L-m!VOvKD1t zfw<>wg;67RxSv*Cqq&pL`W`ObVfz;v6q4o_e!jiAe*}2%%QP@c23`FXcr}Ae zv>0>Y)Zh_H^^FCaO3G_Xu+@++GWhI>yUkC+UvfOkz#~Kb?@mzXpTT2+gB5ZgivIR( z;C~P4sJYr*q|{IhJ@$nprC87Ayz3raTc$l)BS~-e0yax%YV-N>kM>7HDOh7Flp>>` zSZB=8l+$p72~6QRUA+(_&gf{3^qGr|a<3fldSoDt{xq2p`( z)f(-@?V&kt%y26;Q*XBZGsmb7r*c5}$~@L|lN5Iauo?52s3%-$6KqlNb-|k;Sz}Y4 z_9r9+I~yO{Y9mQc{8hE|77S6Z(eD21SEU|fGpF?+D8^SaxWz%eOm1sarhYvrydcG9 zTMjmD0d`OVO{4Hh2+w!Rvo4t64K+)nKdWMRI=RQt%S$--RY8?VrLfH$8kn8KHS1hS zx@Anr=mvIj^f!`ttlQyAexYNJ_K}r})pzEu(Jad;39bHnQ=P|$ zCjtJSo@K*3oU+_&kgUSFZ7r8iJ!jFrsDh=^K%RIC6R@XfzPZnmlkx2a-5Ra~5nLQ+ z7YW%%gop$K zeIb^>C9fz`S+}p!DSKnk)rJCq4`bld-ii0WUgUDId=lRF==G0H<}G{5|8$LX2)Gqs zPt>DA`eOXCS@yIK%X%H&`^5fnMf&;OuA{1yrb_eA#=q(F)nbQ6V2_@0A2_2^BI)}5 z8}a=N7#fSoCWnK2Z;)<1y2cH7L5}OyH}P8EJ(~uvUgSE8;NagsiBLqcP5u9FM{kLU zbJkviul~9#*XVW=it7^mVu;4gC*H>WEUSP? z5vvn8z3TkD!vS6Dy{&wnrqj2r7c;u7^9kB+=Rm~ZS0Q@jJ3wF{R#aF?Ex)7;EV=o= zR+=m3>OpbzS!fky@x6WI`vw*%TR**vJVT%u@e2X}=D;28AqclrjQIK~eaCZBPCNT^ z4|S3=H_Bot%{ysk&0hlsf+<4&(a!rB^3!r0+CJlK<-HN)llW7o0>%=$-FxM~w4!1l z8)x1R@7NBVnS{9?u;(hXWpe+R)i&wzBwtP@2xmoQ6U;Fa0D#3^ZZ;|_N6xFHIVXQv zHY^te1QjcdHrXYLC$12tp}<-GzBs6|`Gfq4588nOQd@8;zBFTo_D8Aa$?gK7uP-0q z)}8=?C)Z8D)0uvig${3v@~)oK5%SLPi*)}@+u%*t=t-5d&&50oi!7h5oKiJmk@HoQbyhl!{3ovv^!0pI|2XMo!)_>%b z8j_<>joyPhMeKh|s+qMu+Wc=G;AxymNSrOOs}s~Qg8}Ua4d4x z`aMd)IC<<@dEez+`Wo*qtjSM!|D9w65|r7wMDZ11E%-2p(e1uyiCFf^q00v%K(%b? z6g6Tx-JUZeBwz3MqaRaM3iMs{c;t$_MqgsKnuloA=rNo{rgSGkhNgo=Hw(cfddFEujh?NhSUwuKI=J-U-z2_WsU>9>%1Bal zQQq}fEa(&}0Twp0}m*Sm^7oc=w<(XY?R(?0cF~aH15F zL0NmW=5g+$LeD1=?+!2uQv9&-VOWDHwpq9Pr{+B`Ch2Ti>;4|eGkQ~{^Hi6emm(Ulzb)q+>_8C^SV1g?~cW3-Q3u} zcyuQcVM}rGxfy^I*n#2|d>ve>?r@c29#aUr$?4>TbrPKA#}zF%E7lk@<6CSAWawv~ zit}>3qt5k=&D}iDl%leB0mr<(ES#eO0<>y5-EM+=t7Pl46tiauymCzJUdE*sSVk15fdp*&ho;}&)5x=Vdh5MQuKt>dioG3{T=cJ>(T`uduJNpXQ0 zdNoC~QU&R_@H9mPt{O@nIO^bF`c<6~Y4xh+CLi<$rfB1SP9=oE7%WvYn|q3ert-W= zcQI1RpP@~!iHO^UWysh5M5bw$uIm!f++lR%#}4uE=Hn)_mY-1?MTpTX2nh6eBntg~ zKL8U?d?icKa77D-`(KS^W}WYIQ9u)RP=RDyG$vavl(sD&!@IA-hz6o?PXlPqgZU z0VVS^9=HpIEgDN*WN3w@8@DO%S??R)keqW>|E0LI7WB?rK7K$z$}dF0iA#23Cc9Et zwLh(VwW){O5j2sEp!(yhpL-YaT@KD728+O)1`og&Uusmuyu4kglQTYJXl*S6DO7t} z-D4jpI-a1@MDMCpFl#LARidh1m=VYE@$&hKUQ|RM9v^P87*xn6oDmhF2jS7%I=m** zqvTL2;ZS)_JKBh%24GrcyiZe~h?*gv*$i`Dua>j0tfT$9F^zNHoB?pCF%6tIomlSO zI@rEWK=r@o7U0AK&^PVOTZcm@QYdr3PvZo2B{#74qa_Wu>VCLOgp&|smeK_=5 zag06flajLWW$=wNn)7?|nbZlIdGa0hTfcUFHm?2m+WdOtmX*8q$CP3^91kB;R$H-> zi8ueH3oRvwM>3+gukhXT*eu9K*eu{F0Ze;n;O;KA{(;|SK@MaZe~pI7N3|`^vFS5F z*d*>Znd#!4a6PjGT7HAjklP9}wpuA(uB#NtnYqYew834HaNkM2Da8)0(+- zDVA;3q{);Vn7K7|yBp(|phi}**DQOGr?oL1BDClV#8{A>u}Eoe09?TA9g|T;VVTr6 zx$+4DgV?k}>T&Ue3rIXCW_)X;(J8^_Jvd-KkWK*9T`j(!!nNG=RJ(Bk>(Pv>M)+Z_#>Vz0yL zHE2g%QT#5Z&;Dq?sp>^TQr)})MgIYv9;D)8jS#UIv?w*Mx3~Tr%&$N#b}mAuOfAu+I}!w7VN?d+9s0ilzYz2V-t z{@cJ=Z^Hb7`>6@{P6+U+iS~D_+1>P)p?h(zTM=A&#Fk{LD&H$nN-gXHDue7nQ;nLsr}#THpB1h0s7-NugT$fDP0cMxJ3#&%}*Kv-$-kJFPsU{Y44I%G8DMo?U;krRW|jtuv_~faphe0 zEHbRFTWMZ;j#y@<*Da*yUDSAd(lUPviwvSNg>d}*j~DV|sy$$*{<^NUI)pe93!b3s;W9fkfa*c;pa z&86tfaga@>Ny7Qyv)-27fx?_F&O-z2phDlUo zsls_bAP^!a_f_lK?}kenH|+X{cpkrSq8dgKuB~&`>5NODZ7$ev=^gFk)7cqHC(=d0 z>C;e?GPrp&vhUK@iv%aQEwozSb9&7_UF_^^&fv&b$9%l(>oK3TC!eY7zNe*5K)0;e zI)bbKd3AHfLkHzvrKr~H7uJXNZ5pzf5{Td3d2<9--y|mXgaz(B$I7pG8=ls!Bki`36lM zzHC|IpdXq~KKEyqyb$YA=30fWx>W;M`yG6VsSM}LV>B_l@(`w>+XQ+=Q3m8slC_mv-Q>O!)|8#^|jrR4LjVJjHQ_& zT>w7n#^C8V@2kQHz2qPEP%*%=<*06nE2(X2ngvG!cxA^x)@-bjMM-oJR=3|6c@f1x zQy#SmK}f_#c_1)_7k-oW>4gxp`WAbdG11IMTdn zXN}MQ>P+tJLBPgKV^z@1Hi6WpVP9zD0y1rGW&JDGn}3T-1K`Xh(d242Hpq@cTsoSq zpS{dBpebh&N(XZS!0je@KgTBp9r+$c%;_u3bfEvY9L>PXaZ@XsO|55sdw3K&dIk*= zjzTh^QW8(r{~^FL{#tdS_Txr)bKPv{U*H*KH$9S-*-OAK<9!O63{R8)& zKG89&-2Qc+NMvEfTrX}!`N{j!9Jsdju=(H-;u(b(fZ5k=R@eL+cC)j4a{!wHoDVkn(F8`>*)}GR4X*8`25XckO~|CJguE{)H87v)u#l6sOm7`=B@>~ zbReo)3%sU!5XpN;2%kHFo)aS#98^qb@mwnF-r@tqx=C&aGRjr7Un$G%RGzpg5-W>;X2i2rMT?3j0vkS9qyb!$nnmxRptZ9R@vN-9MQ` z*2SCSef&sR%G8;&1pDjXa6{taacCjrWDB)q&=_4)ubCxXH2h{sv2QKgyoF=RT zp2Fe-Zn2B@=!=SEC2?4+Eu9$6_Vn_rPfy>jX1B=gL%%cQewQ>ovedPDYk2XO|7wbH za}j?j;N0qD467W+? zA4g-p;+o#_)p*I5kpwCTkIl}*hG5|gp><+5A)?#X&ufpEaPl%>dT1j{BN-FZi^UU~uGgYY7705f$(l?g8Y0q|r(IN3d=llS z8&p)fhJi`MrFQ?@*8>)>Ne_SLblzT<@cVNXz}hzryx)+a_51?yb5hB;6_=|<4+M9o zH_OXF@VCpWM+WK^Veha|A0GkFDYiMuWzPfdFQ|mLo_8J{O&#dGTbl*FOQo%;Xy1*h zw8Zh~6UvlNPU;KLAo1a(e9ZmtNky!M#`t`NTy<5?XI!vgLn3#5KHpH*JYHORa}fLC z<$KayEAv}6$uz3Yo4}jP<@*`VU@$y*2L?C0)WjgAxMJdVmU(Tg=*&t!cvUs$(+$O4 zt+991M?BO9T}e1meVA}ek^nE6h1fPH2V#07g$LJ zUoJGoxXp#5KWWfBr9=KeH?A|A+wKq`4q;zA^m(lQo50c$vU^}M04mZC{}4EEbdc%X zqOcjg;}zIn2|NAAIDX9-(dXgN{K`%yBduOq6d`R)3r9Xy%0F0i{l<rqWJ__ydIVyV>QjRV()c}|-pYEAlaRg9?=xfUi0OK1h^)7zfA9Nvh5N-Ey` z43TX=zC5PbF;?$~sOVI7qdy<#?$+GSq{f6|VYOT3w7KLoQ8w5}rQsAOBN<}3y@2Qg zkE>9Y*M2N+oisRA$Fmri=-?gLHotvyahi(L`x__os}_)A(+>R|nU9s~GEh{bK8S+on^d+>W3vXY4pv%Y9)(D67?02}U{E zDggetIOF*@3lZ2V10QuEGvnIjs75&QIi?m%sZbhT60Z%!8Y-!t5J_KnxLx=5r>hv3 zoAlo;RRlAf1~Y`Z|Fma?F1sbyH?t(9`qZMKV?@`J1j0noVEh@bG*a za0}&qs+5BT@yu1UFPtWQZDTH$cHmMkUY3ovu|{Jv+I3sR>v^|CBJ%0c>+UW&YgN@7r@YQk+!Jl#6rk zL|FI>D3}hsjnBq^@Lnb;Ioq#q96p&0ZG9YEX2dLHg7u^La7TA~<)Za1i`KV?0%FA( zN)1u9G%?gErJ#yNN*zbvT&c5s#J>%bDB_2qT=+zAO9Q1anwmU!1P;#3? z?g55{d{rC{;#F2p>LR@i8td;L++pGB!koK8Gj_T#Vh$;E)kx`>=b0Vcnk=%EdsU}7FqEg27#SQ z;CBl|Qmi004XX2mQ;*hT2Uj$)ibZP>hdrpf=P)i13{^s?$<}PFkjX&?ZBy||!S%nm zrvp5A+TS0zS2v1x!wkO0IQjM$ZJ-bC!H?Hcj0{G0ZYrHJUynAUR8S=*4LN;J#NBVF zv6mKp706bdJ;(hGDT&5VNJpk5E^6Yc$g0p5Xk7|xjgwZ#^yXA)7o`zYOvphTdO_Sn zW$X?oPk-c<9-J%K{gXpAE!K8=@G0{KOZa6W)U-~-4XBe$l)D_#8N2q`oQtv%8&ZBy zli1+mqZ^ro8!V|rb8Mv3(dbYTsU(b<(}sEbX|W<8J-;`UGratI|By<>=QaOSQen2p5KQAFapqILsB-E z22z{D!p$!N9x~<~#RD280xBoZJ!)RvAC@MEau1l$Y0t_1?3}lP2#RP z&-`;9`~l3S#c%^54|>t=t;kwz3^YqF^7P{`&3TOJG!Pt2md0l#aRmyS)m``OC*Y$OVmQ$tLYx+cx!vj)kLF4VeX#gis8nftq=Jwl(2B5fIS9K=?| zdsVBv0A^WQvsZCirMoq$WlbP)wR1!AA;yBa0-7a zXa|00&|v%~B6$9!XYimy@vm!y;#ytL=T^EHY%&}Rm3s;61)uEhu?%*R+)uh zMj<->%rn&ApHb9JKKe=-BSlT$Zvpj>swGIMhmvT{zR40Lgaj?DTiFQ((27*6R;?K!=NMO5I! zo3-<6w&>J5@FYtou)>I`X!JUrKyqS`(A(}W3YQazT1tcE7TnnM5`xIFi_NohwmWQn1S=?C;uk!gB#4 z~wUB+f2m96{xg!69p25S7#0_qf~okThl^)Q_|{9un( zVRh)djj^i$eiOOkgGY}{X%Gn0_Bz9nub_Q*!i`@s^Yyrg9e%R2aWhB`0cOpcv-H)b`fPcH%ZVbq+)%$(OHL)E7QzKOEN{B@%v66GwzF zz=Q3&r^M|`-OOVhwA|q+PghG#0=E#EaU2%`Z6JbQQ%l~3HPAv%+7X$&EsV(>2DM*f z+-k5^Js}>{sZ8?-yFyaa2<#C0Fvi9lI(*8p1d7-~0dzVy&`Y@?F;4C};&6X4{!&8F zVUx3$6XXsLyT|ofGsP@G6v(~lI63nxeresMgUZ$o?w5TJpXmF@!JVDTo+7IVti{hB{74Y<~B)|KszTo%HhouSq zGN7iEgR4m>nw_&ZE?f{bn={m1s=a&)vGpUKGlO+9=Rm#Acex5l(^6ojeIG;Vvs#DF zl!^CLq*2gam!iiN841!!u1a813M=&R$Wf*SQTXYVwav*Rn$UxHU13wwkufz={{#>0 z4*mM4kZF&eT2&205+ZVT?Ae2v;i?##Lm0< z&q81(A}N6mzD``raQsihtZq%T!!-}rW{2%#wz=^00hFt~0n}F(p9S4`FA(jr&3$1g z$Klrole`-f-)CEa2|iCQ3xTOaxdw6Z^*9*;bSGY#pQ9ND`Pnb^QD`5}j1-|7LAio{ zRn4j9gMjGJ?W&Y$i~;L)ixtHyGv%sE#~(D$Xo1CW1~4<4^+>DxE=r=46A_{-b2|-w z&Nk;#JD5q0aev!q0OV|1oHD33k{g`um$lDJ_a{YDJEBU=7JeEQM`?w}HcqJwIz|Eg0z%?ei*^QwVW@ zx9;d!>BXDmyU^aP<@P12$~zi9K;{-+H!-i^a~0aL5*aXWG|<|0WF;JWaRQe>N1mHE zMFIPS#N_-t#(1*PcX=i#B}$Z`7%pNE1=8c%CYf_uI#1)pmBWEsVOkN$B+emS<3YSu zy86Jx?CJdZYiB28{yL<&nkC>m;2nRXnMWMP1u{v|s!}X+p&JYVM?mhLT;l3$y@UAR z6V$GNs26jFmsiTYPa#F!1$z_Efm3E+?pe6^{Z#qsLiuuj>+E-mIBw0NP(tGD55>ca z*IS3=R{=2HesnAGaEU4#?!{e7bspg((l!FDX36yrLZB)++qQH_p6?GC9F=gtBuz$f zp*>=8e7oWgG_tSGousqDaNyiFxU4^~udArcbK7;&aAV>|$e4@+-a1a6JD_xSjp@opu`VGzDSDqkC`NBL&`C5TK+2N zI}^%9W{f*c^h;=uK+_6TLP{<~{%gA!a;ar%qa%z}oAyj#Cq8KG@7^?_`n?ou<7Qrk z#a9rk{XQbJQnml^xDTkMIWcsDtS-1xBq*zR10(tuLm{$l82y#s4$3NZYwkZ}wij(0 zar0ZYTr{c6$W-NWQdso>292RJ^se?$_z3hMMrd%hFQj`m8u=mO!&elm12a>nofg(Q zOLMul05wBUbt%e?dm)Y;!cc(ien?0UO~+WTifhibclTo^;=M~H@HENi)Th<+QQF~Z zKO4&b!y({6PEAdNz|>3cYJZ3PRXJ`YfXl?La`N*j;8XjKw{si&Mthro!6em5QV$j= z!1?s(?;7r!7BCTQZwcet9XXy4PW*qJ|f(^DxYm%=>hAfKkf zjJ0Wezmp#Bd)4|#?;H#s-=^lus}DwVuQy4zT&aBM*|^*D4m|$(xvV(%Kk*&R`+m-G8FqM73P94;ZZ`On}s)4+}Fo{-&l99&uvdAeyBUW@Vjy5t< zZ`OoWTylujiz-$fa!{`&Cf4|ElbhO>(xkLKT?xC`Axch+7Iu|`hf49_Y`@ag<9;G+ zV<$6;zqbcAf~_A7W___uqzSB$27vp1H2{v@4;(Xdddo1#||fA_4+zqK_ z{WTI@85T!X<;ZV^KXX(90VH6YyNoM`SBE}gHQnVFkGRyR{E2EnXZ}KZD7TNBr;8_Z z%u9n}&*khQ^y}al8idNvxVK*_5$0x*TsEGCP>ACkdv#sG< z=hTQL|0rx&{^y$U;?~(!S->?^&DKisQmpDg#eG-UY!VD3Om*(ngP=&;;KMyPNkI4P z{N`^A#TKCKlkf8m6bvk-{pOk#2w1x38+CJfY*RnHriCC@fZSjs@Abg$ujlhpCbQHEy*q-1>D21cja9!3OItGX)t0AoLwxma8O!8no7k2 zT6kM`6sjEJSvkW>Z~*dUe(Ak$b#4OYSagawh_*e9)#sCN%DhIR z320J>2Mu44AWi|F5HgvZQ!Rhq=ePPXrkX#GmBIHjraq1^i zQt<=o$!9>XceM_Kk;YA{@KNJSut55wwRkVbH_PXhIeIB=XFg*5$AIzsY;S4n=5FuH z#RaqpR@!+>W3I9e!4{M@%;btY$WNtQ3G8{Mm?N19no z_p@ToG!cvl>q1Lp5F4D7gqf}42tV*mN1h$Vw=vC>DRV zC|v#|5ADmyDLQ5m+x}i?3D$RjtAiMqtG`y%6V#y|^+uoPH~4&jYE)=3I^@ldQEus{ zZ*3G8nP`yMbh~Mm=AH69(?@o$DnBQjO6$2VWzCsd*kZyltJsNz@zGn&G_Vq60vRM} z2)R7tROH05=8P{35tqAEdabLSs7qb-*!b|!p)mlwZy|Co2w)Khu|0DPEIpXNSX7lt z?DP?15L<}VS+id>1Qduzs_af2|E{4z8ewZx`6DK?!x^}cxhjvQ`1T+t>KP>`Fq4Ut zBkRY*tV1L`K=r#D2eVncIEfLxsT+|b@{u;Y9&w)Y&f+{E7_g> z#eSHL5sDz@67Ab`nTG7L_gPHs?1}0(;?$L<&&!4}1nD}gO~|p%Ax*KuwF2jZ^lzdp zZ0ZGOV0qoSfAoh`e7N!1&27LVW0*8%)cSccd(E_;fI?=R+Q&OsZ*S9T{^9Z28Zc7b z>Tz_35k&W6f)6>5vFcIoeKUV!@CT^s=BV=9O%vH{)QO+;tHj5P$Xnc+gYn*o{xX~y zoMr@im;$b{VK@uYO7S-D=P9nx%Wi~gifJ|Vu~d@)`lG*REa}IxM`c)1CDNu`{~5(H zRwhmMAy%l9n87oN0olq)c#}6d?gEL!)qqOABO(Nc3Re5!p0R+@!M0<^T>7?u+xrE~ z)^p=p)wwIf<(eknG%Y4v%fls;>z{SBG_1f@t>k0&kua*3gN8NG)PrmRnjD2B8)L6l z91;Y<;I&1**1z0gSpP}zef+vpLk^O^O(~S6mg+c~b+2*z%W^bnH9CQm{Y6_UP)_@T z;mM)wq8kmaBm!eOSAaz6xZ0x*2S(alL`4Aav;ksok6uy_47~kr1X4s8xJ%G;)8efE z?$?xni%5clX!3#2^^XSEnM}-Yr;tU=MMdyMd+f(l!@E?%R*z5gk4c{{i8c#1{ZId# zl4C$L?e={c#e@PMx%Ee9&u}0;^Z8JhOup&Z+Nr!sQJ&08P3a1AYo-yMh6);aM5%D~ z5$VOKSFJc=v2v9Hs$)qoVmg4`jK(|TOpIyu}iZ8ws>jJc5RHq|JflrEWz2+Mciy$x1n_`etjr2j|0l71dm-fSF+ za|#hPeqLoL2RtIbSMz|`#xE6UX5K%63hl!;`==12o#s;En6qEq95&lGK#eTZE|M)z2K#|j?fJ{S#ppFUUaH?_WGvDAo?syF^POk2l5d4 zOd2QK`v~`miOaoe;)Vg_a80c$W#{q&+pZ63-p^;`(v(9Js7w*5u~p|Jy@T?a(`0#m zv+=jjbTO(vXSy&glkOXUQbkU@FmN}=AOaDoNZ)96hF8B)Q|M@HcSJvoD zwtabCkA`}AMOhTa8*C5a?M$gZzdhz?boi692OLdcUqZwy;>)-yowhtXEts2i3o}KI{{%d?Z z8iIZOHB!7p zuUW)ThyI;<|6JaDCukX%@g9R8@)nQz9u?U~72?#vY)XU8PKoeW&+9`iR;zdK@ELD~ z6zEjjX$CPk6%L97a;FT91^t<8vS9>yduOHZvYnZU=vOSRd+LrEaM^T zN0|udKdR}ibBj$NkBvadkWE`S$9eiY;m5UzpXhp&=O^u>=V$!of_}E;F2U#DWvBzM z6uE-to$Pp0&Rh$F-cH#{%ua=V7nF6}^o^~vh2vJfbwX!fw!`gfcInR18Dz13SD}Ms zWB=hN&w*Ewp4g-Y3ymp#F5Qe`ey|p-a!bSs?BF87Fb=FH5%d7Iy?*fM^LEtOZ1WpU z;IOT#S*(6_LR>qIgBnvImskq1n?>H;pIsU^u_{r?=4!*p9o`SK%~9Xa+IWR3-OGU` zJ!dBdo%wN!;*RipKb_^I1f4q6EUw73-EQJM`a8z9Va`xFYgJ^C`@*pNz#KyxpYm6O zlO=Ljp`l>qKpGBhS$X=L7`o*oPQ;l>X)H$aYyYcp$&SbI&22^O@`cQo_9wW0B3o~K zK91A@t0!<9=)l9smJYX!&}L+$;OOB}zQZJ~Ih@aJ{kUDbM_JWb;2J#A)mwi;>hDj6 z5JzaSuW@PO1>@l74497%9!>qJVpE6XMSJjI@oP5k3)T5i$q`|Q6Zpm-IS9zck2?+| zx@K}kGD?!1qaNEgjjQ{OfRjUs>WFIwSYFQYFaI~9x?66|*3sNbM5LY=w~olHx$XJ! z$f1h^fS@;@^JKu{Ld*eSj}KsZW>qeCtkvnFg_T9I921T&`G?<^%R*?%^X?garI<|d zwtcD@fr=x=l@}9Sw%>WI-nHsA-fQVpCs$kiN}?Skp~jN&3HsyuF=!OG-0`5mKRQ^F zwqMzhg_!TefckX=M^i(krwF_lp>I*O^-qPFImIrp)s>1>ddw}Ye);6V78Tjx{A-x$ zWq|yK+ea~4j)jSmobaKQ9`Fu6r#1>p>3>b?Kbc&MDpQt8sq15(L%+WLG|$?*U5QG1 z^Dg2SeffF3cE>B|y1XVsZ`2R>ct5ByR(b*=L=AxB+v{fgSHPPx>B5QB`un(mX%&OOFD#|1mqud|v(Ieu zaAheAo1Gk#Rq{0tqDSWAkR#j-^vv8!P7)T}dwsh)t{C7EWY5Qs{8!Tdp!EhPBkKK_ zdOX&)93^fbp6kWP*Tm5bn3-T>Fw5BF@|0MFP9De=Du%6TW60<)HErzMTW zo+!NjD?=&>`-smmyQewfU)5wFu?6fqGT%463MGv8V;0OS70FjsQOPa90mq=Adn=3$ zls?5h8Ww@=z0eFptLY+=B5%F0TR5k!vUA5LiKTZw7b5qQDx%UwMFHdI_>Si@Frz*a zg?wZ-(%_xTVlnq+Vlap&tdLD%1V4HHZ|d`(l`}*Vo44SerAW9SZh?0mRU(rnvGfD5 z@$Ku=2RZnVz?xS+Sr&blK7^h=j1MLXX?zY%dNVjY7Z7Yb%)JSjX%DUnIVfgjj?+s{ zZvP>1x6%C+ztS>cQW)hQ^)aUjjyWb9!~z-v(p*RPmucHHqvRGZ;`N3vr%~yszqHcGKq$ zRid{rVb+4nLRvrRSJtt`L`wr#~Yu>(VRbT z{l5kP2G-$*H{e;&os@{3A()?Y4>K%ryE2~CsZDVE{BmPA_R)TB>&y{6#7Ksk`1zgw z)YdT+jb)hXJsq1fe%NxSn3)UQ&2iu>KELdRgX`Gqy@0&#kc}*-q!6_4s-kENn;-Lb zU>Nx#Si=9+Sh#d7cWTrkR@JKtQSn)mWL{HyreF)jbijS(A;V6e+?oq_b!9oP` zi;%loH?XeWN${?J?RK+vvH)v_#8R3Le-3`ROEb5VW%T*iy2TG4_78pH5ga_mgzxfx zN))Q%&vt)YWDGL}SvlfLUo8ZRlB)z~)+dGQkd+FtFqy1j%64oI4y{?n-@iV*!{6KkS*)DJN?6vfe#BshQ84;a#u2jJ zY9uRiAJfqigUFx*F~B#k&TsVk)hhiNaaEqwGrwKn6!JteJZu@W?0m1#2z1NG+GvSD zpMbIY9-+lbLEH&A_~8-rGIAYIIhEX}n8qXnZ;AybrNH0)47D{lMWcZb^jMA%R8m*@ zdJ|K1${@^c*l1jkI}``cZD}`bMz0`Mqkf6p^Y*YT3x!+i3#Y2NWXl|{?X+7>&dLFq zsmU1#roXwLZmp^sm-$UmjBVNtnSQz8ruQU~+1F#Ktwv6DgQyk8PfoEGGOcNI6D~i` z{ww?jFF!9_z+#e^I71qHB{!n}&0tCeu-v;rsxy1;g1YMRj+nN_oyZvY(LKYqdUZfb z;&^qG@tXgQJW67F7y69@s`b-TH0=8>l5&T0xl|kcx*GZjL>_PzQ0=4pm(C8uK0P50h7&n++S=BF2R~BiUUN8t^oN0 z#6etAAyJr`H<#-#?ZqT*WcFZ=vn17VqpT7V5lmIv_bR5@@U(yhs(8pWOH7?($f&}b z=4~sxz)nK&S+8dY683-jY{Um}Cg z>F!2Cy1TpNp-Vy#knS$&hHreo-EAd#`@d7+?eIHPVu3&ZH_$YV#!T3(XUUcI*uT`u?tW z>dI$r4nYQ}fIjHl;L*Tj3@eDXxq-%+I`)Z*tNo3mXxh<)G}ihwQS`Ei>1guQX^GAZ zq%hGQ&$wS+Xi!mi{?75hwDNE>CA8=0AcEGhQ zN*BngS(v+EL_FzCNAZ_3Yg{Yqn;O)DSsGvH6*iH{*i+YR^ut+h*XVXF>K|{{9e!WN z^uD?`&jHVrtHWC2FEM7WDM|8(LBbdV@442NjGh#xq{GzogPENz80pZ&A{)n&N=LwD zLnAX;H0tX?NLj~5Q&jES&kx(^)MP5(f*+$%cFdCr0yMa`vW0$(_HseOc$-}Q%yYqY zwfOOLjoNenee!D`>g%BXmVy)(Z0Sley{Bq+$J^iMpot8Y<`x3uG=F>e^SY)syIqGR zETl#gw1Ns(j`u5Ly-Yq6pW!00X3#^Yva{3mo90VBWY1fog%J*aR|VrpPuHYY%TIEL z_hDf@7C9`BhoPZ`B>fvXe#Sp9duU`Ejvi9Hqx(VTWKe~@DX9Jr(3uNQplJDLl3zMy zEK2#duCFT7{RK`0D||p&>UX44+I5-RRv`g)%zkyY2GYFF(rU>0^F?OUSNNV+W|DS$ zkz=X1@OX(bM^U`?ZB|}|U!nP(IJ0NsE1ovl#}3uOn$x*fGTjk&9_z(PVvHha(qe}# zENOv_JIO)(v>HsR{&O^=7l+)`QwYNjNt1}2DMz704maa@OmORG;{U=Lo`2#W?$c1~ zIq%+QA@a{U4Z-@m6bq>fwa8Gg&rufBaUmC{+;KqmGT!JcFkW!E$)M7Apr08Z)#=}C*XYvl0G&wVtchXY*~ zlN^04c`RSef`c^CRS-bdz}&ZUcqo?G31b3!Ot#nMr>imdiW^ZIIOB3Geli>l(4~`} ziPKs>xZLw_02TL6?|wY%5Gu;Na~Icp1`XYmB#=OfzB&G&^KwuOB=0#0HJVjTQX1<> zJG-nJt%SBPq|)Jgx|H=)F`QGimvUFXn1AHYMSI>U+Rv<7qf?R{g>(*AI-=yBbMoe!#$r1k8jHY5#_)EB zFVm`;K7`0oTIy<0EXt~@oqq3?A9kPWKc#T>dPs<}KO zFd|vLir-$|1f(Q^@4F5BZaxS7!c|#wgg4xPW*ifS?cNli+m60`Mj>M$xRt}?V)j+$rNSON_eD&OZYen6XQOx@A+-Re!c$Lj%1yr9rd z^t#|sN^0gSdQK^H?&}`9a>1Xkg^CGKP=TT{>aohTV-Mc1%T~YUNo{?dohR)oU$IMn zSXWVUsD5D|#EWe%8U0VdgW**3<{A~w!PEO!Z~x;zNYHdU=F_LzMxymi{mkv8c>JK|#77~6b?PTqO2QL- z|0v=^t9vH!q1Z+^v9}93y?L#noCPbRWF!{R+CWvyht@gWm}~h*)pJ9uwG*Q6UMt}* zZP^0{4=HM%{+CGR!A|5!)%|J5dqAg^Nz&lPO!XtXq^U*OlVdrCrt=)iFAsNYm>ykzSU@`#SdYRTlslgKP0oM432*nae0WGVUY z_mbhCUYkmv}r5Z<31$1)zSn5Z`$Y7IGuW1r*b?}Jsc(^$c10!6a_3yA60%3(-!=REJx73v@_Tn+!r@S23=Px z*c~fnx8I0d5ba4VoJHF~W$I`ig_@KW{tCYd%wmrB@ma!JLaZT_;iifaFPnWL+_KUJ;juDa1!E`nuI6gU zd*Pvf-G7q0=hXS{eY&FG)3@aJG^m9E=g?R@Liajl75CbQxCsztW=Vja-Z$PhA@$3FgANCx64_7pS^DdAi6VUJLg zc;g%YdCd`+DF2dscW1Uez-X2~7wh#@+KNX~%KyBR(7P5gZ1|zTMRgG+E{;krOjYoH z(3jZf>wodD$KIg!>^fQ1CYTz-GvF@N>U2Yg%LlGpH@uk^S|STjxxA{tTdhPI+MAG-)2B(|~ZA!6n8eil2#vDB~xVb5pM zZ=%pZV2YWu%uGJUXVuJv*w!aDh{RffD8i7UJhXFdbNBv$EpW`O4|KUY*rjm0V{Pg_x zQabzFkk}m4iws)ifXzt-{+5`?1VOcXLVLFNo<$s?*EIdtktCSSaE9G-S*orH&EA$8 z4>${cb~6z|n1FQab!kw}S&a<3%rXkE6jijz0TD5LCQP zKM;YN2WscN#oKE9Vtcb7@F!=!O>cPR@eaQltgIaJP(BFfw7qH~_{~l>hu%^i^yReR z`<*T7E3TB_9kboC(_xEt>~F0FzxS%ChcnHiTN*?VVw&_qTe%Xm2qT}{u@Xtet?@Rj zA%?AKS)44Vn5BVtc>9G6^UpHu7*Eb#6Qed(i1!CL96Uc6zrBJi)ap&ht9hl)d`17w zdxR7?G*OH)I5H?NR`G=-wE0Q@J$@E5+@r3KFYF5cF>s=ZKB5>xmJ(2Dvt=hAR)NSl zVsDk(>%fF|l1FU*NHov;fmxRA`m8gRukC{D-a}KRJ>xpE`N*NS--RK9gf))t)mn)(vx*DU83^bD74xfEq#6`7;^qnv)kKlA3rg z@JXs(I*zz5R9n>HxJE#=|2*3BNc~9LQZFzhZa!pWoU-v6?zrjmz#w4ZHe{ z{QHESpW-v(E|)AAc$x42wXNr3J?~1nB0ABHiO>d*uFY!Updcte7pYw*Gm`HTaQ-*b z0KH^QwHRRhwP`@*_+tlsK0zU?R0{8ur@;7c=|e%2Ux@hJn-GeSOM8~HrT|ueJIJ?M z(SYf85j1fXH0mjnDM3U@1y8;+Gr$tV%Q7z$NX@mNHG4GiZL49 zQa75aW!#<2Sfq<6EgO&(xTdh;cUTP2r727S;e%Akb>1X~ zRm^Fl`N=hQ8+nl%7Q~a;hK6agIU&^iAvUCVRQz1kOtTI}-mX*Q|$=?>)a_0ooz}Ym)JA& zzMEDq{15VrFOLzfS(;CYHnBN)XZ}r+ygxnfEPw;cqLpPyRCzzKT7qg)XF&-K23`iH zhPF`7<>9jSQ_Wbh*ujG%^c1nv@rmVBIi~`q`k000w`18fDG02VBt-h-zX2&)v?I}3 zavb<=^nocgh#YVb1vFx~;$}3+3B^3Yh3LRqt?jp@IPsslS?F#k^XW>@uDFvk*}PUm zlM(MIE!)1fmh(oz9}XL+-s7CWOd&@VnLc-6YI?2pGhXpGKSVMp?bIph<8MJ}{lTpB zcyTBaP2P+kUp5m*6!>a(t{Mz8h52fb5SX_bExp7l8aTb%IK0ZNqZ8*vE!%M#T zEq05+@>Gus$l9Z%kT!{C@OFc3QHsm)&bqag%< zvEdgLC3jItJXwqAX@^$Uyz49~PAzlVnA{l3C_%irEF&DF$FB1=3Ep)48O%(=iZ9>2 zPIMw52q+WwKb&G2CM-fUoRTtKQjfZBl!LC{v`rLWo@X~l?V4dMtg~z0iRXJg-w?yb z$KxZH9`v=5N(g)WQ`6mavfGZ1bR(_iT!gc|wfw}LMl#pi2VJw2cfN1Gp!d8!NERi> zT$XSdu-m(SqWVz$eA@FyJ@4_g>#Bd*=T33W$Nb%TUT-CmGW|yw<^LWhn6!1Km!;C< zoDDTxA(a#Hf83Yqec-g^$iP-vYF(~q3x0=rU`+1J`V-QG-q=v z7bztz26Xi)2FzY>_Hl1``};*7QdqF1cVJvl413OcS2AuFrnqL!zLH8nCsi-|wwzyA zn*ENS8|N=I+ist4A0HB*T(vC0wW?sY%f8(b=WI=Rtkd|SJ#!;26@rA;qUF<@whcP% zqi(>ZqWPX|CGgaB`Fi0iKfL?Cfg&}ngw0Stt9B|T)O!&}m5uToZ$1u0j||S=Y-q+B zmwa|)5`5nM5;ry`ikRUy@W3nnZP0fy_KaFpNx3A~3Hd zA*qU%fTG$bg%*a8B4E;98TK&gXM&V0GIY;v!aF%q$|H(Ey-~Rn!`;&GLz~`g z^I>T=Xi}%|FAHV%-W8_*x*H#Vdc&KeR42(47Q$Z8d|LOpUqS5qraP>gkF|A;P7N|e zde1)kO;Q}=5K*dlSN6J1^OmbxR{97xCz@z5KKsKCjV><9)TlWu{KT5IX4(p==KOwj zQ@A9n7EeP|najtMP~CD3U!~(_7&iwpaJObu)11wfx0>V(bF2WI0jSLU=KKoifuJr@ z^e_Bex=y7~z{BC>y>@DbnhOZuq4WWr*piPMXle`I_1Ju?gru`{>{=Zf|Gs zGu~fo##`Qe{BdOg(s#`oesS!rz6LuhMc)ixQ}qL$>92bol) z!91%bF*G?gk6(Y?X9hv(FJMJpYrUHQo@6XCsMS~;5sJf_RvFd6ZvlfU2Uy0G{U_;4 zVKn#wp>lj5+Nz`!W8%kAdRkO`wHiXAgyAF6OR-%k=MkK<^P}?MH);?>5$0RJO3_B> zveGHx)#jVW$kxV7s-oP*9p3U;9Ft_5C>c(^?sp-~;OGWSe(EoT5AKKWn1*`~ROK_B zOcs=`xcR7+NM?9Z1l|`MW1>12?_e4fV0oV>p!U&Im1Sa!o#cR31ne(-pfA{mVuhMx zb1V{mYU4LwXhN%@cLF3LeY^sVjSi(sdx<~}$9vNO*Q8_xgr*j_b^;uWQWLVw9lpEh z6_%9`tShFfU-I4n-OZe6M&m_<;QDVTWHN{~lrh}`avjZ@ju z5cXj7I%FbqsbhkqZYoaT!I8E61R-KSlaYI8wj#p7-I1g%80itVsv>__y!oF#eRxf(S=@qj%M5;XVZ?<0`K3-viyT;H0{1YO@{QU(>8g+0BJ z!s?-P9qK8Yh(QzcDiN3@@AY=+uO5i_Ffp*0uV>6}k=^0LQ2AmAn;mYxsX!Vq<#y8d z^ip12Wt2Fr4T=^IhS?8nCM_zOi`43w-_$>Ee)zR6M4F4EwfWtT*zc<_8vT5&8~0Z> zpzvold0)f`?WWx|I7AM5wW8T4-OinR;1kJtUhc926w+^$$m9gBAFch(T%b*qXk+W$ zp6PE4SBu)})Gccvur~qb->FXc6Uhmipn_sndhJ`?cyBNnX!HS);$I+Lw(W7Tntbd2 z>7t!8T$N7RWORgF7s6cn{RrPDzi{_%cG!MjEjC>qF{JgbVpQ>4cCX((v#SJ?Iwlx% zkdQN{J@%dFgaO@e2(zD*c=1&HUTwo0o>HlM3M95Dsr%}h-Z~zY960)xpqoT=hAcHq zD_)|UND+e#RxDsdz#KGI&qxzh-|MX5cv@CRWoU#2KB_is=c_Z$?adSoX0ksd%IX<5 ztL2=njVfMVC(n0Fy1=m~&q4^LzR|#Gvyi3fh$j=3BJ;l?P zH1zX&H_c+7-8Z=;H>Mhcw)GdfScuKAGR&*jDbGS&xiCQcxQWT`o59|#AqGSNg>YyR zSn#SshjU5D#W)-P+;(ybDr6w3n376`w-d5=AUw=W8Aq}yu^Y*HrDX(M`{elr+BG+_ zdU!1rmZ>k|Xgi)cELppPGoem;UqJne*A7ZRcIQe)*qc&V(KOl)Mi{3SiLMVH6#oefJWGUZ_{DIvf0nchdwvzSu9eKiMFgb_ zL(|}+kJRZzaY7GaS=x-aMs=ba+#)}IF^ZkVK5}S^)FoSg85MB1uellwN5v;me8*n; zpefVvlLI2nbkRW*X=OwAopN+{fmS_`u3Z`T9+wGY;z~}|NtAYA$W|Mz-g*OUlj{Ad zKPTEe>c&;n{TbzMJ;4p|n#>#z*|F-qcntwo&P|Iakmw2tD8h6;f8uL;M0euLDpc#L zNFSwWC*iE|fG-SyB{@&F_-?DSeX~`qTFl=0w$&Pcc;3{0*MCf}Dr3;)o^$gw7?-iT zrQu{T9?GMmF+1QG>PS6`pRRk>?VlNIWApro>%S4normQmUy45~J%&rNPCey(*5ww= zwbl_CR1y1dBJ%w3w>YLA2}OeF4CVfBnjsrE9ze@MC!OjN+OF%B_AKVF%15XZQbLGg zV~&*}a3dxwrw564&hKfyIl1|19pCfX=<3JRKOf|F4{sO^{8~#mq;V_vG^xDOMYSEz z?7Zf;2k5@O=PNqvP(6Q_Y}YK8zm_8xXX(M=pKZ;R1C#ZZULMo?siR_b1zhv3?#QYC z%DxbL{mRZNRnjTS8$o_Tg^7t*0alH8*A#RLGHK?`)m($p#+>`J<%P|?9D&MrEev`Uj zl|R>`={l){Nxc{vde8y+)Gs)L`F9nB4U7?1z6a5a=V9<9f&-9UkfT zW;;}*QjoI7`&%ilG`{?hotx3NK@a)Ys7tFRwDEZjcfcc-&&qEVoCpV(DCs!-=N@8IJjB(ueBRI{IQfij9sSZZBsBjUJ{P%z@gRZ?xSQ_tr?|3 zNX;KB%u3tuF4v|57iS@guNP#8f2<0`rAN9+^a}wSQQHCCBZ!Wr`!d*S%I~58qCqj3 z+vz7XT}sylu+z8j@jpKYI0!CDdUrD38AMNV6c(lcL)PnGE?}p(c6xeYR4WsXQeC*{ zFw(ftx#f?Da(A0gYX0@_@K^)o7F2*1lov+=8os%L%FWA&(gK~AXDeh#d)4n-?jY!zMsU89Jm@&(2j0b z_LZ^QBLa77E7rd)HBIOZB&!g54ysO_h3F{TE^fod2LyB_N7$f;m+NX4=r@KXMn zLd9&6+1<~s53iGTo^Q!-Z>}ppWn%F^o+P|Dg!x>K1EL^WSk$e~&rG2%35QB?3jvYP zORMPidz^>;fkoDpE6=#!@r|-9P=SE2*&=p4B+HcDeflwpt86(Pq1pAdFS_+YF_8e z|23gj$RDn4L9!pd=j91?v8yGnep);FPuH>p?O_{CvrKPS(>gwN3yAKsua5bfk&c$~ zJas~Xo!IBOk{O>Uj$05fxfEI%W9Dd}>uuPf!Ph4HRbn{v=+O*8*K2g*_)*VKis^35 ztirZ#URG3tjBd-l0AzT6V)<3-K%oi2wEU}LPG zB-wn<-*+R^H}ChIP-fP+iz_?bIB^guYkOJ4Nj8RAV*3Cpi9_OKk+AEN9`h$`m23)`#R(Ge5a+|{pBpI6s^AIk>l9;>*mLKE^jRx z(8x=k@Lm^(K}g<13|Y0%A1TfLo7B3|_pz9wgN2A+a4&O;`0DPhXPuolrwxUS^1$8| zSn}^D70;aZvu)Ie)$l*1_JwHt)~6VgZ9Zwc5Lx314j2?!B2Yct|E3ppbcZeB>mTa) zUaxh5+`79TSZ|mZa$nB`p=Pw9=WY%6VUoSo9hu%ncy&#tzn$t!uGs2AC*X#4u|(`| z%;`YmrxXk7Letztcb#<6A;_P7pjV(v^hex4O}n6^**-lHD=#*?2%08%isD{kOx<{= zR`k!KDPEE&!@>iabKFKgS!4p5N560-;ohG%n{*JuufJK;`F5VK8iydRv3a0XVJYh0 zMfUs5hwc{p7a2+>cD_6FboeRsGN&06Uy5`~MG8gdj6u8pgzGT9Y>DpCM_-!wr#){? zO~-~E<&?q6D26iAaz@#}?76n#!sLR#LYVy9&f}_Ei#>?^L!%1vII4#(eT{8FiZ3ae z%_8Sw!t`(eR3Ymrde+tx^`4U4NIFOl9j?P!*7eY@myJu-rn8}sU(g20R{b)wGj&(I z??4$90>rRHY%DK@e8f1^aHz;FK(E3WdM}MZ4k=xEdhg0c*R44yTKA5r&6(Ir{c(;U z#%-~RoT~`LhQqyyNRR32_p!HDFfmRyIPr2;MBmk z(Og7Ah{?E68p@Ahs#N`lRkmGufn59q$tMz%pE_IX6l*7DUs6m$$Jf5Eu;7;zZ6g(O z4Bfv&3B*>QXCvH|I?PgS;*?Uy{h>ccxZPwM)X9_t8dFK9T#r!oUwGXiUtAN~rS*+R zOTYIIn-KdnDE>x>Tzap9K+RGUlFU=lggqG9oWo6b6O-*G5;n+7tXbkq^@j!D41hxW z!#_7fHYQl#nDo))wPsA84f_#N;0oPZ2yoHYQ&G2nF?QuPbLDoMLg1knPucDR3hQ)E zgYWQ9tLW^NmT?2>M!V!0O1Q^NM*FVrN!J8ceA9pQGQH&B<{CZ@MULd{nC>S?p#MyM zr{@|^C0VBU8RCChe%!^cFfJ!p>!yEDC+IuwTTlcJh-lF*vSw8FzLmp|=4r2A)6QS- z?fjlRNdERZ2%0&5BA0ieDvUw(d%^z&`<~=wj~y5X@BANeKt|T1$0N818=X>LEf26~ zXn8c5s_jJF;Ey1hH~}1Yd-Z)h$=LztU=$g)t|#G^)<(5u4_ z+0M*1SYt*}48e}?SSCya}EgX*;2d$r=JzFh6Bj#db+?Q&b zRun#_<)Gkyy@X%r(tKZ@=3yWZJamnQ6r?;CG6hftuhv%-ew5LwpCsk)c|A8iqz1Td}yPk8__OfcC=A_u2N@4!PD zH*DCD7^C$w5MMenBQ=L`=}Fd^?9=K2LJNedOk&OKsL4L;`P_S;a(nV`UEqcT$zs+? zhIbV2gZ#t6`FUf@Y6+mQt;V;%tTO7XrpMZRFr|nrfWG>Bna(gk?}Dwl&43!qiF7-R zbVqfvCS-p88nE3B-``K`RWUlm1G0Xn=e9tV=zkLG6vu;geS}wulDRu3AATEnuJmIP zun@xyoAQ4P>0N_Tg*o#pK#U{50Q~SpZzQ z1U|ST+)olNP)*@oBRvLZ?OprB9?nxd?7{FbQ@|zrOGT4q+KJv3WiI)G%LJOZeb&bj zO<4Ypp5e7Zlc+v!!A+vtm88<|hcw%n%y)PwC2&w8cxYnII7l*q2tho|D~(*EyxJce zK3Y^!PH}+ARakq=y+){fU-NmTyhbohfxQTWzui9~p@zO=RenX7f-+D=iqf}acTU{R zwyBWH(uCG|e8X{|u=bAI)8_<+y_01_YdVxt5jh-e1A#^U)$&-dG?XF;MJZ)QiMTCM zj(k+z13s0Qg(;@T8fi4jYuy9T^@~l8UH_Gv`((WjvaJBj0ck!X-Z4N3t^7ciA`CqP z%lyTi$aZvh@PZ{apN44Io!50L``2a&h^7F{N`HR}zuR0u8}>8+*E|{(cFvI>ibDl{ z8KP^~tv~CaL7EjA11q8HCo|qLqG2R+4h_JSDS@{B$E}1DUV)EXp6p** z?YIBpQH-9Ly91jq9Fg#9-gkQnYAmtuI6JdJhb(T``9I|mN(+%}7&{F=unI<@3FFcC zNwrJsZq)2PUSMlx>DTC?KC{bfDa$<1Q{~~7NB?2>`$51x>>7P<=y&z@Jg+QgfMv@l zPt|1t!ZrNN)1l`6cL|2CTX&$8+3H{S8*)uz<0XFTosxD2$#@aGI$syCSHJ#!sQ-jcnq=XF=U9upX2 zsTGG`@-(F@XT1n6OFC_ZmlC5({+R#X+Y6^}In1@kuhw!J=m#EYL)+M3ABunc<^+)d`CaUY=6_%uP*iHh=Yv@Pj#waCP~U|DT3 zRXE7jg3-jM(0N4tHunU6F~F2bCnePF#yQollSkwToX@tKe@a)u`fj@wbQNxR#?DQ2 zRr&TzHtV$KWzV3-qW35&(>vk4+|&Vox7O3Udd}^1!h&LXhoN)&2ii$SK$YD}J<$T% z>w9t>`wZaVe%ifw0SGwai_D(a^d^eR{_-vrG9$ZA*_l|+a|^Bmhi1z=%svWI`|u{- z#7zF2j~jm4+!wa2f)VI7MGyCtAE_j1I}K4ip9jhM`{Db8BOMr6<}B1`tnR*mO!u-} z#8$whP7H$joSIT9x7#K`7>MhTvm>EWnI>m#_NMT&&kb1<}AoR7`68vD{liH57^A`_G|mz`nJg}oU;3fUYaZYz_yy;Fp<^HtpmP9k$B zcW*Q@r%lTdzcf$5QJzMu=@6PQczw@p6H?DplKHi2`Q-dvH>?>=f=ne9gZ2ydHkXGg zf(Tx?Ye#DJvF}+|BPjU${yKuIr%(AD;oGLK%14^{Y6L4kQoO$ZV}5K-qME)4`P39l zL5c+V4}9C9T{p?mo|Jdpo-%k`EMLvRyI`78GL2J?rA*Y|)O`+-+*roF6ohr%$}1)v z5rwToNV5{mpwO4GGO6zB+eMM0li-yzIiT|I3UXF^%M}Sw|5j zI+cinOj5ErD=g5YQ_b|F z#O6@OBB|DDr10c-Rcd4MXH8Ddl#5Me+&bc$4Fi(_f<<%jmTu%p3ZSJjdp6j#@TG(B zUsDjT-AL3hL&_mm=Zn6Qj_KKJ{m{3J2z;3g z+3qMzz9`k>dD5D&r`P)Kc25fNfY{bki?2>f5=^w6zeS(WLl=!A6$#Y(zwj24MJ zpshTp%ioEjL4#%~6q_hQ=zv+j^5nS=p#?$DGlG~KQ>~xycl8Evkps`xPo$_^vKy|B z=Zjqbh1I$@?{UXZJB?FF#*Vsr|Uq&{-8`3q^Ic2M~en!vowvE)?B`z>M1bam(lF1}i1FXK)oOdGmMTVKHWi z4+#AQp&b+;VJMgr$}mOHf7AvRK=K8sM9~N|+v^6!hYT7oT_+4Q@hFn#MP0D6a@}9{ zhx@!-n8ef-^xo;3 zZGO14ikn2}Q!?&&&5EkrzR}~DJVE2AsaadMG+y}UE8HwC*g_%}#CN*({Arrt{I=~2 zE<@QKHn~)5y!-p-f>X9a7`rM$21PpM5eEa*gQRNLtpITG_nS>`o^^-D%_wtU;|O3VQnW<dpcF$jR@8=%CSChOK61X~RfWi_V$*k*HLYOwLPT z_2nckR$u{HU!QQu#)cr610sLNWj)yXO3bTEA4Uxh4II}XL%qXnPR~#KJga4frLlB= zj%8`K4c?eZrx^P2=YwXXs2AW>hJ%GTGTK;{5S-4 z?St}5Lp!9jbyjoSWM17|Kv}z*-`Lylt?H(Q-~010v+gj3#kJ2+Sg#&hLZe>r z@{4#dIoZFHDZi2ru10J*`bI#{RKsd zaZ^Ha@-65e-g)IT!0bzUO{AI?vbdUkTD=U6ccN~_ZG|%pp)GzEI9%3G*$rMsb`DSO_YP0}jqR2;s+Y086*cuc z$iQy~G|d_inS5qf*+aFw3MbKMKO*1tDYYjhrE&dUy5fi}Bj{T}cS;Z>E$dkDHFqc} z))Y!`w|`+iN<6G8v0fxO-p)kKnSFH6UGn@(u9S(?VRxxsNT=%~!_&&3E;Kda8+x7$ zB4^OQFDK?x?0&v}VdDE7Doa_(Qo*Yu)15EVaGFvYy^XDe+-1>q+HS(C)s#m5_M^{Z z1tKJXlukA}O*fSTx89D&#VYO1b*{-7Ti49yZSpUj`I^E>hOc{_`r7dPAN+EFiA1kl zY;*a>o#|vi0UF`B#oeaeFKgb4bV%dqmvIbx*Obm3w*>{b8V0#V;b{d^PVBklC4AF7w zb4!+L=ie9cs+q4r5gb@LAI0M^D(FrLj++#kX<2T&#*n5C|L||O{bOnt{$xpczfxeZ z)5wl?wb`pK6t7@CF!8pwv1l2m9(-jqgx)MwzhU-TJv?WYtYP@v)+?_3)s_$mXZ*qx zH;@!XS1LbCnm{r@cp!X}^){pP+YI>nr)1K_7^Lkx|uIAhwrZ;pUglx5Q ztuWW*M`Vsee5yqR+DB>SHd}s16LV&? z!5|ZSvaFNgW+n2PG$~P~7SeXVSZJ^xk2AWVT8fVME4J>(2XrnpQ4nEjg;AVK;dv8< z$_pV+P)JkQK%Qb1WL(V?39cL(?cMM~BPaOex$5*NP*`oCG*Po|jN3H@HMPrGJzD{8 zhj zH7%5_4F9&D7-{M^O)l$-!AT1L{ea*2L5arT&d0sCF0{yW)j~j-q*BeN6r~gq~RAp^|;< z5{{7wIXZcjO6QVJ6Gsi<9B<%&gh0{%|8*PXMJj#M5dV%CDQ!_Zh_-J5+>&lryB00Z z**v}dbV@GEHEwa7_@Q(1XVoIX`w?kxSeDZpZvF7#yP-XX421PQeFHrU-%C9*5d*Or zM*r)MYV&+WW*QD`#G-<9bpJT7bkv8<`;n_Xm$_{(ROV-$^dN_&W2mJlm^?KIQ`D~X zLO!gMDjD8W^RYKlF8nVqs7S_&ttP=nm%NSj1%48Cq3cM(s%J#GJ zg~&>1nmr>%LV#9-sriNIYYX^|L7px9>e6=nhQ2Re0%w`VU@URtsY1Y0wj7DIlira* z8KV`NO3`Gsw!$OL>{uqN{(0W2ET?8kl{Z)GUB(wFa?JQV=eVH(8+S%i$*?#&>wW?2 z@gPl0p)x`V`5)z7;b<2aCAaiM>6co0_+~Jr`*gVGB}8IUqWro?^r#b5_*0VDA2$46(mopATwPVR3cMAyk@+67w}G6 zOhoPfXKE`l#KPcV77`YOH2#0mLdr>gT~MVi9SrMIYHtQ&9pgE@L7AgbaAfGQnEgk2 zpBkAPO{on$G7Mwh6s4R3Z5NLIxNv~kuvjN*WOLWfN?>w5zE%Z;Y+wb&M)B8g4M#Op zRy3tIlQ5k5+a=nVFC)AS4RWHJzQjiz;Uh{LjOri|M$S`qdaloK+6 z9vYxSsLPTItpA+9bb_%<8+TajJ7|KmmL+Y zoXRlY`5}8K&;{cQ#xE_aYJw0UT2Z*DWM+tBm`s>H%CystF9Xolmx}$3yj$OEe#0O$ z#cSVLef$WlduB%UMT1ymdN{hH|8cRyu_YJC2eJ+(9g#H3WkNf6sJrJ2dW#fL5!hCX z=MK3w2XMgIM5u|axaQZ8=MXm=K418{d);p7@~%s5T0+Kt{2UCUiI_{_T0q+@Q+$dSdZNLL#2^K=3Z8&-FQAgY9a-;+_*&4A6_I`%{_!KZr-k>591 z^`>9@tW-$MTKB~fq-Ng2{~GQGU^fnlnIMObvbxu!Xr^k!5_+qzfXpV8a`fU9Ic=Im zE3VaNY>l7Kf0(ZSGH<_Wf7Oy!UnY;nz<}g{uv8Vbo8Dln+wEz&6|)R#1!IPM+_>bP zy1eB|_TthP--vPO+r(|yn3w&wTW((_dVi#u46ZJQE6@4b5$i-Sp-w0|DB>;A8V%jW zm)W*4XZ5^aKw5y^q)*?Nt}hx0f(pt9S@nWLUur3~7xH>riHCyf8FVN(v1VbAQ->SR zGRIdnn9x7IEHw02H@9@GD{@n{AC)JgS(%DyAQwEIGqj0DnJ?0|9ou5>S~IZ*w@Q)0 zt3?P;M4}Kv$>^LU{cp?gG2)-Tkra4*qelD<@42qIgAgwegNZMy{{H!oRRFG4_2)9eh3q=6$ z%1QbAThM=3zjJ12zfwvwAL6>!zwgyK4wBm3744GyRv(wU@A)-$609DYfzA^4)mCR^ zFvuE2qB0@Z?e#M!WjWmb2nq{nG>eWH#aJ@5MIYJzrK?5mIg{C)Yuq|fEpFqFQZg8-1PmIu0u2_Su9rD$W{>yezR@_dNC5u!OEy94njw(TPgT7$EPkd}=Bsmy|k}Y1Z=q|$}YTT&8 zB5Y}!)au-60ky`44^5|1=72kRZXtuUbS0W-5{L{uIDXVp-odNjoe^95Dricq>M}|CBMq!#utl&&guYEi4bdyaa2w*!`Ht-#7r`YIT+o59 zUp=rs?r!N{(f6NvxQ;O_C8Q`eSxD8;FA7`CvIHkyG0nS0@f!`ZZ)ALL79msreUlG} zc6e7T`gN2y+Y-lz^ZiN%phxhC7*2Xx>b?mJSvVT%b_vTw*3Cpi=eD93fT3qbFmu=S zw6~J>??k%E7YTL%uNOiA&!^z`Ii%TpoZM1%1l*G;lKC8@2JMk7L#C~eZ#Og!?u%J8 znMWT9uy0V{J5nEaa0NK9DXeR3^}lX62o^S)w#+oC*2YVP z78S-5t7>hv3jI*zI_?0dr3is(``nO2$GsCpVyTCwc{2$1eg7VgRZP|pxcKSC%t0*} zrk1IRR)HHX@c43`>MmAvJg)`qoI}vtZ1mTh4%pr4ZGMz1gUfg*phWZ-J3%1}H^!hN zU4dj8A)Iu0NPCW{Wx{fRc;k;iLfqkR$KL2LZ8T^kJDEAv&9T-GmI4$USsl6Qmb?$4 z>S(6bW;${s^6vQ(WLy-8Md~yVAzfO=vbvw`v$94b-ALh0#hA5mou(_e9YKknRQ642 zs;;P`i|q*nqL`V=@$ErLN3;j+Gqvut21cA0N{dD=ZZHXVmQL5ax|oG0bds5Lj=iy+ zZumsy&+9WC#eACaMEUF|$#UmdbOd$j6k$L)qJSbbQySgk{gAG$Nw6_is^K}2J`>h2 zurU?&daTD&3QOJX-8U%T@P8a#WmuF=7p7TCmR_2rTRNpXrA4|sC8VWUq`MnwknRR) z5Tv`LTk-|L@8P>H_AfKf$@|=A=1k=o^AG=T*-yUS1aFIo(G}#z~h1Bq@I*x-i;goFL2u z$*DF*5qkq(quvktyQFy5%feBlX;LZRvZSwgz!rkRTlLO6%L+?eFnzu-9lZd9Fpe@X zE)#*GJVJSAoro@i1(f5?UB4DcPT5#;Q5;TIFZKMua*N2_f``(nry5|QJMroRxc9r0 zXi$(A444tt^cqC8pjle67!3G2F7@0$#O7j{I5yg)SPRf;LdB7xkjUx6p#uHb(tRsb zXq|F4uR@BbCuX;Rjd^s*$fwNaa1^ZEpq0iAJt_5Kh~^mbEAI8~#qw72A(`K(38lg` zKzZXQQ)#fltys2V^C@Sc_0Y(+KW7`r4B*#2S^TB(&vmZ9WXV1%$B^Z_RHJ)b17Vk+}h3)tME%1oToYvNcWRTSCe zblNlnZNszeRYi<=RCjayXwXOt=_QLT9m?zKNy2ZE;KX-7_l0D}16A~6IQm=OzOAD` z8ZoKKF!d(I`TT5~lthRJjOWlEcg*u3(Xr(mg1EuQ0@k!?)EdBJ$WoBHmX_9K2`nXA zA(_&;bTSTv>*RXH=6CirmR(-dYysp(JsYDFa|(Q`k{6 z$g)3$erBx#_DomHH~^&vBbcLKw3@ujtMdE#r_yD&idGFru_6<4DM${f*z+O^*F$Z; zr2AMdnOLSGW>iYx5o({9&erq#sgjD)Da68p4UB?ZM58_0VrY@z{r-1Z#P5ew{HK-x z3nR%!ofk2@V0U&Z2J9~QttL9HA`?vdsc+heafuN|ETC45?++?E( z+^uW_Zm=5tWv^^hJuzJs95yu{`#pYSxNo^dT(Q*gZUpZFm}Wr`!Q#BajqLB3g38_@ zzlXbAQ?K%I(iM*Vc;q#gvjjFliVoU;0q%9_0Mqz|7>WB;9EGu5*8vk`32O{49I4eY zmZy^dF>H{Im{^(hAT_cvW6#;0*k!+?BUtx}@thO67Mup^EX;#QW(nQmx77jx6AG+Yoyc#A>T_*`G7Kfz%(J~}d;kOGxVKA-EQOT;h zB5NIz1g^hE^)3vJ2q{*?Hbr`#sJLCekj5eaXXR1_)+bBVl#>kh53!vbUB)d1`Y z_6dVIH)_(ps>8GiadK+X6m*0-gLesy1-yV^Y5dii*)O7Ow`B68e6Fpb(P!p*{1U3g zPW!J$W*{{GpyL4^dE-Z<4Ftx(4%z+k+Yz)ODq;dbK^BM1 z=6gb2p|INbEwh;_zZx&Z4I%damrS9wu|m?OpZ0eEz^u*5z(C^%$ADwzqsTmORxL|P zIL3Z3+ro6#NNW_WW-zgA>U2%Crz*hoMdM!=5Uz!*dg9hYy4Xiv8PeR z4yF(EmD$ou*{(9FN=9Opf&UqBu*^tKd_%WVanMYxo)SQW$bUvo0@yCcz{Jxl_cC5U z)-tA?eX=hrZlnwbq3gHy7)m~2U3R)3JuClU4G@mV(#gr=&oguFQDE{9e=9DOU@~K| zWyCpwc8d-Q>lYZ>D;Fsovat@RZe)-db9=uD^nEeSif0ZxtH$5P`RuC#lcrkP^Eq7T z@4ZE@Lgw}w{IEzmRXG;o^{tgYU%>^ z9aBm*ka?+|=tkypPo)yYv$llQWv-2U-JQkM#v*~!t(wu*6rIZwE|Lh@Q-3S>sJz%8 zdbl%(WQ=-EvGY^QLANvIN0ma~GxNTr;z2Zi0uoS@o+0_=)yKYySIwghv=>kl_0L;Z zTwlu64ZpzNIpCP<$KRjyE@#s_J6zQLEt)LE;0ly0>*AGtlP`qaM6H>r}wB#*O5(MZ!-vi)t`rWS;QS_M>cqoQ&8#{BZJ zApT_YIfgRb)d3sFDM zpo8JUiuG6_5D^_EkBWm`ja9$^hhW#YGbBH(9+ka+EGT_J6jEzoUHs$5q9Kt8-^khd zdT9U2eYeS4X9i#|38aWga**XV8j1GB)7U22-8?1On_3Qqo{QK|tnwanQLws2(y!bV z<-+%CY^F_kw+#~TkBgt?ZP&wi-(=s8;@eMsii8Vo*hjaNrW&wLynX$j5+6CSf11V*4WP%gpTvaJ!Up6u5Gc4 zsIuX5j0nkZy*>p~ohyG;&%EKYp|r$F##$F(IX|evdPTnxHPz#^aiqyUXdEL8xHmcW z>Q%Ys#{SZWwSKLU@r9T1jZ`)T)P|u5Z)|>BMNl0H;5c3KWFG!)lAEd7PPhrnt!P)0 zxYSv0;C?v*L80_v`}?0W%1cNt-*H2!Y74GqQ_yYwskD5fnou0ER`Bsl$_xnC>`Q;* zk71_Y+VK^jt7{|@j8$qIW3KFD&LWC?b&*+9J5J;i(i?HgVUT+sA;{wfp{FuWBr1OIlA2cH5c&Hc-y^tlAgzAwdxo z5-N@&T@eG^)QvVch`y$&>iX+X_YLnGww7+pw;_y~ys>TeHs=NVR+5pg(G$v?80A8{ z!qcl)B#Vm|GYSh8r}{vwVpE^h5~ji6=N>ZR6K9Uh^T=J_Yg)++1n`RY(h9c9J;0!h z7ii!TOdZLg5t)3n=UAYwmZ0W4Qq&=PXHAXH6nNdCFOW+Z++yOS%% z3gDIxn3g&$T2PH%X`)iu;~@xF7S7{)^~zE)78tYlTo5K0Qc}{(dX(xCY<(ZSNy_uk z-1o@-iQ!c60&8uJ0)U9Ie>NC%z&o`n!l!M&O5j2Z>F|S+ZL&<2`)KpLHkSWes7t3y zqo#VJ$4G1vDx>S>bc^Kd5;;Rs6ou!R(jSF*uXnPh+o8m8-Z6;zQ^)4DsR-Vyl1e5d zEtS3@JW<(A+kMr+Y5ZXkE44zJGQ2kX!Ut zMU-xg{i6sl9{<%x%_DYC%lT@jtDUi0hp^(KYI7xlCu{(i zSghn|3anEYuL!NW?EY4PdzllVM0P+g6&WZl$ku%QY1?8srUHR=Kg@lUck&IE(FfM4 zjV558#f^qu6wo`=S5GH%ZVh8eDZ4{ zp1_Gmw;K8LE}#l%3ccuI%7E)VGIUrFs-$3<{Dm2) zVdJvjvvQ~fkPzC8Sf|#(AgmbxSzPz0$TgLs5f`^Oy3av(8wu>sQHg@UzphS-tLC-y z>Z>xZCI3TpuFXH{o9ON@|MM;mM(=xcmOs5n`<{-hXSM*{_#C_fU8@Iz#%{WE5fRaR`O zcv-nQ^;pfAwlgut_Hum%jS#1FrA)t$`EUQ*+{oh_t)cxGyrj;{4c9#gyivI~J~1{i zcC05<*)BLh@kTCm^I`M96l$8ky{2)srKLU8}Z&k1gVNI_u`h{56k^ z)&Y@vs7kM}Jhir@`?f9_?wl?J6wW3OdhW|to~xj#c$*z)!dZQAVR&`<5p0sLvkU4l}hzsv9I^SAbb zo&2^(ZZQcjhLfM zyUok28B*+&z7Sl2O3OuaN(#-r6GlD}uXCe)T5%biQ8-D#lG#9>%$tDRTaa4x1%?q0 z&%j#gAKS<^bo5+Qt{nM!^6YZcHbwe7ei2&E^mRnI!u%vvZWmJEsLup(r*S}>w@2{o zGIDzCHS?w{AbEjL*Z-24+Vf(VVReXg;v7n*>vb^*NP(#H_Gr&Jd!nB@__#wdBDT0L zq*B>A%S~8X<(j|eZwW#Thd5e| z%=wyAiz_3+Lw_wDd{Db2iO;YFp5V%VUDI+J)@Na=!rM+nZg2D!G8jVVC4ZwoW6C~B z!kXolJt>y&c`bb53Var0N%IGUAk&bQ1)FD*fRpX`w6hS@M5{HyoLb1KtVap+ea_UPI@OwYYy6MQGPZ#KmJblNh*zFQQ z0+nM~XLOB+@1B&^g-CMUt*#43QC;rzm1p?Y3gJxRTHy=@2QLPnDzR!5OQEAX#hu|V z4K%b|yCpN8usJ0DSF>Kd{aIq9-MKu+_IJ*4e@gkxk|67c3ToH5UBH0s;czfnmmoJY zA(ByNzlwBp5D|PD>%yRja`V-8=gSyVi6NE|VC~%Bd?Wk)C#?wLps4}1KN&RnSmb7y zk+kZxgy=@>Hji1L_Gzdz_80L#rGQ3>1mE|L63bVrevLSXHVyTqqld3gGvo>sN8CH( z8hnLwY@HjFPL7e&RYtihofJ%+6s#y7&O{j-0_h|i*T|4U zIdD%0tBkuT77GFMEF4G6_l^b`N9({e35WZo$bo%6BqN`AIM7r7WTHQdS6lq(PZ)?& z|E5j~x{r@f7UZ5aQGuN8>QY5q3HE(BtUG^Ocb9g*I`~_MmRex8HzT8hEnc-z{%{4C z*>Qhm+cyP!dBH!5GZ@XQ=Pw0i>gGU(U2REcrFZ~fhQu(Ca&&8-es1l%B(W?i>aQq9 zEer}Ld=&jD@J!?ZA3i1q2Oq>blqapCq62D|4M??(iV5L$iqDYzWk*wZlisoE|7}gk zgPhAjqm8e}drjyjgCyMPn*0wUCkzskWp0!cIJ)*^`T?PJ5n)E}U4z7#6|9SD{}eK` zB~Nu(u$a}${Nv+@Q>|ljNEsF%GhA?xCCcn$A3_aoD?ulr@&!#0l~)|(S==mtZd_AM ztu|_n&uTe%xv+-Nr9}44ujrY83o6muWY5b!hVhD3MSj`3>UJi< zVH|{}eW`z#0@^Tv=PbeP5Iv@2Qz6(b{DcMkFe)69%5cL5)#Br=+4xa_H?qhPf}F<* z@_A6rl9U7P?^g-%JH2TQ%V4^gGJpgS&2FiYv5><5rWf3-NU4O^%RkSg&@I1cf~C|D zi~`)GUShT!VpgWqGEk=0V#U2cfR|aCaG6-?*4GaD=3w%0gJ&KI!?CQUf@0&5iJ#8k zKm|J_F2U?oGv1cnqVDu(qL?|s!KJ;8G~sYwn}rxTis);77q$6YD$oX1w=F(iEXDl3 zZwI+qijLAT_FKx_Wza*X=mkCD5y@*?RKHV~!>q2vRF z%^N$1cxwifW1rgbIUG37@a>J(@tF7w2{P|Z5goP5p@dKonZ!H&Ho(7QWzK2F&2DGK znNi>W*VRFNG-i(yTjjJ1xhq%xqOyU(X}NVb>}#`zdF_KX@cFaU%4?*?sLya}_rVmN ztC5HgY&JHRBT`K!VBmoDg)FV5lP^t8+csDuQi_b+*lpkT;-Zm$Y^f%#1XH5t(xBXx z9|lsG|=$?}sXLI<=%IIaJwtZUsjAEIeYAY+zSEts+FHIF5 zC@#{Q3Dx`)Y*+mQ60zspgY02Qwa`qn5%54hi_W07&i_#So`?Z0>}HNSFkP)faNkJ~ zv?GS32zS2{UqenHQY9g&i&r|ayfkSpdykSUjC) zk#jqkF7=hLAcZ398;7H`rLYlJQ#yLbnOYDLuSkxYA&cSP`7(e;N!$%ae+fc z;YIsePIv(I#2PNU;9D|#c0qO-Fq+LXou3Ua#t0rFySg@P?-c*~7^(F7UUy<@ls+v& z(u`36gfIYxf@#`1O^t^X6D2zXsrF-SZOjNn{g5)rzj=`EZV6yeqJJgw>T{FBj{`rs z8e~8%fzNKVVi{K7A(g8Rmrz>=oV^;z@L}LdE3p#Sqb=n z-f}6n015zMnj~cJ`M`9Jq)}i*zom(OOLR{nJqTynBtiw#Cp=J*YOT$6sVNx3)?%mn z<)@JVHA<3%7(Dkdcik zmp@8tFBh$K$pHyy3w-`r3@1xsIC;6acmRho`6>-UV-_Ez#ktN;9XQzgDGb0d?E944 z%C}qA1%Lx$W$6@|nCaHQAN9?hcgfNqPz6SOm?WudNvE?RjH5EyNj@w3o#68{lvjYM zMDxSqT1i}IqR=nx_jitnt)V<6gpjsG_ZeFu7R+-`^PtXYGYs-9h+{gBL)x!{BP7ql0%;d z#3~gWiD(j@Ixh&>B1Re!6pk863MIuc`9Bi3w`=Qno!D$TeUfe zrVFvF&kypgQ{P?ptVjbH1^=hs+k5rm@dLT=QEIJc?zgIyHg0WZKPa(UZ$uacOfs+9rX z^;^lcCm(K!ZN%bRP*u!Agi&=lMIXQJsyC=<2JXB@gF>Bp;RRi^(Y7!g%^ThXFcXBR zef$feP4ngKB$c7tI4F-X;Ef29iBIWZ#JhFsu|f>kj8s|s>~#cgK@d|QWv`i%B{34` zZNF{)M=mntpBT)fiAs-jE-Z|YI&U!!$kdz(pGy@yS)EKW+;PEz*07>4*w4t3e)MEHT79X3#>6_dcwb| zH{7ApI-B8^jEbE)D;X5p40VEBU8G;(f>b`9B} znuKeCP9_BIzFOdP>m$i|!!_Nm&Lvraf026d*cA*;tEvGV{Vo`eUR#=Di1z3Cjh|^? z;nx<@W<1jNo4yVQOt{`MWvuZx?Ktg*iP`xzVly;fdpz}(Q2@t{+Gn+Vq#y#ULy8It z(dC&(^E(7D(+|jmxIS(}^0W$zTZ*_akdeQmC>+zv%VV}j<8~3?pGr0^2u$p?G?+fV zBk9O?cB6D`p#Ai=8J<*Bl!iX*%kOR%%Ez4MqRv`g0xPG;q-A90+1|t_X71KkuZ`Ym zRZtDZ6h|3?1j@$Eon^$vu@!pFl>`O-;Iy1L62c^gm`bDDsKkH~sn!v_z8S@p+cm#1 zV$TLg$Z60GwkvVFC4TxwfNv1O8aU3uq?L77fFmuBwX)K6+|{F^4~ z7B(xLaiN^FxU~-Kx4Nr{u0!OOUQVUs?eWJZAadj_7v-`G}2H*J`F@s2ZOZ< zjGato3GodgwqFA8LLSq7Ho;(Z9yAQ}H&VmK{!U0Yt}*XFhyiwdMhF+UU<@4$WIy~_ z<>bO`?nTU@uaYEI5jT28&ajfP68RjOaR)P>Mz@{D;G zT}9QJEhv?O3(tM`4^-P`c1Evfc^#NKcnb602@pP@&H;Ul!J18~DDdHdz7f5kK5 ztDn)|&4i6F*PK|%XRRVI>QH^PsdB1}%i;4E>t&UZHd!(=%B$d@B}@E(Ax*8w_I&8ukz8B&WnXnAUc~|37!mqH3$em>F6&P<{Nf?OvFjvxk0|3GMF$82`6@(j>L6wiGc47Dk3*nvQ9K%}uAG> zc>eNbv40Vp7$cXQ(YqKSB5REYf2<*~Bg}wjS!e1=#Dd--=}5@mgI4f24HIK49uiEU z6dn7b%}`CvMNUoQgNYf0Lw+t7qnGV`meq<9XArwax?@%2bHoZ?nE_VTS#LL&A`0-~ z6&P~GM^*?0^#qdjn9FP!Xe$BL(0CdSeXuUFPiYM%f2{^n0v5BdRC1ul7v3v@F8|HI zdL?amv3yi;pG#7gT4hl!jjn;33t9MoU1i@?R}IVat!MmM7`^y7PF*x|2I`d@dH4^8 z`18;ZD19CdXHX~W-zO26W9OL|v0Z%-` zOYnOBQ60ln+fe@?N6EqlzrT6=w5hV^TLuoZn0}r^cP^C|q9WXf zz+`ef6P+BY!NAJ9w3sSg*{Rt7mQ2rj@|(E@a*u?ZmWJqAQq(kyZ0%M=aRE;N(QPsZ-&;$zQZ5DiWF3;B26i@Yy?MF*4a2Z1-R5blom=LHueWj_F zY?9t&-<^a_zyB>YHru>)2L}0_;+XX?m>blT2e-r|0LK%X2P!#b!T;UdO93}vJvh0m z>VVwLHoa;+pf%3fKWs#Ri<|L2+%cxznY*R8%BZlMNGk*7z4yN~z*)?Th~5N(Lm)0C z2sqPEPR{h~L!bl-kz0KoufrkYt+O>?C(nR-)jjp>7vmda3 z^UIxG@s(V7QMRcAm%|bWaib95) z61Pc&rFK)9)n%4{pL_MN2IZf9xU7?+6dn{%owiv@efiAk8`tx1@*kG<^?jLRQg&qS z8=!E9#J|;`S0&nq%2qhsASTEZkAzoT}k>s4atFVZ9=s!pRFhbwr72Ddp+)dAhB zP$@~YE%fk@(oQ@cCBZ4M)DN&7?7Nb%B^qgLDHOY{aqx0ykQ1&WZP`*GH*Y$Y8RcE7 zBL;ddLNJWI3MW_&Z3BRA1GQldO}Jwv_h>j|A7CLo^HP>upji8fb+js1;E2Y655qb3 zE{0FsQ?2a9;>Wws2SqViyJjyq)|0-hCvNp6K7{pJfHszbP~0(x-ylg4tzgBqM4yI{ z3>HO^V-jl_d}BO&jPh;3m8l&H4x9GIplACTP`%M~6!S30bQOF657Z|(q9_m3wio&m z5`9R!RO||WQ?=skl+j_mQSWK%pQ5XGA&$B+$Gk3@ShEI&B6fdO;~1bO>&AT%Z;VP- zN_SoNZVe}T`kcl7ViDdF!~Xk-Evd2UK3NVknh+)~rd7VQr^dW3s${LbdRMTWw4u?ccrUh;{FU zj{cL7@-plGOHOksJdJhp30^GftE?nc0mh?IE(SB*jO%T;AXQs z90>JH>{JmCRRg$9toG@`>l0svsI|~4qCtmbWll&6L&1Vs$E_8P%>0rR<`FKVy1qo8 zC=B#C4I~}RVS=i1lt1smfK5U}DfGHJPyD4reo0u8@_52qGy5Nu5NosNAW+RHv}2P8 zzt)g{&aJ|j#v=?S#J2oiZ~(%#$kokE&S#D~)qA2TzMlF*w7Kv>Vj8t&BC=-%}uinR7l*K7x<@ z7JGHmIhjM5YV3otT5Y83jn1(irBEnDEej=O(3I=+hxxpJMzX*~EgK0NB+@-m?0f3x zjn6?S*%I#_=Y~}fi+I_R^!8M(DKA&=zr@^=^nQEx7DaqDexH_-?_Kn5))dUKUdxp;sqFQ2Id)MBt2;SmWw(4h!?vQ*!}l z{6(1t*CMNB{zc##HQ7w*OD4%1$op;y8-mPueyA;xG~_2e5l$Xmd0q_+gV*?&;PE9)$H#Cu z=;zwDyu@JoH=cJ;{qKkPGxbu=EDzH(NdFv2bTB9=R`GBtMf?t@!&EKZhiXO=BUuQjVsc=KQx?|M8j>IEMunko#_$xJ0?>>NK#hm z9^@OQy{%xCZv-{}UQywMUIoW(JwHX83pN+1zO)8uHL z&UfS%ffemK%6<~AciOyg*Q<^9P;~@@!}Oohmy?Q?@eglU_RZ!aaw*At=n#FI3|F%y z3+ARmvbFe%YH#QQxm=WNPl#nmyf=%{TP54kOpe%MN~ z7}4oa^ZDlL2Cx&?`}o(u6H!_}-`ZsXyg9O55KL{ExZcMDCVD&In957Hj9tBxQZMX? zOlS0A6#Zf;P&KWgL`To(=OwO2>P0TR0VWJ~SS%G?ND*KW?kj{c`4Dz^uNy+pi876{ zt~?#~#_>xS*jHKhrSVep!cn4cvClJ4Z_4Dy_a(LXkEIK`9Kv(+m?`D_9zxDbNo>4A zjR7e`H=fget6oSyKxQFvrD8h{cNCl1e}Uq#!}`lX8lXlSOcAq*$R@#*ho$ZX%lJJw zxNx#OsXQlc$8-f>=*|}C3hrZT*|QH0e8S{(eC3_#cK*3x--*O|buuI=!Q;Onj12*h z$>p>i0vsl{jfLoOzPw_9+81q9DW`bfu?x!3Q(pzJrn7xCMR|ztPtV(og;r#%NRLNc z18)EJTEtzxKPt;eDvqg%zE<;va>S-?cV9t)r$nYR7& zo~EK4a<|0W=_d{mkbf z6M&Au)Soy-pwj5mM|=3?AB=5?+wD&V#!QC*+(k+TRxBz~yC{ol7Mln&DlO>OuI3Y- z`|tecfAE{RFDOlphf5BdnuUOJTTpNq#Dq3jWgMaRr=j!6N2E6DOO!=K6T==3v0)MI|pCDkm<^xZD!VkDb#9=4RE` z^*kO*l1yiu<;3{5h< zKg<(08Zsh$$J87GQ6fYLv5~};ImKhqZssBjhEOy5R2`uMts*Bu*#1WC{3Ip}5(pUr z@6Qwa9lFl@D`L)c;Lv^QEfMv11xfC`IbrtNSo&8BZCoeuw<%a&_>)7_DO21V14<=l z{Wwvn38|<0<-o&Gnwg>BpNU?1j!}4ntyeJeQW*6peYiAsjGA*aIgu%*!p0?OEy8i{YU@0OTBv-H~cWy(`sW}?ZtHu5fY5OK=iH_CI zUp=P}hn{wWWelk#PjPll`3dZqwVH=IMf>UnL(K21luCr3{>=%iU4E)1|E;Up2SL5b zw%|u_$>%SOVyT6+mRa4(R>VNgFO{t;#X=wi+8ic>V*_$=Q{Oo!U&w{+ePGM;N{2~| zOQA56^)Er#Ur6CYkh}w>|@VCuJ<@2fJbu* z9&MLlUvVdW#dxkusLiYxa=g)f>SIINeMyu?>QW5-H9SW1u7L04c04pob`yey!25+tXne!rS$V) zilK5-%H~Dekz%g;sU-xhWnpYc3*QTimpdpi!1U%&O3W)K)wwKR+&8GBRu)F+e@gWm zg=H|EwqRi$-B8yaW1PH&TRWGUTFVM$fKY|^y&YWRm7id9I~4o?o{k`_d}+Wv@D#16 z7HhBzTA93D6#IMsuzw>5@L27l*AMSr!@6%shUpL6cqzfNt}m(k=HD?jh`{jiYoD4^ z^1ECJ5rWV#r6r1oO=ZuVNb~V~1)ISLw9+6ngdd4B-r;I3VVu_`na*`1gZTI+r*yG9 z5||HLP)sdbhpAmpaGqcAg`Na)5kEpN6=c^WCInjP<3=l&qS~^^V*Brp=um-sdyYR1 zk7w-Alxz1xVAK!(8x7xlB|xCQ)B9Dh(ZDbXI4h@Eu*izh)35Nj=|crnP|f386RU|+ zG}ai0qH>N)t2l!d4nE&22ujqkR>POpIaK!>>@Op22HExDQOEeoRsy+W6WaGR|B8L2 z2BQsuR=#RVxD7T*m8SHs@Kqj<@1FHi{5;`3(wt1lm=jX zIGgHmgI4z%t(-xc_v*B-(*0*jAwRo+Du@$^@EL<})}((YzrWxS-ZJf<<8uyO{dd#P zOb;NSR(?=MfrZKP#t1q=LhelkHo3Ss*ik96yuS8G#y^YGm*rX($s7}? zU2R!&987%E-h6*kv|$6I=(^C@cl?LtwA7zLLKNkksQ!F?C)9=Zx*hIj(q6RCu{jeC zpR7SVO+JT?h>2kexPm7z#t@=HJ&C0fCXNYz z-&Jlt+~7cyHk6<`iclJiZu+}hv9HoqkCg?D8UXV`wv$@X>p#>`#XD@Vr5t0K#O=J~ z;weya!1h5DE+51GRtPJnS=K=-;HDIoUyt?jXSHYe=oA6SoFp+>>Xs*6k1lVKfo*zm z{O)dKbT#=#A=y0|jHE^zxMx%p@>Oj`!Y_d$$UwdvK9!#!oZ+IBm8ul(yM=cy77(Zg zOHXS2IFXV0CExrF;D4KQ#h17zKBFObuv z)Te*Fqgb-~8YiA>cTK-b80=bI77PnLKJ%R4^nrclqYvA{)IudnK5d>4jZr60^sd>_ zI}voh(`Dj~QrxGax+qXa!GKTnP%;h;%7c95Sb1XS<%_WBDM@(7plkuiRWski45O3J zOaY=~jT-eX47CPW8Q(kXZ0_nJu*rpbBq>+AmOXHCw0<7b0p`;!_{^I+`QM@cGw3U_ zG2Z`mcDL?@0H4%1F*)NFweTG-5|#AP|4`!cvJM5Ng-q7=&U^J){q}Nb630xO^n-%I zgXtks;#^nX(NL*t?L-GjVMLril;a&DOxnF*{zAGP>P33Rg?a{{RabkG0~*IKiQb4< ze1?i}A+i6}@3o4r+OM#cf^qx~YmeW~gz@?v-f@@@Z#k&v?Erne&FmMLjK5;%S>2=L zYuu1S#3}N`GK*FaK7`{2O`}$O)ml<v1ehUPZonb;N*r#p$zEIjUuu><> zWK|8q%`4!(k}@@>5(Mu=kt&9X=#;JY>2z7VgsH&Ht7VdKq;v+B%_Ytefd^OgnLK}~ zz?dI-oZyGGsk=Itm6l$Czsl;w2zktSt$K zsoJsADUcfuEci`7!}rw>6bgkv5v_6$!!{_?-#r4oKl>Zg9R7OpZh&3iwGyyZy&ArF zp#e@nX_xd{iEWHiyrz%K{H3`Fb6s5!UbZ9HTs;0fe+*VE&aNmgg6pBTc{g#~+stpo zpbO!MQ{42o;SZ_w6_>WX6=}bbgbbQt z=o=`1s(Xh5m-{|_zUSJveg`Pr@UmVTdUNuf69cf?!mF6&!9t>&dhJj#Iun24s)>Zf2 zO~+L@lU}$9)t;4st@5le46a60&f*LoHOKNqFq@Z~d<~~OQV^T-%-@#*RBm(niCUGy zwl`B2@8@aekU#MVVom+}B5Ec3AK!Q$U-N}>n5A3n^?#_@a|dOon18=MZj`*nks^~_ z6qTlq6EqqI<=m!|pt;>o=h%L@M#d4E#yQm9nDS2pR#pphh%ozP1O#O1`m^J9* zajK=8^J)3!{JtR0kecSdYEIwd2y_mhWFTX>6;wUA;B-zj7-gC4ex(Y$XdUjO)!W++ zQof!J>7as}>=I1duB^qSd-ZCjhu?5_i0|%`t_mbat}cE@p>8e95U6Kv^_};Zh<;J$ zHeAi1Vq?*ADHoG**Oh^h*R1^1GhmlJL9O!_MGlOE|M1K}zBUV&_uf>&jIfiSSRGsR zZfg!Tlc6frF;U=;?h{MZlah2syx+NZ-tVWXX158J>P+1v7KYEpZLqEDN9#stQli<) z!r^M>^G8zQfjFWsy%9;2m%C45iMz-1sjA$s?yH@!jys1Km#mN+(kwD=|K1>wC?!ly z5aiav3ANs_;l^Vag}vAZj`r4un5NxH!AXNYLUsco;Nm3_@-XP^7LGZYANVt}2nPDb zn?J?Vs40A3&2R>(RKIzEDYisMliXBy+k=|k@ZEQkbNjB$UYGP>_+1K}D+Njmjxe^I znnd>%iuM89auNLpQaYbcRpF0)F8`dr(8xEMW1VLii&K?A@*@V4AI`s*;eYT=ISJg_ zD?N)M;8#lI1UPR!#}Sp$*MHR4@%AM7lUCuWBm(qnvGU5g)6`!c#GO32$13I1XYC5qIM;`_N+1JQ!HS3>0#2D7+!!>_72_Ecezc zWbpfb&*jdr5qz|5cDYzmX4XC3*PWWGbC}|AQ_`5(Y2hS%-AAQ~5FRz08Cf*QN|#zU zuJV@Q0gc;T*lm9JjggiP60O~&*#RldOr40U_t1eC3Y^KO+S6$|!{TAl7soQW(q0be z*hwzu2Fg?S``e=1$eA_sbz}N7{w-1BFyJT;yu$T}AKN&)=9iA12l2dR+3yfbw$`V0 zs>HFZl@AWJWm5oWXit`19&USNdUrCE5c@>y{ftw2$Njieh?Q1;p^~;Cs%p)wTXWfg z{UxO5@lMP{Cogb>YUtsZ+nM;)$8dl9syC<$MeRbndML;8zuLtN0+w*@)3C3c4VAP& zq=^Rqr`{ka=vM4cllAo6x)J)8^1r{*kqUEeV~W0{h@O3<@O!Q?o+MWd-KKBGV*5kh zu;@vc@^Q#Md23gM3_Fsylb3#?S=Do<2H|dfdtas5#+RhCOmcmUANBZ3;xy&V+T$Jn zABuWOf-walx&VzHe5?C8&6nGiI2e}Fqk(_`AI{3`be=ifc%?C}pk4!fCLWYN}`iB+EhhbROR=CR%MfF&bu!&Tex1UO%@mWV}N0#VZ7&5y? zVu!kIe|~l34HHo$7pU+Siv&@7TqBZT_y$*%j&I7K?BV0G5~y*K@vKUJUfLTQb?Z$y zodCMO<%(#Pqbz-8jI9l?JB_Ckv{%G}8fR7Zg4n-G^$X9*Z;cPl*Y+u7Hd1gV3mX6= zqxy(aarJ`ql7Z!HMpHPDrpCohG=T)i==AoWMv`NYfpISo3>>!=IS%xqv!0&vn!`MPLs#BqxbrfJ zSS&yiOpgDS9bjssl<_B*N8B>y_vQ%$AZLS;G?AfK8VD!%@}2hfo>35j6a+q9*>P{RYn$10|IboyKV@?=9R zsM7UXIFwa;fN<%gVj85=-Qc(IGphrcPO7C;PFJn0@FEjOx3W}jQ$Ipy*-HGwiQVdE;@D=w^AFblpH>`*o8+&Msj(Y!f6>bSk9mqYd`SGXaL0bbS!WyijxWC)Vs(oz89t-=T`2q z8MmR@ZAA}r92{y8Lfyf*m6mP>1AKVeFmn_f$wgtil`*&}au)D@Va{fYhduXFn{x$> zJlGY~yz{WXF;8XxK!@`b_}}jL)mxid5{MSXhM#_n8+|&hp49y$X2_sc?}LTQ$vl=uP^au;oMRr<2kh^j93xV7~Oz=iBT zAqm_xnsw~MGDNY}2XRCp7=?cEc%(@kTi<@X?C2bbA)yWJI%2i9*L-VDYM^VN5PPHq zAtn6!d`fE+lxhQ@k61K5LkZXXY__n0xG3a}TthcUx`YI$Y^%sHR8t0%-`~rR}T+gbn5$AqM2m)G=&mW!Gm#M?y*o-@C{SeA6&q>_29+;#I zD?lRp)QsB-)QoH$ob2J#cAV< z+iwy=+OPwadbxAFcnsCP+kPm0kPev4iSR^9f45P)d98;Z^AmT{Ne9v-yp5x7Dbbdx z@*v zlGs+=fjb`WoViWMhYW9CV3|xk+$G*>gPLE6Zz)GF7_)%->~XPZs5j}Zi`Di)F_273 zuf^P7Cq1QqZ3!T^KGVk9_q$WS# z66zjql+#xLm@fwrjUkwy)(13sTl7UDsDP?gvX`-Wt)8=mUujpCgYa>&CkTh^x^>EC zKjmm|=M)eW1fz|RR9cLi{$SXkT)Fyhw)kA>DW9LRTn$4g*iOVME0S}RejF`0c^{|V z$_)ed?_1XktY1^f7fJSo!%K@R@Xq~c&dGRNRfY9e(a8a)eb3RZ6*hq|KR+Uix)M3}+v_v_4_E!>@27rV?+_1S zr67JLKv!8_g+&AW6<2wBPE8ijwf&O=i9O)H^%ToFQ0J?+0i>bf;3c zdWO&%6TN-(xnTXadE8ehM1b}_9qtAN16mS?0TmS3WidK_dQ*N4yC$u6VM1Q(EQ1{&kBb*@XX!s{(5!K(NEgwG7(3m`a8eSCzeXtBTKU%pHy?+@lI7Zg>cX(Ef>`5~wqyX`XRX)u>g5$fuQ)I*15 zmq2QICu1fZ!jgFnP*m9`3z51?_1~%T|4t4fxJ?=N1&B*3u-~fh`d|{b zU1AfQFtX}x!ZMm&rc5GyeSHUX3>4i--X#5UY*(E%XDj2Ofr%5(IcS-kD4&L95TuHXUT~NBc40!T7A=!1st{X21#@8?uP;RMyE4cf& zb_Sqt!Tn<<#dHeHZ&>^UF?44#()_nKt$_w9IKUiuBE^r>jCve4hv=I{Ey}H6zU9VQ z3NV}_L(TI9;BM*zYVvLs8->N$oE7XR&jZX(tqrq#l?S4sSUZ z$3Wi}!PDUy31C3f0sEK(@-9E)P{j~$WZ6XAx)ci7i24NIBu4~jp%1kMQjS~4`-{w4 za#jmW`6t$4%j4*JAS`o#t9>$;)aneRWiz8GakK#||GpJ!s!I`6oC%a9p_M*4|38yt zQn*)l7fNt&OmIBXIxTsee_{w!FVMSqO*8uL97GGJPP#gi@dF8sjKKQGkqoUfRr166 zp2y(ouPKZF=)iRZtsy2Qqg*r{-2h}>vGIyMZgi%&qSkc(AL*V>gK9F%`leJ%=XJ<} zqY8;S5U>Af^*Gt>FK7Vy-KqM{FRAqVjl~_c1UU!2m@^z=^6wPbSTwcaQP^-t)vf^4 zZw#29;z@(|AF(~Y@s$%aI9s6XVF=~g^{44%z0(?Q?c1v~2bA~$;cNi%8JNcVaVNwC z|3mZOd6o!Cdb#3vQ9%g15qJdKXl}?~G8!aE(oBi9RdrJ$IUjYixTQ}d41e?|lk0rjoJV8RqdHYU`|?OJ?#cwxqV$SXWW?UpRP(7h3ZHzTX8QXnb6 zqwTH{$smXgC&^^@@P*VmcRV|F)ED>*8VcJ@mKa2~z)r)#O~8nP5NVkH^7CuJGP8x@ zit2f+(O;_SL2v=;M&(GP!SfY=r?Nvw^CrT-%X@`J&KwWTwg z$DhSHlEF3St*l*wPjod9Hy68q6r5KDIY@Y#tOb%?<&OquUaIU1!L-z|u%Rbu93$Vv zYbPQ6Rx3WIRD=Dedaa_)>C(wUYkVnuj?k$}w`PhYAX7ua<`;w2YHNlj4ZQ!>!KD)l z@C;<46;yeW^VJ6HNZQo16*Kto(x`!Dm}mx+?5coY41q=1;QD*{b3sa&C`Wt6}WOmRFuWki6gnZN<}Hu2-M_u^~Koa>k@P zink;E0V(YdE8g5M7&7o24sY!J@Y#-67zkb$3gc7Dm4JK9Fq#Y)+RqSPZ1{%kh@1bt zUBV(`%4H)Pmw{&#x{Nz&HNb0|@H4eN`5|%lbuVh4 zrQ+I@HlN%&o{(OHTN!{|((_Hf7;Xu&9lywy5XXN{LQe~;L&y?&@4bn8{av5X$jLWf zee?N315mVltEV1mm^!9^TlmmGUH`9+08*|B6@s@6#Z9~nFjvN zS8Ls7s0N{TDWcFonnwhS#%NNB+Jo$ty+#MP#aYs6;PnDu18J;zX7aESDyld8jP#;%>Aq|Xgv+MF zJN4am{i`FG8mz1hv8S<0aTSRMnE@)*Q~Se{qspW)f?MfK{m$&y148#LIizK@`WKEN8+IJn!^>%Qk=fxBfj&)JO_aPp#^tQk{E0{Net% zpi~G=-s6B~nb%4cQ+UkyI%e7}dOEv7m2pC`KJyJGJ=o~aZ6h(aNlCvXtfM=yyqdIV_%v;m*enL}< zJ$YPPlNybV5(eU0T440{1pT04`q=V}zVPy*%_3}w&#dvr=!_|^^uD01$1`Upv3Dh$ z8rSNUm7z~rf5)xAWz$p4*gW%k>e%$Sf$DI5Ju9ppgG6z7l_Mhzyp^8{Xjs2o0Zx%p zPdo`3CZ1Jn_?=?~QG>dJVSb9{b-ptNGhfAjI!0h|7&4*Uf`nE6-U~PGzRQ2OHP33J zNnFwJqbJz!Ka(QFgk7;mkKolGeM$dyDR5KoI8ClBq_xT{%Dru-NOv0JWm^HUNAGO%oIWiM;KH zB01wihs5xxvX3?%qe^!-r9G8^u z${^?32472y9`a70sJ$>Uex>Ju7cuN~!Sq+Y0jg#lV6MFzIeQ^~0|_Lo$bw62V6@9P|gq(XYr8q6%Ew#=Gcm2SzxeVa$CS4lWw8Y+DFcWFBu z8LE1%1NjZ3vJ5|9Nhb1;HMYVU=Ie$D!lRG&P!)e$Mug-DFz7Q(hRwon6w`O9X&Gu2Hcs=PpS6)?nfIX)+F;j0s2upZq8C;BqB`2xKi9~2PTGS zjZM4kNXpP=|K=ef*Gii$R5lt1tPe9Z|{q+;Q#^@>{E|~s1tH=8L8UDdj z-oXc@n*u}snS9}3sIm@W^oV*LUHX>K^!djP=f&}OoXqYV& zTkw!ud|NA-)G4xhga~!dV5Lb_or{<5@?LYXe-Z4+BOqgTlL)*Zdf%UX$t=RTyE$Gk z7({(=$#ZzHdAoE=0*!8fA$<@;4*~?JPU7eY!pNlOopPRoJGhEojr(3zy1kt`mOe9h z_26oMhwd~~X44Ifg3egqZ}WDX^-cn@-O!Xln;Ijhs%oN6KYp=gX2Qvqxd-AX3w^?GLG=F5NoM^Aex1_q!RhLjWL zh_6)8BsD&J+7|2P%SiR$#Gyd+W{v8Tn)oKMoLxv94_XJ7Z!-MCk4MRev^sOpjwd-{lCHw#^FA@3BEI)6(%>6SbwHkXX@wyfPr3DC9dakDd^L#w+4s{hWdxMk`sa` z=|-MR>S|)hWmS;G?5+`OFk$B=e2SWqoe*h=BuzL(F9Jar*QkAh7BoCsyCY735wwnL z1b*j_QqWlzwg`vzTWiH}>@M6|{1+BDkI~(R9}^=No*a_>b%(jNU~6WLRPfm`D^Gd6 z&C24BMOw8L=YfvgaE3Fdb7XJu!pkCKIedSSfh9nDCs;J67O_;s1G8wab3q4yv*#_3 zw_h~5xxekh3N%J)(PpooIm7RIub=Z3S10CX=!&oh2mI#qxXEY4I-p`R|Mz;tOD1ZEdZF=ehD+)6q zOitBSLpbBMKb!dQN{>ANO&`;Dc%|kx--R2;`I}EZ>VTn5Is;4-n20Ebd4@qsjvom7k^+$h&#**fkgDhMt5Sr{KJ`nCH9=nmI+h&@Nb-=&=F-J522 z(*gx7LW?@0ZO^@6sdo|mQqX3_-)wM=Jw!riM*E#O5%0H#o_5-sMi(HdC6;0Kq|R^R z-VT&XgouC0Ff;LmL`2CKsa>fu9`ocsq(9O-6XZPt>n|?-SMK_#5zPp z1f3b5&p)_x86Pk+=;kH;+~g8Y85kQp5V1I^tHsm{61Y0g@@Hef=L?dA(xmou$ofiB zd-Wc}nHB6l0Y{r;b4hwrq}Qn{@~>^vaVwvm-jbu4N8eUS$KoW+g=Oo$ix;9%RsS=ccvOlkyr$ly9-;8L4SjWt^8#sfYHE~zd0rzf$kp1O^uId zCW01Cc{UKa?>UeWe>_k;e3$BQFC4PAj%-z|98M2Mp0icwU)|9@U@%J^o`2NZ=h<|t z?!Mn?Fk!1SU1ktE%&E>wl|z39lWgw1#^FmUhDPG%&`$Iub3NAMw?hiJ6zLC@J-F{t z_b#o2hu;5gpm`IeuQO7kOu+g9=062aLwW6%`s|IT!TNbZL~?RU8*0tqmVP7Af z1oIQaP-ZVKMi$aZS;sUg@sBKhhJRc8ENhi;HD9;miwN1kBZ}nhn-lxpZk7K&qJ<^^ z-2a%*`$?39Qw&RZf-h*WsLstq`-2OouE(xpA$?j)zyGmF95*kP*l!^U>fOxDOL|U?{)OG0M5T>F7Iwhdm&t@uD{wINWjIBNlNis zXr3FLP49TFN^lG1_aw!`WYNB17=*J>HqGsXnxJ!6=4sa}V{F1CRjjhVSQ8#s(T|gq zg7RI-qshP!#AV|iyA_t75iuwuNbM;G_Wj2r-*vN;c!tuVy7z3~#UiV?#;ZkwpEqf3 z*B&@0?j5fS#B*`lc~|!|fAc5DrJ+4*CmYVKmv9;eSmE4E1v5zquIO~(7f=SiM-)SJ zJL&h@)E3~sNM{bK8oJN&WBWebeq!spxc6i;BBH1>4VZJLe&Pr??Ta)FDA|M-3Yn3q zCV{>0b~p5mKHU9zCaEl(P*_>A=gzVw1|~<^L?1yL%`Ob|-Xa$F^Ye9mb);3sJUUoU zlPQ==OtAG=nBPBP8$ zYWtQgwI7Fw_h7K+wBJnV1p^Aet-Mhw%$3_}X|@Lp=i+4oLnWSaW((>k&H4@**6V7lRjXs;L+dgv|BjcJ4i%l5dw0A1WG^^4`8UX-m!nrdnLCZ8r)fc6v6 z{}(Ayi_GrLo4-OjMOTh+yB3V2hMHy&+k0L{FwTfN1jvbDP0YCn0!*M7GGg|h!Fs@e z&3%<|+Op0nty%G1(jZCGC=5Vnk3`Fdphon)oP*ffmvxQ=i@upH?t3P)T0hFqqcmIj zbrI>V?6Ds$6j4e59gxTI)ctkcc04GU93WgQ#}!+^Uo@XE0RGbW(z)sDR)fzBt7(~G&3s!x^#zi*&^sVquBE)7v=sa0Q3+V?) zQ07qM9m4})bj$V~=0GzbKLONd7`-`173df-obkx~)+<6gJ(QddR8oGMSHo6_nYCi;Wf54?+Xdn1?9(r1NyM~TaT_6u~9T6Yg*jh2) ze(9U|20ZV=k;&efmt1Ww$V|#6F<_vi)4I4|uSq1!v0t@(j&*%5{666CoNWW=1JV^l zsQ?=tY_6lAq-To;Orq?d3|)}uP#uZlXZ8soR)+6UQGdNl^_(HS2Zu6FUmmH}5XOJQ zOaDpwKBkK^D~>}noToixrvJRe%R9QpMOixMWI`)c1aExh*8uS_^c2U)s?BJ~cNx)- zo4B4_RQy5P*$3VZ@;99?4{hpv5?P36*jT7-Jn- zNQ4}5!U{NLO5FK<@&Emk3Z6PJqV46~+9U}TLB<+C6FsbSn5!p}G%EZ4Ws%FJ-?(!9 zKb=Ww_a`qUR8|BF_82-ccvkg`7Ww?sa`)OiRD>dp|IXIfY&7lXZ=BHC>}=fRu%UT5b4*xC!}` zVnDZpD!%>;EMnuaw-8BlY+o3Gx=T>Idtg^J?|_cSrE}r}&0Wgb3*W4jTwMtlbqbIi z@pg4`gC!BQey?z3zT51oBZKum8Mw?W@zh){O{<%*sD5?bx9e57f%X3i!d0E|(F0*# zR-7qUXj}!ec*I#{_3TN!l0^G*s-{fL$$X~7u^7K1Oc287=yo!<# zASf!jryR7lF0=ZQGpCh=G`5BMT;{k*{NX>Ze{cT&6Qdm& z4uhn3=`_G(uTc3jHDu)xl?9n4{#s4X)NJ>V)vGCL5d@)?zo~+8_0tVfs|)ZIlI38o z`qO;vHDe&hqz@X9sPT3x9Fw7fGx+W39P&lySsID9#dql8TZF8n5~Vp^AD%aGRHOfRG+l?h8 zMd*iiS+%yxe@Qu%#=(FcjjiB^j11UPHS5y67k6p|e?US&`n?Zg&7r|LUfxiX`ZSQh z&vY`e|2(EPN^gjD$+0g6~YKu1L2dnLP#KxH@a_}BSPzT>e&UoB&6?SgN(>$Nfp;C|<_}H8H4VXmzFO0T6=)P^R4iq8iHeUEMjRUg>&x$9 z0l(Is(t(2f9N=&EiT#dZ3>1AOD=7LRmyT6WWpZ(k`|S3q#4RHJ zIPsox**iR9sFBD+BhY z);}JF&pE{8whJ^;ONbJvGt^MVZ&Wvd(5Hg5>uPw%Xv;-#S?wMU4tPllD)w;$j}%w( zqN9(g=9nf`9wqx_+&Dm%h)c|JuX0DBU^G|OSAPC#C;$oC zve$dX`;2cd1r>Qk2VwFNmCQ}kDFgF<5KU;Ut|4guUW;?jrQr_s!zJ3?2r%gajE2KY zcs>JMt>^ZK&(i;uvhc}92|W8Bu`v0~a?#KTge*DptOy~bK2S33D|(Cj5zR3-p46(5 zh25URsC~8ifyl7P#y+&j)`Xp~1VrS0SCJj3Sr^DGQ-r8l@T!~oDEWzge7*_VBeLT0 zC5Rs(a_JG2>=F zgfqnncq58>bXnnNy_9kGE72YC!y1mz zTl={wM^+z55L4lh6)|dTW>58N;!<29c1^7RYKfpv@iEwzJ$`cecP{y!{d3F3yjS6>gv63;ck^SklzP!W zK17ByzKo#x55imrX)$Ce(cU+hn;3!Zg+1nTyLQwt11O2Vi(3a1s|?YRSyF9-d-weY z>aL<4c<{+jI+j0fOhqbezYh)|(5F5b(7e^z)DP-sge>M~%Gu0$iv{!ZZdk$DI)us` z^!jdw=yNAM>3GkvHq~H<*(MWl*(Xv;q$*kBwp%6{lv@pTCCY(0dko^t5vzKJb;yIo zSIPpz{yIcBPc&&8_ZLd{RWN!*kN}^z04u;y#I3AsXnaufvlLh|08{XT?=-3EBPCO< zM?jBc=6)Btj{dYm?IjHQN~#cm&{YZFDYqB9@rnhVqcA!I7gPfN-WZ`Y0urS$+ z7lcbuS|%9fi;Iy68WM>7>?{rU56GC6WNT^@ku5`I?C1<) zvf*z)o5QAu+0Z61IBzX*bgEX~86_kma*Ttq`DZFfj*1KwHaW#JywmU&FK_`!+S-kq zDMRcEjpm#sACnKzvgURJ#05wQ<^a#aHU1|OD*9jF-Y6jvTY1VJYlcW&BgL#y;PQCP zlt)_^72MxW7rRX7(N+|-hNq?>GZAD}$Ao?TpputM?)#Q_k7zeqDxctgRzNA|8aNYp zA@X|tcLumq=gRRcp7Zx|Hx1v0ez*vbT$`;rP7J$AGO)KPk?;$+J&QeHpFd;=x@;%m zBhO|$D>ymTQb7XEM{c(DUK1!w`Z{R-YBh}h<+v@g@(H<&(fx6oNeOD!8_pFZSZ|2{ zS0ET_GrTczlbYNFk#S@&gFwnKaRzYL(Rx0TKkU%w$MCp*7!}EuLPpGb0^e_*pd;@mU+V<%pP2RZ z%7Jo!z-K(H3h&QYX=m7uw7JaBRrddJaPsYEu$k6`0I6ZRp<9Lz&2LZ}@U!R&d&j1V zz%lVhrffanBisA{J2Oeu_^OJy=zg@7>R)`^dI(FT&>$OXcA;Xus5K2<1IplO4;j3b zNC<74N)lMlj5|qoI1tE~)IZU-69B>cdI3j=+@2#&nfcg^g3~oH0APtVNX9e5Y3hvKnY&QQZ{SR@9?b zLs(3OrLxsk=FixEuy=`hKde-3hsSvqGV4)Z&QCIIc)cScLP;hYd3LW6dA4uD75hf6 z41qu&VC#cZhgc`!VVOCnl&QH0mqd1yI|o8^2QA3_W+Q%fNr_j>&R6|9bLcsXVLUi~ z4}sL7{rU_Ik4=gSEddfo!Uo02J zXhdIKRsVmu-MODVpKMLM)qMhj1--~$$+sPCkMzI1Sn!L7MdPMEtXvh;ebWs5xsPrzoR@3^JbZ^5(0@D06O2O>J^GMa#mE& zrd17VR)P7}v%84OuxUX1a;V4K<_iL*A_1Uj|~Y*=ANT_Elxdx1b{oMIZ`? zd{~`v!qBA2v)+z^J2=JoAx_$TXr750&O+t~+dQX6<{ZT6dhq8B@id0-wEwZj!>3*! z(qW}DkrX%JRT8ne&a`qhNt}46{O-5BvQ~i}>y1|qxnz4F!vXM}@vIzIN=uX8{{O^Q zATUhrys&~}%y<=HDNP?p%;lp3)zUCt5FTHfJ_l$aG>$ z<@`^H69Ie)Nqf0EOYqd8-vSU{2=LlaKqTR3ytv~?A4Z4sE(k(_q0>D<`91 zz)(0Fdil|TG!j1|g|6x#nSIkZ?)CZaa!NJwNb2TRWX(w5nqD55DidGm#Mw4tEb~4Z)^mXR9nsP2Wxb(@W#7+Ld@un4{{pZdS9wn)0Ts#mFgO)A% zsyyk#n1~RXNGVPT>qAypzStg^uEDNrP1IR85SPPa1Qta{k7m>mMUPm&%EX0;h6eM{ zG<;!mP9X9)ZIsM@#Zby;BHqUSQtkP@{^t_)jR~ASz1a+MdLBCIG>66dV`{aB>F`ac z{>RV3yrs3ezXj^)#y{z1cep?XB>V-T1;scHjII%ZB*Aw6M-;#?&S90+7IVSjF8wb3 zEksG-oTC!kLN+D0#!9shn;h?C392U<;3l7D3SKL2h8@gNu4rYd5oRp+Ext(kpfND! z3;_C;n9n~@2-rWX%vf$Sj|~##$U5Im?k{%)Ib!*CnH0mpW~;jn;^^2;(i{nM!Il(z z$0a335C5LfurA$vi7bNzE0e}^j9N73hp1ZiBq}c{l$Fnh=qYROk+E5>EbUp|28=ro zxvQ=a&XRDEg1=1peO`5>Q3$~s!MRk=6ZCrgQ^8Uaf7kc~6=!L)+* zp!c7#&U@vjB7N=I?k0tV1+R$P9EzNeZAPxo2ckVri2cO2%_pEf4E5q4dL~zV0kUQo}8T{PQW> zPZ;s6I1%y6TRFl}R8mToxE_+!rAtov&APulhA4P=INittP=o{30zHp0XCIGdv0E+w zYtzkzZ`yM4z~<@rge!!dlvlp->`qg)Idq4j579sf2sQ#Lb^gP7+F(DdNwlfE| zHUy@~)zaKLIog4o+-)PDzPdT~ zO7ZcBc=s=_a8wzGV1#+@S*Jk2AvRiq<>7pzF?xjMq3RVe%xQMSaaD)Y3WTo#@YyGk zXE`ouMp9uz^U)HjcK&+~Gp>Q9Cw`~^AX#FtxFf*lJntFqbw`7W1zC1=L3#IECkJV! zILSPOb>9L7(DBLDt{kfLx%^L_lluNORWnB+DT}$RR zA9^+~U2S^cCSV`iDe%it$e{`5xJzZyf)O|p8MratR|l?C3gA^tSyo@m#Xp26bce6a z{@_$4-EUhi)w;=G4^7RrziF{MHj1(@j|%x?@f!dJ-*QxdB>hb^l2_6AwL%KD;|e6q z|9%p|+@)RlX!ORPfIbokZHljX_e~R+;=A$1Ve7VwWkh7a-v9nt=U$ovYhzy8%yhTm z{W8&L@#RnPbF2hPOi+ppOv_q2JIQ0O&x}{|e2LhPljPXHHX7jD7T;CRhuuIQs}V`a z#=|%0Zd0|SPh%#O8Z;QPju$)%Fb=axu`^^0$8FhUTrPBDn zJq4R(&CT4V&QkTwmYR;vQ3ji;C-83#AnpMOe)<2gWU1mX+Os3U=o~FvBLqlDUbIC~ zNof>Z62iEYtZw9vjIw?FQ{ozj!np>pKga+z0P>BYE!I%y~+(lG<|1 z@k3k4$)Tq9JUvKWqINXj@dHZLe_TK)9lRZMf89wK5Z?J;v6smeN*MB4%Cvn4qz0O; zyd}fU088P+b_$WGB5K5~2Rx!)xBsmFZVy$QpA3D`pVQgeEAbZFTL8Lcz@6Z;|4=$~ zUWg$mfljHn<_T3Jw5P!w^&sM>)YFLP9>;ubw`#`Tp>I-~Fy=-Y#d6}Rob)~=G5F!! zSyPCzTjcSDGO4wM=kqsr)ND`3q6C{OB2!<^B@l?1OHx$mTbxFGk#-Nv_2PC~q)9OR zM*^v>>LdnU37y<^_T5-TEG58N_i9b)HYEVp4C!gWmq&5{acOfMK29_ zthKMI;lw71y)9w*X}QH1;8AUX)>bpIrWlUrcTH40Q|_zxusSyq>*=FqM3v z=m&!K?AKX+j)>uMX8lOBFNZg--s~G#dMG~(WF!<8fJ+g~U8PP$ zP6Km)WfLBquvN|jfa&T(@M3#*ApWk&pyaG)cQ>#?OK;RJj<*}JOI}q(xe9* z3UsxCu_pX}ZXjSPC`kjDjs95Nga>XKlP}rMAiF<$;tf|97dRa2?=z~2T~}Q5T=gF)9))4gpwJu~@-9&G{m5Vd zxx9vX2O%G`lOc)|n<3hzL4I4r*fY77h|lOlTjH^|>ES_8&|m%G^4{46%51|Lnb=7P z6S5k=FDd1hWwK8((4V6ZdOT+AEN+{JHm=8gEs1eZOa1CU4L*ic!wMXUm(I>UV5)Jv^h{zrao6;%Q{kZ~FWz^A@1q0Y3<1EtdE6KNl;c zs)Dq5*iJh*<4;eATepA(;?0j3YUml>oUFLL+vB>bce&3gwOh6c1knTKk9hFcAvHg! z#0mL;lc2t0yBXOYs$RXm< zFzr7%;K74q!hO4T8{zVon4m~m_IpZ`d#ZaX#=V1FJ%0g=Y_Uw8BJVOI%erG=TS}C4 zMM1%Asd}a@TeGTHua4K_?Q##6`u+W(&iR+u?N^u#l`nrtydQMPFm=O*NY*tM(zRwR zVh$&+tA1YPB)4*4rxtzZSWhnuOjt6gCgyx3}m6|X38s2ErTGbnfwPBz2mwHrCw;mU}>p|Gj>x6$iLC5S%>P4Tw_MDVRQ}3w8N$Pq~uo z0RSAg^7iwEJGX}ZpLhu8jz!K6+z+;3?xIm1!RYsVuAp-`401=KZHJx>=;SW6~j_Z>er$N-{Nd z9S0G$UdPnI#1HJ=zbk$_!G!NQ)>)*57A!XDS63zC6kJ|Qf@Vdy`Vp88MGo7*c0v!L zstV8b`df~WV-~8)hiXjo`mu2vpeihny*-fz&Nj8U&-Hq5=dw^ZU)jCvB!;#KHSzu( z1()FKX6&4<(tL3V9sfYCnzZpZJ1T2^iV`+U(G6M~{)x-By{pd*+M}k;NlmGai*^TW z1pOrh;y!4W(Y}by^e7k`E{x^(0{u6j=OTez)Zb};p0_mUli7f4JwkPs=FaAV2wBap zNM<-hB6h%kD_!9{vGTWuLa;V|gO730x-+C2qSuCp`O}L;u_JSn17gvaEvu^pXxoGg z&DgE7o*As5M2wEjT@T?e(;}hegfZ3O5|MdUfRd?Ar@WnT-I0Y_hKb`5>ZnTPl)gb> z(~|Iyc~?BTPDO$VpInqY5_neY^czzcD4Y3C7ZaCq26@moYA^q6lZ<2k)N6Qy@$}!b z497AvZ6zXQH~@4&LkrV-_F~D{N@eshZlaemk9C4dYDH+trth-E z1B`Z;y?(!5>%160*%cg8n3J@IxPh+lGJV8zd2JH2B$sy+1cE#$5+^^n~xiak#010i_!R!ZiH7#Dm`= zsJ~aJ>EEmlF=KueR<;rsBrzJ_=N5DVgP0A9s>=XGixktg`GpIY69E(5PhMU`l3z@B zKdDD{i4`wInFWo`e-zw_`DaNm8YgLY8#dKYpuQ#p=19rGuc_?Hd##<2mk2J%cGS}5 z_npnDhIFf@=#r+UB-rP}oWbREPmn+pxrxFDD)LW@L^*SuS}o18i8~|3U(5>@zUdnm zrrk880h~r{Q|H04A2&Y?ltujq087J=0JkKL@v%}W_BeLLqkAe9Xy*W9+B1^MnsMp! zL@*2FM1??h{7_m|;ruf*a=j#RPSGxN|5mPR0W9nafYWj3uvr_MRri~E$J_Q8`!=xk zjL%f#&c#9IyfDw)PohxFOP{cRtbG9X7|W~afeTQxhfS1ZfG@2Ncmc6SOs)l~j0CFYNWl z?IV4zgQ2j4^f)qwp584p%7q=vu-6PL)8-GIcVRKOAoOH2DB~l_>XzQVExy_Mu2OYs zhCVnWNw6plCstY`7c1)f&(?&qzHFsv<#|Pjptraph9A#)nn_aDWtrNx#Z=hhAEUEf zmCuox7mei+z<@Z@42GjZPGJcQUw&=== z+RL;{J;gbq+q;F9{xkQ$6Aktr?ic2x#b1#&ha!VlzaO+dKn7V^ zPav}07I}<&A#ubrB7!&5=)n)F`CnaU8V*(5_iGPa1aZ!u)- zOP1_Q%-ovDKDM%F-$M2#8oMHfAxkM_ClbQ_?|JvUxX+vO;#~i8o$H))o%7qipYOgB zxFh~?@qnZbl9KqzK~Q%EA_!`Xam%M?sEmJ4-(GO7>lIHOUj_(sX~J*rz%9{><2boI z)(S@d(fqyaqE`F(7b_;U-VE$Wm7#y^-YgmM4~;-ipQQ0@Q@c@t4VBip=*t(`M9bCW&Je=rUW z`m-Vix{0D1KeWp6w(u^Lbkl!cv+@@*`Rdk+M&eO@eqqyoFWm25*b+>yY$W{_cD?qn zXXchqM?Qq*xmw&Ln~>4IJS4&*JXQn`#--$z{b6(6?D~EV%NJ_m?7iK4L!$X&gdhR{ zA?Ae58k7Nj8stBY+2hFgejCHJh29nJAH&9*-#qOZ-36xlb#iW@Q80j2hKgvbZvadN z_`S+51;@RtIzA`-d+FeTnMZk)D9>JOq?>~x_59b*yq)i^$|n~fT?wCW%ey+&#e)X){LTb;5${PJTp7{>ufP}&JYF3Nzs=hpEaoRtCiOPuT$pD0m| z&V?TQ{fHHpDaF04JFlC`m-{n8Ge}T?j1lAa#dpCS?_(fC;ponwGg=au?^>{9v-!Jq z0h8>?*kA&c$lBH%b&q$v!~5j}u4ds0PltY;I!Ncv4>#VI$zbSMcJ>Ii#xRjI&1IRHpm;%VNK;MC!hy1|q)Nu-MU|ygI4Bv#rgb*cj`A4I$uABFs|1aWypI{GUvDsEr2cD zbF9>sB`h}WN*Oo2xWC8HQGU!yF@9?qGBTh$MAof+ow0})$SI3mJLRzWkrZb9e!-&i z58^gF`s`BS1p<(~Uv#9{K~ajfoSw@gmvm{B30i*qzxVnzH-c_4>eO1~JPxmoUCi0@ z6fgQ>>jUYcEkG1$Ydg=0LAzc>H&`F!hKOm07r17m>Rv9~B1pTN*Ak}MDk@?sQ*s@+ z6Vz?*-Oa`_VU>627wg!sWp;En7I$c@EnFjJe}0oDz0x_H%{uVUU)4Q3Vk&-+h!71a zf?h3bUq<5N)nvwB-mCkiGC0tts1~ab3|Tr*$Xjx;Lu{_sQ>Yy*vc=I`3pKm9Sld#Y zm~Pum6glpm88Sxml%wUHL#j$0tBc-$k+z+%v(6s5YL9Sz!RFdB7f~I^rQgjB3j#l@ zkyy5qgC8gA8{>Zhce#RBzXF-~j_GdNW^N~@IAXfF%W}IAsEc|hsTX0(b*g_|1q>4S~#kVMO z2WBPel9EVqpV(KEp`zXkL>7D^ZOg!Ey(%5v(~JnSm4!f|s@N$@4ct5msVeAjSV{)Y zEXw>!FvKP>09q;3pL2v6wkQuR!bT~jlbq8sZ4(St^z3KfNsw!`k3p@ZZ7XP27}wq0 z!UB^TzCRV6WM}Xpqs8+-c3<@IhgIQ{Kq}XPh?Rl|r=h?4|T& zoTEYC$Amoi>*+#t^Vq;f7Z)P$EUrn`&ewy&`>?OBEE_ap7>|eh(A07Ru=z=O5vu0Hoif#w&GY6sV=_qO?UFfn?|viF3l`X1OeRG0H9 zIbn^9D6?)1Vo`mAwzyugjRDK|i#PYfIC`3ck!cGLW1(%J@D2tVWLIwF?+Jl-699cc z_H;iCUul!SE9IN}lwGJ6S%@oW*LoZ~K{G8nFmcA#p}ADT;@s!6Qx2=WPsc|68{K{+ zr)4>CG*X`IAl=D6wOhxILvGc<-E(Rel0+Gn`6S0ZJb1d-s#cjH8sQO%S`&}Luo!+* z#YbN(>#$>-sZv+npBDj(<%RU7ChVK0LdPFpo~1)1^NEV~)@eH08YJQ1F&g*9G(JoZ@%8LAIFV|J?C0kub0utZ0IDE%Z-s@ zf&200x?SWxywuris!Eu%dbtXke<$-`<=_+BkEowH-yH*9?WnujUycj>zWwe^bP^?j z-b8xbfNVnN7#CQtb>zaDewFTy!&Kua1TB~Z&=0d_KB@XF|M7oxe-?P3jrQaqtO)l`A zWP}2HppNBu==EgmWo{#);QQWjiucpN`fxdJr(IRk`Ra5lAk!AQymQ=`ZjuqeDU^&_M9Z8p-?p~`Xw4(LQYI!gK5UmS*7 z83%dRk*w*S$?3FH$TyD;M}2Q3bFktN`NT?sFU}8b-l#fLjGSkGs!12{u#60^sYnvZ z(uLbf2R&d3Ha!lXRQxmc!6wg(CHVS&4y0wcR;KEnVy|xV?$K-*AN!dZ$#&ojW_ke* zvAB4V*mNsV)*vOoNuaQisj#_QqgY(7(pz&Qrh8_oPDzVn&NFI9=i{6kY}}90{Zpg1 z*_kytqwLDRhcSO7$Qr4H`#kijQFc**k7->5uc-1O{HyMqJL?*8Ps~LA`en}|GEm=u z&}4F~&(%#fX~r_K9j8(fSE{ZW;7;xe7!1wIr_)ok>3A+vQ#HHZWVLws`{c^;BgqeaZdq=ZBi+HIx0q*|BrhSM*0ZFO%JQ z^%QdAq=_qY8CsrMi>}p8=u-hABXLJwUWPvIY zz-kXfP+xYl`_4a=@o;tV@P3yV`my{oE200gs*wOKtdss+;6SpF+y!GK}DYqtE? ziGSZOtfZpBd_tV~MO3+(+$x7|d>>^gM1E%)kHWk~kK`p8?%OCNBv+&awJS1q5!5mf z4ocSNt(!cSTIFVEk?SFaF<;Zr&)xEzQu>K81VmRNf15&z3V$`*_rB8=b`$T5wqxb` zF4nu^dtR?-jCe?j2L$UziPs;{hHEj>o~kRvop#CUf_|!<`9(2a(u@UiC5qG=>l5b_ zHvr*jZl7K%8+!%E?D{amXnx##$SvAE@c`a{dw$@U*+x3-PB2_#OPcjJE1Tkvo!Yxm z_)IxmgoGH(ENjoUW7N{gp(>@~4XFhk>9p4dOI3lXhiL;yBY?M>#gP*=l3+xBTY>L= z$DjGT5()w3%QLbvbpBj^@lHOr1-&bDHK8L>JfdtP049n5Hhjo!O#op0iz5BiNWb2K z7M*XM_Od;;w~{IxYlnT-NgUM4I;*Ky5=A7FP|N_Wqe37*y_Rx{LPDgtbciQsS2=0G zGfPKU8Se5XcjCTntpx9-YevNb)PgP*j?2^QM%0!Q%x$a9`<_e4Nw$QB-H{QO{f*b6 zqfk2Yj?@e1v;Wz=q^4Z#SPB{yii?ZCm9`FMTsK{1yAWA{mhug~#QyCg(FoYz>|65S#jpF74|DHhAxnul z=1=5U&Me}yg!8$SAMVnNvN%a%qJ0=TFWI-2@bJ$y1+GgMWCfZCl)318Eepxmfr9n? z{1|4=J#HmF%zQba_f_dcj^2#^#?Rx)t;Qtr=>^tjZL(9;wr`INHYuJ**PyW{9={WW zpzLW!)ONy}o=w4bv$qlLBQpdq~+wK5lY5# w@+xvlDsTm9xPl5Cezm8&^#3hz^Kfu>4E)~ Date: Fri, 24 Jan 2020 17:56:25 -0800 Subject: [PATCH 061/168] Adjust GitLab CI setup and turn on TOSS CI tests Adjustments were needed for getting the CI testing environments set up. This makes those adjustments. Also, batch runners are now available on TOSS systems, so this turns on those CI tests. --- .gitlab-ci.yml | 44 ++++++++++++++--------------- t/ci/001-setup.sh | 2 +- t/ci/100-writeread-tests.sh | 4 +-- t/sharness.d/01-unifyfs-settings.sh | 1 + 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1351251ea..6b4632af2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,9 +9,9 @@ cache: ##### Templates ##### -.quartz-template: &quartz_template +.catalyst-template: &catalyst_template tags: - - quartz + - catalyst - shell variables: LLNL_SERVICE_USER: "unifysrv" @@ -56,8 +56,8 @@ cache: after_script: - rm -rf /tmp/unify* /tmp/tmp.* /tmp/mdhim* /tmp/na_sm -.quartz-batch-variables: - variables: &quartz_batch_variables +.catalyst-batch-variables: + variables: &catalyst_batch_variables LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -p pbatch -t $WALL_TIME" LLNL_SERVICE_USER: "unifysrv" CI_PROJDIR: "$CI_PROJECT_DIR" @@ -81,25 +81,25 @@ before_script: # check for sourced spack || check for unsourced spack in $HOME/spack and # source it || check for cached spack, clone if none, and source it - which spack || ((cd $HOME/spack && git describe) && . $HOME/spack/share/spack/setup-env.sh) || (((cd spack_ci && git describe) || git clone https://github.com/CamStan/spack spack_ci) && . spack_ci/share/spack/setup-env.sh) - - SPACK_ARCH=$(spack arch) + - SPACK_ARCH="$(spack arch -p)-$(spack arch -o)-$(uname -m)" - spack install leveldb && spack load leveldb arch=$SPACK_ARCH - spack install gotcha@0.0.2 && spack load gotcha@0.0.2 arch=$SPACK_ARCH - spack install flatcc && spack load flatcc arch=$SPACK_ARCH - - spack install margo^mercury+bmi~boostsys && spack load argobots arch=$SPACK_ARCH && spack load mercury arch=$SPACK_ARCH && spack load margo arch=$SPACK_ARCH + - spack install margo^mercury+bmi~boostsys^argobots~debug && spack load argobots arch=$SPACK_ARCH && spack load mercury arch=$SPACK_ARCH && spack load margo arch=$SPACK_ARCH -build-quartz: - <<: *quartz_template +build-catalyst: + <<: *catalyst_template <<: *build_template build-butte: <<: *butte_template <<: *build_template -unit-test-quartz: - <<: *quartz_template +unit-test-catalyst: + <<: *catalyst_template <<: *unit_test_template dependencies: - - build-quartz + - build-catalyst unit-test-butte: <<: *butte_template @@ -107,17 +107,17 @@ unit-test-butte: dependencies: - build-butte -#integ-test-quartz: -# <<: *quartz_template -# stage: test-integ -# tags: -# - quartz -# - batch -# variables: *quartz_batch_variables -# script: -# - cd t/ci && prove -v RUN_CI_TESTS.sh -# dependencies: -# - build-quartz +integ-test-catalyst: + <<: *catalyst_template + stage: test-integ + tags: + - catalyst + - batch + variables: *catalyst_batch_variables + script: + - cd t/ci && prove -v RUN_CI_TESTS.sh + dependencies: + - build-catalyst integ-test-butte: <<: *butte_template diff --git a/t/ci/001-setup.sh b/t/ci/001-setup.sh index bdead8817..54b9a8689 100755 --- a/t/ci/001-setup.sh +++ b/t/ci/001-setup.sh @@ -205,7 +205,7 @@ echo "$infomsg Set CI_TEMP_DIR to change both of these to same path" # storage nls=$nlt export CI_STORAGE_DIR=${CI_STORAGE_DIR:-$nls} -export UNIFYFS_SPILLOVER_SIZE=${UNIFYFS_SPILLOVER_SIZE:-$GB} +export UNIFYFS_SPILLOVER_SIZE=${UNIFYFS_SPILLOVER_SIZE:-$((5 * GB))} export UNIFYFS_SPILLOVER_ENABLED=${UNIFYFS_SPILLOVER_ENABLED:-yes} export UNIFYFS_SPILLOVER_DATA_DIR=${UNIFYFS_SPILLOVER_DATA_DIR:-$CI_STORAGE_DIR} export UNIFYFS_SPILLOVER_META_DIR=${UNIFYFS_SPILLOVER_META_DIR:-$CI_STORAGE_DIR} diff --git a/t/ci/100-writeread-tests.sh b/t/ci/100-writeread-tests.sh index cb50c2a32..32139e704 100755 --- a/t/ci/100-writeread-tests.sh +++ b/t/ci/100-writeread-tests.sh @@ -68,7 +68,7 @@ unify_test_writeread() { # Evaluate output test_expect_success "$app_name $app_args: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 10 + test $lcount = 11 ' } @@ -86,7 +86,7 @@ unify_test_writeread_posix() { # Evaluate output test_expect_success POSIX "$app_name $1: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 10 && + test $lcount = 11 && if [[ $io_pattern =~ (n1)$ ]]; then test_path_is_file ${CI_POSIX_MP}/$filename else diff --git a/t/sharness.d/01-unifyfs-settings.sh b/t/sharness.d/01-unifyfs-settings.sh index 154631dac..815e8f9f8 100644 --- a/t/sharness.d/01-unifyfs-settings.sh +++ b/t/sharness.d/01-unifyfs-settings.sh @@ -20,5 +20,6 @@ export UNIFYFS_SHAREDFS_DIR=${UNIFYFS_TEST_SHARE} # Client settings export UNIFYFS_SPILLOVER_ENABLED=${UNIFYFS_SPILLOVER_ENABLED:-"Y"} +export UNIFYFS_SPILLOVER_SIZE=$((5 * (2 ** 30))) export UNIFYFS_SPILLOVER_META_DIR=${UNIFYFS_TEST_SPILL} export UNIFYFS_SPILLOVER_DATA_DIR=${UNIFYFS_TEST_SPILL} From bdfb413c927a54ace796d4a6a0b8283e00bb63af Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Thu, 30 Jan 2020 11:17:46 -0500 Subject: [PATCH 062/168] misc doc updates Adds three unrelated bits of documentation: 1. Describes the various build configuration options available to users. 2. Adds information about the resource manager job integration scripts. 3. Improves the documentation on short-form boolean command-line options. --- docs/build-intercept.rst | 60 ++++++++++++++++++++++++++++++++++------ docs/configuration.rst | 14 ++++++---- docs/start-stop.rst | 16 +++++++++++ 3 files changed, 77 insertions(+), 13 deletions(-) diff --git a/docs/build-intercept.rst b/docs/build-intercept.rst index 1ce63b710..6e18b27ad 100644 --- a/docs/build-intercept.rst +++ b/docs/build-intercept.rst @@ -16,6 +16,51 @@ In this section, we describe how to build UnifyFS with I/O interception. .. _build-label: +--------------------------------------- +UnifyFS Build Configuration Options +--------------------------------------- + +Fortran +******* + +To enable UnifyFS use with Fortran applications, pass the ``--enable-fortran`` +option to configure. Note that only GCC Fortran (i.e., gfortran) is known to +work with UnifyFS. + +GOTCHA +****** + +GOTCHA is the preferred method for I/O interception with UnifyFS, but it is not +available on all platforms. If GOTCHA is not available on your target system, +you can omit it during UnifyFS configuration by using the ``--without-gotcha`` +configure option. Without GOTCHA, static linker wrapping is required for I/O +interception. + +PMI2/PMIx Key-Value Store +************************* + +When available, UnifyFS uses the distributed key-value store capabilities +provided by either PMI2 or PMIx. To enable this support, pass either +the ``--enable-pmi`` or ``--enable-pmix`` option to configure. Without +PMI support, a distributed file system accessible to all servers is required. + +Transparent Mounting for MPI Applications +***************************************** + +MPI applications written in C or C++ may take advantage of the UnifyFS transparent +mounting capability. With transparent mounting, calls to ``unifyfs_mount()`` and +``unifyfs_unmount()`` are automatically performed during ``MPI_Init()`` and +``MPI_Finalize()``, respectively. Transparent mounting always uses ``/unifyfs`` as +the namespace mountpoint. To enable transparent mounting, use the +``--enable-mpi-mount`` configure option. + +HDF5 +**** + +UnifyFS includes example programs that use HDF5. If HDF5 is not available on +your target system, it can be omitted during UnifyFS configuration by using +the ``--without-hdf5`` configure option. + --------------------------- How to Build UnifyFS --------------------------- @@ -61,18 +106,17 @@ build is desired. Type ``spack info unifyfs`` for more info. .. table:: UnifyFS Build Variants :widths: auto - ======= ======================================== ========================= + ======= ======================================== =========================== Variant Command Description - ======= ======================================== ========================= + ======= ======================================== =========================== HDF5 ``spack install unifyfs+hdf5`` Build with parallel HDF5 ``spack install unifyfs+hdf5 ^hdf5~mpi`` Build with serial HDF5 - Fortran ``spack install unifyfs+fortran`` Build with gfortran - NUMA ``spack install unifyfs+numa`` Build with NUMA - pmpi ``spack install unifyfs+pmpi`` Transparent mount/unmount - PMI ``spack install unifyfs+pmi`` Enable PMI2 build options - PMIx ``spack install unifyfs+pmix`` Enable PMIx build options - ======= ======================================== ========================= + Fortran ``spack install unifyfs+fortran`` Enable Fortran support + PMI ``spack install unifyfs+pmi`` Enable PMI2 support + PMIx ``spack install unifyfs+pmix`` Enable PMIx support + PMPI ``spack install unifyfs+pmpi`` Enable transparent mounting + ======= ======================================== =========================== .. attention:: diff --git a/docs/configuration.rst b/docs/configuration.rst index d4289ab24..e1e68b539 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -14,11 +14,11 @@ certain settings have command line options. When defined via multiple methods, the command line options have the highest priority, followed by environment variables, and finally config file options from ``unifyfs.conf``. -The config file is installed in /etc by default. However, one can -specify a custom location for the -unifyfs.conf file with the -f command-line option to unifyfsd (see below). -There is a sample unifyfs.conf file in the installation directory -under etc/unifyfs/. This file is also available in the "extras" directory +The system-wide configuration file is used by default when available. +However, users can specify a custom location for the configuration file using +the ``-f`` command-line option to ``unifyfsd`` (see below). +There is a sample ``unifyfs.conf`` file in the installation directory +under ``etc/unifyfs/``. This file is also available in the ``extras`` directory in the source repository. The unified method for providing configuration control is adapted from @@ -173,6 +173,10 @@ command line options have long and short forms. The long form uses ``--section-key=value``, while the short form ``- value``, where the short option character is given in the below table. +Note that for configuration options of type BOOL, the value is optional. +When not provided, the ``true`` value is assumed. If the short form option +is used, the value must immediately follow the option character (e.g., ``-Cyes``). + .. table:: ``unifyfsd`` command line options :widths: auto diff --git a/docs/start-stop.rst b/docs/start-stop.rst index e7e5947e3..648f81dd9 100644 --- a/docs/start-stop.rst +++ b/docs/start-stop.rst @@ -100,3 +100,19 @@ use ``unifyfs terminate`` to terminate the servers. Typically, one would pass the ``--cleanup`` option to ``unifyfs start`` to have the servers remove temporary data locally stored on each node after termination. +------------------------------------ + Resource Manager Job Integration +------------------------------------ + +UnifyFS includes optional support for integrating directly with compatible +resource managers to automatically start and stop servers at the beginning +and end of a job when requested by users. Resource manager integration +requires administrator privileges to deploy. + +Currently, only IBM's Platform LSF with Cluster System Manager (LSF-CSM) +is supported. LSF-CSM is the resource manager on the CORAL2 IBM systems +at ORNL and LLNL. The required job prologue and epilogue scripts, along +with a README documenting the installation instructions, is available +within the source repository at ``util/scripts/lsfcsm``. + +Support for the SLURM resource manager is under development. From b82e33ee0696077081be9f4ae4863ae5cd7a59b3 Mon Sep 17 00:00:00 2001 From: CamStan Date: Thu, 30 Jan 2020 12:00:57 -0800 Subject: [PATCH 063/168] Update high level design image in docs The high level design image in the docs still has UnifyCR. This replaces the image with an image of the same name, but with the name changed to UnifyFS. TEST_CHECKPATCH_ALLOW_FAILURE=yes --- docs/images/design-high-lvl.png | Bin 57613 -> 106844 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/images/design-high-lvl.png b/docs/images/design-high-lvl.png index 897fd9f6cad8d759db1febd0f57c55cdc2db9aed..a3f23dbfa1b91bb553e60bdb6c26fc02af147e22 100644 GIT binary patch literal 106844 zcmd?Rbxs@yj)*Pd+TNNxp}GfDBu6Ry&E&j?El5D z&W;;-aCm5}=2o}%p69;<;J1e+D<>CQ>KI)pI*$<}W?c8axY&~-tph4e*!jml?@7i2 zPdC(cD2E*_xWgf%gCeS#9?m}+SIxkGQv4y$NG*uNlAfr~oWf1_pPTw^kt=`5q`2qD zzj$b)l?SB=y1`iI#YsWn{D!#v+rQC|=Mf*5 z1$J`KDI&solH`obKD6%ht5&>3NB(!jvy7os)y?Fk7eI3KENUvP%q=kE<~|VH*fYZ- z#R>jO%Nn(@xzB>VB>$0XF$m1UiVC&ynS50}jr7jk(DCvC9j@+5U8Rsb6UUe%|CQm+ zvdH%sN7(-nb}BHx1gru}ofx;}Akv(Gc|gd~9X-zv!MA6QBX%&>DL2{~eM4tq2%tfQ&+A ziaBTgd;4cVvsIW1FH-)eZ~yC;m^HvIP~SmSKmM=N|6^j%5CIMPES@+=_3!EnMGL5R z7j4Wk?Z3BwL<3YL$viyt!Doue2ib!L^{<_PdzZEHF#aH#@`P9w3kKgS48AC1Cn|kJ z1|K6Pfu3cRpgfhRo}OOJ5{ZCO=jhlyGCC(0PI4} zdxM&%_WKr1RicB}^vKnghZfCeW!Z2MH@#RWEvc;Yn&t(O;t$T~-UmEBN_AdR>kZPD zZWwO^&V^y-iYna;u_C8tKw6af+7`mo8eJVLrVVIz?LA}|s;QT5+4KWLeslFZ&zXbw z*J`=mo--P%1rfQ;i(2`lGm~uQhZ@6=z$?s$gXswpD;4l1C;%;Mwl(V~Odbcf<>q+eAnPY~PED*xt763n z(PXtI3cK+9mAB%HFFqI1S=<$=`o|7XZ#eN&(e^N>3z*7TX=;6|d$Oqq4_O zFB(`~-~e;aL#?c>&8jiOt@=r~#M|7+%dV-Eo*`B-|1ht%SqZQZ8MW`^#s_h1+{+th zR_lE8C}YgX792m>emocJFR>uIZJeq;?NpcNk+$I^nCGC^?D%Uu zF@h_)I(oW;zGo=0I0yPGNna0BH7f96cv&#?*%|I(7*%JL#Oi4IkrJcP2UYsTM$oW= zMShO_f+$KuedOeHw9JA`(eT30RnvVPg--nUr7X1RW`S66uH_aCBl!|-SqYX*am^_l zLnAkXl_G8&l`gFKze~iLaZKc_HB)fH|FF3BsPz_kXMs)8$d^iwt;7LhhYJp+IagG> zs+G710bbPnQxjiSWIPsQIY=p``auH3dEMCtBlF9@&%BVT+tzqV_0b79bd><|N5$+XKuy%eQ>-Q`l35*@;H9pq$85oqFqE@RxW#HrIQoHS zve@a+$SP8jqvn!|vPDdAURB))qc1m4SAd<=buttrP%T?`fUrHfQmHEcZBEW4MG}O3 zKqY7`_^zIKUiu<&mudNMwvNjauC6~s^SP}*BpGunrn(#S>jRYypyn}HP_=jM_daNC zKYpAf9leXbI-C&ik33@D#^$be{_^A9t3~3wly<3|r#2m)QeB20O+j&qOP}&9jk!Z* zB823_ue-(j!+qs(G{!0g^y@IfX65WALgHjusqc zYf;Zj)JO~GnkS~r#z+v9yqq_`nVuLOD1s}OX--v@xf69cHE^W#3r^#fF5TLAt9Xpq z7rkb^pTD#i)J$0tu}5}8dlJd{?{%q|IrWK(4s-5bgRU z?DR926ssfiuOvO)3!_u^=em&N$0EdS!Rs3PbpmPsd4F$XxV z&@)iPhgg||4)cUWssn87zN)D<7W^iS5BU6fC0NIt>fWyoiJts+8NbqQ8(vt(8INf8 z7t#q3%%-YliLQirpun}if#~U#wvkITW2H-s+Afq`hOY+=VUgp7LAUWI=?;!mDv=1! zxyrrcaBoiqq}@}LG8~+>7%c{-xOQFK3&D6*7iK7M=(5hQ*0alT|94f3(U!ByKx2Yk zuSDxP_qQWOA&lQGy$r0*8uy@uN|P}8Io2eVgY7S`b^&#?f|6}9WYjr1#lc#ZLP}(h zQouCULXmAzmUT!N_pX%)3HFo_Sv(sRtP5|?EsSi}kiNuyb=Wy13u}!M)=(Tg3SKtD zM@gx*pwWVCC^DNyv)0vBVUlKi)y$sVuD*@(+S>C9mLtVBi99D2vd1g^3$M8EiCQN#eTROZ$7x zO>-YvS|O3s>c^FOsuz*dv{Losjh7R$jOW;Jf>-Zf#~N)FDho0@ExaN}X>M5>X0$MS zdFyZpach~?bhtW*GtqWhvRc4V?Iu2)$#`1D^SxQx?U-|pN&u{4o?{OOH6C-4$a3{o zf~0dYaj?@ID^X#5*WVk*xI!6_kzc@xdNpqkX%w7L2%?@+vwHC<;;^RBL#IE=Hirvz ztBHBc&I4A30G^C^^n(VCbjb@&W;lz0XLQ+CXj^^KSkMpEqIxE#f{(p35^nEbB4L|RsPRdLseEU*yF=ca^|Fc-f z9>Jr?Y-+S6ls26PLRk}J1{E*}tZ>qdFIc9! z`8u{wQPr<3kolkj_Ol>zladhJxA?}7j|^;caQ|H_Cd#}fcF@(}olAjy?4mzU$Dx7U zPj@q-zAmCF#A8XY8ce;2N zJiTkWDb=mBy}CTT)JR{&k{jRurCw(yuH=76)o7LnMW)eX79cO@Fsnh=WtJ8}Ji)le zQ&kysmhC5#^4H7HdWLV*RXQ)s+#3WT_f@R3&Fjw8NP4XGks9|D-^a~4#kiAdvi7O8 zs6EzvL6s%L0zV?t{?x2AOW$Q?x-Hvsm6uL6x`|*0x$dB*qcueQqbct#lq5reGuZs; zSfOpRhE*1PyUK(e=5;?LsE}@7t(bA$;HZkiZ37aajLLyt*qM{FoRIbU92aR-scZ!Hr;NKWt0X0dwkb@zxM8jl$5Pblg%E>0yHuB1Z-092A3OPiJl$TPu&l=;Wma2Efxp$u;EM$*}{xRI-;L(g4GSJVf`5p?v z;Yg7?G)iEa_e*`a}upi^oG)ut`oyNOdqo6N&s^N z8uS)K{mn|!B{Yj>QH4#OFv}TDP}a8epNPfkeU=3L?dIx_oE%Y$&vEDhyI>ixuu+6m zeW?uF*cwDj$8yP6I*C2{0|G8qqMNN>Qqh(DEp@HxO7;2QyE~1=jJ#i#zNl^`&n+dd zEm!lOI1|@3e#~B8DlKV6|7%p#AVlBmQ~0oaB(C`l(+a!0`*Q^QV8M226VoWtUDNTX zihJs=39Sc1R(1kTuQnwdgI=rcV}Fs*GvzG-#msO3Ob@MkfNkLKRz@X!LL8+3gb|^{ ze!L=&y1UZ@#p`aqgWPRtN8Ne#{`{A zS3a>RBU8RcNhmBkguK{S+jqakAixS)iwd5MV1D=I<>31Yky0H=y2DoM7tn}kW*Qmw zVUnyhEp{aFRLjLim=YPdH08Fmnp4DV#e=rH%TUr3Dq&GLZ+8vOk6XN^$NcRX0ipi{ zv02CPmVeaIHMgQc70O0Si3c&9sHF4?AFLAUN^oSwqq7fUyBAJ1qwsu2u0d%* zAo9Nad{wSV{_EJ$Z{7SNJGH<<|DC?YWFX`)_tL1gJ63(BupK1P-iNdNZyMo@*0+#i zUPh*{IgWF;JZ=FoIh2<9gQ{xN64M6-0|8G|lt>B| z>%R|QgE~u_H%vXLzC6>r=&>2;P5nVzRL+Idzmrpi3{K^qRIbl*GA+RW;%n7gbRYu5 z;Axc#?m+g}Un`^VrC6HpySal587YhQ&iCg?QgAjTwZ@rYD83Qq&z7QmCn-0ox09d>@j zVl!$ETDKg}miNbv@5a-s{oE6=?0CLWA>?J1g2;wzOtW54ZUzk(hvzHkkv_)}js}m< zTQA)vvU;b`CK?}7Pc27Eljg#IpEH+)Ko>~cpc{j<7QUcOwa1At*yP6PVZDcyPT`j zRsDyn{i$*Ko$8oTl$GohS(`V=<*jQbEEu~ zlAm~BQbxv#>%_%{J^#fZ<@UJfYXYvGtGE*XcEa8=TW4ZKXi{ow2?7X2Z5xh+w%GQQR`m}O$T3G@dJ0XGiSVZj z#7tQAXz`oUXSNANz5$+`EYulN5d&+Ix=dNbLOpKv9V8D?a!&gy?Jrl|U`w4+5$%v- ztXFvinJ}&e_LIZwIepe>;hXQSZF=`x@$Vg>96fmErL7xUZOVqdSRXK?8|g2qdzu>F8c|NEWM)PjMh zom89Qj+blJ{pljL7IwCr*n3Q3KQT^&PB8HUt$`@wFDnfeR z5ESA(o7Hv!DIrcy&K<9H?e%f@}Xg5mG$!p0A+ zMdkrP9Cii9=g`v| zuP-<%tCf+zaj(K6p^hZLuKgQM=2QyKD^$jDY+!QtjyM`7@MQjc1L08#edWiuUNAAh z^-Bh9N{T{OdAZVx*RAfe@2X)4@fM(DNr13vDe zn+QfGgfiTJK`uq}M(#oWj6(?D8{YW}l`FVavE3mbOsk-p&SF>KKK)=p0FE0e!q-c_ zWV0-z_k4$%nDbMLvf|>W+=h_0s`~5Uq3!{#hEVy>l{>Sub?(Jr->G#?!SwuMw_c)d zqHjXidTpu-R*H%035g?ncG?QoG^$e&g41WjCxs3Os-m$eFzaxqn}SFJVhidTR-7MK zsIB4OeP) z4FW-(ciAys$89^&>T`j#hh-}XJQStx)$=dM2FnHw-utyBH-^a;5Ybp*3<6(~l^EH{ z2Z=U@V69J8a=}-_@vWAuJzI+gD)Z+!vWlq!PC!UDL@lt5m@`2cEtE4g{Un*i{t_OJ z@El-9$Q7zY$ZL)C<1|{xv$nL9>6tc=G%t1iVKi)7{3BO;V*vLQ?vseJ7XAzIZgNtR zyrQCF=i}z;gg#c9RDi2dAf-hctq!fa{yR8cBqlujuI|2ob17Z=Khex?@Qn#bmxiJFKsglT&15)TadUy|IGnDA=DGWG5%*ijya$&5?On?ICiaE6SxfIOd9X z)x((dIAuJ=Rd4G$)lcKY4>*Q?S>LsLZ9YEMO4UuRLC)CV+a-ir&}n(StOcMtQjL~1 ztP~$Z@XeDE5j)QtYX5<_0V7g$UDo zYvnVWOwzsrUCYS4NtY#^PHthL>^JGVPi^jX^|@y*5#Y~3#KG-r;|3ck`-l(bM-s19 zK^tF>#?3}KfxJX42HtdwSXxxaG>y73;Erv!xa6%_dh4Wp1JPZ4EUS-aj_>kv4_gYF z=CY=?xa7uX3eqg} zu$OzWWYh7rViH}3i2U__+$Yb$bXwx888tPvK$^#Hx=WJBY1;=9?>j?$QSVkJ-DdS> z=VJ!bG`rwG_9M)Oh}xSgrWtNOJ)h18E@nRH4~3#r5}f+23Sa$}yclMz*n^lcyL+CH zA!YucV;96@z82(hznxq+5t1UsrWNDgFhJsAR`DI4-{-6gcKe{D!kTNArEv{{|Mlte znme}K8My5sdnAdlmOgH2=>|%K*t96voW9Sq2B9L|yRe*<#_>)+$kZ!Aym9O0=6$n@ zlJe$a#mV!})s3zbIGDZ$>N-DFCld^~T(&DF+w^I-XXIlibL8(0CgTr`d7SHl&n9(~ zRanzoo!G-n%-Iv*-d{PID~DOTzfg{brKkk1ZEUeIKNJq`>p@n%h}fu92xYhrjbY!L zjO}u)N1r91(O1}k%C!+Zb_TY>b-I2VGqW&{@UaT7 z&!;|hm&bmvBr)$rpVgNnD}THIrw$)OEaGVtCvQQ^0|lAh%|q*jc7eomzj66G$w0@z zR`5l%&y(bmaAl^g&pdDCF%GL<>2W5p%O4dK;VWe?q8Gw_wHNmKG*fcZ7zMAoWklu>Z2v%+2t7tkhJ)JsDd zK#6twiLpB$ePp)H@k;iwqFd=t=x?*Vj zffm_@?=E-8=U_0TD5lVaoqmhM4)Lyf=2oCiAiN_t626{Rdq`pXLpBC{?nOu@k|2M< z`{jOD%Qscfr9A3oW44)DYXja*RX$uHe7e4zRnb)m5w!f^oKWgn!O*@J@1LF;$w)y2q7s>GODt4e5&@4QwEgKqf!J_U{}$aJ=?HUP$u^KY9B+(I z>XodHB=%1u$r1L{+0y{O%d%54;L{2=Yf>E1&&yRB8kC3``1xWM|M2JmRr4Gb!mOJv zH#;rvEV>_-)#(N-Ci=WSH?X8cKIUecRo<_y`DX>#b<)Go;veWxXgvUOJNqoYiiV8L zQ@K=$A6ljI>-$<^4G$kaIZ~I2q`m=T*Nr3sSvl-3r4%Qou5mw9>d;vrTd+@vS0L%U zmF*L-xulKu8{PNQ*U1`-j9-s2)5|V>asR#jPmLF`g;D|k>4U)296yqVhy}!jApsTWv+ny7lA)s zeHJ$iDH1lG3=K632_vPqArl}3H8_!H0dR%886HY9u;k5yi)=KuVm_ton{2LRcwLfk zkV99{`?kq??nNi4&Y%50nxw4X18xYysq*`9dg=NeHztnCWf_a$*Z7&Eay zEUGf&8SfVCC$J|E6d|Y$+~*FxsP&>RCi+e$k#XR!o8@CfiKIVN@=dv@LulTap|*UM z>$fAXLl`3v+KkMC*j?1~IHo4`x{CEY@59xWLhfSm6ZT#0a&rQVJnte{s7U)+H)0(n zVh1O>0Gi(KHD5_1j2w3Z%ueLHncUOxuw@H34dPpg@oj>fbGFsE?nDz|@7+Qmeux-~Zn1F#^BnWbjCw z{pb@SkNJR&gR1OO!Cb3_Hd?5|h|V_4oVt7x!*F8!zG0qg;x!5cBj|Fzc+YKd7gavD+j>2#t@r?;4aI{=5sKCZY1g|*R@nK~ zn4p&wj>}dgdxwf5G^K@bAp)QCs>#+r+a$lPtDLi*_=qf&?FCsy$@VO}!`Q z2`aQevEt&Uxhlz@(}QNC)#fJq-_(u@mDnXiI2-1ss;#X!k$Ca< zx-NR>+hrGv3AmiNq98NjpI-)|z+vIFVRO(`@1CV8Er?|u?8h^NjXQn8ABg5|7galM3|s8h81BlFq7ni{2cDRMgR>B4eo+e6wzE%wVtYa87`2X^TR`%Ps>Q! zP@cfzzV^SWEuM7$)S%P0r)iBDKzI>%E(?ajnirH!-VI%3So!tLbHKs#c(G2v>({j7 z6)B`tgxDzGA_RX9!~zISnk-6;D}36`xQ*2kqgK|rFlZB{uf&k{o;Pik;1TXu#ApD; zAHNiQ(gU$BoZ4Aiq+S>=|1yN7LT;~b1J?s0eZ1hhb9A(dXca02JbC8xmI0@0=fW0M zr@A`dK!a+X$)z$H-PU*ATmF&Hah2o5mCnCV2+^WqG>xrH5H;|K{p@e;EVE*aw#s_G z``c~lH{u_1{pP2cq@AxX$u8_M^W6(ddsYsHdD=hm>a(Ck}KM zc*zN}+vzL{PT;U0CXe8o|A|clO+x5&*C1Ape)0{@#1lvBGievi#A={n`zzWrQV0P> zp7ImnzGWln5&m>aFNnPCYhy>zGS-nm|5v!B!A#RE2Q$Jt6OgO51>Bm|Qp^2v>R}}} z)v=Q`(Kiq>44dk7rL9I_8kZ4XO{|y^lmS{c97T6}l|~6WOR4D>?N3`j9(PX&A)(M^ zUy$bl+5BxkA96<;`0EOgU3?@|?fdji`SHt>-{f2hr!|R%4aD^| zw|PjE1i$MoVgAOYw(*HS{qWf-6Uw!5hqYYTA_CDE1U3r#X7I5wAw^omUL4&9;;&}+ zx|YI+w5cMDO%06akuPoJT9Aho#uli39S2TW&W*SxQ;!7tO&Ae>Hb!kn04HBH*Izt( z^%?>2@=@KGwy*b4<32=aLlk}rbbdT;uwd`&ASZz7OOBNfQg@j4tL5R_cp`+t%d5$T zmX{9c8SNlHfGzqQS}fkH;y^gLKH>eaU*KCnL5xfdg(er|+=u;P&~AWuZfjl)0>zTD zl}VE5o77qtbd?1m0TkV9r1aS-U9(lxw=2#i9TFkH_m&XzN+OiX4ZgK~bGrF(B7X~G zBYpV5s28o(IEzr|-kg~kq^XVyhGU~Sr!qqk<*twefOOHKfHe{Hto7ezd0AoP7> z6ixo5aTCQ0$r-i^f%8rf2@9(k6o^^x#2pM_PSX{?letMcWbiv#5`|w1_gnwX72TrQ z#S8i*G9}#eAD2}x!f!mE1wV4)xSqc|m-CVAbA=?Y`~h=L|2;I&U6GD1>f8_(3#*66 zl3d4dj_b({@R|{w4Gk&%l%YHsK8Oek>no{rPkdYbLw;kfFKSQD>ko3f*kz5GML$LT~QK_r~ZrdxeQmSV5pd^ zc#jfZsI0$=pDamU&2iIihW+B#0`4ZJ2pTEq>_&<};!s95j$bEM^VGds%9B#}-yK;T z*e_RSA7p4VTys{OuR}<^e+T`XFnuCfk$8HG&v~6u{nlXukYSLZ$gY+HlOI+sfR%S{ zv%lQ(jBdjftE=~W7npS8vjuM#XBfrWY5QaO%?glK!1JG1M9CpIc^>z9sew-w`@#YH zeG!hi=(ZPCd;_x*L57un-E3@9KLIX|I;d$mPiCErnxK!xm`2fUl0!IORYk8@M5awW z0h1)`JQJo2<3K9$5!19y=D}%M-YPov?vM;$CMI4lZ+4~Fq(YeUwTN43Uyn_hV+N&W zT`E_LnjpW4GtbiahpQ^*79JcZ;3nBzpV~r$YGFeXK`^Hf$gUEy*)Oc|9aZ+!3WFO$ zxAYU3Vp~E=;rCce7_lFb&qK{uLP{X^W{X|pqK83hL-L?IBT71=@yvF@kQQqT@BZ_AI8|@y_6xZ2Sx6;55iA3llqW+zD3V|hQ z39`RH0$QH(9|GseD8Z9L%3tJ9YZ)}@<0-Qtp|+J0q6RMAcA@N&h5oW3!IyS6Q~nEn z{V38MPsjCC+T}#ng6O}Crczf!7K;#Dt%O}N=~;~i+hePY zSFFsn8nsQeYtkMm0&%YHsv@P^S18IlK`TNd-+ElV6<#=SsqM8Nhc@wdq$sqwmWcCW zq2~4BAT+7B9RVxRtSp`*6ryi~$89%@jBDwJ6`k$AoAA4xNqoQ#!uHQpYxC;IPk$wW zFk6o@kS$cgHcxy{{FO|v>qa8L6*3mF9^W$QcGX9Ji77P4{}^^@n7|G?6&9sV+qy}b z=H2(Kpr4Ett<@7seuYrMFFe?)&uH7ok7hf+e)umh|F+5N?Go|t#2G{3>Ih**0L{Aw z{<8(Qm`|VJRipUcxp#sfM z7b6XPF!W|GE227(bAh<%D@;!-XZ~wY0kB@TqNpG=i4H@yoEz|u(1f&Os5~?gsKI!F zjkbb4eMZ@5Id+m@thM}Fe9hu|uO~`HMc;G-RsJBQ5R?1yP$(B1sJg07ZSCpu$l%d; z!L_U#d?2ddqVP3+;hxi(lA?l_>O8zQr^U-DId`!LK!zZMiK3jg3mn9-hUJ`>4D|(J zje!y|@GWKL4eDjcQira-W_sPAKdm*p?-qaweixVRRKE>E+OX>FPnx%C_$%AF9hsXZ0vufjExLQ5YN*biUkmV;Z^*7lX-W7| zhtOYqRlIjQ+Ov{5``tbO^qTRRY+fb-Fes-A9+D&-Fan^XMVy+wbpjIY-W z7|zjF(a9_6!Nq%AKlAFKZ(ZAD9GvL`C>autec6K_+tv*%OAP9x6F-nGQN~?91!amk ztoa}v^(p7P%Q`1VX8qIHv+#aTyO8}JQRBQhW1bawhb`b8==gVj5(Jbhrcr~p{C6htDd}6A4cXCm3@E~2U8(mlfRa&h?30Hls+Tx4vmD;k|0w@ z&M=rmRDUg?zTM&f8;>2!0QC4{)o%J`f>vW-Ns$uQ9Tc5z{&4#%X*SM2=Hl?IiH(G7g+m@6m4>vb#)r#qOe-RtVY8&=1yZS1^H;d~@fN%Z za*-99W`upg>=p9BUW4zl>Sjs2jD9NJ=4v{|{#V23n#bF=bgPca_z}n)CEr8(16j-q zf3(O+p(4jp>aC-NmMJ+(f*An9>WwMiaOUUrLCf9Uapp^_b{wLh-<#JJB8i?qh%etd z84gFFXV9V*c1KERZr1Yfq2Ozt$5?X@XI>3#au3HjX&|2dbhZpIg;c*qpvdstmW%8! zx_SUqo4@@a$CPkg(%ST~Yqnnhk!c{WopDZ0^}WyL8u5V#r>=2Nqb|x4G=*(2W~$gl zC}*$2&!e zv2l|O^zHO$90{DLA0IM|EUiGupMCZ`v~m9M^`N(%x`6ANR$ne~zvxW5YsZ9?%58SM)5i-Wfo&|MjSZ2Fjmwu>QSK45-{_g!zw_ z{g1l+_m^%=gS`LmmOlOexa#Zw^{<|gAk!#JVkbZZkUM>c*Wn}EakaIMazIg&ihf*e$KxVyzvdplA>X$OmO@B=hr1T9#t)^l=?8wx;j-gU1I%o z6Pt}BusO9qO2~qMU0MT{E%an5ObE0f^TY{Gk6EOmlCr6siDC^#qx%cvi>2Bk)%;$H zr*@Z2k+|4c`s?eMn0Z+;wl{r5`e*$C7euqAlUZD>voP8Y$ChW3$4*|#waQ(nF&#eO zF3F$$z_`D^KUwpElv`&JTU_-xgIWyRotjmn6K8E0uY~vnM*IMkuz!3=C5a2w1JBl@~W9 z&_WLq#T>UL21ZI)qmnl^Hv0W$!r#9>`v#!Ii+DZAb4ykpv+uJLLC>intzS(3H6T`? z&fQD;rGiW%$HDcBU-8i|<2;K;f`{`}@kI9Xu>z5qkx<=0KM`a?JAiVpP+GviY<%U} z$YXVlxjyCeILpszFyp0(chDq}>e;0OJD02%h`$>A! zVGo-Qa%q9ZJ)h%7%=d{RUXS}`R6H1=fxtXcCAcC1uG%s$&DEsZqD<(e;*(V*9kva<3h@>#XnxE#LYJy#dq*A zF=rMnb`%5=iv}!Yff&)`K>uOi`frG{QHknDV#F1`VVi@Ir=as5o91+7M_wduU6MeztZKseQV-i-mMUUh6R3O-e$Msbo#8)qy!mFKGAoTldEw9Yio ze50Y3!;a9OI;~5O@`Bml3cnmuTQo5p5{~N9fvUsmDN7JrQZQ=~ftyb^sjAAh8g!Xt z2}+StK|%p?e3T@^R!c?Xau|Gw_jmQo)J7aE4k7L(Z z3eD#($2;XScZA>`Z#5!JP&!2@KW+vPez4ml0%_&*qA*Tw4fjX^(CqYetc+edUHjtb zQXw^BB_CnIZ0PhW66v&bF!EmBPA-cjYAWZhA?s{7dZi37xI+@1+v+(JY1kLtAsuptdcoo9 z@aV<<1LY4+pb#tL{tbd30yfTjoAJ<}1*0%;`ckfq?Qdc!feILw*f7xO55PD?z8Qzj zlSSrFC3d`U&UF7NYyRLI#Syrs%YVB^yfxhel9nRVq0yGYT^Cm&Kif);--;afj;6a> z{8LHoK$18H(*!rC#R=$#%O#Te+#F2hN;qymZ@>I6QG{idJz zB&=>9HzpDIk8*Gmy$D4hmad9{yNsaQ3HQZWKk2kkTX1{3k7rOZ0xpD{z$KiL+-)36{7I+LKxJ?pR{F)U5kP0PNfUC#dAY#Js1y;!%H=5hWJBNPX?(*(uPA>eZV z%i$aOCa4oe|1B^6KV2a-W%o$< zxm_6XulYi54N^uY0MJNEnrzl5s3?I0I)KN~-vZ*W}y1u`>z7z(3JTJtbJk;#;N);D@R_uV+4R0C~)%$XLj5V(>*UzlHKoH6Z6db~5-v zu9cow6EeVB{Dl|3l>@$(6BzGfA@Cu4kJeD1iy>ML^JzN%yR`sG6haVk)PTal*?LcE z5L^bD6alk#ed(v&AVJrmIO+=k>lg$Gx7rW@k*$vh&=&3L>gw+Fl(P=r32o~JZ4OS( zfGF6bXUbOFlm19TXrB}qZ(t1tkXwmrc*pWZ6I@q~AUH<#n0 zhaoRFOE#>;g3f<7ls6R*0Pbe+@Xk|&xNK_`tqp4|i)}afTS5=tL}p3~fgK-FTxu%e zgg{AIS>ndV=eA%wRK^Yi+|IU>j#qEtXwQjII8M~J7}75kj!a_xe1LRrJOX%%J^})Q zf5r!Ob*gBgi%={a9CL8GKbO7ksQpPOT!(5{3Ikfse#`SfW4CzftU=Mc;;G9m)$SqF6nBHf~{$^i4b8HdBG*c_ti z+y&@fYc&v>9TB8S!wAsy3<}mM><}yO)8JoBoFound;I@ z;s?E0U=Mp3U?yiv$G_;Xnmb29`2)*iZ~g=g)Ei_|3Bn>_nlLN*P?1Dv-6)>n^WxTp*ih$mIIAe5_q0j+TIp2S5Vo=iU?k9} zgJQ`3i@??fZRoVyK~Vt#YFASTk$ou7_u-L|>yDn7{i%XzBmyp3+tHda0Y(FcY<^se z?lg!3BafQ*@)R5Ngg?@ z=wJZ=)b=;Fm=H*RI{Tj}^IO);v~SsEmANMv*}|F^bI0}9vfFNYL1r|m=b2vwAfw;fdr0_i=D%zM z1i1~ydXR(R;f4@gaDRKC!&hL7dXe1?*hL!o&i$Tib`5!#UP9qR(lS8V0L9 zZpK>P-t7@Y{HjZMl``&EdsTh+vB0|Kb@RhO@#U-sb+PN)Y=@7}F`N*P%M55&^8_|^ zIJ}=+?*Rb1PBCbF{f%W)^cdscg>vHS<_QW#G(;GviuGN7TTma728QlXgwDj$PipM9 z<=>buM}$vVH^4+3z#>HCh@p-wKMi#Y_Oc;TD*+zA7}*f3Zj?+lL<{&ab`+^)r6 zO=sVa0Tw|qkRR_O8{4~{=eaWNg2+TaNLU0ReKWQgvY54(Yyu=aJQUz=@B*3M4@FM^ z_x*su7Wkfj8>@{5f*VfI!0)B&qn|izHd62}pesLp))KqFKSPr^upo)Q-b}q0g%9NL z6#4y53acOG3#=&HcRhg5p|Pxtssf?QL1*v%;vApOFu@CFOF(l8zhpZmVy zTCA@Db`diayuT{GQw^*3u#)vmsQ*$VrUrp_W7TvtrA|5!cT5^ea7=hFD}W$3_1y9W z&?)1C>7qqY<4N=3mj3I*hxq&Y48#`r*a^*4SPO*ikfv7v{jZa)0e~kSbTM0$qYGP# z&vb-fvHX>kWc6#XC<1(i5ZsmD1zoIjV}iW9c_hMdPr!FciQ->xeO?PhF=ugxQG$eB z*Mo4)&#{re_>YFR`%zo+mP%Ik)|DZn7=}8SRo|P>jIOmTEk_*0MbP3%6YtpcSUR5U(`UmTmD2dCstm3P zZCbOTDWy)j$T%gid*9=I1>j3XWu5P|SJ+;GF(ZEygJ`-CXvk+0-*7cr)qBp@S7Wq- zPc5WQ$PVg+mw@GV0uTNXkQ=iQ|GIVQN5ERlj5D_IUOu5DDZ!}p&SSldJ6*PR?Eg$P zy4stFEJQ;O4BY^fN$($CbXSO?xpXuQG1lZyqn$YW=8Np8Of%X4!`NF#HT{Nv+{A#< z8z77xAt4e{(m6t;1l77l-ZK@Xm=iQ$KmDl5JmGbC~TK@N|wQ)u}l-FH;{SN>cc< zV7JxmV6G^96wC`5$aD2ut&yJ)XM$O*Wf;4y5NF)|2j&f(RzP?s4GK>aAFy-c>70pOCw3#1EURf*bOa1 zjxy#+Kb+J=zmff|SUe7HdpFddp=~TX?rKK!dN_r`FIDqQ4RK7C3s4cwZQAlD&+{=jzP^>u>4_ zK-_F@y9LeoMJ}FE$2v!5JpfgX4U5|1^HI9qt`J=cCzoq0Nxa;bL+bI(K=EgBVx6Vm zq*m!GT##SS`HW>vB(k7jW4v1+>bmAJ%^1?m`WPxqzr)M8@_zHLQYvJ4D+E@auwkQg z6Bf+A+{ApAv?P>gd^%6cA>nC<93~x7hQTX&QT$)Ga&X?TnL|&kGs_B7;n<1YwbV%*;w11I-7tKv)665PwPHPGrj!f`%9nL_9k`i+l zW{+dYm%(db_<@A;^~(js+(W$60pQ6v}1dU&d4NS+$Rf{@igt8y&SZ zwfKpDV;8l&xu$3#TGV2w)9ARB%PAD53|5{h(fezBGI5^Ci-AL)bPx@tW9#Q++jv$F z1{zX-EY?bIS#tYFWF7iMn~EbLL!_!fBgnS>kv8;GOaml!Z03vOhs#m=mDc{E)gi|0 zMBesscA5vy?wY!n6B@use&1NS?f>`b9SWTdVw0uIlfiK?_^TB1T$qr4c$qI^emZF5 z+q@XC_?=gJ?7-_P&XKGua2CVjDt944KW3Q*dTaUj6~SC4-16gv6tpbtW0p}p<1(b? zoY6S($~LC6Q(F=vvF9NmD$3BCT1j@c7WIZ0tyn;$P&kABQuN|Hs>o;(vUr5+Ou#td zwO|?`5ZcGi3-U&_5)HYXMx^>)gw1gUF7t4ni|%&CdzW!XcMIZg!BR9Py-zKS_^38S zGV|5gOHq>CN-+?IM4;J$%spg7oQ4ad9nN*3tSiEvb{Ay_J7pJ^>n`Z}B$B{fJlWcf z3thN}u&OrtiKF0VDi*^v5s^ZDEs#{WnwXRu2-mV8+qP!7eLT70oomX{lqxs+R+~dp zHy3kg3^h?y<;~(%(p2L&G^Oo8HeR`q2pGU{1#=58Z@Py6z zxQ$>r=@bQWFJeF1=B6zMt_tfk3MS{yYf`+ot*Pe%qLzfJtWDZ^EYuJiB1Y?kY_ixrQB$>%E-hB~{lVqoz5V>vOI|j3@TIPmSLY zCy7GyZzYHG?zF#R!7pRmt2>cg(Q%pWnMNWEqSxcNLXy{b8UBDu5C)W>y0iwZSPlg- zbeDH44_2OrM?JLt4C=}*5|+46dR;woTW=DurtdN4rlaVFR@<-)KgQKZv7!mgqggm= zh33(rl(hsF_u@6=eP|o~dm0>@1~6*HN!(QOe??Yb-2|cGDLzAT($b53op8OvzEYp< z6>-`ATi6gW53N(=DKE_dZXtr%;5p@}dJZYyxk7l{?LU_*X_?&W1Ghi_y89w8eX@jR zjYQQ^B1vKbn!hhz;rNS~{#>Cyk~|kg@aA9H%_(hNF;tS|umm}b7VJ8<{dR+>J!vAV z3t|m^(s@L4p3VB$Q8{Rj<`c_=dZujKH@LBbi87B5c;`$ECe)iYVN*HMGq%XTq|JjbS^Rv-xI9F1|&vOuG0LW+9fvDWxYqN zF=qL2r6i*k2APA>F^w3JQVbZruho>($dWvDUuDvIvm4;5+c9w?v5(Z8s4Yrny9*o^Uof&aL^@pUMDCtr;TBwJpE6oX0sa?T`(i+vF z8Eu;+y2WzKywguZ6~-AYL*pfI-FV-q#5JTP$>G-7AQr6d2jwm z_H+QBGt1pkTbw2(z2X`vgq{+i155PO6B>fjt0?iOkeqYWk##ktTc9@P#xpQkA!?$X zyfs4k5Fu?`B-5Bqb$+9ZS@5XiqIa7cnRj1482id?Jpxl11m@;|k9Wc`HBKV}KdYlz zs8l85jL}0axa>HidzB?>#!8z?1<(G>Zj`CQFZ4+=sMxsr3{Z5BG-hj zO{}-Ztnm#)BSh;vv<~Fz-9qLc3Hpq}QT1vz`B!4msa?AE>0N}Zh}s%-mW)aK>o}7P z>RF*X4T{Yr7r9Oq9h1llgWDw9tM5=2mTy@m+caaFNMzpoWbe4tM5gb`LFsknHSW-x zOd#2AyX+3B9OW0$41EY^%X>9DI*jps>1G#fkloT=zA%Fpw>G7zWIa}Ue+Z5?b!E|O z1qSfpUYKa)JRVb9?XF={#nMZz)=AXgaisP!$qA#-#uk$@X)+{~sl2ReAomT+C&h73 z8@_#ePI%4RN8Wdu*iG z9ZHMTXXx~MUYNTnfOCuxoLDVrSB5o0iYV7)@djb3l@vRdavbdnza6`Hk-bw<5tmr4 z&O6Z(_{>dpt?fXk7M`F`Jc$e2e*FZp#T3J6#nu)btRrc_^$)csi-^PgRM^?mxpgHx z!+XseT2D1aO0V=hG@{OEfaSeU`|a2acP3K0Q9XSnCHf&fCTO0eXu$LrH(JCT%{=>X z{xSRdRoAHqpO@p(p^6jO*rIbK=k(gWu+KjKa)u=uL8n@Ho^bn}`*if^1pAfz^)&&zrO-Q&-T1|@pkT= z0E#+HPLy*)cLO)RAfVNUA~~FRA_Z416JaVNl0cCb zuSZ=K?YyR%$KwBjtR^{O*=}_PIm|r3g7R2sq|$+yYfeU~?ZF%Bq-AI=jEksfeYsyL zK4Kk$mj5#O|iYSB}spd`|g3g$n6DA9Bk_W!WedLmdw#fic31J zYdw_He|i(0ntklvi7!3{0itI_V@f}hZMpX$Uta{|DrJbh-N^Y(OW{`dg?(K#r_sb{7K3#MY3tb;!`AJSnfmV z@r@%>pS@|bxsf)t=V|HWF%+J<$T$2(?KP}@cw>Jno_WebW-94e#>7n}N*3Db$IV?| zIrG(V+aP3DH@Tatosc1g)5hL8 z&iLg{rnfP{Ocm;=_IDq*!ex!C^WtTam^AQiFWjB6E=3{y97(XmSxk+r>0O#6)Jx37 zu>5QQp%NWL48yLIWv`)5(T-8PKAlZ-8&@=Us7!3JFQ+r|6Eo=>rMMfD0R1Y zwbSyHK;e6azM7{pT!EXL&tlozKF;AZmH83w7~QQIN)oux<{RX2k7{s_w&}Si1rRAV z9%x%lWXybIDwC9}K`fF?3re=ZBS$e~-~JtgecLm4aUN7(2}aaY3dy zGF=oU;u#Rgoo%BQr^(KXG4iMsPaq}x!m|T$hFvypg7#sQH&35ADxZ08ZgeWf-l2HW z2Xbf=a%yEK_-fni$n37a9Wk{LM7)UE+P3&jx3}M3xG|$t4oXK`W!t>Z2`^xvw6&2J zIk*aBf46$~jV@^hv)AyYuc(*|V~3#Yf_!)8HZVruwy;TH?zh=9p3?3y^J*g_&PO_q z#tG9qleXR_TRMkA;B|)FwNti}ZKxbb5nonPb-0Fv-A`%P=+;PhX8(>xM2Z< z2|8V?li`B2=7ZCq?EhUJgU09xO<*h(1jebT zEa|yb-f!I(;)vUH{@iq1aXbj;A5#{w%i#Lu7Ghf{9m6@^rd6c=KAeY4lidYwB@R`D zU1*m3pD^7w`ELSln^3QZz$!Fn5?dF$H9uP3a4aGu(lU zw{&h(QZywP(nRiSF?+1GBa+9cTcZKFU{$-Nhf8`p?r5=I828&q3T8u2MY-^Uv!4R0 z<==1ZAj;v8qv56_QN0l*-|Efk_iB=oxn8L7?ww)eQGt=ny30$W$ffsQR_a96R6zx< zL_4%?lsWIPaFUYmb2C_hYFV#Wp1$1H?X59wfnD;;g6oVfi|W{F&Dyp*vE6a$l9jia zVq>M){8)IaB@CFnyLQnv&&aDh=fkBqyyYm4tIfqBm(B*bR^xF6&@bXsbp201iw^LF z*9!o#_SK``Az4ILR&!cmu|%}u$Ki$YMg!#YH1FvThP#sMa7NWJShgZ47sRE#otR#0 z)MIPLiS}?7uktN)|DylwW^22FDHPuIsksjKJC-8;ThXP@RTm3}Kk5Z$om~=eWi6zm z(M8S7s?$Y|x}Hy_P+A-d-H+G(6@RGqlM{M=6FP^Zzcjr59@IB>GN6!aM-j7_-{^y* zpe)#nIw2Q%bNPMqKG-f<#B$RfbS#O8VM4^MgyN_zS#R8 z#s=ByFd&v;F<36+hAICno)=jb!S7u&IcEl05b7X>VjL((!6ob8gBG$rmO#NZ+zU@6 zE=%m;E*Rwsh6Ml3Ir{jND2H+}AcET|Hfd%QhpKJ=dl(w~(%w8Q;)s2|)Np-5YWDngCrJT-8WPKBTf^v>HvdIwUqsHouR zJG$f0duIVLrZgll69muq=f`2?1<}4x9wtBXOe7gYA)M_hlbS??kh2?ZT^(o2~5Eaj%YuZLicGA&r39EgK`x1tI0F@kfyK%xo zF9%T|*QVm?gi^~I-^)}*==vktu~A-dp@+(k{}gh=Xn}+g!YLmWt4t-=Ru=6=wBho6 zFxY=)u(IaX!yyBWYKloyUTip`oP<{l zjtMaB*_0?xYkl9gMzCz9)pHuHv(<~TxsF}3?R4D32<^S~^3vnqmkDkJDW;uZIGlEf zbRZO?jBSC1uY1Ht)esNfOkmm5E{lOWSg`MR$Rk9?`A#|Eie)X1tCnDyOr&2z9Gw$- zr+9_UoUX_(bqb3ehTZm*=+5GrMLE0zZLn7bcubDdV>-p`SvMi7$1H1 zzM2p9O~#%imt0$3VuFz7JLnD}DJdpfqv1k8DLV|y$UR6_KSNw3l8%)TsYD0`pR!p& zR(Ah6YpT~bvE*Cl2QR^X7{yb}b&bbRcsGy?rha9aKyh9`h0jrNU0eu{bP!2F07W9y zX@DW&gIamCgrpIIE#eyXeUKEhl>6E*p<6`~pZnZ-JG)6+(DTKOTyduDMwDi;B%S;r zo^B{{>xYx0};CB#eTFo z%mjgnCOwzBTWRfl(yJ8+ta^Ei_+dNBrw)@<;@D1n^(m%TRSBV-ds zoBq~ai$vTWYm0P_`qCo;o7Vlc7Rp5vnp(Hh7y=$e6O1OR+y~X?BT~wexRIYe-Own;t6L4 z!*-O!QR!be`G0@>_c#AYGMNj1@BjbT_y82}e;)29SJG?+0E%zgjUKzxSQ%EJPYDdt zzrPBv5!G!-j(i*XAoQBqo22o!lXK8+=1k0G!n1`J11pq}aH%EgXNuTe{VYND%QsMZ zL|-0Gj7>Z;!eTOSU!=>CwtbYR`Y$&QxW(aN0j4Hq=l504J~UYNcZ*e~Q^g*Yzck&0 zn>WGBD)+891ti|D0OcNUugoR_o4D&WAS}6+${w`NJ`(=?DTUShW^elKPrSUdo(lK+ z+R7h~IE6L5zkOA1UQ?de^O9Wmg29viL)M+LjxbU2D}NpGQgR^ZvEm@h`5OcHtquK0 z-JjL3oA_!~r&n4EMSsw_YWYfBey#&o#@Sonaa>Z)eW%Pzf@htCG9~O4!Cf-u6T_`% zY5Bp<|J(M)wS8myv2N~{T(58D1>H|v|1i@$Tz>6l;AmS>(h6MS`Um%nxZ}8obhcAG z0Ce^DO=QSadlGkD9=)zM8Kl>O(FqrA$Pl%>IAeJ5Iwx?v+T`|a)~!Y*335pKmG6Fv zWZpgh=uFB4?@E`Xbrl51qd3AB@pS_#Lt&V8b9mIy@5RmgZ{GK2cS@}LWpDQVUR(f7 zIHt|czx^+Jd|_GPD^3;Dwo&z|Qe(pBS!ws+u6(`<=0OZbd+?m11M9!X1cr-PWg5Ih zLDi@Q4!80$ZEEUH`3-uT{3Av0eMaYZlehnM&4f!pvP9%YMK;WV4LzQzuC7<5P%75t zZ=zmn3>d(Zo+h@J_?ejX3k_G`xVZT25cK=*YClxKGIy5!C9$i+1Z~4>?11A`2pg6D;EUKK6g7>Zwr_^ zf@tE3Fzh6^?gFfltI1dPfj_{CvsLj1{%Ba%75}_z*01Tjl$3Y8I|aK#VBkA{r3+HL zC~9CW@wDJey>-4f=S_YOFoqg$_XT{?uB0ghu2W$Vk%f4tPtfg3BxJL-Wj2EPJHVc$ zJbui;ITSNqph>|h>zbJZ7-mPMuh_=}Z`^poY^y=3Fj8umL4cmAoa~rByX=vt;x}4m ze1E;VYc(HObC#>X1sE74(I6z$2q-rBlbw-!pEEBJ(_UzYo@^gxjk|=YOIx==CV`t{ zij8qYhcHU7=(mXCOXz>^v3XN3Gj=jsPs-7MnKa>Yr^7n>?q%`II zuci{rwQ37da^t0j+^hhtvJOyQ_s4HLI8}dcdFDNLODdtVizD=SYa2KNV%eDqNfJQY z)#4LN#Zb1ws90#>TOIyLN>_m}-#WVg^PRHyj$GvruI3~NHOA8M+KhAOeOE10Fia{l` zxtss$@w|+uaG9ujV)Y@ghbw5k;3o-+>jXF;wnt43dmQqEUvRp6D%9Qh0~|m1fWSJ1 z+dciWam{)4>FdWLH>*b520{JS>n0hyZrI9XP3c981|xza;e<5%*YGv*v6g-A(nGzz z9C_1ZxT+k5EQ2f3MQW)H_ZHADoXW~d@e&3w^O`aZZj6JnyGPe4>*S=DkHcnESwuR10be>D{Kxb-zQu+Yf|o)tX@7F_Jm8fQ6T3KGiemInMDr8)7*j--0AYzp)@_P^a&j`! zN(^yn(|>NJO(rMa1=9Qa@7f1RHsZis*Ed2o$n&YPcVr_jJ00io@oHR9{yCP$h~!rXTA&yIBrly|S^)-TRv9>LuG4uk$#y|DDew#k`r zo{GgOm)B#wJpf)*JkpUeoJ{_Qyl*s0dY9SSp>+}D5UX*{SLdg+w6yMAY#Ue|d$}vK z^#E^$4Ta?<`beXgKm7UCq#!vZ%|Xqx5-;AI0*H{VL+8%C+UQWlgWyiT`l%|%FeeuT z40S)H8aXH_`Jo{E`P)#;Pa3Wk0d30~0!+mb3=w%dfiH>K3-6+}?k^LD6cGyBQUMgp zIUZ)8a|{?rj_TxGA_!I3gA|d_Poi(7$H@rgoqwUXXuPzghXAAS^+uwP%=a$7Cp0VG zWja0t15a+&^oJmdV@ruS3ga6(Tmi+uW`NVF&~)aQ{ehTXj3G&v?mxvOILg-YC^l7C z;5abW{Nx^uMjbInJtHv0bd>lsoF?Dgd($WTW%=DB7E7NYX1E3{f-B%VJeQ>2f6GQx zj+R@+LI;9-p%Ez&?LvT!>0IX{=^k}ODQn5?A*l#mj*bq1o2lRt+K!VFC0zVfrj+N1 z%{Rx0Zw1Wgslw;7oX&j(J+@b#AA-$8CZBZxgcKNfxSC9jeP33|hW$sr8G#DUN76+tI)L=`O$beDLxS`)fUFas1l zOwmlX-SQ}73Hy40i=xaO!#js+~Ne#%+oM6h2|8uqcM zC`sW$g`V*{A`RB{dXMHqZBVX0D=3cHR(mcPB}GT@115s{=7o&f0IUu zs^>I<$^8S&pwU8j2vP&L79%ZO4sD3(dt!zQ$;>FR4s3lk(aRQHBg7*4& z%%D%rCyt{?OgX@|)j1VxmeJ?1{MQTydW}f2j^LS^qEo{;rNxme5{Gsz@@U4=jnPTh z@QL`rA`gD!ZIf5L`wZ;xJERdqfN3|w|Blso)H4CstZ=;9?rTTSn+zyL4(4)0#}Cd& zx<#djfI$>=^Kt^MA03JhuXE})R&hhpwlhJ1eXkY%h!aop^Bps@Bmn{Ks6pskh58N| z{Ui*?kP%^U6d}iV5~f4NTo|)BkS|`w7lm~~P@mFj(>hZ;BQGN7lEP@qP1FLPqRC=^ z?#mB$W;kd|3!pFUit+1GXhPB&lULG06%Z&kjFDqCG*))VtPGCc70YU2=lR?Vcy}#; z;e`(*tIkyXpPoEOS;*#ktUf7@!hjztAoMV?9*qCX<>i>r$P`E_WH+kP7Y_G|T6RG( zlR{Sy=e7m^{*d=3b%l^Mpp44F4wa+T!O40N)nsZkkqxN zY*C^pLjt6_me7)FeCt|8jp=B{{fkzQ)2MhU*i*p!hHWS4B5O}i{;Vw{M_H(thazo{ zvoAePz(&rTfu;LM0;+=_o}TjbKqv?qBt}eVOG~;a8`1Lct$&o{m{R5VwnI_wJAQ{q zj*d(D#K`*C8fr?M)%PY2DaEGGShZ5iioh5qTrYcG4( z{wz{6kRB1WM zi(g9TCh#NEL#9Qgi;JI0*P=gwd>v^Q&lI%p4mV}Tn<3=YxvrKu4*#;sriYLw!7Azn zNK<6%HI|<-Hs2g!EtQJ72-{zAB2|Mj;0XBFNoOj9s5#@Cdve)?L%EsonAPr^er=K= zexg56c0VLUt1H2OuFtNXq&m z^T%t|Qq^^LHJX4s3{rUQIYu6O9ah2XBw;WJtWgR?LkZiLQq3aCX{m$;*@!Pb4%r4z zMmUJ!8PGn)u7}&Jg`AGB>JlTg$Z_@xz(ejc=aLp^e~&IzAV>FJ(#BNXOY{U@-g~hd z7{=i@hM)je<5u*7uNMKxGX|(CPViLAz!A@+{PfEGl5~Dn!PrSIND4Q1d`v`&BQ=ZI zD7g-QAHp!aFc-#_RK_3WS5)?$;O*UfEMitU*l86K5|TDUF?F7f0-eDCH7;B50l}r# zu>L;+C^!P6Z9z?q^$k-vVaN4WqNAM=LtkaSQ6tGdgpq(H%~=xXH#F1k`lP8C$XtC$!w7RxYLeT~ zXH8b^sh%AsGe^Mga8=Ho3boRT&k$%>^z8=j4HSR3KKQE$;bR2RDqo>2gI{*X3nalr zIl*7y2sn0ADiLT~lW(*Bk?1G|9nXXUI{sB&^{@I8Tad;}Coq-s{O2h@q_KzG zynS2PwA5e>kaSB_{;b!ev}?Ha&V-&>PSk&-^CAyk zQs6|&7xyN8Y@afO@Awp~$zvJOa}hbg2ox;IGZ-PgJVJ{E`_K@urZDG6q zAkj?Ndw*KDSL-VO9+>Hp2cz=!sX9XD{XJZqwoMN7L_4!CRI}khnN5YL^=MUEZ+6;W z^9eXuYU5g$hl=WyZNHI;j*_LCsov{lax={I3Gh~OWt95_bNcR~h$D{m36$%-aN&Xv zzll(j9XNA?`jq_c7+PAz3ltUS9=WuIX!PmSgzI1}z2K>d*X2+?=IpLwXI=-%EsY6x zWO$UGK^|v<>BOIRyhz8XSn8FLvS_lfl?rREuR3^Ci@V0`DhP+F&-t;2!x{Qvmx3Sd z;L-DGXPIrgqKL>v$+-0M$$yW0M|tA3Rn0$TKWGB%(ImG_`|4OVCVA)9nNZFYQ^$Px zW7P0l-7X?^;bBhKS0|5ljx21!8UGdVAeH$ZKVEc2XgRGsGchr#?EK6*`q+M6-sr#4 z@EkH*W+d=9NqSeh{ZJjG*#6F82@!IHy?Pz$sw2aacDudj5@h2d8__I(Z(xLlB0ubk zdDYgK;~ya-;`zOe^Fe<I z)^Yjz(w4Tui{cc}UUU(?W&J=4K1YX}&wF6XENyF4=f4>&! zOH0hYz`*cGllHm6Ia`)E7^PHbn*GJ0wWR-^ER{$v4m7W9+&|pt_<6t6mPK>_N;y5< zuIbIvb-wy2q|xo4X~WxWziKmKH5~?Q;1wE}c?DYC?D(|LJ)meSGS1B4ky@-u7p{5Q zVR_c(+db#gn-y((zI24hE1(8zso!9F@J9|g>s5Mn>6j*PbRI2+C;NL_egQ#W`8)4e z0rHMl4oHxH6p%XZ&c zUbuSk(uCCkd>e2D!tDgYHsqwIbRNB>s<{5Ki`bT;ciQ&oyw%c;g&x+On=&jO!f5FZ1mEoya5jcXxl-iixS|NQ|;ey!oUK z#D)6Kdy0RB3sPc?pzwv>-bSRyI7_Kc(Q}%~kX@hQlBIG>W86uUO}47?Q9@Jq>z-eERh13pd{UxEB@WRR;ah zt;StBYFFblGvah(m9~LBHg&z+#0IYMIcR@{kCpXVeYqj*?#?88 z97IiFWnwhX+mb&-eq2mh~8e*J+p&~)<_yL0TApgNV~ zu--+At0Z0o-y5|c1$jH(azBN$bN%Z_i3nq#qi0JiIj1NOu^-Aozw0u7ly`BD$G}at zkWtEK>!wq%)u~raN+-uIkDokwMMkJ;XC_9yeUi@f$@k`6fg8@3Q@&p%PGr`yiX|*~ zQ%{Ks6JcK7os&x*JZzi~m*J9jYu1z-M(w|yil&&p@1!M}v>B&{cy(3X@K4o&K(8`c zJe(Z6kL$=7k#+v^BZKKdr0(I-OYvTvjh3RKvu7k(Oe6j3kJXw$4sx_j7;2B0><}Vh z_7|s7)dPWRvd<9WL#x+bZ#gCPRo@Wg3dT~F z|56U@xzv^09jN{4ma3AB95F^4il{C%#8S|c4{e`qy;R2tOX6*wt2 z9a{fx)NU~*Jhz-a(*KlxI{4bTBQvMtr+&v*O0VQMpZM&+*(p)kL?&uo>(XY_Y1|c# zd@eQDs2ppbZ3PncW5E>ZLU;o6Yn^i!d|pbw6j~e(x-r!LNvuiFAn!Ph9$GocdP<_jCEdHTD0rEmtl2b#j3P{WA7kG>xp@Uh2A|MUC@8|!poH5RAjFG zuhql!3+{sXhO);EA3IXMa%EVA9R84;`(#*BTUl6bD46H@{^zvjJc+-#pGNT8 zS>?Lg_ZKQQUb@Lcra$Eoz&H0x6zrzBMeI72&Oa=pw@qK?wr~AExlzsTe-c|LP+;qpCUyB8DB}&0%O2@sZ+$JD$R`Z5A^67MOj{Ugn@-SO? z`NM~Y%0+sBZ&cguTzS;Grkx-5zl-_(*ec>r_5HQLG{!>MK7RR+GBDIW^d<@WGE--` z=_kJ>dE@uBm6G~D{hhzw+?#%NzT*kcuFSnGFmQ~wC&kxF{N^jlCZAagrc&^85-Vxu zioGRlttB=3Sy`s53a%ba`zdzW7O_Vr{kq;gAx$)}%d0&C>_kbI-6{E#9l!tY!hY@1 z`+e}3pGJ!DyI0zDUrR8$(%sXwzxW>-0PWhM zKP1E>_&C?VpCnLF;@sB$cGpOWb}-$@4Txqfo5~W`gUc%oHv|JV)^1-Yy%t>bWm%DZ z!CR#CPK(=K`e93mkZ^y@(g?-OF>cAUmKpw(N=JmDDnaY04f?PK*JSa8z-cx&fCQVzq?UO?VL(=~M1cdyg&9BB? z9WuE)wRb&g?TOs?^rgI>#~?SYp%Qpv%qr)JWqYZt!`|nMbE(TyiwpdesT>rK%JQa`yZr>t>YB06SM#d%fF4PQ^6k8e-BfP#z264`X9nvC23K zNxZ?y^(Vx^;hs-Q(+$fa>-Lzf!}Ks0)uE<`(G_l^<+eKDYOd`RV9(-+nq!M@2W1#t z0px)7SN9e3j~0t1Q+}@PYgfZrpv9zr{;c;US&Ee$vw;9h~ z|Mw@3Tp%LtTj(ePYW&{;`v3VCDGcJot8q??$p7!(KMx?}_mf$Y4epbGejs&vKv24R z?b?qNO}bQ|h;@mkuC6)OB

Pgs>116GsNydmYCUbI18t8%nj@uDW(lvmF|a_mQsfZf0&7{X zwVi4_FE1}2BGt<#GjDiN~Ac1-@m*nT(5B&D-!Uf8;R4+5N5J>@JY7idh>h#Cm#r;~XE#Exq|9HuBj(<&t3}Cs82=kvHrlfQ zho1(}3pWVqhDOl4hIE>uz7wn$7eHo|pngkK_Q5J^VBtwh+q#E9>irSTVwGZQAYZaI zNJueJYZF?rUooUbee)(+>IMI%yX#N2cX74|Jz0G7>>P02zr<~+F(b=#;#bFC(%FsX0A;`)_->)i%I zyaN7i|?3|p959bXA-CKtc5*XQ5Zp(0#B=U*3q93rS#JFK+IH7m1y$GN zuA;N@67tl>X+Y8_P2`P|H&1+Y>VE1@8zc>n#MQGQ4#hD#YqV`0fB@ec0*>ius=&o?xxu=OSi2wDI*kOeSe*r*|zzubiYN zaL6JG17T!#b)3lMR+YA0{dYn96VyHltOK_la?ej#KCm48)G82xt_AX1H_&Ey0f_(T z7D_W1H0&SRtX(j0>%l0C&A`+PhPf^hLL-Sl&0Ui7Zwi*)NugO2u7ESH@G3|y#TQ$bh z`AT|PBmMga2)t{Vs&zN&GAJu$GJZU{I?J{{PW4B)`hArq>R-k3FY0NieF5+l^ zhovi-Qe2qIIh6m0a1(CFhjwLL`^{EQ%i)3RB|=tovu!0ZNung5u_a=N+J3X`k*q`U zq-9d9dZQ7b4I5e0U97zNw=2?kZEcCR&~`I*>$P#|1o2zfn{N1!zBG~%u>J(8ai~0z z@?$;~y`|reu4i^N+M(pI#Wl)s1b|YGuoJ>ypUJt8%BFlCGT5EU=Sy4gZ!;>+cL^M! z{)H(=gzT}*B}ecL=&iK;^@QN561a)qs8q0r%TCNwuxygq_N` z2)##eZFv89MN57QOKwXog+00oJvvu_;BJqrhtM(uGV70tgFf z2A?#G85cXH5n6I*VCGMB&085N@1|F|gejO3L{$5)4E;m%o&SP}zzl!R1uJ}Z2J|0i zucd!GKPkGwaNYnE;2Lbl-QcTZEa$Wa!bfQPg6i=reCS?FSe363E*LpHOrTpuu6 zSJE3i0cuje%_+588j)S=gkDnQ zAAw)FYGefcz13fhX%N)>?{2};jGxwL0-LxRtQl8 z4c4*Ve(LEi%$N6jxB{h?BkHOrhl zO)QMfRoJzcxC^!}iocDGC#T~O0(G>bLwH>l&8sJ%OIp@*)X*9s_#i|UWb>Y6rPeO$ zJ8d$9E!V^vizkP!hA*D84NIc~ok6OP*>lV$EW8u#$x8WLuVQquPmsLN=D!rIv*Inw z)a%Yp|Ezr~A|C~=qVi{LPa|GYnp4YWv{pPFstHd zZ(7OFLDjy7p&=ejh0$mT4=*|GkSacgOPj)PO5)2{mV|TKyYV`go52afS80b<05HUZ zgOloJ`&yJPFH?C8#%>BsMZu1aj=;u^~J^sf^{9^C_b{^I^Bp5cPD>Ii~G( zDezw@1J{{)m1=#xn?7}|3uc8SfpurBmr8@aGf`KRiKNS~$q`NF1f3_EpqAZD(rvx2Uy9Mf`Iy9B>$Xm18n3V+OQrviK*) z#EKB7#Uw5LB-9w?I{1_Go#PmA$FTStw1-^k!)tXKL_{tD)7F^5CAlmZ=hxSFTRInB zH_TK0%n1TX|J`IK|Gf5sm^HFD4oCxBUPP`Pn=38Te9Q@n;3z;JJ$I9mtIP{nWEGvy z@t=S6Dz3MFd=RXul))P5vQd24h~NmQ6Xr@84l>uB!_u1x1o5XM zMFg_5A%cR+N)@%QRvV!n!HqZv6?yyOftD>Bj%uEcHy-))SByQ8H)%1t5`D_nMXZ7cnsoLy^$^tf@*JB1M)@dmDnYP z;gO-{!$o$^dr{J|sPl)H<;pe}0Cp>J!Aljz#_2PtejhX*TEDSvb>yM}r&-&MQWUtSN zmqmCpjJPTHZ@sa*qV~13d)om`=PtJ7>D@b%MwBOVV@ze2VLs{}3dx@$0WmSjF)3{} ztPVME`jhMG2Px}Cu_;csM{agUu#P=}@SD0LI*_!H!1+tpDJH7IcFa6H^&QIi0cJ)IL zXsUU&B5)j=N(B8_?P$kj5PSVX5tKSZ^bne9*(|Y*R=hL&K)^y_Fp_zRCjnpld?ord zSFeD~#X&loKAuDjcX0_%3nvLBZxosz*oY!x8#3w&l0h{*al0;i!g;1-6@WeRr6mUY zKC^@VllYxG1xchY)f$flwuOwgexZLy9HBOCLcmNW_F(-ySQk@lZs;nu}d%PMg9>=rv>|cYQnQFeqrL>#t644M>8cCye<)B z{+6`K2h(_FdQpT{LGwlVCwu3^4r7-iV2)8z<}ReSb^agT&N3*Dw(HhF2ojt?aED+) zLU4C?7zhpt?ykW#xD!0UATziP4k0)M2<{Nv-GZOydEWC?eW%{vrz%BF=&EG8d*<$a z@4eQw?D9jSt6#@PfTCY{*fy>JgqNl}Y}xqsmt!cQFdQe-NU~d_7uVrwjI*9WORo)C zHqImfJVl@-4#N$AbiYO8zp2^^{*#NQzZPxm04PPgSvsjgM0OAE~$2Y9{gl=OD z@qm}=WlyIP#;_u^ri{4OM*Pzm@>dr;khhcD3qXAEfG|@FR`>Y;_i4h zngG~8PK*I;z94S|TfN1YkSb;nuM#u4l0xoDRdIe2=}d%tjffc58)M$JjbSK5DEfFU zjlz{t-)~mpBVh`VjVft@UFD1=M=#3=YRcdnd%>|S(j&k7wrtezl)mM*46HMw>C<4( z*>v2#H*tu`RJl-hPj%=vk{Jabp*3Dy65hEml{&K3$0`BU7xb#Pc-AayJyvW?8~RbQqSc|3ET+R9(v| znG0x;rCb$eOD!F_sht<+6{}ApSWWpD=Cd+={K*~I5E%FJf-Isq5-w_sa+FbhlpN7y zfON5hNDe12v#Dk!^#ZMmiCy(Bu=V86CguJKN<DV5tR0DfS1(=3>kt(4gKuG$cC)yR ztlYGLZSTMQJ^99q{cM!;6`52*(3t$v47hMzX$5U*W{VRz*Ci#oQ|hVOFx56#@MH4_ z_!6B?2=8zmp`(+~eD%X`jzu-*IldOnUNliQz!$H|gfJzQK+Xwr_RiCwZa0&s1IA0- z8$V}=Z>8L?Ez^w#e7Rxd7(p21JJU$V#N=Dk;xW@J62C>;o47V+CJ#j1h;-6)H(T16@-f6b@xaPl|42)J1f7r*&#sx z{@ktt`W}~<yu94S~s&A|nnspCcuKR|&-BTbVDn;D+wdhrHg^m$%RjT~O zQ4ezuGrVM)sAYh0APTu8_qrFF8iF!?1g;~mm|)QPz{+E%KQeUU^Z3O5{^>Uh*d-Ch z0z9LmXFIZwdzXE{kI)#I-~T|-ShrUY=H5Q`84K>uRL-dJN7MxQhSVUMw^pV6*{HGS zz_ok~niC7OmHuFlw;gtl8`|11XHL@A!s7ldcqzoc0VZ1Z>sSTQ@OWL#S!@CVqzN-5 z)z{~Muz6A>;^UbG(cpB=@(1-1Aj4v=@-Nk&((^j+;v$Q(-|H~?h59%b5- zWvUhOYl${E8?@e6eE{2GOq~}AX;A`8=vm$F4B%P@f$lLO@?lTZtEI&GaAMBM}UP=e5G5p9Nhj7KeJF|k!CU4OhqoBU;8?f93n z{N1-MX3gARZYDCuS+gt{j3-rbl+K;g5}N$BVzqaA4Bedv-$l7;sI$<}&^7@q1#Ys$ zU%+ZnX!qgujo*Y44_A`~uaEi8?>v4GmcY7rfH=l4wF2o~T=ZEl0_xlpGX}jO{$5!= z8LQn>4sJTQaN_)`w8}Yf&U?e6qXpL-J&rOUbrL)1(7Bkoz|eGg&5gtR<8^9u!uD(k zTsPJIcUkIIUhE)o{Pr-Xz$!bz9}jK1*j(6y7%Ieq0!5u9NP)c~Mt*=We(Z_7U%-4M z)MHeC|9ddzD_J4U+%ozf*yj{%y@4n>vU*iMV)ub4rer*^4}h+|E&!?P1=1X_&V6r- zZ?yQXSz}yQCKzvsTuE)sX`&dm@{H*Vp{t0SmpM5LWyT~&-IgUfZQOXv?lH!nQd=3> zQM#?@D>UJg=)*-jy+ei_)0$pm$wyty3F-Ge;;26AsM_5DUo@LjY43C7N znj(M0j3E=yS`e5%4;Cso)%OQzg_j?MlYKu)VuL3-Q3*;y_mVC>B{i1-c^QArfbBT^ zLO!3B!M`^l%M%;S-VQYRcbZco`eE~&(~nZ0Mgv}ux-??u*P~O+!m85C-gV9CPW#CG zv8?qFm1*+9+f z;at;}Z1{D`WY5oJ4h(9HzmHGMh<@q4IanEfFdxZcd=9FHxa6H((k!hG>o@SNU*_(M zUI($`S-yMh3){C~(3TuhNZHxIqLfWffnlo~ z6pI9vmJnV=Hx;i^hYl^(EjbgxHya3-=4_#_7G9o?M^{c3Gq<=cwIHmLcofIzPjLj^xc@{`pbP%Mry?%7sTiH=mSW<+a0xUA|z z$~7|8nm$;P(5)k&aBMXa#Qb(p@vgut|423Qq)T^qz$waN3J86=PNm=&UHZ#q)TBdL zYkR`r`1-QKK7njkU8pk)bSJL{A3htmn_Lgf6u0OpDZBZy|Py~#?#l`y+uvu)%?O8{q zqEc$Ea(Vp0JFi>ZLn#-?c9a`(kLjn*>bKnRlKq8dn?vIue>A}^nua~7wWxD#e0L7$ z?(z{}g~kI0rluZ<=@kubCQgD2fWnvn8T>}UlaG~j(fOk?zVINZ1aKx$qqmf$`_zLB3fJC!nM6D7EGKRQ4<9wmi%4*2;PGd_}>&^1V9L z9p5Icp({fAXD~2MoDYTz*{mlS1WV%|+A%Mnyd)=JU7<8$RMC~@j>*6J=CS-i3o%)P zrLwk6heJX5=17I_;Us~2ID$LqL%H04>a&7zh_-T~QjE~bBRgcbH(z-AnpZ<>l!fRGpKQZy-<76} zw6*lyrnU;B))57U8JO47DoB0yzwI*OF!xj;nru;xs#zsQq^T4DDQj8T;e)u;jnX_} z2}7HynrY8O{HeIJ@o*B(f2mte@^H#Gt|=Jn?tJs$BJAZTixH#p7aTPLt{^q(^e+}s z6ZK=uOY#7MFH8+8pyB+gc>oLgwuhC*dnk6{Dqdcb6Jz3!6iFVeU`dXQBCr?l$XV{r z2Ss|h&eiMmrr7N2dQ`#CM0?JC`$c!TIFXGtv^yy^FQg!N^O`~=Rc9qOEeVs9bvxS` znk>OhYklO1;n6=aze|lAjb~)c6jRzf|U!99HOYp3d*L?wka1{315m(|2)`x zKg?lsM9NE3_no(;Jop1PxxeBwM-~iZ5GCGWOr6-oEzr@EH`;Y=vv+dRIOt})GpLa@ z^j_8BJB)TIRG6_>2Myo}WU@G5h9f41|JeE%F@6U9)=|knmFDNBRPJrANMO{aaOTod zQzm;>R%af^2~ee@F@LI=+Y>OdF(wiS{{?w_45tKP5}8a)8l!_C1bFvV_0gii#>sC- z1#cr3zbt{6=*krej5Uk*2=Z1{P-MD1_+>IS_f>#ep(j)xzg|uhCgt)jceC^&yr6s)@&YJ>kFR zv846ast_ou!#U-=Ws)qV6bO9o2o+@K;kK9y~7f|ipIlpO@=bPlKIM= zd%y-PR|_}+ag?Ar83zc;LHfCtf?pc^nylNoQ2AjI z_wkYSh4<#rNv8pIWUn%34tBIwxh@q!wnwvKCcl&~40#Bg>z}B=%Bw)0@0e^kG4Kee zpzGelS3)y6&8D3Vj(}$0kMvZK>_SP*HszRo8X&rms7qjG(K{2kOr`MGqx5a|h+n@V zDH5_)6pDJpT@+_$+JUP7PBz|9*4De5Mr@@Dp>gmoik`nzr6cvFoq1a4cqKEIOKg|` zh!{}dC*R8}xeYQ?sHDk#NR=1cezg7&`c9h2sb@Av)ZT;*gy)kac)DfUbkiyXb`eWe z7OPS?Cy0iTx20X|@F+^@xFrmT_)Dzzv%Vg70;Z(hAptT|)j&$a$DA)VHIo7pbS>Yt z^UdD&g`lw!XNZbNN)zJlt%FNB>-V6}m+B{(8>Zf-Pt`9#vK%lki>Nn&PpHx=Lv}od z%uUc7yhy{AMJx6FQhNqTa_|&1KbhPC82p8=&F1#U%oxb6GU&eUdLEr2(girs#W5ZI zO_-P4iy19a&@v>D8{JjxjIil_L`>fPwd%UB8BC{OKJ>u$NIR2LUkb>-r8Efzmf`If z2@e^U?(EAC5{P%_{7Ks04{qdk6guJ^zaDr5o`rR2@ObTp#qO1D$2}BgJJ(z2RMEDF z%Lw?glT?Hx3SSr14UE8*BXy!Z&!W-czt~Y7+-4fRfgW}hN7B=%)J(Y`xak4#5h4^$ z$Lw>Q;aS%`OUrIkS|mH}lz@Fuw?lraQc zP2&wQ@Up&1{Jqzcogb;7oGSO)KRMfD4#YTSeJ4}Djt#5JE{at4p5Op|S2JEkeln~v zDNn|)OvRyT(6*vx719O-GiMW6EB(Js76wlar5$5+g=AWvWhGPr$7Bb|NS!-*Y5urQ zWu^uh{%?d`Qo|KSz`%opbSW(FBU8VcLM5{g^_~+4q_lzw{B%kyQr$MH_sZw;h*>4pUuk@@!p3bdLh78ck`mU< z^|Vy8eERNraP^l0zF1?5-I;$g*x@O8TQej9Tm)2iV(2Yf)N;BA5xj%_yZMJ;iei@d z$U*C-4RAJC!_QnCzc0+VP949ked9jNU_1;;EWK1`3|8NG$RuWdYzvnMm1;kQ6U(F- zdcp+%_D`s*_8UwkUUmHlOcKR?EqZY|Sf9BOnC6$b>!Ph7HuJbB$NhCiLLofoZ+^mb z%pUF3W=hbJ-oP#BeB*S+KQ(b&o=@G5XWyLC(&QQ2&K)fyOXFay^o z1%Pt05J@B2uZjk9(WVEwD(`8}Q{R*x{}QtuG7j}C@(H6T>C@&~V(QGOD4u@g<950^K0XfjnI%w)R6-01Pk;1oheEOa z1I+PLrx-}5S}szH4a9h|Aa}(rak0ga!qj}zb-ki~i8q>f5v}OQGwV0y5rq-a2Ny(=jO8Yq)Tb?GyVH>8A1X5!6bFri+g$eX4TTUMaa2#j-Wp@O&{`- zT9Ror>$HW`bIQ+*!DA#p2B2iNRrs+hixVd= zqu1`8)Ox~!>UNv`2n zl;l$x3wkGzgAT-hb6`e3YXx=1e&j_{HQ+{_ zAVlnpp(bq)rM=#kQ~RnyB-sWUP953n8J7w!!H@Q`cEULVIcyE_*AmUyXJqs;01vhC5QQFcX1Qz->{Vnd65RK%VBI~2#YUg9TJQUDf&VK$c zsp;WlGBo(csy^rfBql{IhbUELTK8Mo9H7j7H+UZ%zf-1srv z3Np}sf=H2{yz)bXX^Cz_nt7AyXDQxugblt%L_9Td7e}eg=~e(;^ENq94E`$x{_|RD z>5qUNiLHiMT~byf{is`~Tu926d|b>L??zHe`=LSg@cX2JehxaQ5|gdr*ODiVvdYsm7G3&0nJV02I6PV_F?v+twsBtH0+X z0?MUmhE+>TnfCg#scxXM?x{?P%~oAZUUx$xo97gdiBt~)gRu)2lSu0A9Pu2<0f=2F z0;i&);-?2lDw(&rVNVwSF$&Vb8^oHPKQY|Nknl(|1YMVvs_>!m|Z=Ehm&k%ztmqBKCOrQ=y( zHcX+|b1$?8&8&tr@1q#{R1X(d9y;u(<~$|e5<<1Z(-0^9ryKD{ zNTb7Z8oET7@K%3)qF$3bS#uC*_;Wa%^5g5&X}w`vp(IhT(Hzc49~j)F;xjl0nW|+V z$sQBU;Gjwdi%RhOM>1h&Clq%V9Ed}h78v#wrUwLGZw(f&Jke#NaTIpcL?JQWm3Yhu zG9JX|k=YtHXiO2h>|{tNn^BnXbbaJWTPQpxtH`5I-?bkYxrnGNG9jS)houAf%2BjR zKrozf8=N1i5N<|>(vnoJeFd|GnyYEOuOCePM|%3Mgf>My+kJ=|he9r+_L=O?@i1FDIpu7ia3H%F|L^ zPDue2@DWlVdD`2`i}f&<*AI49ceQ+jrj7;Lz8Ous|(X)^&VR2KlaPD?0~Zp zszj7496x!gH~1+~4@Np{#)Xpn8fA5Lp~Tb#7M+*bg`%iZ5LULI2@G#~?iT02`^(W2 zb4AK9AVGL^HC}dWR1_Ev@UTRDbrUEx-Ni;CFt)EiN-tg{B+WJnB|U14o|wZ?_`~P)Dj!2J|Zx$ErG( zG`m1p8{~P0@U@Gy&ys%}Xa%)*?M`#;5~~~S@I@iG8U`W$d_DV)d{wkH8oUMlIm?%w za=1R>uBHTPn)ijNr$Q|5esYo2Q@&^cB55;GnjQ7+^r19(RG2TVQ9OgfAG;j3z6unh z^Oddk4buRtt1KlT=4uQ#n8fbfRUex&-1Iyy%d#{aP=m*U3`#ry_*&GCHkD={ERl1p z?ZobKbu2y}DIKF>^FIC$%vrp1_{{pVWLji*IF46d%*&XQ0mWq%dg_=jQ=2u`^18Ej zHk~7fN=b$BI-qm zDw!?XH6$DS2LXQOaCltcbZ69ReMnwH?Jz7~y1D^yrhTko(o0ndLS~+g-v>dnG@2pV z!5nCJr#`fM%)!qrm zs)CT|p}#OLd~4LIqBkpz9Ybsze^NenWf`oiuM@$U;+O5md5TGq^h{`+xDX5jdIBEGV-|NOL*0*>n?Z2A7r3H`4Zl!4EdiSGlSwEssy{=bfYe+DC3Gmp1o z{kM|`cD>ku$P|@R=YZ+I9|x+ZNNGE%41kpO-_9=(@!2>U_%_M_Ac*}xSLXSzfiU2B zNxAH~@c*39|9TNG`fQf}ZCYLWzkdBcSLwGFaJ(MhYWW`n=Kngg|Nl21MS)U^OpW4- zf&Blzzbl#_7T+JXl4+jhx-Wn$<^D}7HV}A_BO#JnqqBeW&3vlsjORuC&!vd~YMZyL zmm056i3H>S`Lt33jv(TmocG87{EH?iyi>>f-G`6**4FL`x3}n*E781bHUD0EAbI}s z60+v}Kzr$yT?n&}Cw4V+M$amEHW~*aQpOABtI0>6tpY@=+t63KdFtJseej)i#{qy{VN83xuEqhRz*3Gj%KJ;euwp?wxk521G}WtVlP(|E#1^GvZ#XRIjJ6wBb0!!`1T#I zx`9098B}EA^6%nNL-*mWe!3jl303XXH@;^y#$OsZ(2dLuMm4V!t`sq~Z`{}9PZR6R}w}^Xt28j!K`p#!s zUuVuk@G3>5nTF87744B=TV=RoYvtn7)jptK?s035T<`5x66_L#kT{$M7oG-(Xv;** z6(Plr$D}Z$AhnCcxKJmCDg#<`;-j5Pr)IrecRkrGo3p_aS5-;@Ps)7KgnFs=6;79_ z`KxnUbKB$?Gb5a2O=(RlnAx>W{@v@=&ScWzBLjcm*F5!XSNoJGV-3yX=PGX}|AZSu ztH%;hg4vswW3jtu2G!@7nDSuO6UTKF_>l6sr&RbjeQzDprOZzF99tiF?5Jy08$y{Q zr&bht_T|T52|S<&)CC6RVVdw*&Y%eQ5 z?UK>fniz&h*yzSd1#MM&z@a3K=je&EmX0nh$qb8z^w?X*gFh)aT>=AD@Yz!3%eG3E@Gxrpxr~~+2o5(@a-o=!&?VnsV?sLvgg`2X5VjfP#+j|!}sm&4VUrlFh zm40YByj9I#6_ViBv>M>wU7=CIoOhEU;sa-HY!^lK9cuPUw4IGWDGIIkPt0NFIF7s^ z6v+J207Nj+JE@o*qi@sHn~r6v)F@-_Q3g(24OFipe0!w=DLG?gdo@#FFe4LrwrJsL z;QXU8aJ*6`Fw)so*6JZGvE4q*)MJmb$)LuxD^eXl^-zO+8Vet(;$=%ksAUYZ5Snrv zp4Vok!;>~Wvo685+kA*1k?d_zMH@*!1+6)W9qtl+*${l`;9&(W~`Z=7Z=8a&N zTydS#Yo;6HaRcYQ*ctr2H_x19RzU>464-9!^JY0BR2OQ z47RePc2mHIM{??Bx2vHVDg?>zV|Si>ITZe0ig(p}ZF3)Kd@951=e;4Pl2Y-&KRDi` zu#rCMkT7xG1mZWp)Zm`rJhUtzBouMMKg0!?KG85I>Vci^ed!2QLh?owK z&#BMa#RFVsyx4q<>=_yWA8boSOhs2-CfHjfeCC)BFf3?R;{}_qJi)1CXS+w;~Y40-j{!Wab-#l+^+c zR5f9jO)>5Er|G2#i)WtDKXW{1XtP2BCuz7VEEx{~ke@{6evU`|uHFm^`KE^|BZTCh zT6ms15M{K&itOND-+Fs$3wkQbh=u)4uPV9j=AG}2J)j}0X>(o+>@wR4P(C|her>k9 zrZZl>AbM)Lu`mAk%$~KApj?$INXElA(bUODENEgPW09%g`>efNvA2`VI4TUtU=ZL5 zWpBE!a0|PQn6w1gF zzJw8szH=Wg;)8{CV%df363u#Y=-b^j5rVp!Pr>Y335N>Zy1WuSfKG#EZAW$flJS&&U{IHRBYSR?N|_yHKC z4r1S#Pi&z6TUp78!TE$R+*wY9IpcrLWYW8y)!9-W)r^H{^3U70!u^rGobhI6@*OWw zkZ}A#rlK4VV&0PPBbVsTbI*zSC_xOl+afSRV z>1x9dz{ELT6b;3=FHUT!L0js^A!WP05j17>5z<8s?d?HW*_IOD8{eVM}Z9b1Ea@cIs6S{5fN}?KLV1OXz1XK!cEbo^ZhD~ znd6E$q{Q@rn$_jGKA=+tX;8JHH6%`4HB03@?QuUp>Kv1I6VGB&faZ<#;VZEFcr7}7 zrHnENPyNtIvWvm57#8DGBq`lStv>noMZG?OouY_tLlIi1xvWK zl!)ay$5&3)jn28U10)fb_0f%$SISWggmGwXiE$}e+@sj1qf1~Zf2Qa)Ccn%B~C!6z&BrLJ3lb{des=G0-6!+6_? zWTb0AlQLD$^z%2wdztdG4rcXUO^WN*+U8=uO7DVNvLIvaWaY_wTgy#fDI+Rh&4vI( zl4GQyLFX1LU!Pf`JAMeRA^!Kyu|D zCnhBoNTtZ?T3JCvBdkXF67+er`fTUxl=y0O@o0Ml%6JSu$JX zTX(GzQxaGRD9++NmxykAM)P|0DySKr9xgm;oIkf=cE$EanNin%UP)>Fo)btXrO>0% zp_bEhZ91Un@mc`^*Ymbq?(9Nf0&KIgDv#Cc;ju48W6YplFYJv69>k}QeO)P2*fC#% zT{pCuIQPM#zhONNmDmaJnFuUFM+S)Q9yE|JiOcd&cQh44;=%={n4hw*g1%h$WitIN zk>oC8yva2N_fSg{O7X|JJJ-D={*Yzm#b@Q*Z?8A-+TnqE3ZIT?DtAmep#lrY6l0-^ zkI%SA#`wAhcyN*d zv$zbm{UU?FK~-=o;C5TP^wtZ(VuZskKe1@MgY6)tCXG)_^c>nR4LJcc_2iTkIdgOK z(#olfQ*dG7P}y9E+~|?-#zh;6`jl4b;{E*rYvSr~0;k9Hma0t;*9VZa{gl!2?U~By zl&5A)?DTs}r4BFqR&2+TNg9->6hMk4hWa95%JXa5axDc0;v!^D{`U#g>zBGW&adj} zNa9vy0W9!ZOU?jjpIA7IVIiyolRf-N<* z_mj@XV``STvuOn;hJLTlmSY+6ji9~*pa9RPdY>D>S~dXi1Q`I7wS2$f%9r!u8p|2QfVndT%;!mT3aR}7td@GI*)59by(H0UG1npBlK+jP0C1TT76J2g zd6Q<;=&)+g)U!Szv$8`s$9L6jcgd?HtC>EeKo8v7tXWuaGpVBV*{{74W(XsjR0AX& zsb@vyX%`2vzm^={`?h42Kz+fSp34bg;0@#!^mmZnz*{p$Mn~^-*>BTdrTJg!`O1Ks z_DwSbLkIK1j&XgYOa$;)zsN|B|MIW`_sKXbk-ru|Xy-3BtOQM^Bp-E#$e&PCd zJS}vmRJm(uvwYt2X2~JWD)h3Z2aEag4<_?TB~HPw6rWD@d9L-e$BWva z(&ku#4T*gRlZu=tlGP@q^dZXAAFL0V{@kD{z!_@S0gJzi29mUye?F2N zLlWs~R|L&)Hdx$n`a z^#6mx9|ZLMw8-S1#^=hhr(^N6ORUA-1PAXre(6@^zhkDGO3Fjv)}wFlYg}j*M5TQf z81gzsb)$M(yS$wNsO=@fl!r`c0!``J-48xy+g}1+&mo77hqnbXahMT^hTLDwWBKk->?m&VHkebVmhq)&tpkU2+}EvuT)q42ae=C6DwZK8IJ3?-dlOl=Nq>{ z)24$p7tNg%pzb1C36uIhG;eg*(5FxnGl}-_;VEipXia zyi#e>zI`j`wP<$7G&lczh75Lsv|}BS55&u&@6mEr+LOSw5Snb4TQg_O-}4A-eSu0# z_k6W4f+a%NKWEODp*MJcr`>FI`c3;g1WqzBNts_RN*YV(BEI$hY)B6qFcNoM9QVbL z(EA+LrztC-!O-SDj0zTbFlTbxWwei!s1dfl!!`=1Fbq7Ga2Y>%=9Oak)RM~nK`AkfysjYR} zm<0G4qr@OshZ z)*1?x!A{6tgQuuqwE^m% z588#+yx?i=e$dRw)Up<=n}*$NOI@c(aDz_4f4=p@C!0Q1HnScZu)SQ-V(Y&kkxRbe?}x;I!9CmD#jdL9_nOz>4-QW8kIY9=%#}$YUaIDcG^IFYvAIqo}oe5;f$%a@9LCZK> zNu+HE20vb<&%6zkqkcqC(?7N&+UTsMSS2iyoGcob1&QpbQdimJ23pX34V zMuz?^;9~zRjv`4YiTN^eJebPw86ET-%J9q^0X_$YGADq8PT+QhN^l7X4%fY(1D5g2 zTx?t8j|YJ9WgH0GAd?E2mu`FnmMFGoA3HiJuj(^EB}CZ7_q0t2}_buXW<#_`Lt|mTV_^WtN7Xnkoj=i$@Mn&ML^JV0zihU9wf_Dr5r=jg8b9ce zxs~Ku;M^T6>e}x6d8Q*kTRH;jEY9hvO=LCMd*RjMNefqvY^DFqkojX0w?iU$q9nmI z_xE(Y@N-`*KNkkI#`mxt5)^5mU@4;{?mK{9#rfY$M5#w0h)ce9w-x~6E?n!zru7_H z0HDzCHDKreK+I+%SpWb9G$JXBC>%3@eIhTM}1{faN;qVBUG;%`aW+II}}^a z%+B}jYHCadJ4sQ@A$k>H){L(aQ5OM-pOZKE>tQ_aG!7lzmU6ttd5Wc-+mz=zu%Zb1 z)-AsJ0-fLpBLpTisjZ626ormuyvchB#AX??>k~trah@5*MPnYn?GGPEDI<#+Uq784 z0)>(oz?gCFZZ00adS{RYC4?3#qp^dWzN3k&^2V`nd(p{ zwO1E)x#!Z7ovGoZm)lOb-+p#W8{dzJtIo;s142M|HvW0#v)CVijx(JaUh$kbLq><_fTR91GBOg@C`9cQpBk22{|o_^TcV0!kfjf{42mF$5Vd4){S{06 z7|L{d3}kKnd~?0K)Y3re^6Nbl*FL70n*0PMUS6l_`yiZ0l6-9U&;{$#a#%zxhno0w zCZ+4`S`^!KK8~Qtw0V9h!r@OOiGD(fEz;;!XlcdxXQMXn`dIf@+D44OTg5r$T?%E7 zNsBd0GHiDtjEQrqU0MH9VJXECuw!ac;3(vV)cV{`pd~cz3erK$N3oF3R!I1EiahTb zzrr)sF*W~^ZmpDjoO(OPEx@%d)V)rO^CwnTuwxO`62+L*^c%oi)_aZvT!U;9a- zF9Ji}#PEIvw#G`R-@~x{{2nf~+XRTM@4izFB7OQzm!^-4N0gtC`c5Z5^lgOnqd?WF zhvh8eFlPk&TPerE4H9Lv3K zjAr_RYsP9u%jP-O1{n1Lv%VE?5B3WD}RZL0ppo zqN@@Sf+zx44%kVQ%eDX-8z;_g94zMyM3iv8Vt=G34(`TN?>=_IdJ4;r4%|x<`|Uub*chr20+H_pr zkwQbyp9Cn9EM@d~iz&Bw)r$>mreBOcC)W5skJkJ#XI}s$$H3=O(-fu@%)sVNeRCp~drZ0X+%Cx%&4fNO!`QvMOh#pxGACIj0rtqMO=XJFVx(G0H{ z+%VNt9ucr5$?;~S_sAY4muT=7r6t;E58S7-#BWO45UXv0b!^ftS~VAyD_K>L)GhOV z;+WdRI99A}Iws|k!%sR?pIcDsZAD1}q}2KXQzYSy`3s6k*N9$N*YK`DT#PmX2HOZ1 zdO^et>Z=8|FFD!P8gZ6>Mb&m*<&kjZwY$?buhQ{AoRZc^NwFusdIf$@lMb25mSAdK zqA4PGmfi_?N7cd>jm^3|D~`|QDy(<;l7iVjgBqjlltOp5bnV*PlI1NxDJwNcDx=4l z?XH%oY6#%WD9xWIP3SCaiU@}iaF^jg^a`gJ5Pwsz8p2smF66y~Fq%i~{yK060I$Ad zWMxe~T51U=gOem`yR5Dmf3SJ@!;c+}$+v-RK+r$0S_g0$A})?sjN1RMp=A??xs+k( zwTT;77yU{}PTnCKJv%#dJ+;^9K;0J0dNmn+@;TBu^E0+LMjk2##0v-ru@YSBs5Hq# zo$=~__8R38Vg;`dVq9xkMtc;JryEi zU$Sdld047qFr-o|*NTGrZT?NV){F#;K*+{y{Vqe&`1gXYz5s&iLY2E!$v~UD#0Ml-~4bTNWgs@;tVTf=yJ^yY!y}v(!&2 zz})qKi2j01pS1z^Mb(f-R}_okky@sdOsFI_S@b$bGGRha>SM<(*Pc|dZlNfbqzHl zTOa4py^b@R#tW~x6W4xd^DJ!-+#RmQCtfP_=yixKH(uin$bJT0O6sGjbO`C>_*rDQ z?zhA9SKOrF51{Yf_~f9Zpph1P}EusrWcbFaEha?G*s#3_-t|! z@fydEZyxk@norF0W^tM^FZ+*XpRN6+-yp7+Fw|-=lxO!AgUz`&8e2?$zeKzg$LzY!MW{KO8)Gr)i_OA;-h??rD0D$+CJT@ITs4uJc zfsVse{WnbBKX77B_TV{_(iwnbxqJF8`9lPV;lOuP=z7OS!ZoPs0>J-qy-LOoDZ_wR zU{_Yfyo=Xq6l+*&>#Mj!V?(zI2CD-c4h$nREUtE$Zcbv0?n$`xyQ2eFY%V%YeeYD zlb+iDtbmR{v`cjB6_xz+;HYh5*W!_$kS{)yus~Wy&sate=4_q!0PK=uHcYhZyR8Tc zAnRtpw{SiJPHC7MAv>a>Q-VqX6$x;ZL@Q?AMv)^JYAHU1+WWo|#~j*KOi3Bi8tfs+ zn%Zau(9iEYmd@j25v+C`)5f6dQ*y5w9=BcR2ov-bJmT~QTGB|qOMhJRoTM5Y9Zfr3?;Cn{r?WdM z3!X~y?U&VV0HmX(Ahe2yUJ?gOZrO#dQyDV1WOB?9E);oSPud{}z#DH@C?$n}x+DWR zKSuoU&l`KC{Ns8F<+swwJ(&=Gg~EFkA_jzh{y^L zv&qiCvLn3u&7QBfhlmGLzStaYOu+Hef&khN?$XPcju_hI$Uy2s}K(;yME`wvO7}O7#*SSGH-5(##>Ob~K|z6iP|Q0%rn7NlDwSxMLkw z9hM!1Y|wR8&YvSm#b31sXOrQOX}yfUVjzI5<+Rk*-(|PBmeUYQL9Z8qmX!1Y z0OGP*`vmkb%isRt;@-l3gwxs1Xt5&rKa9O~RFqvCH4Zp*NDeT7G>CKyNQb0=2#$0& zNVjw;3@u2Bh=9l-3|&ei-7P4sNP{%W?~Fdr`@X+#eSdswxm+&PiTghHIoGxKzVd-At|c2%)77Zb zIkYbIBt<5b5VsZRnrB)!cz|RxHFREw6>(%2Y_4({pTVsaQ>7Z8&u<&Jz@;0xHgv&h zvGWqe;+qPlEU04dxG^ejapWX!mQACCp?ojIBX7fX+9sq=gC?Fvk=8Sxigvr8A|@&4 zz5p}Pqm_OQ#4!6+^O6r$lO(KsSc&o;-3OFE$VkP&zlqtr%dyFU$0Mp+;eer$Rz#VT z@$#iEH%z|1Tt~y<<2SJ=(~F#l;T~qi+T;il+UZ+-4u7sbT|1e1>NfCM6*Tyc_b7A^ z0%sKU9e@91=VmRCEs=78I*dhG=-y`Adxg693v2sVA8%&lgcM!={>(-dey))i{Q22k zWyZ(XP*{~xK2$N?N`^uw{H!#H1siey@1s}hrZEZK?JT-+vR@K4-hVt+srjgz!CzyJ z&*#QnF<$_)f{=AQ8>iTrj~2?s{)9{5%M0Bko@5BnQv4R}RGR-${uKdL35-CMVdpaX z{2J<}2u-c%4W4L)a+i30t3%35Nobc^-|=M}pLjc0ORGrC4RrOWN{Qvi&&M;*77~1z zJ^NM&_06lme7_x=_`P&1?rxRr_o9K<*oT8PrwrLd1wJ&q8(FboyG^71)Jwwqdj$!C zdBbI79kFySgP-j??ih#sI_egpHwbK_^XHwdjvH!9_<)|)BLGFXR({&zH*>r9p2d6z zdlbX(x<)q+K6MUv8kw*!=-)oT?IEE^5>Fy?TpH*V+DzQF9vEu6FF-JV_w-fQa-Gqz zY2~kD*0pIh+3AK3a~om5y+ugN=5pZa%zig@=K7mD#A6nE2K3!^R>MeFaGFA<8$E6o zWBLgxmuUY~-*$0J>GfyV#`iu5OW&EbTB+TPiE1wjqh$6*eDjVC=1Pqp+udHAn2A|533C zLNnbSNp24u#z*aFO`GtRW(=h!H!)bwf0cGTqBmx`ZN0tOvXw2QW2aRbiR*U88CTkW4tbp z-t+Ql);^E+HN}U$-LJ`ecO+DVpjY{`pG?z97+^!H86)D1xFF^gD8-W8AWZc7X&Jq0 z#V>8TgF9*-_C2QfBHEtZU$!wDVaYu=(bv#it@HbIiIt1nkl*)ebBU?#X3%m%$@(|j zMGn)9_cN+VU_W6SD-0wyuwms7hj&b`!YMJ|=!rP>M;~pjtz7?_x-tHHES}=m+V_=v z0(TZm6B)F#dDVQSCu@c?xu{J&^FWVT;m6VWuL-PN99|?v!tcxd70GroNI~BLIe`i< z%Rt<}moOOYEebQOUJVrxncZd=7@LkUv#WPmXn?2vbG9qJb^GF**TjEUyoZj@>Y%Esie%Cs7uoGWXSG)c;L|5D@R^aAN}3Kk)wL3 z!DE^|uzxp|vu9W~nBSzyC!{`Zg~Ie0)$=d5Ww^Er^p?K$lUY){7Bn8O{=5}2_-;`HSWuNs zeF2!v+tRIimlOLJ1#h<_1iQzsBuskm`?u3%%B~U83LMC}7g}fgNk3(x!E7rJ;ZJ=E zXs_$Cd)d}@cD?y4327Fe5dyCMh^NZBQVm-)|0qi#K;77NIh@J8w(DOy_$~MXW;Ojb z6c`{ZbXb4iDeirocl9v%*AL?H@!abTa$<8h{nz+|i8uKWQD#;*Wo%%Wk7CNuPJa58s()mNO2~171flrm!aJYw+ zoy+%4^Vu1tevKFN^w5*|e!6_xh<5ri-kZ;>5@GDDxV{eQ_N)7T2&&oFQHKb*M<)8> z8ecq9w>{b|zKRuy-}x(KHg^KbT>{1HoAC#IKgyQ+oG4;32Hr~@(XwBQAxc%R;X6?9 zw*E(z8+MyHlz&H+eC&f>QS;TFLrVIOXU~$^OSm);IV`0Q`I3hg0@PBTDQ`PrQ&GWR zzov*{U-LLs=6^b#&xaWNPR0{gS>?Gpv!b2;+4Z;G4EOMzU5{B{_?e~4t^HTnJA;l} zs*N_qGDXuPJ*&N`lPN_?BvM-fvkt|%$gTxN$YR^ePrI?x3=FD z=hjaVt<@EIK7jeFphRjbj=aJtXCxgK=>Oq{#M0{cY6!nHcN}z}ykq?3q_)a#APeo^ zZK|3XEkDLrix@1*&^MIJ`!q5dd-1D}HbH z(qCl=81l`1`{M^M^CTeR7r{1Bca_692SSuI_$!yb-el9lj%3D?aqI)2n?QKXylXpL ze#=~8*+qb6dg5sW6##_sy2xw`550QXB*PrN8J>xa-h{0~HU*IRV`F0?v7R0xUL__M zp%-r&um$5=U4J!t3#|=*1^yrlH zl3{W;CSw!8E2QM*@dE?Lw?wo;N>Gs4M{@Vjl{p?4s=l_iW-&etyp#5^1p980zIz$+ z+dmVqU6j<+8Zjgc*5x0LQvfaU2;ezM(8N?~74a}S*Lr!hr_FT}abe<*8 z6xrvrs_~kg6z{nRndvM}SLF%Dnt^w5eC~VBy@s|vu9{w3{S$Jm#XQpd_)@DR`I}Oj zoViX#Zs!k&Z`pGF;3tYJ;fLUQO{4dV-|s}{7tm&Dav{du&`|0-2L&u@#(9` ztOy7^pbHr=9;th5=E=Am{GSI(2MgHryuSpjjmMlrv5N(*ODu!ED_%qwsCJ?HFZjtD;@Ln1($!1&fN;1#4ZCB5H0`Wk6yC6#+`Tm?`* zPJ(D}nr@*WfDhR6Mjed9!fGlI|_i$pwE> z;CE-RDzeS*M$n#o)yI-@xzqNnw+a-lt*{dibZ`PKwoD~<=fu5!gMLyxo-EEQx5469 zz}acXiS>HCL)H_YYO**FeV*`AvxaBP@UICA{=O0y&2g&(o_t9^Y&{T;cw|55KQa>0 z2(%&>mWW#w(7Sn=JU9!SbP%s4m&T$}l9MfhEm2`6_&hwMoLt6jB-GG5={ne(?EPc- zt~M3Wc3cUcNgZmRMGsHUj;W|9==q3F;1XZw4lTj_0S^czX6@pG-s3USZf&I2p|G@= z%ut|>do+LTW(8>neI$4O32cV~FEe&GG)fG@bcf{O2=&&`YsR-<=N!6*&*!Rk)bMih z-7CUB%eBy4ciP;gp89k6k1DmE-*K?l8?2mh_%8gK%c}hwwOFOOgBM_sr%bWjuR%xL zCK*NSb+!)&y%#=3o$@H20Y^A?*1HKJc@YL}G#1K{SQ#$hHtvG_5)r32?d=L-a>sz! zynAFzC^MYuo$6We{yGy>EAKc1fMD7~w=aS==4zVMZiago>I;P2%h3DViyf%>yZh@E zapw=U8Q#dQfZug5;LR_38NnVV#ttwyE%!PXE0Dwhj@z46gfZCNtciClFSj!I&G^90 z&jk)W8y>(>eiAi@4g!Cg57C*JTo2)3chWpYT|BD<7JD?Qfm^R| zs5{fz70v*ej+DNBG6?LYb-pz(1`>Olf#7Sb>9Cj5fqm#?0=pdd=vj%*aOiuxHtcYc z``_MAGu#~zhurHNY+Oq>S_N&m7{^)L8o*}BJHMO*nrHAe4D<8S$JoON-X-9`<^cV= z+kBWn#>cEG9~l|f)*5z=mGF5D+U5!LHC$omVv^xf7P;eTo^(I%y#Le)nC`Z2ys5jS zPir|uV>pXFZcRMMt+(*&N-D^Ox^d}Znj`YbMJH72J9*4B+{B2e?i-%a@Bv2+`g|6jt zVL-%=r*U&8SFt(NEcmSJbZ$jMA`+V?iSz`IW8^e6>;>g5SX(h;TNCQ-oNcztk9ZWy zP)f%>P=i+5isdWmX$> z&JGj-49Hg0y@){Sa|XlL`T_?f&qs!DQU1E@I7!KmAHH*ewK1yE#j>Vzf4MI9@!Qj~ z1W|GysWkg9_8pBkHj98VMNj7Usg-yylnAfDDKym7?rXB$jW1eNgoqg-s*%r&PxaH7`MenuxIl!?;5jBJg-<{#J- zJV|$ug>b@OqtQr}_%NDC@}P2y(M#ZqMoTEp6U@9WN2-vU4yPU147s|C6peWH#G_0~dPV#^=^4tfuIor-S@^IfC49%ylO)^Z zl&FKl*d9q6WV_%suH^yJgENOq7w_k3hz+%!T;_Ig}%QrgCNy07N9Db-|d*7J)0cTxm z>c}t)c9>e4iR}1_S4mxkE+_dwRqHNuGQ-@)a!tw_FU^iBR<;6WSY?Yi^H}&0c!Hw_ z|9F(o@#U0M0$jeq1lswD#pTHGLyLxRtTrYuc?f2IUmElh*2x2JdDU9j>yaG*P-a9E z{d{4CaBLTiY-?7?VS78_l*9~G7fVN$Z)}u75FVU3On4}2?VcMCUaUU|ZZ1U~w6V04 zBajlGMBVO!C~V8K>5g!Ro+M`5eN1k4?&E9c0wBtQ^;UFacT@t2fjZp2eXN%}-i-u0 zQ=eCMfR;wiH+ZfAg~EGQnk8>z!n!+K0*aB((xl`M?ZO}aeaxzGg(+YzB6K<9AiA5ARo9NF4;Et|fLwyZ)X z;Cp_I=7$*~MSLG#M^#AsH-;%W_;{NKly4o4EX^*Wx+hFEJ8DbA$+~M#WG*ypeO^jn zyP6^z?ko4xSXvYi^OPaJELz{zTClG|eyiAsJ4&tLi|m)3(uIFsF^x#-17$%{J$RP7 zdgkjU7#SHkND&Ik%y^h?C*e=UmPbVlct&r|<9=X=wTVC(g--W?8FzZ`B5Lz}Qy=Iw z^oj!}!d?jV2w6_X>{|#@I+7>MXv_>oG*Jj;wBCnEv5?-VAT8y8;cK;`HZg4xkciF6 z#K%J>W4|c;#qYj&?^@x{nA*h z2i-4uW@t@Oc((Ku`IH^z$J-qyuFdp$prd;*O(Vp{liv< zIFlFa9S<3BUs!&lh%E=8pIza9$JlE-Kp?LKY0*nEAeq z54%!7iklQ=c)!9e+=fWAmrU_#;TgnDyRH2@vT8^uOqQ&Gpy(5vvi0DTy8m@?F}$#R zjx&QeKWXEV;VpWo0$iy>nOHD!k4EM&b~v;kKfKFJPiwc#WD1m4qw^U{=MjxBvTDyx zNXq|A0Nb;UjC5zPcTSu%P`WA(RR*}&SH0&Vst{|3;W<&&Z(6xTN|qgw8Ha?&P2L!a zn!N8OZ<22|gYu8S=y7w8fJY8XLDswN7%g>cClo$#S!twsEtxw>A^v^@W)-CpAaU{&#O=Q0?SBtx<(*5Oa5{Leg$5swpF2u$}^<=Cb`2YXu=f zk9of6>jj2JN&DPohRO3N2zbTD#L|sYiYszoV|*WD^T0kV5CbZXcb*WAn&5O6tr-S> zjp!DMNfN&m3#%~S*(#sUCZ83blj9M~12!eu@AraSf>Dgt*L; z*KkHMV^JxrJMZOjRYV}u(o$N zdF*I>XbT7)y(|5f$vd$XICPM-djWw|KPg_5iD~Z9Mj|rLjZf${S;OsDLX~~r8r^_! z1=(5-KFBNdu{t_DYJ`x3AHOR`D%g_pL^65F(I`Sw7UMYt z1)_AvyQEL%^>*r4NGS9$aUse)0?d*MNhpFHYU?!lW>z?i*^QIH$H#&fB9C)u^}Kth z#vG$UDdIUn zZ3x^ABlonlG#WB&vkAI6t#B%Qo;0wSU&cgXLj~Z**to-A074sobl?7%I|#mb7Cjf? zoup6wu_uQ6Xp%924}t>YK=brH(NSFwtX2@oCC<~XmZqrC=pz`&W9p_)Gj#rEgKRW-Wp+*XJO(Z>)X?_i>Q9mtRW@5M>X3GqK0F8+6RIdV={W zV|~#Qn}`IOwMPqaBB~#rroM&`ZASO)WZpVM;NDm4b$}1g@^Mj-Cjg`tX$G|}j+H*- zZvcY)fg%PCYrM{O6}lZM=bwhzm2Iq#f2h!j%`nllsncx=2l=*kFUbDhziu`92~pcK zuZ@g?SY`sWj3(#9deFYh_j$MSrHf+fH0!sNk>jBVk6FC3VH4X>qEbsDw)` zg3$`pS+$|dkA}epemddY81qS;t1M5({Ic%3I3RibgKDyk%t0ExHY}ItgBGH#etYtveHpgmf-1 zLi|`--;qEk^9xJpTq>qIMkf#(elN_><`l0sz|s-Tr5bh)#hA~jCWx`#3#1t@! za{SZnX!;?)C1!LS?hIV^G@u_(#QTt_M*L`g9WMw`RmMJg~ zB#lEa;$@_=MV^HLao)Ln1WqOf;$QL$O>_cVSeXDY#fOOd1+u(Tzu&$c{1-u|VFPs| z4ks_HDc&S0FZjH7>(yjk92D`(Ht!!q5={ly;5CDf%@HbE8JqwUeOHrZ@d4~u=wBm_ z;H_(5bDi&K*6Vv*yT^jCU@J!VQ*sAek{)&vVft1oMl^iV^qtlH`+j(2a!Hn*stR%z z|1!j}%AnA&bf22}_JSo8>2do<|NGYg*+J3{FP(4kC6>6OskGPVXU=3w5ODNoH;Zi_k58-4L8wgH;_L*>8d7@Tue>y>B+D9JE4x z&>m&k<*~6)mv`FyMRMDbNNeLoDo4n$343ZW|Kn5{RB|n6(w2wnw`P$+kMU!dDv;Ct zM|lA0=jG;3wgL89p>2w9nvqDZjD31?dp#^5>Cl_s9-|STo5Pl(V-Ldt^}{l;0#=z4 zMmQOCsqU-C2d{zQjMDcX)ToyVYs$`oL{rv%dKWTwI7QDB*+1ehn!0V0d`t{>+?O5& z+JnxUnDv<%>1jQM)VZvHY?h}L^uU!T1zdV86uenndZztCuvK3JnPCoRQ3bSu-B(oj=FYfB!obx@Pc#E z?Y7O()+3KLiQ@j$;byka*BRxPZm!#j%HV)&c>3Z~Q9?MKixzv=q3SCawCxSukTZK2 z9)u7W5uc>d`m3wIdt+y9Z%l_%tM*;3KFG|;G0O5d5J?zZudb=wXjasBRC%mmak zQs3?!O|n3Q(U_iz;8dW`G|S>WwYbHvUMZ(RlSMMk58o8q#TN99^UTD4c&Ka2?6Bcb z>Cq)5u{of(mT6}u(wy(>RGf25x19p`4eixoU@E0x zKv;vGx0vw8o8E8xUhxrFZR^$y_+2sJP$Ss3An*FuD*C^+@FyYqGoB6_y>U%E zhQOp_VCooVIqGp!&#H>xUjphP-qed;uq@oL`k~T>)4uvt*n0e##PW!LeCOBd@^0>I zgTaiFRB?5mLGqG=2}J%h4b#t(`nNjj-0y4>cStfn*p?97Eq?3vGbR4U2P> z3asiG7m5MrNOW2DgY=bsCtWy%MX41|Gn=wnoBdcIe6&O{b#2u}&$_)*;cPXcZm>-O z%Sgj~ZOWi^bLCm9AjOTu(tWmNa|30+5&kQVg9C%9h9|+#j75a>hkiq#cZ++t`CD$4 zI5-$R;^yc4*FEUprz-q>Ytq>`Dk_R&(F3sJ^$R;RjJnP~UtR0MO)szuQe)~@mvtTP zFO?)q`D$c1B5d^ZI{d^?5>2C{PM3)6EvKLAaS8Yi)253YBiZs-hmLYjzEEsS((<~_ z7iANXU1d)NVyW1P8d>?kT{RrZnH5^`rRuJdbd{_E6( zuHnVdh4!F-Dmq%(Dp8sPL>__o0HLnVJ`hZo_-C>UWTN)(Ep=FVS)-<1I;r>1n?0so z#QW&V@7O^KG<;O46Zz(jq{6nobkK_7`D`K5q&fb4byWzK{udb6yLZXkl9KUmUR@b+ zt4GC;lSgsQSV)Q+75~93(clpgcz^_NPXaYI$QMS3J;&>Kt!LV)1KtcxE*;E!4)+3L z79jJ)-zJNUx;%rQ82vUrFNoj#mgdxLt?E#t9mN*H+4s%?HWoeJR`hqjWRb&&Mx0*4 zl6RAEkNeX+Wk25(=D5QV2c_p%#JT=VjOE$-jrA6hL8M3`P^QYk~GeYTG*U&Fbpv+ih7XFd+YD*B?~{ zrK-q~I_r}xKp0D`ys%0rp^D!N1h)6|!KX{arY6S5Jb(`dEVBsv-P6+j}Kw0wETzp{yK5R-5BE_cDLu8(rg#rZA3Qj~9V$-{@@< z8E>u4G2eRQpe|;Q4Q0FjuDo)A^~=v*I!mg)4$KUrQ#WCD3FvttuwCP^n@1^zo4?V` zSpC}ayeHeU)Vh@#UF=P4*k&lI-nYx#Y|58~Q=+g(P(8)@MQq~pksfGrTD?CMt$kz~gjqW000@QYjQlgumU zO>0TPd<~n_E#_XMg%4G~v_`DPJ@xD=#=kv*Q^G!3_d_&}nm-)`k&oahgQ($mt7~hS z#~@qOQf#8)EQ+W2ziWDswN-m~egE0BD&{FUV}Bq7E(Ah-k6!i+RWKI+;R$e=u6JLt z-kz)R7*pd^1G8i{K#X9kdj!Jnv&`qvq};TkNVxwAn)?Pxy~0?dF9faPqedIOH{(fv z0;W?OX&97FqpEJ6sgW1}yKA5&z^qtk0KO88>i45ZlR5K(JY2qx z9)qy5HK2`626(Op4l#@kFy0?F^r|z)Qpjl~969~hXQ@S%_e(r_Gq_jR7sl5vZ1hv6 z%MyPw%xNcWCqzNAG%y~e8z6bp^f%SnB}7vNC9((wc80XcwY?i=9Nz&kLsOihmcm=Q zt|_*a#{r`USdS_lQ|t9uDeQ3bzCO2!8WWY(11_gJg6}eYqX<+cC-sHS>ePfp$_;Pi z7~4P!s)n23zx@}q4!-&Zr^0HGh^cUI0_fKt!{O$$N!A7gpjxW-1U4 zp<`nQ=deM`sr&nUlYiFz6T@H@?q((#t4a?Tw?uW-RD1z5jn226TMORpS><}g_dLLjBIFs8{q<8%c&chhGA`4z;Hb#D1-<7n-9v+C zZBOR`tFRVM$R|J;3lJ<_bQrA!nDj+3jkkncNwVJ3Wxhs83BL#eW4s(qS^7+OOuR;w zRbKf61%XqZZ_|S9T@v8m!vdM>4#(cHBn?~iIR?l$E!izu*_PwRlP`hVKpMXWXmN2u z{rp$ zv-o7V^nDIL7`54Hm6*(3-e9_LsBUFJvt^Y==GH`i(2g6@M`t#a;`-#SS_<*wg~3Y& z8XGH1U&DJCv~=QSW;{DtSZ7o zd@m-)qsPaiUH-G0vi{)!Vvn{(-lYC`&Z;`-P77S@!85Y>_aA=S?)ynEA1RfaM?TsD zLJ+Twu`alX?9UNJ?RoXl+o{Z+z1gbXN~o$Vg8ou+)$NbjCV) ztZu9@S{6@t66hQ+KabDH_~d7@dj=r-RveWJfaIf})-K0_rz(PQ`9Zr7J)N&XSk!If z-Q#&r?eT}m1!N`Y^7=prmw4QiJS+5<{+Xv(R2Rl6rja=#4Pt7P+Y95K<*l2y^l3Hs zj>JxS&Hz~+`Y%^uF+*+Ym+vkeaV5*+51v3^Rmf^L)4^$&kcSBt?87nNRmU zi8ac@M(yoQ+0(^qhct1#X@wdYuzpr+UVLc$-W-?;b>xX(tUU(y8m>z}qEYgcRRqu~ z)P10teN=N>@&Y;j#_PxfQSd66d15E;k&&ehw4gL^>O;gS3lAFxRz3BAtZn$SSvo8| z>WvnME9v~B+Qq}7V^|G`Q26Ls<4YYvR%aIWNu{g6>&7Rj{OCauJRVYyij62lty>7^ z3}_jlAe!Wy7ODs*Y-9P}MQtXspIAX_OBHv$a-y9DwsTmIlRjG!+dYR1)rifbbe2aP zkty*|Zu)mR-8txC)na7k$j@_IxB|19@k|iG60?@zN|2pBS(`X*F&im54n`cDIyvEY z+gP%r(1$SiWta+D<*7HH`@C5>Mlo^e&7n6d#L)E98ra^2o6KX;vI9X%yt`<<(B@A( zLZ&TK=G&q^?-i6;x9p?1do-~|@d&xMvf9Y)r$F+K|A93t&Net!)`hl?El^LfDb8)4 z>Kf!4XmY#=ih)j0^kcV=;?>+f-xcQXnJvyUUx&PR@Y0T3d*sscG4MkL%ihM#N1<*xfYvE)4i7B&V;vLey_eR#!dc}eed0q5V;S$zna7JA3wigmm6qNW**s? zb36m*Yj31=qJv|$g~M=pkf`5bMw$BNQHXEmkBGUM>Jj-&Us$Ka!Pv0?7O5pS34`QB zvn{^1TSNp^N==FFB>ux7 zOkH0haAGbsiG{;U<-em*ve(~KY_M)!Fv0N6t#A$0GfaaKIe~5L zw&R>r0HnI%0}k(gbnN}I7z<`h&F==FE<$u&pVC4Dr|ly%C@U?^9`h3sxY#Z@&JNJL zF6t9GLcq?v8jQI-W~NyZ?S!Fszcp|Irq5Q{WqXr9tC0z}@i8>a@+Vos(lJ+qlol>( z)qv?HT0ue=R*_Jf4Man0);5r<#CLy=`KL6BwE}(ivb_h{H9S5OJdY2+eT5bF$kjw( z`s4C5?%ImuwmlS+nwHi=xYApjByTLJFc?{{ctmL-2sw0yj7N8oA@h(}ZFo{IaT&al z1b>m#opwqXTox>nOicSAwWuQt1f_Nzp~3pFSE>K5@z=>1GUZjC9&2D`zzJ1&zdD{U zTi5M#jMJS+mio`K;K@OYT~?-|Uc|#gpo#O-gD;WHu^3Kr(@Q|M^=zg($Si= zO&BsVYhn<*tP=alBr^F>VOkSTZSf)62mbfYtG?m|Erp}T9GN{~9I-WS*qMiY+u;mK zL7qvO5~8XU77GcvQl&awW>po%y3LYw1Ue4a%BK(`;w3?!pXLq|IzDx(M_h)RZOsBZ zs?|dz7XVcOrLH+ys~wf`#Inr_&gPBI8|~f1O7!*np#Gqu>c(?oLfR>OiX#?BRsoS= z3$JTJG_rir0B00$N~Hp?eH}J+w`r>~?&tRBx>`Q&23%xfqwnrVWxAP2MGmn`BEIi- z$VVaj<#B)Z1EyniPlS?Cf{Jv)6h@S+H6xxUP-Wv0QbF7!FK7m`bpg%$xJpM}&hhlf zD$J5sp<++2&)|hkOi`!{5us*Xa@n;KOHg|`R>jJPZY_pSpx|=MNo?V+R&7vc{hw%z z>lh4H^;NLIt)9ddPOsQt-q0ABeTc4_nZnj7Vm}RYJ6#)mjk(DZPZ5rwx>kAirnVU? zT@y^>3G7_N+C;Wa!5QUW5J1P%qHP0eS#(i%K|K5sC z9CzJ%BD6s?T!N)4X6;%cTp67Fc{yHVY8(+TWCXz5SwUYa^!Ur2l|G*d-Mf=yhzi!2 zu@P6OX00ygqK@?^5 zCXLuPQgV#8N|XeSc)j`blZhA^OGV=Mo=Zvv2L~Q2-L_qFj<`8!g~*>h+Y!IB+Mcq9 zC{Je`Uz{ILv?u)-2#@Hh?09%8sz9ZMr|1VJ8VHVngm$Ol+l@?MI?_vo6~_qi4O$Gt zT?N|_Qko$y6F~NOLC2n+o^;kyKs=P%b`6s~d9Y(ng772WlQYmStbtkA#Gt)4-1Sfz z>?)@>oyr`wi<=djF+Qjl=!-t1`D$lJR=9Zo3%cjg#@MBLdP;&xhJ!Cp3ZcuiL8XW7 zMa;wj6Q#=8`Sz@9*iKy5yd7B^6&)n0PURnJee?)3&B&DKHreYl9zn;Z0^O|A;iQP9 z%WHIgK!C>4?`+?G9Ug}sPd@-dax@Pma)hmWNaYRcfQF`nO{7gieBMoQn=RX4t22uK zYDDc7Ce~%@;kM5OZ{XIB-n{*ixiENWgAq}m-2)Cj18Ls$V z^7w?Lt=R&VmYh5)7J}&v(B{z4Pn*KKzG5|d!W)P`d%-Ib%)y|dtF7}Jy7vuBveocw z1@1$ObA7TNNFYP`G|J26~6udBX}{41Or`z^tMS%+lkHBT4=UOc+=o4iR!6*DU(+W|o-m>3Bk6!oJb|-g;G@t;9nf zQBGT8E`1GdflfOwU``q3sd79Np(?AoVKlqGBu)K6f`62>VLDhdKT(_1#airC>(bYD zDW6tLDj=2NmK{e7%Vfr9s3JK5r&yI49zs4hx(IvW;7h+M%v!IF`&A`1K+;Pq9t^*Q zP2Cx%q*9yAdRHvqux$(v#ZJaz4g|~ufigejg_6Ze;KAi%WGnPT@8Skgh>>aA9c>KBWI|akQB1)Wj@W6rx zF>rn}`2lOLAs(+PG`ye$8^!D{Vt~f|g;1h}t)eX@-`b_$qp&m%0EbeK8b6i9XQRYy z{$!yzSWu*L>iXhELIb|&8ojd=d91`J-pA5=cF!IJrdqd+5ewN~G)l=Qn?K#PE~J`N z`C!W&){U38@u7WpAECRv$;)m&p6z5}(SU>-vAthS2q>TR5}%(Q`{4)X!Q^ttTLUAw zCG^wFxdd1|O?_ZrxtG1(OQ6${COeO$mck3MVeeVh=b*%^RS)}gFI)@OKb?|O(aH#DMN^Gim20Ja`;tqass2dAm?G3ZEoCcqMw({2;�) zMeD`qpJz^!m6hs}_8fVU!VLCef~IJCkV!(+h```|N+#h{-^+pvN1~3N%1|lQfOp*~ z*34llc-D4r+!PVMYNceYfx7<6&o4#o3fYQn!NTmUM~q~&8ElGjbA-*XlSaHIYZgvn zW9KzWFEf327X(>6w~KsP#!DJM&li&u^xciC9GRX4A!2K&A)?q1fQP>+y4|%F>DPN5 zsnm>X=s95Otra2n(pno@Y)H0u!%1rnnGFG?9uH}*YYksz!%u0e=a*KWK9VsI$t;jU z7r*jfzRL4Gj-49pTTYZyetuWI&x?1^TlZ4Sl`1 zv`MTHmLL~w>DtXl$_4F5{i|_#-)Ki~l4IvN_3{`rhvxl){*BlWGbv)WwKDscwyC1`6)yXK;Cx0&HJuzO2<; zFC`4QdO7pp^Lc9`oTRWZ*Vbyg;e_L`>iu?+h^>)=D(}27SAd%YL)AD<4a2AI-*&Qt zlbxzQH6ndQw`psHyArwxftVBVYW=pzdtHzUc=r25xpMeLu+ST`j}P?knp?d2dLHM^ ziTG0x&81@S++t2@xi})?I+2R$v?doSiv)c5(Nh|N=wd`jKaExa6U z_~v`%4R9tfsVKAdsEDQ%5f+e}(0n-S5^31Uk=80AwI;GaAKPJY&c&(Qk7R$N;2hd! ze2*9cHeTO+WMb=JL%Dg__d8Xtlw2o&cF9A-z?C}l=QR1*6BV;mfj7ohyDlzmf#k+0 z2eHZ;Hj3{7HDc|JIZlU=tV4qv?-p!R`y3O&)Q*U5jPWnN9^x7Um7yPMt)#J!M)7J# zeTZXo7k~uqrG%O3#ZP_-?|D_L=)vopUlu^Mh0<)z-AeP<5bF`%)r1;(G`_K2e_T^2 z$8%`@6&&ta_b?2UXC83#^CcC3J=d39E7aI^oxQKy;5~KY^5E-FiN#FtBFUT>D;Q1E zYNmc6&opzZ3=bMfNV6A|xeIy zaj4>h)0DZNz407&nV5voj@QbV(&2;8%SM73lR$10Tk+q8*Nb`@!|zM%v~CxAnr5AU2MiVk!$~vU${mm^RV!!NoAhKUc#cyUzx93fH0Wi%qG1W#nRw_?dH*M2ruGWM*#H*)uAuSFef|hQ zV3y&(m0?pmzkbyQ8(XW$zU?f$UZ(H4NqevfeB9UjlI}_4#_5$og3b#Q|2(RO>KM?w zy$-kNkLFaS%5yk!x*R=Zgy#HE4x~+tukX-zQr~e(f++&z?r-F{WXN zjSRrluD;zD7lAZ?(SdU?2G_~S{0dG-!XI)p(%j8ey9#6=v$0m`W-$5)4xHp?r`5(S2^m9ap}8YXZ9ENcvn(DynByFi__dM zQ=c730l@U>;&69CyQmV&rEyihsx2R!ThFdcgmz5*Rw|o#OC7#BVW}E{${=K596Uqq z39nXvQfpPs<`{cm_Bll_|Ew?qN37w1V+c@X(mef_-;%9o;Y{eZG%h7J$6Y<%- ztz%cZE>1WiFV(TK@tJyp_03%#m_Fhi&!Fn5_Aj+Ls4f(GyuqQSHmtwYlo#C6PZacZ zg_`V(EZfKwWLZVzAmWgFSm4vp?n|!cg>*I>=lVk5QfX6uWHFma7HI^#Hd+@sNg7I< zAN17aa(wY1AowBc!%Qsl{$@5_LUFR(;({8rGP{lbefWL8>}TB8XFn8X;l}J)H?)L2 z8&&Iea=VtivR@?-ZS^!o11vHHviCB@+E+EYJ5;lG>n1zh!~VJuWZGj+{`C0)YYH|g zZq_a&0??6;9|o_Vt?$KPm&`V^)vlYbG_aZfw!T%`+$i=giA8$K^=9((O^r9k8F#rH zG9CS;C*T?!?6VN8+vux)K^iMO+H>v^E&iO%-Oek?TEF-Mj145%>P%5m_jx=KKJX(D zar3Fc(H~cS0NfNB43>V^xz94De93RYnH~_pVnDyjA>Pa&-$#uu<_xx`L+@|0Mmako zsT+7r6UL;XiJZfL(6YzDTzOe;aF?Xc`rFAU8_PoByVpv$)Wv(Z)(RWNZjBeHH{Z|} zzNNLN=Pg{;*sRk>0M+n0Aq%>$z4tk+m9FRUtd8=JS?g<$EYu#j{rznx5bHjmazC zIA8e+Uik{`WbJ=M)Ao14*aK_Gb#vdcO_rd?PM`R?49C^fV2tC)IOBltDUB*+Drn>; zOZn9?ch)0D`3c*pOUbYb&2)UdHL1Dc_lJItXR&dPOHN-{B_POv#vg!E7{vZ7T zc!OA5tgvCP^+x-3HkaCg<+-<+-~A;2BB%qTs-PRTUFYbNR8PLy#}T>kPn!%r)?+LV z46*I1xj@(qu=xfjgxn@N7AL@Y7O_T?GY)a>4PN;oAT7#(#hQ z&+km?Xv|^ksYTuY@5g^SUGz_0Nuxm(?Q`esf7;*wE3{`7oK0rRgd zGqECHA1Vq-3@}GAvX6!@_Q(1p}=rhQDBlBqSx?UHy6R`+%dv=j@dbLlY78 zu`ZXG951e0d>L%JjXlRNYibxU2T7LSa#7Vij<_8}MR_%saK{KuLWc3;)z6Fd_N(im z!rUyM%Nah?khCt2HS%JOCY*nf)BOh4^x%xZ`}|o~nbSNh&o$v6O1kM^Nyu)Dwz(w2i?CoJ+64VAHRlCAzu15gDC6YO1-^VV&Zb8{} z=r@+ny7v879mTB>lvK?3vozVOq$on_9^IA1r`ur_Cx8C+b;wUCr^?bv_5?qsNu*%4 zq%It0^VY4C`~ouepk{DTeJ!Z__X7_I`X2)~7)^{gLq6xv{^TqpK?aRL#*vDh{pMdb zk2Dc|X`bD+e|nn(4)cE^gV5-bTYq}{ieC8N1>BkrxdW>kFU9?LNq2Gs)_q^HitGk8 zPqz-!4b*>DI`~7Z?em(a+TxS{qIyiG=mk2RiP)=u_wJo0;vVPyzW);V;6DJpo@zjV zr2IcGA~#R|%AT@Lk8U`|$sLqfROGC6no(XZ*kaeRU}K zQc$@S_Wym&|NfbD7cCCEAo2|Q@6Z1CBaC2yox~6Wk@0_it-qf_?*Hw}S1UEMadGt$ zH5C;dDe{Yg1 zlgghP3!G>zt53GX{y)0jIx6a|eg9WNhIBw$Qo6eYq`T80rKL+skQ!1-0cn)(?(PYP#Q$Q-=6cFbDrnBKI^yE`O|fVnf-q6eee6euGjVVx8_Csir=HWLh%BCkdG!p zww3(fb(QZ_yO*s2o&=Y_zr_DM33@+lV$1&V6#lhb2Y8u2`UhXgeN_Kik8xh`1Elu9 zefj^hx{`=sGxe$ElP;^5jeVoevTSIBEdTDP>oMZC)hgVYP2v$PZ#M%LqQv#|QW_1} zT?DV{YDY6`vI?m|3Jn{@J2?btKVNO_>EsXpj}#SZTkOURm`Ox~p(>W3w#i~@&Q8^n z^uTf2;n|**?;93l*tin1{_o}vq2u(Gt-5Kluiw~qqD@Dk3Fz%kX)aozX3@F67>o~W zdm6s%sKkk3Ah1NhqB{s+Y8=mv+zDiI%A|%$%s(pJ!#pp z`V%LAK9y&`U1COFG=FOyO3nRoH}Zp3LzYT`;ektZKI zMWb5Jq+1%$Be}=Y0!+-x2<+jn)u4y892^xo3-b$#i!!|4zrHh_^zL^IF2FVC&BLC# zE(ItdUskaMv6xl%Y^h1RQqWajU;!-w%@u3F8%3#coN|^*0gi=aT9jy*Ms+|6U;Cz zZmy0I!!(3s^zB;|ywrQ#&VJ`J&s|5j1_lkB+US~^nluJ-RAh3hBW!=oovsX@{!~r8 z^(3d=Cg+ir#o8rxSO1#DhJIDdn;)w*>O2+q_J2dy#L;u<7!-6DR^W5*+Da?t91DgJc#$1Uv z=G1}G^Z_0B3#OPJ(JQ~tdH3uob4U)q4-s@m_ze0PeXK)+>4KlmmIei+SUf`U0vtIXNn%sdEHHY}R#|s#uH@eU$@f#K$2<1VW`veSYNxLBJ zNsw8BQH^ir5WfFr2SSli5TOnOl>V)&n4E-{kNNF6?EpZ)qDJ@W?}M7n&ny*Dfw7g5 zWh(bf1^Mk+6Xc9j7+azbXcn>eCKps>9d5he2w{pvXTbffpCRB*33jN*IA9l`BFmCa zbDrMI#;rydDTix}l?%Ccxwt7r=7~ii;|Pi)sFLQE7702fx+!6R77Wi%8hw|yR$1}i z-Mdl;rBaU{`1@Xip+fg;4HynO>D;yownGfPy4g==I6x}8fM4u@lc1!1eXr8RBW7Z3 zGQsgTLRMf0E>!Boiva=`eD`h*P-0B+rFA+@C4R1Z`IBz92rHXR$)5}mn-hREMjOZo zVxBImDk#K)_bEm1hnWrc-}55G2^kIzpql`acUE~43d1i?_gP7vz44M>Frhn^pc3w3 zD6{S~T*ADg zZ#Ay?&u=qv*}fZ z3$4{^%!VGoi5?>BxH*aZONcmViLQ3f^aX-NtQ(#dG6iDe^>uhpn1jV$mcKVeeyX(H ziuH|!OPM@;_%gR0An1&2~vWlJQ$(ajBhOKGxxif?ozu)ls~Nhpx+O zq0lIEujhYBfykeaF96s3Vh8Y+F$Z zm^U9oXF|p!TJ_vnpb%qX7maC*06n^Y-ehrQT*x==@1gq2Dn@CZv-z_`qQWnSOHu7| z7rqk!E>eeI-LEUj$Mz8>!5qDeKt=TdNG-mh+;@zGdzegA0nUUxplYbZPvxt`1ONQA zl{D7e!(NlUm5z=BFg4oR1j5WTJ8KE8B*N0DPLB-V3*6o=8TuihlYGIH;L(A_JC3$>NgB}x=@^1=6tEii!N7oU9MA>ts!Arv|-70 zZf&saRyKh{DEr*lBX$Wy1GsKG2-M3yD*em<^q;f9ETWT+=mq0D(d+OiF>jIO3v=78 zh4z{zJto945Fz`FAbO$ww%)di2_8E`^hV>$moJV}9NSQ;_qzzZeSB-H$W%gH99|Px zmS_{`kH{8x?DZyn$tmD5-ONvO_i-u{d@%%51$4_9FbG;Tb$Ew)TF!a7mVMLd4TSlcDNonIQK^(4a1b zP)#-aMR;1J&)c z&q(^~B~+x-^=qZ6Y$pN(^ZFSLjTC$1D$?mGLPDz$Pfx-^1H%VOCP!Ja?w?S*PJ&hq zAQZ!X5$S#;YHqpGBeL$kUs+zaWdnv?+_SP*qSFh5c*uW_-}xsOm)RW=pUY`EUsyw9 z^o>TlC^bboB{>nIy{1q zqZ;jp!7n<0S&eQCCs1bGNp*1ETJcYfYVJ7uGj~&>=Nr zKChq1k#kz(4w-Xi@5O@)W40HnO^A>uEzO5ixEQ5!<_`zIV3R((OxRJTvdSwOm3z zJipG>WD86GV&T%f1}lZFrjyb&&h3arVXJf0 zKr2VFMPo4w#b;zJdyiSKqg1n)s5Eh}xX-&>PBZ>+YiC|Y1wYI!K32-bO#qYOA4dzK z|D=ReS^BBYq)d;6&9GiVV1_?=v~JS%U4&$_?j&-WjYpW4y+?uGvGo)ts|;o_E(I)Hd6!J(4?&UCr} z)O#wdG~rZPebrC*fbtNRi7HN91i^tqAcy)R`ihCn`aBjLUb4EbJ_uPE0JcchWVkt< zL}s85!TPN)x&`s7RMsDk!- zQxcRc)CxdDTIuIwg&jzWQCDV)c^x81Pfu{B0R!6l8a;_;cn@t>j&TR=Q;#4U3&pS| z%cQR8>MfAf3<1-f)z|KBCT(c1r=cCYX*5bnt~JLY6PX5Y#6?R^%IzXdo1XFwj}*AY zB#!7+8+-Qgt{S@*=jOjuaP@dOHFYIGYz6qPtqMJ#u)XW<=AfP)LNH03+c$dH9Xb=%aJ|P#g;3+(B`uKs8)0XzB z+dtQ`Kr1MtrcyTL9~iCJO}l#j7i0F4xge@) zjjAQB1+@(S5I##VlDpt4{{u#UWhiDkoYh{01OzGKpAkiB6+PS>+91f8A6QfMGzmuL)bX5X4uOg5i)7qjSHGKjl!v4 z2Uhag;ukY+6w4xJ6%@T=EH=(!)5}xV)g!ZMVK}Bb$V$|##_9}Mp>K?pm~G8i0x^Te za3<>A0}JZ|F#6%kUFZQhn<7!O(TGoL(01 zpI#NHFr_2z&C*xAIV)&AInL-;>FCdQG$#=PZT$F+Jd1Gb_4nn^E3&PVnAs&Q!0P9R@=Lq<>%|b+PgF(oOGmU<_&yOZogoZMLGjYcS zsHBPSe@CEVOv5~=BCyDxY9lNO-(5q`&MT4NCa7880Un#rQQPawx`QPp=8tsCwxFkF zW=*wgm*e%+>=L}f;9i^uB;ondzfQLwXZ7LfQCVdDgzB;0f8gE~fu8-yu+yY2sMIh_KwAaHsO}?1 zt*Y!Dms0aE=E=67oGy9R{74WPAZHXadSB~%%uyO8O;FnpFY=7wc^*<`IP_!OC<0UW z^CPD_+mY4^5%l_E4eEYx5F^IwubOoVruId@u69u0AQPSjbrRx~rk4sOrG5n$& zD4e2~Tj-P0JSEp|Z+y>zFc)rc>{Kb_7Bm`&UC&BTJw4L_ra+0MFPH>GPeKqfgmN)Y zj{e}Nn~lnTLOk)O?y!$k?AV&=QqXX|E{If^btMpDrf^$ERK*1{o@}K#rEXKlP@M}0 z)AW7=iwnxXIqC-z$cf^WJ6%_lUaR+Mh$RjJ15m0qK&c!LoSq~lnIKJRx``qiQH?v5 z29MMC;@n7!siSFrK_HK01-tB^cB>w!&nxY$vy*VYE;gqz(M#ULY&M8FBj^XfS)EoRnxg5t2G>o|<-%z~d>(x7T%oP*+uYu6 zz&B`{16#M?$s_i>r%a)Ra!S-v%D8sEAVN6ohvkX7zy3Vx)+VIf3^y{LWsh#}k89xBZ`0YUl- z?wGbHG`)0iJCah((tyz{^RBw^*C=ZcNsLkhDCH2IHdV4Xe&BDCU>XQ}7pkK<^qYwe z@+T-46`>vymKponYnJzXK=$=v|Pi=y8t5Bh*Mqs99$(URmco zT?Hu4+8!!qwh57_i0nun5ZjK!@+888#L-IFAQY7`f#bN$@IX2g)@e!+-Qh_HR~9L= zwde7AUnmiGT~{^aG#sftoCwuZNSM@2 zkiNibmq;i~;VbUBjW4pJv%KoMZVBT}v)5+7{Hh5#Efg2YsM(E+Y0uO&h~ok(y!t7W z6hTV_+Uv1HV;BC0Ddb-E^|Gg(dtq==E#>!5`fuNi&g{#-jYf=x?Kww2@5lIm_Ls59zU0t zQ3%Yn*iv%UFV{bT9Y>7_8uMb-a7@Kh8v=P4OY@bF`b`(%xqdfCwRKd@|QKSm@g%sR^?>bx$p*_10}j#~M4TkKCP@G^|TY$eZq8otyK#Zl5qM)^^($3u#r4 zkfJBf*9}QZdsf2Dxawn)@mpg|6ICq3dynZ<5C(x) z!jX~W1?}?2nPh}3IpKA&g>C%+I1Tqf#XOMSQ1~r zm_Ei7HyDo@BdcN%C2Yh7!i&Ae01KA*L&dE(KiP?jcKz@eYgQE+P5P)&pBv}&N3k-v z{i<}tGGTXhlzr8J^kbJ4_M1u7)gI0^v!|N`d#RK$5u37tnU9_9Vvezq8g0R*f~z-L zQ8{B`akYxonuczprY-+3p5E@0!SXJ`!!H&^lC^7r-2Gn9Yj?kn zR7>3XWPnlfo`QeWz1;Zo*XQE;dhz=8$|5?O{mp|Lm=pX)sb#cmd0~23)K}(f%`$sw z{iqr~OZ@i#^l8?U#G~gKhxd5pQB&~D*9?tYjXy`|rrF!z5X6uT!HDt-D?en5-4ZS| z_BT$TwQo#EC>R7_pfvN_d{t&(2tEef{kwj+3L`k2 zUvIU&JSA$NU$%qlbAGa!%1;Scaz+heM7fWh$d(uo;ysa1$Jq55;}8u~fAwC2n1yMO z&O3q(h}hcQgDyy)Yl597TAB;qHJV4bR!9|9Bx$e!r)=j5tcKf`qYE;~M9TNQKQcs% zdZ9CQaPtc{JnBX7vB8yzDO~2Q+`D4#tNuBa$B`kpH@Bl^3jX?-bT!r|9s9}Ktg-Tz zPf)12F)7Vef@$udBN1Tibv?n_T7{MB9}sbeD`)A-LT8|t7`nQZ z2Z08pU+N4OoL5Y+!e41;@XgJB`@Mr~I5Asp!~)Rse6wu1^$Kg>kIrS z&S!wR%{A=sE!4ki{kd~olr?fye`BOxB(V|MFJ5z!(!i1XYjQ-K?kH4RXwuXKL8X~d zfsN9Br;^JdTCY^XRQCIyC@y~==^m&ynM8fAiT7#h^r(5p&xFg8Nx6(AP$RN9V6cIc=s#VwpR(u+4v|QhC+_b5&N8 zb-L*!u3eV8}J=$xI4hj#x>#j!YYT#(UGBZCm*Sqa( z-b#3G%q5YJOHA@zs!fp8c5yGL&#W(kpdXHbgdiWkymrzfm4K0QTZC7+rUHy0u4U)I zweib73wXj9!b6NGNdYfbaBVWI*_o{zeL#@cC)IKkU*uu(qVrs~=ozI%MHe#aseN=_ zmQS|ObwgJBr(_f04TO=i2QLTB+|%thx$h$N7^%zFK@hG zrPKV;>53#A{z}@7Ym9kNPcob}O8@n)1jzwSk6K2Q+v7b7U;$9hUYUpj9yE>mc7BdKpCN+!Y#sv@vVi%7qHjV&3Tu#EL$hgwAZy#%Rey4_6J9Pqh0{sPI; zD*0Rx9xsl7$0SR0CcAJx>b>ssK@l`N+D7>`GT0Fc(e!UX2KWs8M2_ni5@cQOm0JKX zZdLejdyT9yx@#S6TS6Y5Fdj#zq)h2TaUHooG1k9Oe|=wxtZ{)HL|$_HUc0?Y_8bY| z!w)9z^_2_^nF3aBfU)+XxD6@{!Lq8Xm&_N6yx)&xigR1t`ze&z;I{i$r8hi;V4-&F zzVcogWh?M+SESf+t`H~jd15a=Vrv@Eca5gQFum(Wl^wVxAvHy^^Piv0ef6U9mac{sZAk40yQ^EI==gIxcJ8%ojBhLx5HyUgl*0^y3U><{Ui06=*A z(mGhG+C95d0Oaq4iK8Z2<3){gM97Y27#RPhXg zR#LrOn#zOjNB^ooNNLBDor5_BAW&>O%;AU-c?GIi$f_{%-GgjB&TQ)s@}9H1dW_|?BJ$M z(S<1K4n4GghJBq@NayHs&UOd~(pTksPnc&X!M%q+iwa>8w0V~~x3(3WDBG%q`0J=7 zQ7HVbOc6tO8+EWFE~~*4c_&S;CB&6 z!lep0pmb_I4lP1&Um0XJC4jId8M?kjWUZ z#y*ESzn>z(zVkt(0Uo$XIh9udN=0g;zh>nD-lrjeE~kPehYXbE?`2s;fYXu%#*=aa&s4FPtKZ7yi^X)U_ha&36{lvK3a!WsDsRC3kog853MDE+RoOoc#lGL2gnHi{h?pY4Y? zU>Y4d0mAAC14&*I{Qca5yj(gcGuZ@PSk)c7HZ;5cUL zpm+M(0ThPpRRHF5>nfBefSx~%vup|)fbu(cYCnDg#bEU)#k(P0hZwof=xoth)CY|7 zg51&gCNmIPNDmPJa6b6H3q>y_StY(?ASO%-l~7!F1^yep4Oem|Lu!4nPg{~(Q1DnO zV|AbU4CUopNWHSHLR#VDNHv(J=;;VgokLeKB%45*m5fmyFqqy|Jt9r_ZvPLK4Xhdi z_!4@L<#CiK?jU?Xm-nzvQY;L`*ZnC&>#BrL%=q`~(pQTYM$h7R3Q{c+!6j zg;ADe+YkwXLkon4?(ovCp524}XF`#omk}OJz8fsZq@983)H0?0MJcofm(ulN9Rg`lP_u-xzy2K5KOk9MJ+8+dkw~2n}{DylDGTqESmZ&&vR#^Bfd&9+npY)??EQWL{&|% zwm&~o6ETH58DvA3mZ-2KI7y$n6}r@6Zq6+6b3GgO#yAvTg%4A#^JR5FN2u%(ygP1S zIn^iW+jv1D{8uCoEtCTVfEGa8x2&!*CaKDWjCR}*>4^I)cpX%dBdtLpO98g-k2;_J zO3ji$S=D@ii(+~3wF;43GCPdc0&&hMu2?4LpPoscmesVq7Xt7^EO`RRdI<- zm0n)SrCe~6Qm|J4(cj|E=h3f?4u~~hCoZTV{8X&8p27S#E zkk5@yABqPz$>|LF30U~yUq@)#rx3vOh9%cX6IDdCtX94<7usaPldVT1et?;01H=(( zy$hBy|GwO8H1`BfSjro-Rc2M$_NgnL1kTdkQ$z7$l5Vk9oG5hvi98FmhtL5OlQnZa zk}jizL7I2=ZnfBNh0mMH`Sfk@KLu1R0hoF}IaayPSA}5fS`hN^ox-Y5Btmjfh)3K&TfPy_MWPk5|GIIBLj-(A~%XAaAlxLYG*Vh7`d|0p4{Q||Hmf}WBF zI*_)HqL4+if>ZE&X>F$3vlni9B)p!G%9x|qd^K9HI5vLKj>n)t#dbi2$1f^wV%s}z zQwq!0Xgn~|HKd&mBo^uFyC(N%$-4Ve=~#_Y(BBJ1$*M1n<>uxl-7kMFdPQnMYiA4c zaHOJ!4ndcX8CZpbhAcvX3%Zxfx{$X(1!mkegWWw|tW%=gdED;%JGF<|`Gqj3kw@Q9 z$1hO$9864`9*O*Ouleg$lZPSH?iPFTMmUdPv-?GOa<0wOS5QydYr5oI+Q&CGxy{9z znnoT~i+lMeG4Z==oa-Mtdmf6}%g&Y=G%So*VPg}lf60^xvPIVI!F#Yi2+j&7AOU)d z9uS6PiXK=-4FYvX`4Pyk6f7IYBj0ua(o{Lf1)@6r_vIAq&k^Tqg%p0E&KL-vweVnv z4uHq?8<%@_=K@++p~gS?NDfI8gm7D_aX2*SF4NdB2pM%&WKiCU0HTh_B0{$%*H?d7 z>NkD~ha;VGUk3+FuPG`m_WDq=CaZHYRNm_mgbqV0RFXNf64@4B8s>!l&}^4QI+nSY zX6yB*X7=&5(gBwG%8Lo7vy}z1sE=Q23jXuN9ZLdfZ2HIsaD@*4d3G(O%l47(m(lMC z7CM7C`=@Pu&k|zX0`X-cY0Wq9Rmn#-G#i2m$Pj1@Z3KTGP*mi08WL=x`|d{!*yKrX zK-i2`I@B6EUd4pEyNYX$#;yg1G+eU^mX?<8?0rPR2&k2;wZy75%7=Ok6x?16@3j!3 z4G`{;1jHCa;MS)1`>Ey_&CVInzbW4$hEYrIb4I2u5AH+4bWHNxfM^Ya^*y3VMV37d z$8r#aWA;`$X9E!LK{nXTCLk{ucTT|@OI|O==J2SiGP z6H|VoMIeF||yDY?ZFxF{Iz9II< zGkNK5V~*SWju-fw5`pGrU3CaAY>G|cI!`fyRP+Y4S;p#r`Lo&ARSi?&tcl$`{G&FR zZ;GwMrjtOYsSA`S0_6#{8>CIEU%6Z-j)mn^pey~d{c{SdfCv-+`Sa(64?k4skoJ=R zUReI_yuzOZlPsKU@EWy*x>2i(d+>91Xw(ak#3XxF#rn3?!(+h)L_q~bIXnc&I`Vx= z7>8chCu+g9ph70SF?9y+j4W|MZ@{p#_;s?0jf3qgo}L*Rp~E2k7)CMkTz`R{H5CCT zn+LmDG~FEH3dh=z$Yx+wU9;9?-0Zby00NWwMH;aY4Y?l3Pk zoGu*oo0v6Bnd?N|WI2yewbE6Sh)B&nuNi!XyG)Aq2O=dKm)J(KXS?Z1%xDJ!AV%9$ zn1rv^U|-y{jCLFYWmWfLucRV16_=p^s7*#~QM>yV>F-5!8fHvQWhTn1D`vbG3Wzfo1&I|M==9HXNWszXQjSPZ7<_v-d!M`*+4L8sS} zrnD*B7wx5;(5W{l8;(%oK6*5Lk3+Vw?od66(6<WKn+Q+Cyb-ON`&f*09x>>^G(j*nDX9kksI0$QjMF+u1CxX~d9 zl;3CFGbUjk8CCLzpmeW6`8q*;HZ7~N_hMG;Nh#M}S`*&$+G`1lYSH?Pw2?gttn77? zu6v^q{xHNkz)hCO2h?JJRfL|;N!1A+1${$U-=}W z0Ztwdbi+KXy^L@>GUc%<#^A@4pD>9~L?+2hW~O24NAe9r*WjSbPU=KQd-q=I50|#z z&8yfO&ews;AKN5#%M2LpK8u=7@x?BF%tz8p)qlBvE&s4(iq0ckVOD>zF$%T!)B90w zy8-)QY(ak!IwRJU^GDzQw0>iDAFbeiY&)YvplQQDt>}KM%|yb4v%A1075a{Xs^#an zng{v395543U?zyt1QO!G8XP~83Ef$zh`g>Q1N;uQ(vYNB@i))_nLAL1(#cW2Ds8=; zP17LpiO<$}vt(xw)ot1^bP?2?`n|Tt^TYb2qF%jsJgcEGRb-;m`-J@n3nPiQrFt(| zQAt~Jo4XlwQrkw%aq4g|EVxT1LU8bEOecc+-jA}Tt~w;5uqkxh(7o+(B$VUFPE~LJg4%PjUmG!eMZoSGGW$Gviw*+>>#enadB-a%x)@t=J4%rnbuxLx%a2 z>n0{=P&M_n+2|-Tv~_>w>d#5ypz+V7_s`74MDvNpx%J%Jr=np*?WL_zV$4;Cz zUY0L;Rp`^MHK#npo<=L|5cg?a8?%sXcd9My>@b$0=Yd;nm+vXv47u}1yH}F-?@{zK z?xE!Lyd&j!D4YV1!T}_}EsRsFYSNIzApnlewR9ov7kt$bs~Nv#qyo z4VCI}8~&J8O^u>1YLHD(2}9OQ(ouXHp>*}OPrFWTG>@8UcfCmeAuIdJ ztsB3(-ZHo8%z57tR(+?Oc!0IMrsk14rzcNEf!;}X5W!L&K9J>bp3Kd|V`m<}HjH0d zWTF>*`4C3^%X9B9Bvhs^dhYInIh;i|0knwcGM~ERFltij>oc{1vy>~%4mHHGS@onM zT2YZ@B((@5P-&)nq4b^G$Bn5T$647sK0Flqq~^Z7g8USBjb;jxX4%g}iU~J{%i2A$ zNsj!p)swi~0Y4)TeFQ4c>z|mQaL;Ns${F#&$Jd==cW?dGmG}k$5YXPm_kt9NI*$|Sz0&t* z2xk}XJMl^rw;T*vIV?9N1s(OwUt2l@H40QeAXGO-A^qgfWJ73(SC+{w5t^XJmQ6vb z|EEqwDDV7{vfvZ3%)|-%N#-wpdq#^=1r|P0XCcFOP4kn9cb1-DEOsRD__rzXk88v% zL);f0%f}7V7W_zRZ@er8)*2UGkPPQkz+@z{mWt6MN? zEAuN|&8S#T(_dq595ZBCWk>0zhRK%5kokRll=R?4?j&yV1cPZNsv$9aV(0v=w`70; zI9%g*=xBJ9$UvRh^y`grN{4gRhK-dp*U zuf7ijt+!z6j%I%GkS@~rx}Y@W$Sji8W8q|Maw%yLiMTSZr_C^{WATGRt`aWkyt~{le^7><0wsbjW8lsNQe8y$MN2IJtarV~)LerBnMg?z--xWOyb61JOQ18!U zF67mvEX^GMPCJmOD`b^^Jw|w=FXd2D_ip!jvLDRoq@mys8MVu+v*udkMMc>T4b3LM z6#TK@DvK(*vk6b1y=!VUJC}6v{J|m`b0fxdb&6R8tEp67DLu7)b9X9*`ANVLwBeW* zb2IHDS}tjTr6vQ6&uvtOty{zZHv)*aD>szf<5Dx7E(<@#KH zj+!%Sa)kP(r-D*(+jmU6m4GWMyaN1;kPR0^jV##!&;gq>u1xrLE4?pmK!7zInCBVf;_IHeBUB8izHepKUzRg zznz%5Y`T?ey{PLqfkQ0zrcMp@et}{22LhW&cQ!4uJ^I8C7D={eU9PBh03pw?Lv+37 zNm*hi1JXg#)s=lDx=DJo(${{c`VlDq>hAT0Z|XX~QzZOTCb?{8jyioBUSj#<`!?^c zYKX}?8;1rLqf&axyj3z^yKNCWySm2bl0<1HtAlVI%U1;~M8QG2W%B+FSHGsuF_n!z zH_L`_L~%7{W+V$oEj~vTsG>TCrk~fX{_6VL=KV=jNabMnGszpB@e?yu%()07)Dr!z ze4g{MrV7oFGVh3r=I)iynB_BrpKs+MPk_p}uGl)5H)4oAOj|qjIYpY=!14b&?*CeX za1A5^9g5SBjNkYt7d1Jn^V_v$qN16d-Ig|qhgNRA^e(>Lnmw#O3)E35;0h~IpU}|t zy9oZkmV$dx-t`(P?;X{l+I--d-^iEDrP#MXSN)?vz+%_Hlf>>xxE%EUNk17i@)uv_ z#CE?leGZkRyKOP*A2_19ZpXYSegj#|>JFO6G%DWYE(seyUEoCz30W&w_5SRmRi7QU z@MgLd?am$f_|KYr+u!(UfTJ#vH4PD^myL-aTr~a@+vwUdSi6XmctY?hia^!Zfr#Tu zDzW1?2Z`4%re?i;{z&U@2u(ny5&oI5d8ajr_u*oGM$Mgj6fTzuojHWlFL;>S9R7d&hA1u`?p6+a>HX@{(yVq?G*T*5@2+IkRQmMgcRIDvg{TtugC7=O zjtWh*kG5Zivau#o<$8$SFz*W8Wz_MVJ^f*|>%OG2b@@x}1uMaRVyc(!Cd=06T7!qY z=j#vAC2@OuzA=1bm#LvoO;&;i9FeYLxi9Q%@A~YuRbTvmZm|~mr=Y6Nox+_}Xz?eF zXv}p%?j^Y+3QM>c_@5OjEh$y+4Hu+}Y$sDbf{Es49@ZN7SaiOAr=$W$Yqy80_ex$y3z`MvIgtI(fnup+ z&`SmEHp|4*H`ePnW3{d->~dZ(7A;h2Ci))*RNecm)m1bQJ4Y;G{cVcqmr+TraqEN2 zA{=2#rZ% zNue>?YvVM6;SJu~g_cUy7js#O5<{<-bM1eXK3J3$b)G|xACWMpH6Q=N{P_*D2YK4@ zg+>;NgRl-6bIP92#^2tftx||aNtzF&C1ZV!cP)i(Rb~t zR!hDgF52$Cib7v{Q~0)>iEMe(beG=#uqt?-k_&}!Mg^+Q>@ouI?b z{^KeCaI4LdEv2Sv$rToN{OiWv9kH@eTrQ<1sQJE!l~aN|3@ONR`*kUPm|Vv# z54W%uhg%;g@tUh>8!;QccWIVYuz9pU3Z_Ob!>-B){}akDRdQvOVb_?2|Z zZY|}P`&Fi6uvbbd>3DWfKW}GdGC~`?e(^2VU96p2p`7izqr>QmM5S}EUgOJd0oOBN z`b`5-x36EnHqdf!o2xIj-q4(26cbQMES<&=`v-;auc7|EI0}@}bGJA;HI~^(hNc5k zwKNG>IfqQcqT^TmGgupbrZR_Jyi(HRdvaXsC!u%>fAK8-B;k~{ZqcMJU|x$~+51J@ zbX-q?r2N)&S|K?zq^ASkNNv zFc-u=x$=^Px_3G+J7Nf0Z#fplhs6pjy&$z;+|^C0QK?ofvuWP*dRi3nsI(HP-DL06 zjGn9P!OMlkqu&hKxP7;5shsMzOI(_4p0%aFe)#wEjg}{B8sI;AqhQe&_AuqtWy8Ls z0YmgfnCMDDxwoE_Cl6d>Gn9b)^xH8r)@||kqFBqH-xz|}Uw@gNaGA@o+)eXLxpPdt zE5o|&XqeG)6w|xyn<91oeV?GAmgD|qRkT+7ht`ic@rL9|$LTL}WivbrG0hY&CGuNh zMnt9v8^f<(dw0ZZlf~JwMk;j{ZSAn9o)Bq+cSlq(zz+ljg#Y|H&MK$Vzo@$cDs)&l#dcE(d&ce``h~7L zA;^Q)GnN@V(Ro9APC5`Kwt0Ay%dq0usR7OI_dKU%HcJ*=yL(pQLeB`W{o*$%MQLxx zUc>sLku7o}6uXdA-H*Lf&O6^~R_{2~?YNY%Xg6WV6~*zm`cKVlGM9^gtt_x29j5L* z(!1Gi~n9j+E$}?YgeMmW3%Ca!JdS%`+z#vdkX&Mxko&#@Y0z zLI|HShX39eIiQ+nP85gDpItWCj@INQ#Lfu7uJMbGYP`EKC(k=Eb1%^wP=hnyLwV_Q zQp;v1?SA?EiG#1uGdYzh$RxXNwjPE2iBCVg;DRqls8K$@NRRj_(B6V zyi?}1OoQp7*bD}0l-atWVa#?r<#DC6COW7kQFYKQ+vPDQl9HN{td`_P`4ApI&P1h3 zvhMTb`0m}6$4ULPjfoP{T9;x#vE)!TeCN0J5a}_Be2IORC>6iMC*kPPT=-(9QU*^| zeU!!(Sh(C=bJtVBTvIv5d#Fl~aAY2S3iHAm`v$;2i zq_36&-;*I)DY$byP}$6@-1Sh?Jwu-nr{TdD{`JtgpHF7|<-Sh_6U#!j>`}Ez@|n;fC?wyjLftgiqAP<`*t!;j+lK{{AY6OAI=EEFwheiXNhA!ZBzc5d zI8A&avC@pYf5sqA87n8xX@%{zFomsQ-^rzjjBeL5d!aRw_hy_WtAo0U(1-Lnqiv=A z(1D~o9+HXall~%huf)v+Q%lqvNy5g=D0BY0Z}WniERVDoKdvl>n;6$cz)MI!Pd5Dc zV9c;oO!(60SCAdegK5TSf!w}uv8RrZrzyy8ezl#F7>QQ&SM^LDH2DdvH&Zu>-#l|& z;!g>R2{bck^z%?&ny~jxp00E>7;a z$PO}oI&IlFbPmUb%-$VEG#fY+rlGSCi=T0}5Hh&$FJBkuI}ZQ^W=%wk~YShx_%1fyqIP}Dww8K;E2M1W(X zwL~F0yGcm(+ND~wI}Th@K>NLE8K6l%GHe|i@>G}$;`nfGJO79|cui#30n@h&SAV=f zvu>JOZ+oxT9b*caAhwvlOoON%{pna(5o3MQg9bkxHt!2wt|-nQ_H}rlr>>NXmeC$Z z7DwF)6Pwz{_bb9pLaJ!uNibN`>MIAkZcW}=QO|FW$gDMnEvZ47sY$7yX*SnCyp*>= zZQ$j#Lzi_Lxxke5`2i;P=Ue-6@l#LXxC{2~6A*S=A~FHpn2s&J*X!pmHd1;1;Bx#- zAp2MA^3UEGOm{R`^2l;D6z}y}v&&xHBeFBWX94az;}YL|3PgixI%dA~7x_&<5+&{Y zy>y=b^i6$>Ub`3h=ejIl8oT6M_KJq6WgqwX9}f+d#O~2-iE&hUjfY%?^h|~~S*xjQ zq}yYA1ne>taEaf2Vd7M7>4cTn=;f&8W_5(Dqm42qLUp>x?4H60+%#lOyx|`8{h*k~ z2_~n>L|rG5KvCMS`{*h>(kq1|1<_Dps?^rMJwK(tc*yGeG|MM>q4^p}iW-{+-qe|!gOHzy9&W~0wB=J~# zBeqnz%WV|+v0-ITGXKBg-aDSk_m3Zs$S5KsyMuG=QC4<`tYe0Q%#=fR2+53utT

  • iEL$WA+zidlF|1%p}ar8-#_2KzWveTba&2uU-xyruGjUN&u39-|Bo}9zQQ-1 z0=Gw8JM(sYy?ElKaUL$-c+k{xQEBjtp|afdMw>YMuflZ$v7IVVCE@ySuPXI)u1f#b zHx!U&4zO~pUpJYXit$2!UPg*6v3ZS3oa$UXQf2|yv4^?54#(RZ1{T~}i@gn}ZCom5 zEO@8zv765kX^gLQE5QoBzGflQb7)EQe8l;w-Bcsr-w98%9??V?aK058-TNjfe{b%r zQ(iCc4#`y{{CiIEbe@JzUO&pogpiC*)I~@O-v^?e5?ht9xEeKT6Gpy&7z@qVeVWKT zT*{yKbTA4=!NK%%NoR)*$5p!tt572~HAH9WN&G8j?vA9ldDOTK+D=n?GtC1C{&puS zH(!Etq1Yw3(&2N2^O{(kkNUmkZh>Veqe*r%OVgIq*6Akp&1@=~fR62NN|3qNxX#3l zCg##dzB7V?(1E#iMvB?TIV%ApWE^MMc;^+hHhrT48_Q-9`n>0+C@z4c?MvE;oV~{5 z-Hfd;-r`tzUFD#;Hu~)+1`Y8CrH>h;+O2AhBD{&VW>@WsbMT^NX+00R6CMnmSpaTx zh?L;KZKx^ngf4EP{6`66P$D`n_V5c;pXBHMTAJDFYuBuT7J6JM#l12}S9%4W>Lo?2 z4H_@7Qj=a~PuSAnpC1yfc#FeaKFN|wN>lx#(a?0EUBzLE><)F^#AfKa+J#dci$~T* z<9TlkBFT>APC+gvP46ttKKZ{x65XKX;x z?JaOGNUl1lRCrdN8#H3g=rE(2!nzjRTXW%RNx0DAR#axL&h@l&gpIiPuq8pNN6pDS zG|IaM>~RM{nSn{BIM1z}vKHOiB^EWd0?!S5c31gmT`r7?iMlY1 zD0}_(o{u1{RN5Qag8M?MyPpH(#xK_C&1|Gwi7laKCKpppb!*JMzW7C_rIH4cR&p(6 zvAn)7AIwORNHMKDBK0{hxBrcSO)g?Zr2h&T2MHN|Eu`>qN02~b|D_71MH?UcwD%N5U!#ijl3T

    ^#(vn+=g7=UR=lx(2E}cs^tS9K%!!VupS|2wrpJOj%)*G zFbv|)oh$HJ6`|9VKHuQ(_wUd|`QINVm;4Eje#Xp0zE^LuP_Gd{L#|oWp%;z9&KA>> z8vb#m3J6Uf;cHYVKy$J$yXi^J!XpP!5Bw9n@sqpGZh1t{88D7uAb(+H`PuDFyBb}H zB}*A7MG{0LabWSUVNnjE28v%&fy`Y(!U$E;urI|I7G_Slqrv_1kD>!sozm^vyenzl z5_eQGTY+L9e$Tw4h5mjl3we_4+0T}!h}A1(t*uCDfa~WP)V>` zo;Fm~G%~La2ULDNa2-#ug!<+BV}(F9NeL~esDKbcD5@t{SUUIhg5m1(mW%A47wOne zi)=QlxvR8xn@v!ShMyoLD7B6nKtws}BZ=+S?=Z366~I;-3WRjw1HVcThYqppkNjKN zkn?8zz@ZbSqcbxz362UR5+!0?SJ&41g2GciB1(S~r33yr3WjgZV|Qj*Sy^F=^*{!m z@Snq}{I5WB_#T>qT(wPy6$JOaY*50>%j;mSJek0>)(j2VGi9f5h1cUD2PM51DHG^S zgIO$>dlnYZJ)ds)JKb($0DiB-?G_4q0#G;S+(~#2ku%u>0Ab!QQ~{u0fT*TK0{>cR z?EQi{Gi7d-(yRM>RSAVBeNko|51=Y%w(#taZTB^~RoL|#b^zeATrEZEZa>@GZ*EZS zS2${pOLmn9aO`A4WH=gj{?}aMF?>Ka3^-1gGp(*rwaC|!1+AK^igWNO*{3}Z1#>{hBrOo>ljC7uPXp?_%QsIeX1e#&Bo z+m`Z=wha6Zv^jHYSc}U9KI_+I_YV|z*=MkRn>X+7V1j?@Cc5eFp3wt<$r7jWbLlX{ zyP%WtpO)1dj)*_rV7Z;2iPZdwSad{=H8~i#KrNct-hNO%1hC}-fdJnY8_PC16LEM1 z^`3v3Kir;En_+1Rk~Dk1FL^!cy+L5Z4*GxhB2I|wpI_Rb*p(18x;-gG*8om8dEo6& z;iGM7DpAi>xfJpCB>$e*Ys!#=o7JC?g`-MnXNO_mDtYbCLQvDx#zMtq0Xu|pnVOWF zK2ikr8fy(R=0_SHMMY;CjkSs&hsQ&Wf|r3&euxA>4_^Q--v}ITOu!uY<{+lzxRgW% zD8CdWBxgz`fOWjH#yaJ1Jg>J`q0sMVwLpqGHDCiGD-14=7Gm5Y)Tv~kI6vS=RtCc& z;iAV6ffSeeqkmgYrCvw2H3oDIDEcRor-yg-67#?^v|9IjtH93s`ue)TAE2nsk892w zCFGPov7@%!SI;k5m?O<+#!s9gZ~v_K}}?7FSBWb?%xV zB7jH|S;LlCSvLU%$xZLl1`shMgc1^F7)PtqE84KWx5rHZwiB&MtUIBXx`1A@&lMbh z_3yF2XEtjHNW}T$fJ)h5ipFE$4(Wu8%SbuJ8E}EYWq0|5{FpC=Yj3p>&{$FZbG#bD zeYw-b`$|HG$l{R4`4&>cHC9FV>x%MS=UW?aoFwBh`cZS;pEJ%sRQ4L0PTr?n`H9pA zzG`W6)Lx%)<^N$+BoGynAAt;C+nTf(YPlH--p=t$@t>JhRus{{_iO<;+i*KN8;J_v z%y;u04TF;PZ2$8bbcd5e4<*6<#WQ@v@FQX8ml;P<4^)%7-pMsAA~*;UiSAECB)fE+ zpI_m$pho@XmM24FQN5i7M4)RwJ=+7mlQmZmNQ(wD#DK!g_De^vG==VK5A#hgIkTOS z*e~^7)|L{v>cUivN!Y!9j--D>KE0&qk)K%lQPEhx^*=X=MUil9UoRp0ALoSB{5rz< z<$DGX^D621Kg7Sy@xu@r80^vMWc}^^f_y)g6R5}BCEz;!U>DURhIZZ`xH z^!{v3k%E`iYE^Crhy&9nfREP(M99FX@0=6_QihgrSS(WN6>|W91#HgI7d156ZDe_h zz?#6xoL}BhzvSnPu8_^gc$$t<8loKdz(`3^yoH$M6DK{_{N}WL%(|oZeSK(^9`Ww)^+ggOO{+z@U{se?jW^n}H9A7`<3kH$7 z^GC%i*Yu(_jc=zcg~_3(i{#56NqfLuu*Y&Z9@WfU6fU+181C!0{4xArSMM|Tk(|Vc z<3?RcVK(p1qSJXP0;H_a)meXM94FbDjE-*;=zcG!bZ}e`aBxauj6Ib`M!VO}u<29w zj}|)7w?byTvOKjw3Fk2og6U4Ajq~?Ez+FqY@c*u=;4pkv85kUms}=gmD8%l6&{Wzw z>y|pMm6T41Rm74p#HM`Cm7@cqJq}J)9?0^Me`ZxNsbE96@J13r$n1Oh%iIMu!}YjH z!E4dT+AF9sZqEaR1m+J#fU?hQ0WJC#rDOStGoa)$y{Qz!y4Q9f{?vKyKZ2m%(N%YiQ{d~Hx-$cMrR zF%)>96nN4x>zw$V&F`M}OE%(X3ioGDeO$2Z&77Rqd+BTKnW@%YqMUcxMFiML56ZI0 zn{#Npi=46^@#~B&)9kd)Z9U#D19n>*pkzzz`PF*X!K1^Hd*`l&-adW8DF?vbpeTv{ zKjA}`^#x@oI+7Fu_6E!09bFg;iI}tG%_~90-Bcw=Dt3=vCC=vMP9pi?Y?_%X<%NQ3{2Vv?Z*V*A{LYT-&!i z30-MXuED-NzQa{{J}XOD^}d6@VE zuZWL$aAjr1Oc!xj08b7-ven3$y2@8~DK;dg2oN?2EHnYX^C7U-6U;7xSur6Mcw#Du zYhAdAh=j<9ep$y@=6Dqb5fc$XfEN%5hd~$-6U&YklPyvVFj(bwfj3eGU<YcwFpd5-zziA+lTDLSfP3D4G!j zz9&kYR6!9QI+mR2YGGaxmV|Yg@4~ocD*W&G-_8SVs4np9g)^7i&x=V{WlCYw1+V)k z6kuwC=r2JjKFhD-hZ)6Xecywxm|uysmWF9?qsN~}=XA%@UkuAAW~-$?BY&dIEsGi5 z7$oA>44P9IPS8XK)wTUOIERgiR0e^^flB1xyI}JEO7y6YO2tJLfWw7;H^Vn zH`JwyX;+IVs#HsoZQXrAKs_y9M6|#Vlh8+UX`>#cNO4r_7^+R1+lp<_@dDpBtJ(N% z%6hsXjecwmWe>z#=*zLbP}R})$5Y;(zYuj!Q+j^GZX9`een53Q%||uJ*dC$(UV4a( zI!7b0M~WB&TofD$FuoxE1CE#m1Wb7Rs{(N_B2Xld|DDG{YS6fY1kK+}30`kUpKsQt zJF!g`6IG6dcM#g3ixG?0qxxh3Bw>dGA`*cOp5B-(F zTuH^8qpr3Bfg3zs^k)5ez=r&ompy=1_SbF=L}#%`41u=FH{jRjN<2>TJsYGi8~H8P z`f8opaQ+ojmngseE6ikj$Q@OB&GfHqSNsrn#!|%U;X$pvYm>a*pSe(-;fnOLcvp($ zI1~KDd-GqOZE)_9LTX;Hto0Zwpn1({q$nJA48P!169MZl!JIv?k@X+pXDJc%&9AIK zlOWGEwV<#*QI)~JgRxHydX1Gh2b0dy5NLbt(`*#JS<~hbk=S?kZIW@?t4Ww5fpFu0 z5@sNa|3%U{aIO@6{GLfc_@*r~T6o+nP#Sfteq{KI6F!K{(<95AE zH#(cA{5aZkE`_>AmH2)l`>GXW|`o^M6*TikT;*`LL-MPI5Wby)B996tTC!Y395exV!FgA+74F-8Qbva(-)Zzc<{QbRo)Kr!0G(& zEOLVpO^I_;ou(m^Tn9(%%DiWEh$If9<_hcH*&pl_S5okLURr5fU2pt0pFiNXj(1C} z9IjLCx_9L>(~QBzr-}_u_6R6wVqH1A+7> zW!75D=r6$TEn}O`Ve*gh{bm6U^0|z$i>u{H&4r0_{P&9C?cLDtx(2SM!jd~##V>M? z)4ovwe{(6^FBXY!&X>}ahIT1#FBettj?Y-8W;N6F$5~7j4%*{dhK;#gH@3^A zIA3}sK2;KIR^%_13X$fYVZ-&TFB**IB@o#KP8d<+)v)oxz4->E_{hRpM!jPEHOMbk zjw#|*ctH=~ZVG(#xYE==r@PZa-IFvL+v!mkah>EoAX&B%XiTYTs`3s#s1z|uA7J9 zmGfG%2}#3>+Pf4s#p@&f&iaJg79I%SaqtUTNX;9b;p)7@IHDUO?=cms8&~kbv&TO8 z&AIPco<$xk8+|6JiU>8I)I7Pc6xX4V9mZNpOYj*^H-~{3dM0k@GaFH%)4;g}ToBdA zI~)U1;=`wijv!gRTyz#t@Wl89#)hxm6Sn zk1tSFBvz3BN)K^OodS!2bC9|Sv|?4vefN*gZt3WAH;3axsdJ8gofDg_u*$EI5%mWz z+aKiFV)qWtv*_QUE!S+fK!~@x?MdzoK~YW3%9q!4_W&`O)2%x0f;Rx2-zUJg zA3^Qq@8J(3Yp-BmObjuMD3+Ma9EQ(@TV!WgzioLz3QnHzA=(jqx_nBzXO2qd)|5)aj!wH z{6(b=6nqpSG=9#J5;uX$t+5*BGt+CbT2NU6tps%UNx9n_2BTKjA)WN2esPcIV}tLF znC*Ln_>Rec4=*@qZf@>uuHe_pP7a`1JKE^>rm0F2C;mTga|{F*A0JXCj$UqgC%1QI zzH^rr>6g8tOQJu>T&eO1r}3%jrrXx<3O@F1oqMek_sRhp14l@Kfobuq%Q#imn-z*A;bMvxTPxP%i*6&cHT;URdd-$6F5*(-KH;b;#fLYcBBi5^?W)@!j7JocGNWbU8#TBdoggvyimGo-N}c7=zQ0 z!0-nMvvI~`WEm^No7{Tg2u*E>bn~&se=ZUBAHG@A{9b1#nrQ#g;oOs2(j~r%!^3eZ zN+fz1ksyI+5`H3PAdB<8xG~!(n!OKvYNDAwd(5|rY?DRXZDM(mj)WaJJ@a1%Cs;)n ztE$=RfB-f7?p9-XhOEWzo}BZh5$6qk@mgK9`3iNeeyYJGf!krWGAU&CWJz6p`Ybck zGE}sYNZW;q1!JW@Rb^rfQSvX*=Qsnlyvu4 zrXPp=L@u!*LkV@*^Z@HO!{c0dc70QG?;xT2+Qq#2kM!x--(gB^F8Uhmo;EV8!>Vb} zs^{_>&1?R;J1$Sf`99UKtx>ZNXVJrfGO%BUPiic#Yp+5*pw1R;$uGCbX zT;z$NEP3#1Em>1>V7_r~RA(;{98~r`Q$!vcDYtg>TO$khC_y2Q%o=mc)Lvd!?^4ee z510JUw3~VV`+N1N?YB%ZyRChgZg{Uy@v=5<)}rs)(i9T1+&Wn#-3#h-9n?lnv_x@j zQ7NLeQZd+zg&>`)^|ghDS1Ce#_7D{&#C4$+VqfNhAG7ZfCBCNFGi1XN(~;+jPmmKN zn*d?7ir8H;=Xb2}gR=Cg*c39mWvFv6L{04L&Mnpmqah9Kvap>d|w-Boxn?0 zCAzd;4X++6JeFEEQ>X^@ytOLU+QWK4IFshLX})BIVS7fW@>><(9EWl*SC5oHHkf{<%LO}aQ4xp{Rhz~Ji(wK_guK=EYw zIeiD{-4pl>*q?gm-ja|o{hn#$%ru56@pmXNWMkWd{7mepL2^HxdyUd$l}+K0ckaI2d&$oCUU3L&XfAwY{6366oZhk$4_kdxD1AFN=ZuNF5>DQiuoFe15q${_>PC$W9#k%=fxW!~5K+BI^n;?S|5AhQc zge;anw&G_FZDkgFiWcAp9&b-pAMAhBDhI2BQ z=Gu=E9Zp@fO(4f=UE4}qzi?|(&F;sQ04d2&tkj!=B55P$y5|ZJ1RZbezS;uHb(2yCWH6&$Shpp!AxBm|& zF@^XLahTS~*V!LlMLy5|nDs!s_PWVQn(aK@UeeB6xyGGXluCC(&6sp-c9}a?(Yb}9 zRHlzn-Dy26vNXuwN~J@hdhrYr%*R7X(MS@bxe+(?j$JdyZmUuSfzZC6x*2P5Wj$M? zgKIDGX0F@t^EfDr{i@DhF;A!r3ce{`dDra7gU=1@c~%E7r& zvYin=l5YpWCvb*;4wU7!=%F{^4erqh5?UxWQn79H(3X<V)JxZoy%=6J!O?h0dVw*sL-y z3dYJMZV6i@)uC){1>Sxieq}gMUUtmM6r!!e_Lisb{ERbUiJ?VD+2E33B*309ur8&B zOF6i$2ss>W*CtZ5YK>%wnTR4Z(WE?YKxhWn%)8Zen*?7wQWLI@!3 zrUA8EB$d}Ecu#KU;1n3`7e`(ST@JZhhK9J!{SB*p$Cc=~Y zZHq8pp70|=nJ(rB)1#-=A4$kZ2S53t+e2c)AvzCEO$rQ*ppHw3LqA0TX?rxvDju!KdotGn3cw5?P94maDk=eDJ}|6b5<{7d%2B5@?E&|_iuHnF+{_OtCxscWzmty1><-%g9J>eUw%W6!%Vs*F@9i>u zJZdCjyV)JSlDx}{sx+Hv@6*tfDag;~eS2XYQ|r)ldEq?=?-&*2-s&XO z6VrnnE-aM#9p1Cmg>9$G8}OHiL`sLtRkO!9d~!Yv6Qe8m($fMu{?&i9B&*XS`KhZ_ zF5{p-_qYtjXVS6T2KS{)=K1f!_*(wr0T@NUP;n9kawf|k=7MgdvL)pyayF(k$X$Cf z5*FVlXSX<=!!@F7X|m}5UYGyt>kswOPEbzy^IsMKBJl9E_g2d2TYGmUdGWerNnGU& zgqhODLRaTem9RK&(rC18;<>W*Pqph54PkLFC?jQ3HS4H_p-V>{(a3o9K!KgAh4P)> z@kLhXk#ST~>V))aYY!IFRWH(1W}~jOM32r5F4thH(<{#nW~fJ6__Sz?-ZVz$%}COA z3IU#UWTU^?B+_Y^VOoa}2^r<|l~HWyHom@$GFod`0DzicF|jL&~8np8+slrtBZODIhRcsrq(#HODK% z%yA)(1PqQ>d&u6m(wu6{=D_QCs*If&=^%m3A^Lz~Bn+;qKkE12{8D1-5&gIZ`sH3T z5xtSbZBBwv-cIVtMgK2RzY7()8qKv+;>KqPNyo?6FZ%Z@)d4O;?)(QNe_Z z*>L>T#=~T4a^k#p(EpO(-CoEtAlK9Kg~nBS{X|iYlw;kg;zr3de8;=abUJjak>eRi zJ^PcXw05hoC&a;d1^D3C_BR~4X;>6OWSZn0G;kL};E0uS4JwkltnTF68;>S!B@e&R z29>DPej7_))C_8B!n&%%`(MxV-!1>=w|W>Lz{QOSH;xv$HY*vg9?zA_5<6LJL+gCD z!y0=k3;MxZu`Ihm`z%=D;<@1-d#2hYkr%()zNKQPpZ47{t>ev3O4RI0NO%O&(ZQkj zgt|e0z?R|!o4?wqArzjHcKoz~YW0<*DC_Z!}c(E_)ohtq|o*|qTr2|~bnVT1AvpS&4ST+afJ*|Da+ zk&UU$zYt~ncevc=U9zlRn)XUanHBhg1!RPL%8*AAcXhS8)x{fd8M#Qp9;QpPzX&2S zJBO2Y#lYB{zCV(w{Jb0b}~TP8f)KFS~!JqwAm=n@_fEHsMQ*yzv{qSl63PLicqclX75q7jsmZFp^F$Ap-yFt9s=I=`)wVuYh^5 zPt=(VlXe7H*vfxM*IYmr)6@mY1~*o|6b@1mgDwl>WuT`828q@m(aR(1C~fLHAXAup zORfcCx(x8Yppk z-VXAgZyHI~uwn-imbS}eW7A#r-&#jU(eK|CaYQZ?^?g1_=cn?BU9SiBcShp~2yJvM zEJ`s6;7cw5mh>47*8=$!4PLKq5@KS1CyQuqfB-E`B067$+YRq`$h!FGX!x-V_V}l} zxs3vsb5^v;N6Pz>pkweO+BU6x39gExAg~ZpO`@ z=vtG&&J0HLUdyIPf>dY)OU)GG`wo`FHShXxk3G&9`^CYOug{G|h4e?{9)rzjw8PO0 zHJqeFq8ofj^<)I-+A=vvDI3j^{p z~btx;Iar#A%&c)JNbhJ^rUo1W{#-%^N_{`cy!(11vf~Ny1 ze>r6J1&}C)9i;QBbfSD`ca2oBZp4vr<17B6lq;1JB30{nUg{v`(t=f@GPhMBbS#NF z@_q9Z@s66E5gJ}EzRQkVk0Lwe1Ke=D|S6$MpSlIj>rJO`%0$bqC3%nhA}!knJ{lFuhg`kyDjrjAxR{ER}z>q*4j>{dy61oGPU2H5Y&^y3dq8?YT0m0BYK z$VHQt<;`@RHUtcMKt2{ho98pjeX^0;U)-fl84` zmdk60FV>Csp5)jn{+XL)|FgcO76tGTSMV{)aO6G7==&4DIGc+Jwz1Y_OnwoqUWdxn zsiw^d_)UblRE=?ab`bz85vwi%b`{+tqk`iejXKCToc^fq?DjG>{Wzi2$KG+>&#`=c z&rY%Dk*#y-k>gULpIq+%{9Fuu_Jr&)UEybVGI)mM0?9{fPG`K*=hDk~W%Seb%-|fs zhiwwT#c6$1Xjx#zFQX;Ca3ArZk!84?ogHY*;5ZjGddFjQl4G@4cc{1i(*#b+VFzeqgNNBL^Bx(9%i8!xPFuU(1yusKc{R!4ZkOG_NB(Vn+lDC-`?~ zNwdMQZ{vou+kJqPsPK{A?|Hzp#rtK(4Q{Ocr|R-$1Sj(P0iP_da%z|Q1s3&l|J6Oi z>{QO*M~{&-4BF)E|BL4OI3el;0P<7s&996f8X{+Mp;DpS*h7UUK75vyrXX0pQqZ;@ z*X!<)({9P(+-ckADQ2jNYvC zdVh3!6A|`nR&~fkeLNpQZE;Eod;JZ_q$UyPBDZEqwHb1O`M_L5VNVZV*5F>M1!Yo3 zh7_yyIuWqA@ap2CCN3=CNKr^AK+J?W_K$2tc3W)Ii&q$^OWp-@7Ts*SIPjOEF{$3V z1S@lMtjN!OMmFvgq0I?j@GJGIWMI1WUUPX;NE707hv0LCfim>s5z%0EC!-Ca`#B+w z%H8{nx?azfxuu4&Pg*2+lrBXRDwAp*6(4G3>S`frYcNy*A^?eJPvQBU-D3+1_T47X z#w}(*gksxHQx|pcZzU0j(I=wYAK0x^Cb6FV&C|!9(-focZmCF>4ogmO3%=G}U$8rD z%X|y2R zb1-Q>AwuNk=MSx{q{9UHaiisK8pl-qBUAbYqX8&kT7G7u6Ze-s`5u%JNQJ|<*~Z+c zj*0a07+i0sNh*1hiow2NF79+94t1-}#9HbDAp>@|0)R53_pI3k&UV-@S{T1yB6N_1 zV*BMy#8X38un)AdzbnNuLp1Nk_$V|fdh7&Tl^ns0-y4$As@VCy6oa`f0Q`H6Pusus zWWBKfFAe#+gN^;Mljdf2dzXr|VQPx}+LJ=I)aODKMyr=|4Sk?e;^M;4}mibK7L)*^YkwEtwT}H

    4M^-USfLw$hJy0Dp*iUKG~pvSsl&Z^K3($dN()hgw?x@A!?RKm{ec3UXRPaHYCM8c6)wOW-;rePyWgTrnmyE>cdvTC!x@^?b~WD5yK5|gZ@rNzJ=2A%@U4ix}7 zBU3=A2_#Vp5Q88GtS==eBO_CEbH@>rkf=MT=k^?wRbSH2&hEIjKLrrk0T78gTca;% z1}r4?s;hf!)aakwLrn(%=LzuzjQ}`?!xCHXHT$>^mNG+qZ2K9KyHv#JH4t+cdUd$0 zBdrR(q}*0jr*3kT`J)n&8w)$8mKTCsnaWIL5BeSF7ibpC zXA=bl1%^g?fZB!`a8wKbN&_T!eWn_{A16*M=%=3u5ws_{$xBnliK54{hrw>;{a2h} zdt?`$`dN)C6*$$rl?{}Ou5>HX=b4J1iY;xS)=oJ|u#_e;hJAz!*1_@<3*HF$9YOc{ zdCH>CNR<|9%&Cn+c!0ZOY)NM2?LoFAtedq0mf5{dp@_*08v-sgkCIi8tsJFHp{oQ> zq%_u=e`Fi5$~)|hV&tx`x7co5lsjMi!~1Etm%Xd0YO32-QmR9s!Vaj4K7cayT0Du1Hc~lg7b>!kym&OgeXV{ zLi2uO1mGJ&*Ls^{H~}FcISGkK2^=^W7{rw_FgHSNl)v<2N&lQ|0E^j72=JTc1CcY+ z@VqH#PHe-;C&P0TK}(O+6aSH&jvt7GgC&jgdg?~H!S!Aoifl*N+yvH$d~meFb&S2z zPf(59Pnk$KL(sHkUqc&SMLbV&4~rm5b;lEREgnfhD^LPsb{NVpH~;NrWCKY+rpbOo z_I6F?uubEix0vNObDlq>^5DiqJj?iZ8WkrE8S?0|$ezphs+kyDN#Ul>$d!`1W97#1 z8c8qxFRFH%=b_%}U*4FiLmV9KbL-}Ani(EmLNyb5Z%a^d1RZ4mQwYZ_Q`nr@3m@R870_9gF!$gLe z0H-IVrKBfS^F>_o9!ppXHzNf&bN=MOExS~km9XIjOa?&jTqs}gAh zH9)Rs0%Uu{X_Q=P#4V+0w39QDtct}J!pYz>d);3v6Jq_tTq!Mcf|*sB?V`0)kV+gR zSsJV1a$~#5QhwS)hH8s(L)gnVv3#}G)!r$pGM?*NeEZd_`EAU&xft8JJG#`SNWOld zJ!iZV)7|@4p|(}3Zp^jn#H=S=`~caJ{7v>DU=D%bDJ}`a*L0?63Yd2+RjV~{01}f% zAsMhuSDGs2lrgznYFcRe4!JykZ>7Z%OHPlavz7wgLIaS4DfQTK`3LJ`igME=8i%_C zh!Ar)TV)4`A>FJwuO^En0U(~c=;7f(YBv=(h0z4A@uxm!z3LA9{2F2Gr3n)L z4H1VtDC5TRfezfRe=JWI78ocMYmh(P#p5I)59PD)f0|Hv#@{&NX*m4#30Goo>QYPv zynWmd2OVyPFjNNKDi!y~^5jy10@%=VzwW)eO>dMtA+L_OH*Px>i2|DEoUGXGO0A1} zf)Ji0SPt8lUHX#D54cg$GJfR;YJ+sHhKIMl<0a28=aDE>e8Xxa6~^g_I_J+;0|tAz z9`ePRl5KNYce6(09bkAxGpuLyCL#|htLkMxZrLxEe&!rrI@>DFFx0%B7NNF5Pk$Q{hL1AZlPHXU6EU{u+BY;< zKB& zo{Zuj$J`*t8-A`@Nx~FV2B#CXrD=O62{49A1LndH$utL?c%~rW%^#{a7m-||mSYVx58MnV{zeWvv&;Z2P5ZN2$R9U%OGvLWxV86PL=v%APbMte?1K%Nwa0HS0IB zEujA$jRH$#He>jwPr`e`3HKpGug$8GOM~gJB^aLF;+v=6Ag&5WDt2~8c`w8>0+D&` zi7CM#jEcFUKz7GsJ#>tqg-N-7DCX%DgrLZJotOSP=bhM_8R^iD;#Y;~A`sdgKBSco z>{{5z`??l*0fai}bztyjx2%F`wJ4>*nGWyq?|Q>vX~^{U(FR)N9VHXb`6?vf#A@ z)O)vz3DU$IVfv!;XX`(Xg1hS%fscQA}1V?733KU|>QD9MXIui<_VUN8qsYW?FK-Zf4cieDNI^ zqRZp{OE=r)fSDcdr)8O%B8{H3<_%()NEny z=Grq$#5;pc@+^tFb)~oNsYu^Yy~gZOA34j>(FWElsXyM{4Mg4KyWb0F6yN2B$xg~~ zL8YJ44BtM;%``gqaJI{|eKRN8j_Rjp>8G%rTHgMgNgr@g zob@ku_Rdq_fK^MV7|gD42=Glzq!6NG3XAkZ3TT5_w{zdK@g7qr7R70X^7#tr9kdiW zoOB=O$$VAmG^%eam3HuRlT%PY3chn{Or3}tT*MI~C0pMiJ;6|lOOUexMu`f6;kRZo{vZVZYK4ajSC#2dYDA$a!Tjdw35xY_x>Y zb=}TMIc)d7Bs#SU`m1AnjmLCj5_gc;Fta7Mx&fG0dbzn;M{2kGU6w=SgO{144cY03 z(sUteol7mz{*3eEDv@GWyF*-WS{-Q#unyo7fXHzjtsd}uPF1;&loZRF}A29Y{s7`$jMVVx;c`qHlT|EbU*W(mGj_? zCms%63M@e!BtS-;`??TR7-FcZe_Y1^bc-{bJTy_Aqvf67aZSd=In;Opco6-I#5Z+s zTWxi zGQB%lVggduV!5w|OSp_mS1Y16y_TD;v&BFQOL_yLO(eX4KFK#fI~$k#aVI`?eEZXt z&k^bixnhx&1}3;-LvVLD?1mAxH87GxFzwDbER%@ZK89f z^n9->F=_@nQL(Y52EP{I7O;}~Y^nhr>o=*Cp$ZU5b+vMT%ovtFy5k*XF8VfL5n zdS>R$QuoNTe!RUov?Qp-;Pexr{@I;#z8Xw)jxp24)91zItb!#2$JeQrSyGm3@RRXY zq>ic>WMh1`BJ?q|HEFX4L*?SBF1cSDF*Fe~|Lejg2K-q(SR-H}f|;HPY&Om4ZavL& zUH&Y$o)pJY;n(ltCkBD@}h{jKIj6>Z#%0t+Qgzry8W%tM?j|`t*-)8eM}%t zs3#VJ=+bAQhn^P+WdO&$fobfmq@8fB)_uvP`G~`SW~~ED#tlvv>MzmM;jdHVEMOca zzg^BCj1u;2J|@V)`sS}8$YQt&Y5^Rrw`A@nz0^2oA5mv#Te&g>C5NR9toqmeS~6(; zMh|v}%u?&W1Q5d-RVkGrGTy{W`H{A;u5taTNirCG(Fqp|3^3sH&ALumIkG0N6zdJa zriQh#rLuY&RVGKsDd*Sr3Cpsd%pqR`Pl=)A9nj7@$7y8ptp5&$6Rfs){~Zn>J!AMe zY{BoajHMgi)W5JjWQ(rBsD zF>iA|m-+k^t^fcmh2_pbH}-RXJd?@cK<)_$4C`+Nj2+XQ=N*EOkdR!jo_8H!;e|^P8802U)yL5hQvfOg83Y7trp+eveg`4xY zV18h08zU;p;%s@+nB*K1Mj*kUp<}EXNSaNC+y&)a4aajz#Tujp;zSX1uZv_dCAi&r zAZ#ld1}fC+!Hkhfq;h~y3L$_nTaqIIJ`(Z@+<0(EDjJ$7tMv}@X`dH;x|BcP5JfHy zrt-I1uHZxDkLN3qyMUR#Sbzux=Z7$4dI)YR(`>GilToM|vPwstf-J}a=KiMBn9U(} zf64xZ;N>p^G6~SRfrh|<_Xj&3>tq6yFFRZT1%jUS^%39F zidC2KX(s|*c`)wBuEBquwx4$q2FQXf!9XduEFcOX67eHUO!7dZQRM*p#KOU=TZ(|> zXeP5+vND~uD}k_%rt(V95!q+TE_HXBi#52_tEo6E}$tlOi1A%TUI=(djA=xszB_#z!^#c#Yd8sWFJgFO60`OoIp2be&wbK;ihh9&Kf*ILThW!CS^ zBn#)06A%ho4RX5h$w@j*POzJ@~&$~61 zC)XAFD~GV%3&(Qd5Cc)&mbT zgIS|OJgeq)m)0aYT^D~&@M}_s{dvvW_Md7i?MK8`x4uXEF7Bj-wGuB!kzBYf)A@l$ zOZvjX-IMgr9rGEw102K`OiZRxqYouu$mR z^7gOzkiPE)=jji~{VWsM+RE;BvrgkJ1oy~xF$OF_Zs+V+*IL$j=+AP9Mp+c}$#j1u zKnw<$6kp5M(&(glS^Hh8Y;D~A#X#6_OOb~`1LKhk?=<}BuRhp-13p#9%|Z$x34m!W zqlmbr;&)OB)8YX9z9$3S^KwZir7#+Wj{$fniOIrK{+0-dSvds|tO%p8k2^85-W#-w z0Vr&~?_U0U`{U{GL8P&!z*qS%6;Km9pDx8^x$onGbNutB0m<_)L0%B`XzXtfPpqJD znETzly2W4=?tjh~1<`Jp&K%}|J`i}32|N(Imcwy?u^fEPB4tM*(Q zfSZRhp<4o|@H2B2i9w)E4%>AdYSg!ABmr*3?aic6aRaf$7l8H62_*4lcJ(W@IKXgX>4Cd2N-BXc zNKjWucjKidx6&=2A7hAxvV?~N0?<_=g?XW}KrDnj4naBnFe2M#+1`}LWjc{Bw$Ham z0m7kH3NRSO!k+_pF&Y&dAb1du^?_T_IO_CBS~ayg&(EI+B3COjY9kLx>UyYBvH-$` zYLy9exTzOvh$ z{dC*w*G$9G%w3j$wt}wzIc3gI-1*Gs_kD*$C7}VKmcn*j@Oucjj40nH2j8`~-`Is| z=YEdH@QlnpW1cu*ZLNP(l^z{@E~=M-1ljG@!YQkDIfe7iWv%4O4Z~E7L_ZQl?K;a5 zT~W!SDTb7%3{gFoMK~K5kldE}nJFZzltvX?4VC8dBChpYw^YklQ9^Z}8 ziC8uAcd)?n{Je-Nu@rG<+OJGCTb^F@)mlJvK*G(~8H4*$Gy*$Y5b!4my3-Sv0UdQD z9L+tRBu;&r{Mt*0G+TOC60(+Kg$p%VSf>{X>aV?)iBR~1tTFY(@ZjJ;0$(EftAMyU zpVvYFFpjnMC*a&DWBwKN=@4kJ$hZMxX>ZzdaVw#y?{KS3ZenNWk-SpLl5R>san|)N zQB|;qNKUk%dn(^W3=kYei;lkUOhH3fK+Qa?{opGF=El&+xD^qw1FzCjLg7c2A)O@HqZ|xqXtENie;|Tb4ZGEh!-CjwyVX@Yq%f@rur-1h9EzoI-O)-D z6F^~NWcaRQ^P&XO_EUknhL-??P-#M5e)H=PO6Y(bQ&)i6u*2U>T>$Zh7Tz$`{g4A% znHudQI*MVd%iadR+AwhS%2K!i>q+rKduG6nKtmONCX~t51xG;S!ofhG^k(&3`mh5< zbX(XG)*vkB<2lK1KODP>5O_cx{)dq9sB<9>hPZD^qk*iL7_Gu8c!0)KOkck!U`4_lhUylwaNY1@TM?4~} zjtvIVKbb^oeSXX^Az%#mc}JwDBXX}biqCJ(exm3GDD+$*BhkI$9pWi9t2$=o_4~%C z9AR=l6C$q2U9`cRZhw=^4xfsxmPegK_g_}B%`8sUg%F9K-M{A|FfI}v$29b^a7Xs& z&n7=Sz4w{L-oAgXA);I17otY>y9|;<;`(frn^fbn|4^kRR{RFfak$w0ygFi66s&)r zkI-JGlJ*QoUQB*-oCT#uA6^W#PI-xyU7MW}WX(^=?GUa0_l%0_fZWoOUq-MB6{;c1 z?HzrWqBCYzR_U@MYj{=BK5BCBzD@H@YoGCJ3E4N&)r7u3PmiAFhI#lL3h9US;h2Bl zjR%Bm)sjTbBR)_EZcmAypHLY=FXRPxY0$uiS;ajY%0}+4=*>bbb zG_k<58v7$6I{&;KWm``ILtvD=7}#h-xs$h@5O@096&G}5-up!YaA zRn{HODLCK8dI{UmmV;DyP`D+n{-;%YICU!L}3m>Xn*;S2LtK$S{)RPX(W; zOAgn7$qFJBQVb)=g^lVR&?LlT8T@d2qZ_PWgblPwh^dY79d*EZrAmbn+)#N+U#@Wz zV}-c`&eq!0QE>Z1=X_D#kql09*r6JTDWKAMrs3bUSc!>=!Ba{TU`+UaB5FMFN4NqI zN@J_j5vhbOsC$pHfCP@F`y#_YgNhOjNiWeWGj+8B|jj-R#O1m{Y_31QsMlcs{k6}pTES7 zO~mCz{PXX?o@PZzErrQ@ z#0ls{LK!jAVTCD65)oLwiWr#KU=|id3_>$rtbw6?LsA-Y`k-*RaF@XO@L-M{_+TnX z1CN?LY}(lcFQGev1>wTX10&{yB8)4xeOgL2nXi?9Bd1~&9@so^`!533C|fXN=E->t z;_-7yM3ua_*^PNVBb6czblLtqFN7{fj^#t=S)3r!+U%{QwWeuW{t8M?8fdjQ*P;H= zV4?IwkZ4}+1xO9`0j^0AAzLB2FU4+paJ9)^&UGjd5vgsFSZEgpJi#zskp2{2K}vFR z)wRyN9N}`Hrhr`pw5PGFwNjUc_(qovk|!gF0xl-wR=Yi-B>NU|NPdk}S^77-FupwF z_P;|nFMQ9CpumDVB&C?4!V_ZdUkJE#F?+qt_AtE(`Z>i4e&Yj$oRXG+jx*RCa;q)^ zK>s@&mcN0$YQk@zxo0st@xeN*7sO$L5D%DBLJC+s{ezBw{azprA;TCRV5jtjyFir5 zFNn3M7+9#3f_yw++-F#g?}_MvX?6I;cegBI z?_AGYS}ee?bZEk0Fwpf&;dN(Y!bBP7de?Oy{+s8b_eB7JWq*4zez=ML@f z+0PJGjRb}0X11}2UBvCSqJ<)3-a3Y)+!bo`@w>lw`dc_kri9RVSOA6 zkl+cdmug66S0Q^%z~azSvY--S-7hS`Ua@iC0|CZTW=<7UFU+v;e(sSSLvQuJonn<< z2W^-joO@9U&-4i16U?zVs6Pe&RlmuGuu`K|B-SE_8@dio#aTGEr{kk)W(Lk{xkhcC z!jk(%$3u`7QdM7@031-Ua)w4ufublp_JOAmU~!RJ#fBs*I01xCdF2kJQB~&Pfa!~_ zzcw6D>G6zY_dF!L{+Gch0U&%5m9!SApWCp#J^r7dA&=m8D}dL27Yb3e*<{xCdVsG~ zC+}HQw7;IC3%z zb)-z(tY-67?LT^D}v0F(hRhw88K&o#F%f?}!kj#q3XHptfS0?Em8zFt7;1w^# zu~2iOB02ketE_6?i$8e)g|$}Xp59mX>T}7D=@3z{#Sd>*ed&zDq^eexM#+=`zlBc5 zZ7c;<{oGI7TkJPgkX#^0$-0^nwe^wpive=2$I3L488CeSFOw%jn2oM*`gEZjX-K?< zU$yz=B`w#rGrS~8-NtD-VeaX8Y`{5+_rgvs~VxP-L#!1_QFofP&$UE%7+tU;Y26}T>my{iV#oB-ZEpv)k4p7kZ34!IXw<;CWYURu}jhZa`& z9o3kI-3tPnrm{2@CfQL=5O=OE%7}3|9Hrh@<`MdcrbB$c(}EBZK(+Zoa#|%%V&qTG z>!0(Z)7m_O>wohbcvG0o{PFQkMI&(BA@z&E zs@a&ZCN7(5HGlIc#=dya@jIm2BouVkza>oc@voCTkwys@6b77c?|w5(3XRQ-4Zh!` z`rd-)d;+ffb2D@+oB4{BKl zeZB*`H@EWZSpHHo6y99JN2D{7A!zn#<{9`k62Ad4AXUNin)M9k@}NhKupSv13HVS0 z0DQq>U{jEjp9-WihD^%Ie{LWxhJgJ9E7!jac~}6>{{Fn}qNq@D7&f_TJo+$528GfV zXm7k7CX0c)Muk&lrltgs!x;1l*L^&aWO2-!)wb;@B#>)@{)hlYE{%nWDGql8GcyZ# zvBbgzR*AbYq+~#pd;P#*!6=~&I2@WUQQiG?VDAW513{}C=2OB!38IIBDBj;dk>CMX za-!I9;ZfG63g&Ui)(n))&i7|4aKDId*W)(g&G*@R|5<=$&Ll|2Pav|g*H8b1yhj>4 zpEHbsg_SHhE;-@~K-=bkevzwb9U8>`TA=9e(1|Wd4B|qQASrbEPwcijmQMH z9fq<74qE7AC43kZ!$dwUQ2k08W0AmA=v5D*P>QxNu#&?XVVXYrCZf8G@{rh^-9<1P zhsif;l^el%V{md&Z2mg;gTF#@Zm1c^=@sc4D}~0Z^$E*!lUD` zw7h=q@uPd42>18<7@s9?Ko(lfw!4egR>0EbIuGJ~Lj~0Q!Nnc!eMa_!Sv{2o5yiGS zQR!e%fet}+QM7az>WZ719$I;|GG?0LcAmST2nkDY+F5$^7J-_{!6Nh3$?-PjdtM&8 z`?sD{oj+Vc5mM>w-PJ{`2L6Dj?==)2TAe7!PL|SP=a$hha>px+irDVcfv}2*hdt5- z-EPtAv$fBgkc<=Wz}9D0JT+r7sDkO=vNGR5I&sOe40-4!X8+8Q;r$LGEEXe)A61sP z*aV}yNr7Y;FVf#Ekv^jm$WU$Xn7%5h1UPO8|0jOjOG3IR+($k`=uj7WEMq(ZmmsES zM)a*Nm}pHIfaQ}P&-h@TV~^g#^GAM*N$3uFU-FoN;?*$`dbkQq@#e;=or*9yZNe#E zfHi$2rG)xSQbgG0$U7YdAc5pVGb)K1-Jz{jx{nZ>EQVi%uG*B*fRuW{V2BYXu!Ob6 zE@2tCc@Y|xUixw!^ywKUZZr?}{d+mHu0lEsb1J^n08s^Z-~%T^%5sFv9NIfUio9>fndOl&^HM5MD5(EGI|G;w;<|s?EZp#q?9p?5hlk$vW)9x&JCYs~xCm52}Sm|aE8L_!fQql3)GDF%U*#nl%o7Fq_wcUAd z^6zRg^<51lsx(#_Rv2~Ib@^YsO}-NRvyX^;>Fw$8V5~`bboYp3(;<*3Q}a^XX~k5T z)E}|?uTyG!C(&NF#30jQv{c7B_ml-FFaQhtTgn4#L=bdJsP1po5IpIqSnF(`XByle zUqm!GW53tcAz;$NiA(|&2S8V~`%(OUkno2AVic*IW=PgpLrvXu^gMJpdSGR1iaFHp zWN?);_Ul^@1lZ=o*1&7WbjBeRbYVMQVj0RYf*3J1qr0c)Y^V2239uIC8AaPI1gbS} z+-lMls)6EKdh7Le&7gWnMxgE-_U>r*Yohb+l|JB*A?foPy*^44<<-y z;EWX#Mfkw*@MVTTj1vtVJ`@8tr=Yoazj(hAg)$r<&MCa^_>x>lzla-w*e(9j2CBf zJWy3LLFf|g0)G=69BFRVz)5QW96rd?@a30S{eo0#-^orF1DKv8n1!`$`(7VIC9m}I z4`#i6lC7Hc3aSQm%hsAw+QUpl#OPh^@>Aw8dJ{~ys@h4@VL#?BB(C*C!+~r9x_ih! zqwqccI-0DH0dL^TXpLH`a=`#SJ6n4P6G5LIoHD0hH;Wb;_8)9~!sXyA$heqV3D;yk z=g{s&&b&^(U@Y~>l&}sI$|>Uw31kOk84vx-%8s!=i+rhZ6L zSEi&|75?VH5o>|SihglZAim5ZXlut78#*1z|0%2fVNn_cAM_+C2uOwTmoNQhz^bL@zbfuzq`zTxmG(HYY?gf zI_jo>m_iTJp}WFe%Y6nJRMzjlRNd4(K!?llw(QvC=nlNrF)Sx-90f={M{ERtM_=Lb z#}lJc_`ER4nVTPuIRdP-v1-zkuOQ^sA*W5JmWYqKZkkF{9GN@%5#8oU55cwYw6c$p z+wWlNXBaPJfLAxrpI@U&A#dboJIjI?&GNSl@0Pf(Nv&%_Gj4k&5wwLhVJ@*0){d0P-P4hSA{mM zxf6co3_`I2$JI@K|L*B2=`)ngDpxltY@z(OWiGzKH)vNb*K2l<8?mf7pFi#)y=3vF zVp4D%E&`G;wNoD>@unV3w$a{&Yj-_$q#HBC<@O`0A2VGaHhqvWGMXoUY(Q70YXLJ& z(RieVd{YAAKdst)qJLfFN{{X?``0UST9ZQ;$e=QnClTVUqkcQ>R+Kc>7HZwl`B^AN z2hID3fb`b`J|QdI5^*P`YuGIscJ@S|UUeK%w2mlLy=uA-i<~h?yKnU)oCL^0lPxw3 zs8NUd45IO`Ft*_#LD3*{GIq8GNZOz>z##H9Nd5$%e1o8KJId6n853hoCo&{~1uB>8 z&pA?Fn$&oD$9Yf{Oq5#7!X-8P*LNsW-b7eCPr>QiVxB*e()#NG8GzL z@8^1uQxcPYmYseYy8GDg5ro-{g5U7dM?bjMa{zA$+(3nG=hGnnip+~b2E-P1{YrW$ z`A;vg<0Im6{t@C`Bjff+6MAnsK+;)&>do)X2E>1ua5J|pN`^cTYVzlN^)8Wjz$Ky| z1)U;ci|W`u=@>LLF2DqCHSuVY1K;K*oI+fJ^b<=*0X6Tz__F)3K8ZF}8sFIQf1t<`zOESuVu(50y)!Gyln ztQn1UpXZC{4nI#*Ver*E;wN22_!MJHOX@r+?^#5(XxD%cBV5hw`{EBvV1~ix`kv)- zmrQm$pK+F#>65maJ(|=g*lg}-X8aME0wQ;7x`LiFG>3Vi*vK1iy7)*mJkY)H0_LmI zUzSL>0e5y)*A8;HJ;wf1>mPV3E+}=M#THK!Ij^czsfV@cgQg6f=RmyB0K@Nrg%GNz zLH7plwkOQ2K|^)kTdU=XSvJkg!}jRniZ02qCAF9pW1cd26=MRS>LGR2DLW|$Y&U8- z4apsE;T=e>{I`4f;_o4G!5!iw_cJUEfhe*6gYQA?=_3Cc+f$?~4gg1**d-@cNH`3# za|YJ$5id>ON(71&ybRk*ePbIpxBMgMB%}=tYXeK=QD~px_;bgG3`=zFKmIQZU?-oU zC-aPjO6?o#H~TEA@Mw{;&^7!sckcUZ1l&9)ISZmB~F_Y*+D;kBV5 ze3=we;I0{UXlCYk1%&g05P7Y!zKl<`Mnj)bTYslzD+GT;g@S=5Lx~t;pn`(13W~0% z8i${T_YqD-fJ*Z7`v7)Z?(!7^-@zI5L=y`!N?7i{mjQN4DTRMFFVss&gGL8S(DvzS zO9D_yE(~{}2K0!7+y8WUa&cJ9h{bvchlt>C2aQ0|ARwp(BVeK#oG2VbC1V!tTT~XZ zQSlP;I)$NVG3EPDaX9Qs=f$Zys5ES6~CKmL1MP)JRcyT zxKr>_IBXX0u`nFTC|Pu7cW59LGeNTmA_o$_UEuwAb;Fddpj49XGq5z}j^D8)`;;l! z5Uy^@wl&P|-UCJTZX>Y^4=rtKu?>Na)k#3wm)_U@xXDBkN5xX7-`Uzy&;?Sa*$b2W z)Txj91saN~l5QdeI7I--Dx-W(jp^IAR!Vc(9nr*t}iKwCq;J8j)t^_uJ?pa z>9ScQxBCC?Eu60(gxjdS_KCT2EdHJwf5i9QF?Y?=F5Tz~<{z*XLqo3@(QyA5)KfPf zL0#mf^uMTO+;IA#%r06DD;3L4Lc=k6 zE!nY|U>dgA`3wvtTbH9VZ^14xs&;MbALT%9%#zi;=SPvxRFb7d)TSk!40!5f=z>AP zwrb16zZJ(dfvpR1w?7At5=+32m?UijUnF&|;1+qiuDQC#53G6mApl)McKy{eM9?4k z0re6#UfHEF8?%YgqxfROk|U_0u(d?eX&GHV-KBpK(~@!a@x4;HqrQ-o!}Sv)R!w)$ z99P)>gHH*&0lxTFMJM?)G>C=8pN2i+1;Nl$KPcz7u1U)4U_=FoQ2@k>JHz0@-0EI% z52^G5r?>j`lKBqZtG#)$`{g|3_CZ_Do7--ewBzlGS1my*&+L=$v~mbKPsd#D(LL7H zo>@lHl12+pTDIdW7KTw?LY53*9zgS@V~ljyiXt{xg`-UlW~1KX#Y@m2+BkY0eGutr z^Nz%0Kc6=vJd3B-C(Xfw=o8YB3zgTwd^&r*;-E}6?egPd3YO34NLsjH4!+8KQgkhQ zDG~IMzEN6y)%gb|%JqPP7zY-$^u4z&ZDy2NR5yI<>iXW|KznzjY2X5Gr1O^e4& z98xfgy+V<@04nZYI}a&eKhQ{%T*;)dho7%^(7g*g&;m|4t=3gu>k)N|n2HG+iwXhk zQFrN;)9Z2MX`26)UAkHLHpXfv&Rip&8XWWi5)}|xjz27z=Iq~GiGA>89q_2u3D}pD znQUJyS#hyPWRmy~6ZCHjwc0+CVW}@e886BA551`1Qc)HZS;Hr{!4ZUt1xB5@bT0%? zYDtbJ4ywrAq(di?QtmgLn@@mB{D3GwHQYJQt}d_5lh{AWuJ%l5mZ#_Eh=X+X7|frs zU4a$)LX{>dtH`g33Nt_{-T+Vct5PR89FYwDC$ojW5EIyx6*Vm}JhEK@rV-od$SW%` zaBCgd2GDT3Gf&eXdtuP6ubkIBb)*yLp$}tk)NTtCgwD+va!+G)vTg6k22lnN2aQDH zWYxnRgh#w=Jv?4;BKNKrZeH?X^kzG@UE`g_gd>zNw%k}+K+5xevwDtVMit~|=}Evy zNtB>q^&ca@9}o*h=HY8us;JG4`Vb5ityR0{y6TA5*!Eh~loW%;y8STI!b6_uZ^@opF6avYR@B!*TKz-AznB4*27Pd zv`6Mx@skwK71l5fjM^SN?OCw}bj&MS=B3S@s0?3(%vDZEOj7>Hh4aLmypkPl*a~n( z9F3?R8eXFTQ)9w95ci~k=P?T?={@|E(C2ifg}Mcj3{AEm!(@S)3MrYQ&K^iZ;KkuL zEInKmD03^+SKvF4v%QNwY{`rF$)NJlyi+yu5Q7YE`Rquqju^1r1$07#S9*Va*K6!} zP*!q#^~dH36QXyLIK%gw!swN%Bw3|$o8oHz?H$|(HL$pf(<>(`4sT5EO39GgCB)Bya*I-EoZ0-D z0|z{sPrisJQ=ryh0zut1xpH3)h8Y63dQ5;2ocijXo18YdW_A3qAj13m;u;5EU%*kq zAK4&7{S~G4WB^+qgH~jHeVyHQoj(^8gyIak_3K6h4CD&8xcqL+z~e6)M6BKFWKo3T z^Z!PlaJg<_#V-&v2g7k>0Q^WmtMPkkiP4uR2WclfA)j{8JOA@xeh`X-6ddO_xPgpG zp}!eG3%T_B`#UK+Uhn?i97=Q*(0z-6jZJ0uh0mCrh31$Q;O}D7ic!gBqkJ>QGfyrs}4|4InxpN zkvO(LCnLy_heO(jBzy1&KKk4*;WCj!acbR`=9Ix8u*1qPGh~416$s>p#oY!T);*aj zC4wyaIlA%g6we)XVqOR%RLr}85NJ`ks1}Tx3;k0W`S@{0{NS=d;xK+OX18^SV{~M3YU)I`)#WVHavY46KnzcpoW;)`&2slS+`x14C z#gcqoil$nVn_)ZmwXkF*ONC9Z9lZlp^Y-Pie}Wt@%-=q&C?=WxK&@gA5%IEnJTSIC zoSqDYon0@q(!>nnBWsyYGDS_I&A# z(yLq=YccSjV$*B3|3OdBttT;D$nd8&WJ9%}GGuA!H+qly)Vvv+-%Q+k-w+neur zYpOmw@jNSUx|r8hl3ZhuOrZSH*7lJ^IKZmLWb1DxqBK@O2(-(emM-G4^NF!c&&kZS zRKAU;2_BBfy3*Q(1q-+gMhHxiZ$)`aadSFR_zUPv0$x2wYV#Bip2y?jk0wWQ;w~Md z&Zv1AGnHl8NHiVK5}+0F^Nr|{i_H8Imz0S7Rb-gq)o$G|vIU)G5hug#zP!3P4ckMy zWaSCBdw)k3ONDw4*Trq?73WQy^T&t9ygbpuZQVSTcExjU`Sq?FRrnFvYvC^8ITf}b zjPWP1y{uo3PoRXBgK9YPyd?6vgU!?Es^1M{SqrbGxlN5In)|(^NDR0LvYqJ;@g6O5 zAYnw{ZpFm?Q(_zoHW}D%3$LZ_0bOZS3Ip>awQunRS~KtZKhJvE zY#HL;)waI>dOKx()AGdTQoycmxz-48P1y)57y!63(o z+qK{M!9`07XFV-}wRHg8l^COm1R@?E-3sHy@+977&>+A@LD5C`)=1MwK*ndVA#Y!F z8v&k^goGp_NOU0Uxv)WuDn4DJfaGJii$g^^mL-4*lDaya1^6~Y9Z&n|S3u%+A>Q_w zxGDFc{h^S9U__H#Ar3=KWFpgbU9*j+au+zdlvlWxE6>(D+7^;4UHa0nWeqGlt&LWa ze*UDR6O86d$^;f{f_ATu@2w|Q#>f-0+&dzL~+sR9u}!SAQ;n_>w)E|oFW^pvFW&EPJSbT8wh)#1Lt zkYkH+GZl)JoxKPpcZaV^dUgf!RxTH}*5i|FvlP60Y98T*3x5c!xVt|(T5 zVCrO1GpPJoXhAE;0dRRWs+^pVSc4H*TW6pRQCgOkmgdD0mU&|X#JG|1;2s1LHp!aa zA?W}NTJxU5>evjgW!1P+@gKIP`G4D6P`Vh`3L)0i+Ze@Mi=#~MYoh1x<(v$2ULNYE zuR+78$g9kR2je*AgpjksbdNRn{${=kR(kHIhVWxwIck#b<3jxQP;=sOFb5v(?y|aM z2MPh?z$^HB^Otzt3k=VbjCS-eN8YlEZf^OSkly}K-T05TRkz2C5B7Zno%D)fbm1Z? zb0Zxf9>!0J9#5@(O#RPinJXerQmBQ|^3rCxDv7LAKcWWM9Kk{Np#6zMJ6DX1+-?8nzZm#-Cq96ssr^<-Vf76<{ih)O-4@btNCCL+bj@v!T zTXyPR6MrLu4sjLb^8HQAU;njD)>$jyP=(e~E<15TtSn<_t?%`6i%s1H=jsnALYksb zoI2V4;{kjISq3$u!VTo?YPXnqUYS*432#FIHv&nxBvKWtP)=q>i$i&6dGVhN;eEIa z;$eROaAcM5mQ3PB$@Oh5)^dr7bgEQ(WF^N=KUPHCcrdt{(p4*(%<`1uS;VSP*%TEE z`<~du$G3`b)mSKzI7pu~7^~XNvg#z-caBGz9VRYAIA;ZCQEDWi-rtTP=(T-?(N1?; z@=;Q-e4Nschjv@4yFpMTArVSEUe^u{mam0>RUZ(LtX+HPNsM)bkX9awHan^pH}f<_ zncba!s~Y2#%@PUbi=e9pNqCHoj3_?FSBP9A>>$;vko^fJr;eA#qb2_vDce=#F|8_g;2Aly_To#u4S}_9y1ErkP3)VB_;gIY~5@opDb~#EY+mx~mdm~iY zP}Z9gZnDoNMHUw-_N%{N><*VD-$T);kVa7ZDW*fMZ6qx!Dp#9Ior_xW zU-VKIxaz-G=`Bwx-bL()A@&asE1&Vn7G#HXQd#3J-hbED9?a(ETx@*SXxis0eD7TL zMEZ#Iapj9J8L#HiqrlUk$SY;4nNK{;Q^%50^Jt)K<(#KWy=O5S%u252>C7tv%CH4O+|Ml|<=okztun8YSK0s0bA1aY?S9Qd_Qn zy6Plbh}cM}qqi*+rn`ZiDVIm%rgqDI3N<*PNAk!&dO&wi^1%tNix|d(g2U$u0x$_3 zOgW}6IV+U>|5Av9P#em881Ebfh>6(Om*3|HODIFZI(qStbamtKGIl3`wVdqs1T zxSZ3H)&qx^*mQ;(!e{8(PuQ)xj~iQAmn~E=hI~lsFQNV-vRTZe_aiIrqi#j0^~FT` zjr!&Tf792j8wG;N%1YFh?;(jT4kc=wwR{;3)84sYcn4i zvyO^_6QTXjMNTKVpNC=&v&e<$wUbMPo$KdrTe&w=T?vV-Dq-i%*}}>+EM1;SrjQ%n)0m7=#@5n0 z{F>upqag5glK}XO)_C}QQ!=}*P_xw7l%%l2<>81?L}HZ!^i4mpU0TqXGxv9wU#>PFT4P4GJyg_=O(V+P2T#C1l12B_#d6kp|uA_we0}Eks*O z!(mAFp5r_Cb}*6XRw19eNq!H}G<`cmQZ!-VTuOSk&viS?#x8fBt@2QXYeKRKaV)wU z>R6OjrHe7aGEa6!>YLjvCZCF31>w1NOc3?LAmvix%xYo@%t zUHAf<(dCM`L#J6;uB}za$;!2&Qg2E2O`rPdd?hqY*ECvShduE6bc5+{F`XwqI#HE- z=KHt>D}J6AKgqS>H9JM=EJ#N~(*sZw0|0^!%^s-pz_wxJJW5Lb6#axRvks8MQDEb% zARr)ARwQy*nNEdbCUH>C{wysHN-3wLUChq8kkcv#TL9{k)Of_%(MFe$#6Bw%K{_B( zX>KKCWmQRhrR>Fe5xM^3evnxlrnUdVao1Yv5|<+8us%~LZNN*Unp(2ne-n5hZEkL! zdE7E7iY!ntK|*WPaWlaug%sJ(5MQW)jM*TOmq&`h@RZD?uU8yBxXwPmP_&SGY<22s zptUho*WzJ8^|%>`OpCVukNny>ba&#u8~0Tudy?mXKq0)shiA>+`l2y&;xYYsL1#r~ zC6!70Io+@`=3YxwShV{dbt+6wjXR{iyzR6KIW}3*k0yUdMPUsg-2zkA!(?6m@%Ul9 zBtE^Khq0VRtfHIu#y@FvAsLpXEjY)Nwo^n3H$5)KhP7%acuXV@JyPDU7~$I^%paJo z1E+AgL;`&+L5%X>uy9)%Xcr?T9({!S=^ z_h$)1Y28cYPFj}fPw|wLeGE8REhS}#R6daDL*O#NYcWHvj3@eAW7S{Uk>yH4V66pg z7AJG`Al4?@ImP-}Vl!0xwi4Xw9i8R@r-f84X(ABSJ0lXt|`0eWG?tN-x@{ zq{bPkF)^o^wg`?y6z$e<=#=u<%Jayl9mHbHH2{c@9(iI#~_zKRLv z6ODJ1;4Xw{YFa{WDpYU|cbVIlL{_oP3$O|y=bxh#9JVTJYb-d-$WV(aAC;Rd_N-1=Y?I&6bvPH=)7Gf*UY}q`pg~@R?7d1JJB^r%x%XMBhV-WRwh*xRM zwM1rUD2bhoH&hz9y1i8q%LutA2@i#$wK395Tlx7?qdOiK);!L!r~|n?4Cpe3Kp=ZG zdF)kVzgl)wW@4f;oht4_Vv;ztS3c>S!rLfXa1cJ4t-1c!!qO0gq(})T(w?%?8G@oB zZ|yo=1A| zS-4|+{8f6{bUFF=!h2E5MR~^p$`neCH57YZ0%kPvJatv;U!x~`7@|&h%85*yj(dho zB;r8a2f;VO&y*&;L%DG3%?!z~YAu-y6-G8574gC$x%)-2#z~5)!C@HUR9KFA${NVN znEiH%Q5xFR#nMi^|vVFu5dD(z_}3V#BlJb zOLAKD(9E+f?dgfy#6n;hFk0B>o6P)Z{mfJx59LYLkXwU5AXP%W$rY5I*#0~}2G z{^Wv~_yE54n}4XMEFEv?TV6CX#@?Qw>cmVR{%)dZAAExLDA~P|_V2r*MHxKSIik`5j@j0 z?4#9Me~#)sH){E^fQJ;JI{W9CuwD|C2fXHO*WYFo|A06FKvyyjS30;l&0or_!L@R& z5C+>n)1XYfXN<~b5I+vDf`HZ(5P)Ty$W(SY>_$FKI#i*7I@{X|yi|WeITWva^2aaU z$tb0&l+t2_Qpuo0RYE_2C1)so@^9bgQ4|p3X1obxC+R8d`&Qx;J*CniC*!uc+O7J2 zE0T!OKuOE&-ty9|zYk^U0}`XuPo=Z#+$0gB@}xg$q}Zzc=__uaNb`JxMy_b{LkGX2 zZ@s?us^)8+vP&V?exwQEPnI=EF~TEKs7I~89ERh`P$uG|wXYjw*o~18PDTge_bnlQ z|2`Q=g?ka_10`niIEX42>+?v2p*|CEE0n|B10@986~88Up>b$a*X)M~9|1C$uyzwb zXUeYoaqsUD!zgd~^`?;~5BadmHg)5PE(t2(m?Ap~lHUeJQ$-VF znuKzgCOg;T;Y(Y6Zdqi*1dfV_X5rZ(&f4z`)1}R$m|e*~dTd&{uF)DIL&;0cmQ%RX z56da=G#PI`l-)2t8NB#AFS*z;L`2-2s4_hFWAZ`)3ZpdZQ6Tw||1PDiB5dP-{-78} zIkC*9VKe1)Up|HPA|huq738(5QQQrM-#c3BDI`29P#eivX;7-(sbg zHj1o^5&!cq9wNL0*M2=p*ZgwcmJ~4y5H6UFhhqo178aH;AynRRnZNnf5f+StfS<4* z5W5P$qqg|$4E}I?y&6hFnE5p+xRR5ZuIpdqjJ0eV;euukx5Na2H$5^sm=&{e;c<=s zeaT16eaiQ}%j!(&M?ObiFK8!{_>~5^qOm+ZGxb>(r*s37koVlcu8^77CCp?k%Olx2 zA^KpRq+$tuy0l&@W?+x3Mv_RW1^(%z==adh^{>zdQ*&#VmDfCiyY6SZW1{bCU3jqZ zt|Kj)4_@Rfg#}_vddIxnKbsBJ25f?G#by?yJgtF$FoF@%H=RUNR&jR_>likZw5h3E zjr;MPupB6)*pNxN=j(AFG?Rq|Z@pjO*hBjr=$#!*=@P(70sApv8b$ zEg4xQ0**_E6-zx^sBC;^T@my0wRLmhAud%p4A-#XVOPM+buPDIXNd+v1tmW|C4C9x z`!M*!L4eyL;d^l``=7J?A%`nprq`gLUHZajkJ=ShF>X`NmY72IEEMO~y*v674{Qvh zFzT5!Y)=y2=?M>Scl}i2Poeo53)7>|?9-0jM^Ve9c^sHFedIE6~l>bbA_6iu0-g{h+LJt{% z25)31pfTeROA2*KkCIek<49I?c-i@zzcNzp5X58?lw2ydv9i5ovP+v)75>>xqGj@1ab1l7aUL+$%ECy?DUNBD|T^i zvKdCpP+pXJDgErrO@AEF91cqspH;tm{I*tf0c3us(Fk>^yhrDF1tQ31x zrT+(v{#(Ulf}>5B+rN#csUkL`2_pancG+W3skUX#0zD*P4cb~;C?hSclwvYDIeDIAQdTF zH&X5?5KE9!v)M84bvPTQmrW3nhejBC1@xZqc<%~(#(yF0w^aycG8OhgE~N?YJkrk; zD~|GDgx(1|vq?|9V1u|AX$V5-5~gxEia3c|Y+UOWtEe)q9fpx~mJ$m0i;VQAIK=FE z(z6wc>Aek3kFmlHx>#-^PvVlkxOqQY(b3TnvWU!l7e98Bfh&3Zw#Jj}`j^0qoM+O~ zK~a{2HfUsQD3hDNNNyvqhO7G1FIrtlT<9xJWiib^5y`~31fcGUWpn_K` z@fQn6FmR7r4OUwU?M|2>gm-=qFGwPM;KtlQhj-StV(H5cmqRi+DGl#MF~tz-oo@d1 zHJHZv)5^fxY|2QN-YXHJ`iWwS%zC?PhsJ;kJbs6D0rL5d+FQ%_E}G}p2ptg409Ro|?KIs3*Bm8YgE|I%NL!t3h3j1!+SHNuTw1IR ziYWEl8A(Za()uerO{rtTH5XEST_ZFv*)&&!uVzTQFqJ^TmV&g)Ro}1tHhINU5csR) z9?aKX&!|LYT>iH`xJWq_!=Kl_V~LLe4O&)<1d|ch%PzaGDr9l_@i;%boko)#`J--~ z9byvcZOXX!emenIxpWnKq-^2tfNVB@EStWXD3z460z8U}gUdChuvHCeT3ND=><*P! zxK%BOjj0pd#nQz}Ta*xVwtKNj=5{N7_9x2m(>dDX(};s4H9Y@nnIh7r5yY=#ua_$y z_kXof5G*P^o)pKpzfSK)ZrqHQ&y#-}n$VgA%HV&C3kzZ?c0BE~1m*qvshwXRP~tls zi7!#R#t534QkrWw0W>i&OFBkI=wSxW4q_lC1r8owasMx1f0RV`Q#Yv4MK2htejW4l z3s6>@{VS`JW8Ri3BbxV1&sE9+qclVoS95uKt6#SqW>FfRFZ(eelA}WsxyDRP_PPNz zpN1yN;~92l16JdY^oVjzLai7(BioERCfbDfd3K|rrkU>+gGMLpyYH#n?PS;=orum3 z;c<1VYIZ0F08r}#9JVRJ8B7FWUAkWeja=MzBh8g>QfLf)ZlAG}lGP28u8F@*+NnyR zovxl+;}E5cbX5NmuPgzL@+cFW1RV5#xVz5_{-Jpo!~%ZuRY4~Wy(=#dHty}4OXWMS zMHKC!KBl>ozUsqHBPcMXnmtU(*C2cLfQ_`Vg4$|B=|@c?5Do2@yb>u)|MLb0ZZRhO zfkdB}+BwJiUQsyU%oA~hZno7*(O|IGhO=-=1>$~`C}u01F(v!rAZ5Bil}4=h!wuik(hBc3N2`Jc>1R zI4f&SA5*^Q=XU^;_koG0)F}sP#>$LTg0nW?G77v~3u03GWg|afo+_`BrKKkl&Og1G zhaW2{LbRN|K>b@4A0ZMo06V^qfWLD8;2_;I>#=#-9DR;ictS#(p7eYj>+G7j5A19l6;c}|%Rfuaakk_YTIDdL8Vsr>jh_~3-~1*>AzlDEou;=FXONlYMEZc#n%jbK}*2LcRkPS9+2RC zzxFvQ`J_OZ8i7jBK^zzqKV2L(aBIKsbBh1BD&9eh`o24p{HOH!ixE<}nWmZ6{A`3= zJ>*-Zb@azKh$2nEqC{a!M z^<(~^30PLuF3qnLwsFKs2((+(1|&O@a~x6mCKqhsljXUFhu2lWNHT@vBVNRy0#2{y3QC`v2(#(A$6K@z zdUtk-wGV!8ubM!9B+5Pv@2AS&OId!AKALJ-RTsYTcg~vuUrlUT@IM+$ls*sf5TtcH zTD^kWFV>7q_Ok@tik&m}Yn9MJOPn6RPc9fe7qKGbR4n(}{dcIS1T*6wA@*UU==M)quHue$^UO6 z_y4T`rpSM!fxG}&nLob=+}cs{OXOLCEY4cUD@t4%r!l|V3f!NeGiW#I7YxNnhk94| z;_wkZ7);dy6C*e4%j=5DFM>m=Do#_ZPm6+wBcjDC(##A@B(bYlUw0s?iE$*0?ikgh z&xHe`>A(e}N4pQI0#YKW8#YtoqEE~38doIJ0HNs1X0 zDgTJ8w-Azm%#Z>PPp?YEu-Db#H3JWmz9!Y5mIQ7LkW%RV8$tVD!(!=|^Xhcx^5U3E zS0Ps`Pj%I)d04f*f__y)wTDTDHj-1t96@XGxTn)**JFRST6l|jq`x>sAyE174y(9z zll%->X%44>4xv@`pN$G2=Je$fGIMs!E;g$_0{7Q-Dh=?XV@#d;J8eSHoDBh{&wPr>bwj#f;@{Czhn- z$yn+rw7!7)qAV#liT0KS?~xJLVZN)!_cmfzmVKYo3i;|~MVDy|{*mCCg!^`&@T>%)0Ox-)_tG+|_W~ zNjiH1$n~{#7gG!Gx+n zu`4>FM%9~Ml3nthdD$?K5BcRmWgQxo$p>xb3|Fe{Sn5u-Be_5P_DocW0Wv@boFeKU zX{hpv{Bkov+iWwl^dp02hwkj{e)_BXb{!T_<%i{n1jJ{}MS4o42Wv69A#)v^hQ_?< zH8ejZs~VJPBpmK`7J#RKyKVrzzQ}X^>Z+&2(Yd!oXwv;*FAa{ZuZWe;dh>c~N?6L5 zE3hL%8~lCWh;UT8YT9}UH!Q-%C5Kpqwc?&jG-Mz``diE&k1R$4Kas%c>qI!P`z(Jl z9P6Y9?4aaTY)0R(fx)&{(>uO41&=CzxdQF=#~=;7V#=M879oQcp%~=k3?yM!%L=X` z9{~*FpAQ?uj{#(0F~FgF4;F%gLux7rHA{v3dK?i^;#rUm#n)S!xgCg6Uy zZ>^gpW8P5mYAE;RUn#3&>{5`5DjTAUx@K)2_WPCS*u! zK78ZnmorKEROr&{iTWTKSZ#;EEx`=+%&hlD(A)MwsOIzJ>s#FE&-+s?6S$tt0PCd% zvEx|H;1f~wTO;^6>=6wx*Ko(Mx#Cu2BDb*e05EJE_U&2qNne^dn#0;j%4KQNGBvLn z>nJf0GySB6#>kF@)!Hdn@wM;Lk8`}nBl%OHzLr{aeFgbKPWwlzMrLuxr(ZnATYFov zfBk6+SH8Gzd=rJauc+&6uw$!>GO?{eC2%D~v!-WJ?vffeyH(45vP2Z6zV3$=v)@|D z{qEb%E&QW!L}(AzGmV6c{0j@**WQpB%mrOae)~5ZI!js4{YGfl48G3B+CEg4a0D)? z8UdLtLkA1)r--fNFOOB4LAoaDz`1}QrX9u&Tpt-Et(%fSGqCIQ=9|b|sqPzW??7>0 zefJ6N2#Yf|@a9_cxT?hloVMID>Ysp~HKH5vbcRX%QL;BvEI*obLcq(PvkTb#%U3mF z!66{@e!X9rUMO1H|A&_S?^XP-0_;6RUkk$GGmB_zvn{l>by8ANH_SaD?ECv!I_)Z0;EJm&RE`kj$bKg45l3T~t}3YQQMv|!o1M-)4ltHI zRjUSLi+yZ$;aHrW%hx%#l|d*b``Lmu#-ApV#!hL9ojp#N=y^+jhReyJ)1Vw5&P!Et zz4jFmj$jX@(IGL=^lacqQ3d_|a<3+8Y`94Cx3|QsK;W#?6+t|*RzjWb{_FKeH9P+3 zGsDiG@9S2ZI{7yX$xkJ>g-)m&=d-7IxQ&`geMYUY8+EBw$p)JH^@Zhx$z94h@9pq8 z@2#bK`0?d4ym?u%O^^6B)fpKYS~)6C2v}r06u{PUB+a0q%ERCU7q&nwTk#8!PL}Gx zZBr#GLuXD_tHS|JdTYG4jdp2sueqS5IN2xS)fjeOY;urIb?YA5M{FEWdoYZ%MD@@c z?q>rSPx1cIN0T(><6q=y?oX9sBcAhOdo^a^M^=8iWSK;A6XRvtli>Htgo7ECt&i|> z-y+Q`F1(J&BG5b+seW%n83l!nA-AI)5Cn`)WX9qdi}y;+4A)Pwqj7<{=HH*~X|b!< zij&-7B6N35UezAB)u=sP)T(7gsi}sbLzh#~k1ls~TEubDXcy!XpfIo#GZSNkG!!`S zf788g?r#T_PY?lbOuSZjX1Dfcgth0>Bv(=C^6)mEv0;`%x~RFI=&FMZAp`2RI_-LY zwQCVi>6xC!`09CMt~^_@-yQI=EEX`Q+yWT7cFI0W6aRbP<3rws{Hu-=FV|WGkU9CU zS}eyg+;U>lwzn$_OdHpuiJp}%uYUU(p7$949sam^n3%<>;HH7;ls-{qTeh#_v}AQx zL#CEQDE=MmOGA`a>gHDT?ptFu(q~O#74FnlqaYyto%JRx-{dkHACuD6Wf~RDT08Av zFGx+(0#j*P@kD+p)>PY9=;~5ruY8lv^?UuV10r@*7Y}nDVR`_n=Vh=J|tCC>p#>VzY;N^C&Hj)+>UdmAsBli$UhW@rr zXtNEpn;tUdnH*C7(op>ZNlmw^g8H}ya9C^JRx`?_we7ayR7SXOfT?oGMLl$j@Cdxi zm|DQunH=8LL&qhlhLz*POYUSKQbg@FVz5Psr-C( zFR&=PK10O;K8vn5I$`TKkb1ON2{*OK(fcKAn-v|<(PxwQip*Jykk<0sfyR_m&H5GD z>qp}(h>QF9hHmqhnL7!HOIBT4t{q#W)q2rKMc3>TwHfWKdSccS7d3ox#pAQSOw)i$ z4q(B?`4?mLMV^i*qtW7=@_b>YU9i)x=ut=pgJgl@>>Omtzs444pV|_l?m>Aa{R+PA z|6q~=O}4O3z^TXc$xQz6&v?Vs0NM~E&7hxvM&vHRr#zhr__6?b=QSAp-+2JdC*q(V zW?|{2`wJ%vT9%}vG%HNx$T3a9$dp8d9y7l&TvdmCbUYm6d)uN#5Ebm6-JQ+*ll0;z z0f5QyjcEFIParFz+Vx~YjFg%h_McgbB^1`rz@TOE?`A%wrKM$^$5m~g92}PJ3!Nl1 zcw7H_@b{kycc{M&=sy@+``WuF^22`K{O0n!t)Kx8X`cpseHotieZfYu$EctcOJWw) z@W=9ii7+wNebMp^58?fAF+<-5OF2&u8lF>kV(-z6KV^)7dMPjL6qc<$a;2uYpb0f= z3v1Y-8CB#ABN1bk!Mb4w$CRAaMnKJ0JGk;$y@#fNT6vPUt%TupA^c6)K0Wqlc<0s6 zs#oGccoJBBM2PLW91c=t&GMbdh7r#w&GMg5v+0Gvx8=jb5D!Al*#73I3_Y8HG%GFF zUs@_`^rQ5GeLWT@QO{WlFIrV@K^={9u^?X%T|CLV*e;6~?T2_HipTiOCmV~ea0+NA zB4Fkx-$_@N8UVfNp&$D?C<*Lx%-t?Y zMZtCelljU1Oa67dt^c_{f3f-KElv@%KjR~u(*7{^OE*tni<*ZdKs683wTmVc+S)iL+hKm?)panGmg(>l{i5nOdoYzrM;Uf48vT4`jF8De?-je0h-H&Sd|DqK!h zQ!B$$C{furW;N^Zv$po0x@RsMCldV#EPbM~_aB25S_EH_r==?4?+zAdv4jHQw{0nO zA@fQiQl~Xs+(*G1m;ncWxuWq|uwK1l&D|Puk_k{YUBL4^iRmBZ!_Yzd@QYFmRW4bg zu58pi+{alwvB4lwwrfpXRULXRPSoB+F#G`3^hVkqY$nc7bopiI|9AnMe7F>*InB|B zRA1x-lZk4ceph)S>z>DO60_I?7+fy2g?JdBmTC>5xbEMoX?QQ6Mn%&%C!Sg6Jj{w| zd0&ZVkXx=^Yj@E>!Rrx5eNu_jc~=Qsbdhqa?JM?c(E*mef3cDkWcnd5+qKs|!XNL; z5%`DMrJ0V3lr-py^d<(1iqhe!2Q|E`D*LesUFJz^q5geVIHV+O;lJcB+ zmL2%b4RIRA{|kfuf3Hhu|JuyhjmCOQe(UGwe0`(eZz>75C_NH?#%&I(mRu_Z4Sp>6 zwzKBq7f&)OII-E-jmqQ)zsKQy z6dfofxl_6EmpkzYMLvja6Eh^`>E*y;7(p%^DY89&o7G@STY{#(vJ49sie>@-I=~~2 zn@lWRrS=TxJK*rE*J~>s&{uHy{`QC9a@Q{7Sfl)}-m&v6%+i!3KvK{?vKa29iiW=( zo*&}`7{)HO;j48LHv@+G-2o@6u#u!ZFi3!x5 zWc{#yZSF`DnKOUp+UCrkAH95rvq(tAB>!xO4QvdAvA4X6_e`nPBS0)ME3=M`_CW)C zFrt{D!H64*E?V6C=9K5pim|tOvEAzMU=gm64Xgk0x!-xj6BqT z!x!8rI9@sc3!XwQ{hOe8n16D_5jka*tl{Or?{%DvVXX$5Hm?xWnnOF0UWhi$@D=uy zYZ&!O2>Xwy*gBpqcXA7%tz(~aZObu6agO~c6I z*}D0XgYmP-xadzhPiMqJzm41kirfT-cx0J&W{DCNfyhN0gW8 zrKB}dI$!J9ZMita_~EY0?|zY6+jD()Hwc*PuDAr$^<^g~B?0t1!K%W%GUn}NHKoi- zAOxo$9uRVtzqPftE`8(VDPQd`Yj4-d`b;4RBwhXwNS(_KOaPUg;9R<&2*Kmr?}K%D5qZ>jLbHbD?(mdCnMMOC$|dmvyZ z3%&mV58mSW$hoIWJ03Oj$*4D1^MHi6cD1hln6Q0Mwz46zg!j^oY=IT_^MEI-7BxpS z(YIcWA8nVG2la>>13V4s@*8Q|)3R%crI6iGr*o!36ycx7p-=aAYb8|)cVF(b z^cmwmVD(nDo*PpXnb801*)0aH;1m`Qqx3nq0oV%DqkrI+4s`wVVIt~6uSxjFyuJG2 zZ{xJ06)?qclckiFaw+GEzh^b{cB7L<1S=TN!#?J1?K40e3tesk0~-epEA`Z@oPfSk zuNuc=$P>PHAvEJ};hr83G|_{1b6w)3Kxge4^&oXt_R#)a+HGPj0i2=MgYpnFwqvH!Hr2 zgJ_QH)D9!Tv6E0cgLmnab6RBBx6%qrWWlvBsZ|hm@<0YhAi#;`A*Af*iDWOhPBjkD z{I)1jAnFmT$*qJ6!pRnvJN#ZE$O#Jf9JQ*lt5WDVjQ_>^Mn{l0VT^uLg^fPQtql96 zsl2f>id?XEx>&%r+fLY38o&zMP%{U4&js6=@ZMq{b=&W4&WUH=reb^-v?dWVkUMsp zdf~y70Y7=of>&t@UmG?_i9O$_q4x;l^z&vuEyq@<1$`wK3}K-QQDi+~0)J?GIWfFL zl4!fj5)BWbzl)4n#XgZ45-gg4>rZZW}8?my`WQ{E#=`H#?3YNC6h#HNw^g6AW zCMPSQhPlNL8Sa}1s8-*HE%(!G#mIT75lXmT5%aN*^exYazvj@8JPjW) zpu7APx}rkk6sVI%uydHDNaGYTxJNqC5P_0OuF@-Ww(kwpMp(q1-^oOk3zH)Fuzs;| z;)Yst7nklh12d;)xx_zT5|x#Q7Js)odrVvpzF=w8QKE?rm`Xpo1p4xtJB+jhs-=sQ z7Ej4kRJ0kJK9Xj}n@NHK*0@A4^{P7zR8pzA(b6cucv)WSe2Q$P{i}{7ZfUM0^EdGM zl@0Jvk{TXY!_-);!+-vu091x3c1x9vqvIo9YD!8&;89}l=kJ-d1jANt>pq;=OgeD-`ugtf z?lfB?ze)U}5t*7dl1{a+AR?FnFA;I^M$ZKNkt)Nzf&4HlDYr_aj0sE5KR9JP|I$gD0oi{gBw{7n|McH; zG5`ARz)O#Rhyz=XL~VeD%;p?)g;m9eIjW3!bp^6U1A-dwKG3cI0(c}GXh2vc-88;- zThinTDGm0r=wpudsbPPQkV_y<@>`Tj{PqL3p(j~P)i!SpGh8;5i!kY%jdbXsD74XC zaA&jqok50|plJz>gtR!BE1_1LX@;MWt?TpxRHRY&7Nr&1ro@LIgJO8vO2d4mvmNt|VrucdP zqI!TuG$SW5RmKjgaq>$i>#$wU$V&v7n{rBbNl=PFAsjY&Y)nP;M;NiE zB$~pKlD_)-{cfuM;6G3b3L2W0vSRPaYfjV55D1$7$?L|X=XKh!$6+_=`PLW`5)xp{ z2}HRt=y$#(y-GTI?k20F;^UXa%3@J35$QP&Q! z^SRf}bQr`tVvKO^dc7m)e7TaT(C^aAvQlwzbajQX!6~hBKCWycV`7@9_iAJK9Yat& zxCumI!3^s)DglKfz#)@{J_*6m>juIt?a-Pf<7K3z6ua0d%y}&CpRjvW%SHcFPx>|4Ux4%_RP&^)00xMR$M`MM_G`s0f|x1fWO(UZhAOb z)vJ^8l8ZXYk-n%TnnlycrTBIb8KQ=Xm)yyvfEFQ()u9SQ9N~_nr#!Nb9W)QnZ~-7* z*ZRm8s3M76LdWOn6|9f8SPd_%ks=8`N`!@yRLw7z2Cc^jUtZ?`hOW*qUnF~Rcf3(W zTwqa;l0KTj+iNHrhBiekI0HgW3ijRjjU6&Zn_$iU4X8TsqlD3qI6#$7X}(DkQgpG7B%g)$U^WMI<;Nqs9_xwnpeLzB z+LgK-a0recei~%*+K?8-MX49GYuQ_GKx65l5U(;K$RXC;c5AgX@TY;!4)+*2Tsl}h zyPczt1h71_qEzq9Z&RSWuUP_uv9E$6Bg`n%vLF@@bqO(*J z$c`qqf~tT&`xljxC6ja#_aSfp2o&E$b5SE@D|C`?E(9i_ek%uY<5cLO6m9)*@Y{i! zL9;Ctx^1$O7BU!+D!#9JdmCNM>oI+aNIj&qr(|J=rBA8QC?Ej~u}m`4lq7nn-hyzBoCo9N1i;fxA-kAW*QZq_ zO6;-F-rt&Pbm9Sl0K4j6$ZLST#M;#Ng;Uq}hu;DhKm|zCUQ#uygZIT86>!uGKxA%5 zRWYm;-DJ1l8%vV}q}35D0Mg)_WhoU*mJo~}-L&F}xaNI5zI3ZIpNZ^nI(Vz6Aa#=S z{!;wpd~+}vGk0C>_bSs6*N>eTdDguBsSuLg0I)KEg{2Qfti)OL&GxIfo|GV|bo(;e zdCJhMpzSrOd1{LS#H@Pqyj|zKmt60Ge-n!s7#L`8PTtM`HygBu>|g&zjUv7^$)MwK z{#*L)_Mdb{*rU5BIaZ96n-NF!Tvv{{Hi`}| z-00K5MrSK3*awvqSe0_Kh9{^l%9c7=Cx>+bgg7ol)&gK;F^C-1BW8vapwGEnTlP7e zQTGZgqS5BW3P7hsrlCJj$aHUQ!;xZpY98Q!a_u0wODgm&jXs@9Nv~S$2Yg9N!bd%! z!3%s^4R__MnwXui_Tvu1L2owrkl(Et#sK0pmAS~148)016iFh=LNLG`h^<2R&0F}l zRqASw?=w=OG|}VpD@T^De21#9pj2T(Z=?DfaAL$Wi=q`t*iJj}Ab$^(83FhR*j76c zHNk09Lp7qam7*{SmF`F``_s693R`)VOh?5hBX=@YxZsTbqO)*T6*LxHicFrOrdByX zIK51aXd}Mw7?;)vz>#>ufS*)MdXjYE4A>v-v z`Nklf?u8Ls?qH9RktftDB0WGp#HtS6IIVlDfq$gz6(@2vFlL;00eW^z?tL!R0pCbt zU=`bh>K%2C2%X)xj<%b-zn?YlR2UvGMK(36NSaHJ^93?Ks~ZkKohyOb)w>q#Ho3FE~14ER)S7xOf(m;R)mL?!bi4XM;MXfY@rzoTe~jN!X8c zk}$Z*JqS(D_bCD?M0F2lN$j*tVoQZeDzCH4gChw+Bh?+5N~k+YRxS||4an#II4WZK zk<$~AxR1vW`6WCd(9Z__-Di1soRDasMwhG_OcQ!eqQQ@U-g3*lZoyHkjtWVIWphW9 zs0Wx(dft~}CT3>ybX%EB79lazhM{+66a@-Iks~}>7WfcWq&(md?MCY!Ac&ULdVvL? zh|XDjkrJ&B-S$3;V(#oVP1RL1dp+#**ovUG9Ja6D$=?AN|c z#`WveHfKEB7gX2snoIlnT$d@HHmhq_Hili>u7*VvW=<5P;0M`LY)F3rX-ht4q zcvfMGK_WCdHNscVkjSwIL8KoB;^YyHYJY=5fg<;3Wd+Tf#yumt(*2yR=s;1tzhu7z zaxW&h^q9$nTAQv^xc2$xH}XQwYj|z#b_IXCyrn znZLQs2^-KsezTg3s&xL7z%&$)WPFf^*VD+uRBMqDWcr{@N!nB`JfWTTL0R|v)`fQlh<+)=3DHz4N~m5<#ExikXll zgh%Y_#j^r-q7yWs!)f`XN}5a`BrV6wZ$v$lU-k$*4XRlY>d6r9m3sm0}d34bCVJY$}k`UMtdq&$RU0T(Axo$)>&}(10)s}R<72k2&vUp7m@ojn~Y~1 z9GvZo%{~OYMA;H_lTu+oS7+d_x88tW!xMPWe@Pi};4Eb=5lblK>|Bp8y)-_a{dJkF z&k~>?Hh@bbEi~Qk7R}0K-(Ifezh&{{y?s;ORg_}uvwI=Sj!}Z zh+)85aCKu4(wUJ?a#vecxVSv|Ip!^j<3gBL&=J}rYM)o}kR53bCXG};@qP8-L&d-f zFU{X(aT4ZRp)_7iiJa63*tvnSUa9*A%_cn}w-oK(>Ly|>DiBTb1yN-Cchq>O4ZnMS zFh4vlAZQs3=>Mkh+5_M`eLEm4`Qk68O=6Hqg@{2oMPq;~kipJT$O1(GVj1VbdJs#^+{FKz)Oq)A^4)76h<|Z zDkidzTltw@CiLlfaoOvQNgt5*MKXN#5+->$uBtmhhOF$u7T(BWR*w)Kl3L$HVdC6I`^<{QRt!7 zhm*ro<#!aA)g4EXM88BAR{t~fq42_UF6r&!9?TdrPp9~uFU3VAAp^Ve8Kxyfv!QQ? z459jKpQo`(S1CX_f+=~hj0zEgM9-3($WJDk%`z`#7lm8u2)PlZti2;MS0A|-Ovs?n za1!2wRk7Y$@gEIkD1U62%qZnxUF<1X+997;k}1u@%M~e_gpC++Wc|dgx@M_IFi(zJ z7wqpaSMY`o)iB(me7dY%YF}FU!6z$J_dbQ8Z_Y zick6b`yO)q`>S-C#nsMmS=AhR)|%&WrB*V_+-$KtiE3Qon<7@E=+|sBKy7hx zb*1ELuF{+*mhDk0k1Gwx;Pz=$i_PB`Fz1eeO>#R;SKH~Ca)|5qdYejSeW_Dylj~tI z$~AA-;dl&GDZp?4z+9<6(~pUwL<>?D)+Mf*_s{o zv(liugkRhF$PaxfY&O3`UGeLym$Qy>HFttSPprE*u3cSdyr+pRA`v&AaRf=KnEg;o zZZp!7hJXxFF)Cm5^kzv`J}i_3dC}(qbI?*aIf{^*cvz);Scz?<;h9g3Ld#i!7%l&( zUQq*)M(+uy4SmvB#echLkWRmT`YfRfXBSrmd@|v+Cd5X=UTYHHqU6RrQ78L-U{Xj5Ojfxc+m)kDl%U0hxF-7I@a06-i;JBB1o|2?|5&8~6sPES>!@Hj{s9Pi!PYXqz48N|WcPXLubo#u)1czI;0#9*kG>Ee63Lt@xJu4V0 z5p7B7n+Zi3IKJN~qM*M}CUpx4d{8?r$Wt~jAg$_p^90W8*GS39lsm1y&J+qBT|ih$ zst1IDHf z%31sMgz_{0_1Ncs(r&BeJbu1$KW`4nJe=oez|xSWS4mUTg^Uk)emw4PhZ5K-xJDO= z{Dc!x*Yhak!2x{Cd=yM0HI#=-Xe)|kGm2sXi1>aJ=w8`XHO-`i@gvOMpV!|F$kNg{ z%tyI0mVAJ;^=+WR=1?t#b9@Kfj939%TlN&4-!|GxlF@msQv`PH0e})Wuz=b3-s|q9 z6tmeqI~t!Ifs&H)pW@d3{q=rc?Vs)azu0$;7Dq}+9WvLK?CYj*QI2fwRDItTvaB>$ zz}z%NR5Q|`5B+g#?qB@Y1;Vu0_03)$Q8fa_5e zb(bRJKhr8!L-%jgjpDq{K2SSrSiJYeTf5BJfs+pMr$?DLT)}%>rO{s#CA|qFd1mM; z8bOubLWl@3l@=liAP!6~>MRB;nh&eX10hlbfms5)v(2vBMc`Z%ptb4^-1Vx_ufX|Si?Dm?OKFs;#L}In zS;!q3Jc)0V4O}KE7-M5amKejb6x^&eid`2Nrc%X$ILV30HVS33Z${S#Q*uWz6kG}J z31mXiS;D=N95A#Qt@hxY+{o!ni;oofGj|L)$HwIyO~g>rJKlPYXCK~lygr0c<|Z($ zvm5e(q=!O$D1&%)P7Gg&5Usb;|6-Qaml&gxO9kT8m*l5o)YX#MJ?||90tOS`K+{2P?SvtX->v^aA+-v8p*Tq$W3Y|K5(Sm|XMQ7Hjk6RLa*c*5j=8 zHvjr*^S5^AufDp^n3>MA94_Md#3ArOsCIm@G^?!aoCkaE!o58y;GM6pa~xgNv+3sS zqDpW8r_eW7m-n`micsxoZ#QZ-ll=M7p%6E1fH`UYwX!m)Q3;@Y38t zmM$t0ele!rZ{&2^d9G!p>dI3|a%Qf$)mm!2u9p@W+KB=}0ja-5s|^vkm3DtTT38JomK>Z6eVi(D2`l#3v zSbfAy2tw~*Rpd-RIGEK!v;s=}ix?K7h~4}wE5sJ3qShMCpL_HHo<#T<75RK03Mu)3 zSdYkyne>+C1_&)G*I8}~F4f1VEQ@+2sqOn1>^TbLsS)SFW_S12HVy4*-j1(cVut5W zeq-wJ7IAAF1ZGimdH;Rma4SlMP~avaH1n+r*ABhCK4VD*j3`hz)Hm$x?HK{yrn!MJ z9-xRV0QDL;ZCbZ|1IuECc4`k+2&ELoIL(bztB9=Cr=HzYwJWU%?+}kV6ck@?-6r$AkR~8|>0tLiB zn=*A#2HR<^Jd9+&1v$DW1)K1KG$3e$Zc0siLbow$r)76El|E2OM<=t^v7iW`EVSSJv{vUdb7h%?!=rb_{xG?jYY=VmYJn_qKQ*o{eyraR2!{4 z+w-Ut`^S(Z9hol$x8iP;w5qiLR$ln3iQ>gpizuYBPaX!CYJx&bsWPUENK8yDlA*z{ z4+iV_5g>RvOp4*m_TB*8i46b*PfUYzj>K*`Y4}_eZu#|Z3v`EMrL*FZrOeBWN^gLu#5jh z4^5*Oo)Jb$BulaN@CK8b=E~V8Q(@ZLm!Re8Lk5zQ#1$2l!%)a(@Pa_cLnq zoX8|<)yg3QF;W^VSSzh=%M~1)WWBQ^6)y{k+p$Am>W zC4D&j8w{uCs9S@zi~vI~89ZKoB`hq4dxxJta~};=e1S^h;**7XWMBalN zyE8&6YFoEp;cBxa}proc3*kAvu(Ru4uvHn_ItIBcpE)EMN0X>3wcGX)DiMHCgu{IIupKH!Y%m<|Fs>Niaq)^;K(d!IuEJn zGKML0%>b@qO7aQ_{oYQ&LhJYgHG|JIdqZ98*N|-I6t;}!)GP@@(7}9SuwkvS0L}iF zA0Axur?i#>7{M!Fyo^YaSCswqPdh^W3sD!C%c>F8&jw=u4^wX$)>gE2jUvI_p;(Fq zEyZ1eyE{dTLveT4Lh<76?(W4Y?q008yWjMj^M23$Pk6F-)|xWrm}B^qa#Ls&sd+nM z^{Hl1kxvW82ZLxvIBtFF)pQw^F{%kVH0*t2X|8SQUzJItv!90rAk1aF ztr6^@lSmgO_4B^f4N)_NIEo+vkI7}d+H{@lHf|Uynm7v;9i1}VFH+%_vL6FN3*eJ; zMZ@U#a+>Cbd`cC>Zqq}NBiJCrUu%mbd7lL`F%bQyX{+c0iukw^bg^m`;h`bP zW#}|rl(b>CzmPw;RT`oD_z}E3gS7++EigDB1vSFw$45HW-$dpa0U;;^WjnEQ>YG;I z0Tl6iB{$+Bq+Mn#Al_6&H>;3kp|h{HrBaQhKlMGO=^kSfod)nU|718#K#zz{Ps^=2 zv9Gp2{-oN(B*2cbz`CFGd&5zaqu}{2#L3S0n0TsKeuIES@!PPFTF>2{3-xr9RF&#^0v|7f+qMHseoU?}8P>YL z?2K=hU~dk+xO{_D;MWd|<$*TgvK=c4=?Um_Hbf^;oZ;T^)%Efk-~*RbivT_Ad;(?bLwc zUwB*GHmIt-b~xxXiYtW~cz+@fSg_B>S@?irLpLAi)s4O&d+qEuOi2|>+45ULZaE-~LTI;NHDY$qj zXT^U6N({^MD%)|;aZMMcjmp1Vv#9weyPT?XpYN^xaZfUc)?qU+jT8)aX87ttuBKoG^8onCE zJEp&TH_}j90)-%n-7iTCm&)~GUG#B1GTy_jqpqSH*8i>u3?TEnm|T zUf|{8SL16AFYbtf+HGzmHj*oqNZjXNLiEoNzcD5%2j#-ZGLeQj%8_DU9@mo;70NuF zxYCphUeXGPray1y*YuTf2jBfnH(aDch=H*hRl`DjMmSvU$6i`KKHX)q8O~|%F6l2k zP5k)M%8;*nHhD``!J)=IvG!BL`Niks;dsZ?C%gV|tRqV76RDaCgB2CO*S3cH|6|9H zM*TxXv^uPRxH8s55p2E*(u=|EEii!o99=hlf7h|E*oA{7n*8045~h0p_Ro)jeBe?- zXd8l#h68&^A+8m)I$#+TTsobAr*0-C0@@JX1!HmDf|mAnFMpte+s=Du>zae~8PN6T zm}#g%%X;Vfa%!`e+w*wGxxh4qoQXHL4Ziwim2|TR#o<(`A+OV8meQ55D?WVeWsdCx zGH(^$_V75B#7k@1U~O_dMfzm?(qOp={KxY4mYKbj90hYEhvD$zS_upqmd~_sQMng3 zpi6{9-`H$9dE6uUSp56lfDIZsd#mNo{%Z`DzQXLWvKRek1?EN5<6b`jy@zx2K`>qe z(0Ie+BT+|cmLls~10(Cp`w_Q<*Xcl>Z$j>=3i^m!;03}+;}B5En3`F7<%i_v;jt^| z)RbanRE&lY-QF~XdM3T9-zRbdyKpw3jG21(E0M5z0&j=J;LZohCfq*Q%EhHX7}!e0 zP(xAW)NkrK>^J6kpj2tA-}iO7_e^Oc!CpV~ihsSd+wX(f1x(b*P0qLPd&a0DL72Z% z5sB$&*oD6jm^3$%N!Q4{3k#2bB%t|KR~h9YpBS0OA>sm{z3os<#A^^Gz*z2vBHLI9HSNfS(@R`N!2!72B{8(PU28*@#j zi~RiERg?cSc4T5$YEJY%-jW`r@9iJ+q$9JByLLn($$zC2Ewjc)tHLoDX*HslGFv0p z623g5DjJ{~~B1M`+^TeTY>X>q*O8$Hp zX}b-A64_;_xB!e<)AIKUV%KsD+To@C+5JT&YVaoN7|be-&H`hM;6t#-jqS^JF?REL zHc3P*FQc6BYhYVB!VludI&AlA__w|ybeYkAFMr&NdAcE+GXfUhn@y~Q2LHvPwOB0Y z2J7DrJH*vkfgMor@ta9dU=wa6KSVGrumUDNI{vrH)DFiff^5Y?rEZkxbr%LTdC5TH zZ5~P<&dKf@B#?ax#`yzK!IYv@VhIe4qhV;u!^{*Gfv!a>AaJqf&A>@^1@=Mww?O1^ zQkf8oWFSPw9ShO8}|jMHWbh^PS(bIpfTBuY9uX+Z9P1~%sdrK{W9c{RPRZ$L|k zVVKmMAHW$V4F3~DMwNSye=6O^=#lezEZWgR=KCVU^IKU-EHUyANrwBwypdwq^ z7V>S}wnUso{`}Lc-)}kaflA-0X-i8^#$1SXYZ18vVPl}qSeI4tOTYQ53JH$z_5;UX zW%0P_$Ox-&$)y#YNE;212B2iuW-N$KUyXSc&O4)R7PXI<6 zzj$oAR_#dWNXlJCG!qT;l4l=u>axcj)5tt3Ykq+Z91<*SWv;e@T%Y}ZuLc)*N(sH; ziKVxcM{&5(T%1*IW`qLkGXP+y95C{WQpw%e(TH1-R$_n(S%t0ZSWj*{IHZ6N$0+wE z011t@9Q=w=JwV;_cK|A3`Joc370FLY9A$q}tpxas_xe06Q^}>?Bwdm=|D!B1eo^HI z1eLEEl&ow_tKE1Wms5*Rwe@ZXrY4uJ~L*Y*a$;ywkJ3>kku!nccLcoE-Nov#NakDGZ^GI+t|LZ z-uan>1?SFL-ny|kM*carv9R)}^Qr@LWRk`!Mt{FQQ`g4L=ZCsO{%|6a0{Q~ht|5WK zvPS8R)Gt_}Vfk>h3Z6@7EBQ}6P0mdnRpo(8n&7D>i0bAV9@RHDuzR07vN3e*=h)w{ z&sP0Nby5ISxPVm~Co~QYs*w_8RK^&Mg7O4 zpndsNQW#6VqhW%8Jm@lh598tPyR>T47p*W3nAuFU9fH4*6)LsX)#*X|;tht!xfut% z{t+jZHd~KRJqzqFAukE-aOKix)L68n0K*};avsb*aK~_YvdWSM_!fcF9XQj|RYrvz zUXVZ|pV$5VmfrX4mv~YUf0lBw0_hWkLx@-qU8nC1cAgQ_P}v6Hka^=}QanbE)uExnoQeQ_ zbL*A*j9#BCP#a4w^gL)uDiAd!a1+Wr>o?R^DRw!O$SX#|hm@{Cs%FPL;$W?L)~0LT zl87Um^+?mXVVpn4c5=ZCG#!-gR}?BQJjlk3G$Sk?%G2oGMEuL+eu8HTYSp1!44&>$ zlPMNCFV(|{e?Huk^ZndTP&o{-MLW!lBZX-D@z;2NgrfdT8~E{)7OxZG>M!mjW+>E- zW~7nYn@c-N+GOdEw0BK0Hv=h;fJWdqxZwEXW**KzLbL+ntk9e>J3$Y@xVdL>1Kk9p zfg?7UnT!vmaTF`M&)jREa0)oSado)-K_TEr=I9)qQ18M4e8KgFk9wRc| z`_Jy&NUYY4{xWJIN$KCkD={Wp&6ftBx?g4c*{bW_TLi&L$125UucbwPqI+hKqi)h7 zj{F0kZJWBwAaX255WS5tVNXTo(4T^YLFT@~H5fpF*LvfvH#v0Or zk%R>OHq={dN#Zl8|B1%eb)jpV0_MVDCcpmMBMA%h=_p@rD|(2E)e=OAiIT zIJ6??;h0@ZSLm(tliBT+eT4r6)&qp~tJWV3*}5S(_EriG_<$ZRU1I%ku{BVp_-a@J z7a#{sfuiR1<;$0Y#zp;n2*J7hT*K^Vt}k3WJEnMi0uPwthk)ir%zT-8RE{&8I2mTo zoGf^lONlp7L`6Tn2}G)knK#?z^IW1@tRTWpOg)OUuPiTN=P9|RqoS$0q-;t zvfxpf{L!oD3hOY7l3AZXbW@;O&>g$&+Jj{na(iQ|)4qzZFjVhgRnDHYJKgbAXro}b z!pcOY4Hx5^;c@6BlUfQsv%Z{qxrRDtXHRtvUNBryLPCPX9*3#;3dA6u``cQJGUj&9 zM94WdZOoQ_k4s2Aj*K!y*rwEA+4YegVi@=@yc7ocQ}HIjoB{hgxE;D`*bf$(@a28- zCFvSOvaX~NcQc1eXlYpS?L)15>qWtiZ9YPgwwqB!<-JD1j8&+~S3^Jzc!s@j&3VrW z*G3oqsjqpmw?p)Df6ZtjU1}@t$j)Mr(2-{;GBgPe<4{`7aBtC96AFO0LIp3b~WNAfB zgrVY&GLADKJF41b5K+@`t60UeX&0r_4im`rU2sNYE*F_u0Z?a22B4J~`u)oZZPXbP zfV!V8kzp%E;!wSec=kEWq3~C+Q~x+;f2&Uns!UlD?)CWr)-Z{0220=EQCE z=xMBAh~H~RbPQ6aj2#$7sEO+sg1NP8|*U%;QB#@1sv&k8M4b*)>N>|-$ z3q)k7P<-?`2%-%^6}V56jqy znmql96|s3Qfbq%(@SSdFr_AqINBYom?E3s)2@|575)y7VfF2@2l2D2qN-#kbTK+#! zUdP2ix4n8n=fXS|e*)@tEzee})djwdBj@EmdFSZ(t-y30lZOteejY+_;}&dj-uVb7 z)xaXQXUvI&>XPV2!vS^?+fZ~jsk}AS4Nv2zDnAV zyEF5r++{EKjnEwvN`}Y-vqS~-d`h)X@8$}ZuT9K%l#<&D>zj(qF95<{xWxxU9OYc1 zld6}xD0g8%F;x3}pu`%v;TX{LZ#ufv`m%%+=Tu|a2^&&$82A%+qoER@-fllk;P=We ze>G#!5^G^uT4nlSyn`t?m10@01`)zZ4f6}b(MW=+`5-6y0wUobmu(=t;bbJvN_EX% zrl*bBXw=RSC6>eZ(VrX21cYgDfuK9m(vOQp+0TN7KPp5E>!oX-CSBLbojS`F^$JtB*bg$YjL|#q{u|P|D5*jD!q*h*L}f@!7=%r z39E+G`BLXh&ZO@ohdH|N!W)T)k5V0~EFvE_F9kpjfkFfT7kem>0dVBx*&?LbZsxSD zYh6CNHoj-A`_6d{fIb{tw;V!%N^ET+gq#^+*c^)e;e!lh=Rl`D^muP;3yg(@bpwG< zO6F`65oo0u6;+-*K0r&%*1~<{1qT!>=Sw(WZi{-b08I!*sASg69K|8MA;lU5X)vKN3N*U zf)d_nb2~%#b*W0PeFlOLQJ;Zg5xODJ5}0akj_lj&24wv0ur`ZZ%YF;u8mqv*=)7!qTJTW%Q{;yc^{ZG5}C*zp$=hD$hBL?2a zlq$_#&e^Heo~J=oq96>y#D&;k+Yk=Xw-hn9y2b+<&gYXcA*iCg55+f?k&e#yQEFgB;OrLF9C$hJ-4V6*r-X=aPKJrV z(QpNH?0>1nZsm55O~@1~ENYoLZT~^b3J+Tm3Dzh<>oJCdtx&?N!G$d&RNE6mt>cxP zWBorZfXG|t+A6KV9)`p`ddy+`?H>s0v+jK`L6w%?fTn<)TyG3ldP184;WFb`JlM*L zja}nO*h*W9AY3t2VoJ)hy>?nh(N-|j=6!SO1ArLiCEy{k4{&gE!zg1;Oekkl5V40k zz_8%Kr+dk#HPOL8R_YRZF4K2cmh$~ouO$lOV&((HZO}68g z7J6&dEitc5xlOB?qibC;l^+a;!tKccz>C*F3lLAQ{>tVUv`tA{jc4z|z-0-bm-ur#cY|72_j&*KPR739rBx~pGpzyn z5PUf4_eKbHemdHVz+x;{-T7`1OP$L|m-LT%5U--#$K#iXZtxR+pYMOLoMf*N~1PfQt}+5id(DRXSHmT90l!&g{S z62lS2M8TJ6(PX)z?O}Zu>i9Vc1v1;t>473)YgK4nV^G%S~2; z>kvR8SfgaHp8{PQgn^ST90*6ZnY{z+Jz+9U3NtjhS;$pXiMx2RdjEus$Ge^>zrct^ z%L6dfz9-OK%=P~RU8m}RjK~Yz1m^CnyKOSL`V3ggYV)~5mIUWooy@0CzO6QuCUe}v z27km%x*qKZ|Wi{gzi-RcWDvMWt?5qIuG$|(V?imWLrWd=49hTGL-2ce~QcXyYO zYSCKe%|h64a7OShzmU&Y?YalHaZnKbfU8n)2p<7hTI6K{yVHioJw(>kH7pvOaq@7$pWy8#&u&Nw%%6?eD6*t{e6kvBVp}#G7^>3KS)+xF4uM2Tu;EXdR8-gy zWt@Ru3(AF1FI;kkn^Z3i8IVg&;t0U4 zw%-v=u6o40jnisau;~^1W)LG4`Eygg+6CXwx(5>Xi+U?uF8h}uyxBa|2S-Fix(<&H zmiOgJ)sD+_AUHA(*Vi3jKx?UWKAjmmbZ(H&M`WF^2>XG4n$ssBk_eK@DZz+7Zsott z)hQdMo`ltMei#L-{{8;<+$iL>x>^?~Px^|G!16n~aSVR7`k{c3*KLQea5sluI@)8FP)(``AZK{EkCdkNGTsQrz1ALHg5D9_Xq6J9>-H`J z7l^s+uWr~*kZ15S@#Kdkp+qFY=A|ssT4>|o;R65`ad-=mNfnE7w;#b>fljWs9?3Xh z4T5eUb)X$301W_+HbAKrxzQe{AB@=0UnQZ)XK`sd>35(87>3~G!X+r;9r+Mz!znR| zfu9I!q099sQ+xICFlcf0$cGSkkgG}9{9mY|-Y}s`TQ`?@MCO|fem>p2S-@$7{g`6S z;}LPu0wC5gvm*(hf!>-l2Gm;cERoS8MDl^^D3#ZxtVnN{lL=8+91&Tl2jfUY$-S;_ z^xO)HqnO<9C5}cht71M)KC@rtgBS?&AdTHpBqAb0B*OguY`qT~%>-{LVUH<Z-4l;gOLMI^?!cS3yx0z6rs=}nG??m!6kLNeLI0AM#RICaHQ>>x*J88S#7Gb*QH&AJy{lh!yzc(yk1}u!7dGnpmw_(V)EpXyX7wE(o-}jzg2=ky7 z>&jVC47e#4>V=oad4%B`&g!-8a5Ww;4rJhqIt$&%=`7})>S9ZGhhmn^fU*)3w0)Z^ z)7Ly&s$kYmA+l>~;}okWN>9l+IYGecb!K^BXVA~2(fR~gic5C2x%@LYDxWmGSr5-( zIA2IF*r<*5=a4ApKHdnjy8<_t@J(}kOfFq1YLf<`b27ka$Fgvd3nC|b&Mm{Ic?5HRzMex zzFEVCNUkr+%$iqAQxuxuNWP|6R3+H6*Ce!1$i#-X6wWzs{`?e2hhHCczGFPDHI0d z=m_oz23eQa?P}-yT^cl6sm@-TiM45Umu~G(BR%Ls5Nb_Xw}#<%2p-jC?m~Z1Wpc$D z;FgB1@ppo8_7@G{f7*6bl7MR8O^EkN##b*=Vw$S&iTD}tzMV4l9>75`G^&iD{E*nOYJ9=|>i(fBXVOhRou**6;A4P@v4* z0Eg!9gNH^wdbFqNF}xm%*xa;g!mZ52_p1&n;r;Q4z~8gB7{uzkMn!Vo-= zU<+HsaAukNqS@*IJZ=wiHNF|L>7q9Rt`V%%a;e-XJPTh)676gb{Fnoz3QvbFMRL_X zs-BT(Mg)$DLMAK$tWiCqhXII+4MkbGIw=wzF=a%6AHcbY z1thfgO!^&Kg;) zIObhq6wyYC6m41L7Z3>9BTpOc)rCGgo~@57@I7o`5fj^|@bn>!U~ih7gCE z9S!jj4y4CZi}~kt{B*l$psvo3#Js1lQUmXRT&8e&dD&Nn>FNM%4X^<4wcgw~gFORB zea;Q1*PTt1y(0c1 zTI83b*Ipj5|1UR0#CV9G>PIgxugLG;aTl?dz7hqqK{O?oPSq=Q)V)6m`=_&8#9%vI0zXAIDX2ADu^o-O z;*x#pqV|vpszSlx19FKwgt}e?F+`r6#G)itedHr1G=ll{Yaq<(fulf2&OZLW3T&Xi z%pKt`))S7krAEaP*0bu? z(5SU&RfACDR0=ubJKA1%n6tocUQJO@B#xTFJW9Bs!d(@Gpg)D}xW9zjShrEDdZ)^e z#Hasuwmy_l&y$)P+O&!1G(Qfcm|g8k&S{3Pr6B--u$EQ|FyId`R*_!4Os!<+`V5r< z=1mF0_EfdmR@-0f5ybJ4e?0oh%-o$_wMg_x>}_UM_gi!)4qZyWI2svAv#)FSy@CH zsUH`2DKDTSztsD2@8`|s5IKqn_OQUge!?iibO%NVi z0f+Shrr?hQgm87>i?9*edBA7*Mi8wWcuTqycYjTh7av&{ND&}+2RXm9?{D`)x3gAy zU4HLfN{S+04(B53Nccg(NH_oh@oF{{jTo`N2qM3ND1uG@iz8T%5>Xx^=n_}};GkWz z0}eymVNMC>z(9|!uL~FucRl_kcIb5Ax9{HvaOaV_GuggAD$sKMzwyXF#Gw;h$vxfd zXH(3vI;kS=#h^3Rgj?enKVXm9(hECa@WKO5qW)4Wqw484d>=Y1OlgM%e}a**E|ab` za#%Yq0(a$nTllvzyRfQaH&{CQN(ozan~vh|M}hE;U%N#K#xRt;f)oae;6bN=TR=Io z+HK|dZjl7;ANBk-(}sOm3ETg{so$Y01PxL?|4`AIHr9Io%Wh_WtYRC--{&W z^mJdzZ>Ssa_T-AVs=H5~!Zz3RGtBk1)d>ES=g~kxlVNIUUnuvlO9RHrPXxJ}`5}OU zKevM^dFApo!2q>Hxx(?6>F5vejRpD{P|y^L9<)k!k@DZM0iY!>gBF*-DnNGbZbQHK zdUL%4<|Q=aYv9XK)I(2sJo&1uZ2PLW;RxEliME)qBBDus+q$frO*g>@*-B!~+(Yua z+1b=RODY&~Q7DsOy>{1#pj--A3Q0&y3+J8{;E6tr1&kP0VFws?-$5wU*cJVTUz5|) z=o5W~Hz1nI*Z>+#a04FgaR#qTq+u_o>xou$d%$=2JWO1>xd=YU;Rq(jl*pL^P{=`a zKR6aD*AUgIN*K1XmRcMrT~1fjA2q5pF%2N+9CQZOx)eCwfOdcjCc^w07!(8-2X{|) z_BHk{=LVn-tRQIecZOZ%(5eWifZN79f*vU~==Oaa)>G`0^|1Y>wc}Gl0zN%hs{PuR zHZ?P&g4t;J1EH}DLdFb4Cl{348Tv6<9Iuh#By^~SnX%z@E?@+U*vR-LJrIA5@&MyZ z50e}$4C~rLM#}4q2>^MY@19~Xp4(?ZK8Lc6V-MLN!@mEVSeP>$?8>CqrdA@b_YFf3 z#C<-UFFD`~?AUsHfvmywR}EAOoj>lj{V&c!dO>OD8S;h1xKwdw#iyt86Olj9YpdCE|B{10t|6d~+%yWOHxR8L zb7Jx)->i|BpCfi!{rE=R4&U5Jvffb@U~azI;TJKXCp1`1AnsqX0k3Leu314bO^W=LGsD<%`0u~j)p(0|MY!s;zm8>wJQ$|G*J;&aC!kv$ z2@{V#CAQiN*jTz5&C7l&|F>u`n>{frm1bBT%SO*n%wP{G59M zpk^xJ1*b}>dVJ95mLOg@y6yEv%a&{~c;(Ia#caRQ2a)JPPSvT_4S4}`2`l(xX$_oD z*LbkNTC-BA3jP}3sF!N1{43s(Vh<^I2?JMvedyQeED&A;QJEsp*3-*tzx_Q*_dQ{P zY4d+|DfBuJ0B-)i0zFAsIOrS^F_=g{e`6{Vnx+b18 z7<{W3ALxN!P@QERnd$pZUXgg7T&=0a*b{#$l-1AebDL9ktwdio7OB&@+9i@CP*&DN zydxrG72k3$k}q;#d?C&2x6@KnKDNGyZ1Us8M$*mEO~Ta`wru=Jes5%unTyO({y)_T zp&-n)25FuL1Nvpp54y@nQ}$L~hAe8-?!CH@ClMeAM^Kje9y7neorOWVVCIybK{03e zwaGOUT~;~A0K9FLYlV^}Mb^$Ge|4q1S{ZFtK2H1$H>cEW3-R^W;h+c;+Hf{ZI}7?E z65^J zwa|g0RKh2`UCFN5+kkKqK29oOwel}vS=JmWIc?T!dsw{_d=xZ^XOe5ijFuJ|XT(=mG+M|If~n1ua+ z0Vp8J!jUJO(a?I#Y{L}cLpPXT>2;HI#ZDkXP*^JvEe_&1Gyi6(fMf6tw6W@Yft=ap zthnPlfxf_gRj9GuDSrH3m|1LIe#cg=6jJ7ej(xmwhJNE{H{kzt6%7iZ|7j)J7*gh& zRO0vZ$#^Wg{Kv%~XI}>B4s8f&sjab+`fktu9Q02}A0ocX6soLcB@*q72F_^$@Gzo_hFdB>mscxdaUzO2-5Kc4(4)%^z( zM7L|}q|Bs?@`c~KlC{Zz8u_37ZP-0?C^#SN7Cm_FeFZ}Xvqnl2q?MXpOXEf5uPSzh zt8di%;TgCG+nt1Tc09@-+GeiTKZM`ozj+1s<*^UJyOepU{BUg?@a5xnt*xswbZn>K z6vh_!h(Jq3!X!p#%vaUKDEZu2SA)M>w9oe)s8y4s0iHouOS=qXG{Z%4VvB1CG^N_M zU`WRQOW`45BIJ~jTWVrzUWccL%Fq!y+!ucm3k;9Yj^KRRK4Rk zw%eRkqpd8SnUd&v)jk8M&yc3Blc#A~!{)KxtS^--+(Tuy@!Z_pni}e(@^I3G=< zRqwwq7B9fU_s+xj<;$hdi&eL|1&Pq{FobNT1~9w@ge1m)swoRRO=R374#@Uw=xx0$BWJkN%aj@9z*OV1rny5$x07cThTul5Dv}Xc0%JTw(v*Sq zEUyh-3IF&lO{4I9^PJ%>NPcU)J+AC&j=GXJ0fjcBk}z=~2`@@#;h4Uzk%=jbk*kcqh)O?~*1y*$ zag0mUaBI}2FW=JzAUZp(6Z7wtk)?X^y+ zVue5-6EiKR%-zE}`Wu7yEcKZ0w@-H@cq(W|zsxA;@ANYEq?030Qe$JR@-TaH= zthnm$yx-avnqiP?BNSJeHe78$J3VKQQ%NnpDNO$d1^<>*5Okiy_(BHMrA{dwo*o)H z&g5J;LM$q@c#RDWdhQ(BRgq@~AOP|H+-0brQ~sT!^cYlSN)J+cso!(1(v$nmI=n`a zdfNV#mg4OLNGM~wKj&q)^|zcX7P(ex`}7|&pHiO$NLoxD5ECB1L{yl`_aTjw!nyh! zx|4p~I#J#=(~Poe$VPX>yO}>Uv7zC|uz2OapB8TZj_b;RpQ1?dQ{7p7u| z{c{Vzy?u&UbTw+vAo(Pp*p0H^apV0+?K%&Oxd}79JH0)VOrn2(UA%&;np_yxlM{P4 zewzGUzNR&38-qs7s2yP-r%5i&YO&&j2aD0*lFS!lT+;T2u!S>isR%lm@AR?F>Etdp zGhw{!hvWAVZ!j5C!|rV!pui8;fM7huj*qkNenKyjW@a=dBlomts(U>GQqF#pBs8Cp z&fwO<(vT6OyZr=}I&SCBbq{++P!dx~C25LSOsvTjI9-*!E1kwz)&ER}BC$9O_Xn`f zX4291Mw{kk9w;{$O#`fXFdZAliTqo~Bu0*3ZoRP_HnCyU4XE>Tx+>QTTzL{HYMi3vVmn7?vo<4Y#muqF;ttvV*ypYd(CDC&h zRwVv4?yM_j>lEk+(t0~Rf7L;fYb7h@SWX1EqLBai#U>hByzT|rm4Ud^Hya1 zx>pPUyQ&PryYL*sIDp&D)YT(1%0h{HVX%QY`ZtqRkF7XB5^m@`UxK3%R~IFh8O?xs z-nkznCwS5X*}vC-*m`+8@%a7_^b06-Z#TyvFWi;aq<0^;5_2OqNUjiMg0Qvqvv`@ z2gK-C=a8q5t;qyF@AT^CGizs*iwD0BkDWS;yXwyF7YCeXKE(X$e*N11q}!T&|Mr>; zH2bt^AD{f@mQm&;r^K{B8MdRr-@feBK?R}e&?1^#vf1lM6phZPS>8u@Q?2RJ5?aG- zDX9j$mdq5fha%;V&>`npZTO7yOlYno_VMB;=cXYK!sn!X4`dzlhHB} zsdAU_@w?rB4U@y4i-#SiA5G@xEP5cJE-t!WN)5RxR_^3&{UU)S+((1WmAioG*m$MS zn7@U zV$aS?7IJTbdKqv{w#+Pv;$Vp2gN$hD_<@C{T&(+ya41bvP_2Mu(#2F0*gUj~saR4J zKK2``pc02@P8cDdm#~q1hUPwt zp`H{=)Viho!zV2O*C z=GWr1qF2{imV4B@0AXl^T@{2D1y1Tzn;(Za`)b!|G8mOxc8<=KnxfTiX5}vq?1nPD zD{N!9veqT)0TwP_?{4jMABn;&rtd{PL|RmqPruuEL{LB#GE<5C@i1As9=H6Rvp}uW z{ZTCxlog8*j%GUrnRxVH_dAguO%3g|NBjjx41NbaVG{V8sKrH4dwhk}vMs#4?q~`G zAb5AluyR5neH)y?ICvS)-)^9xDY{wcu`4Z86d3S~GY&$HL30#>rGExEGGwthY_L^a zL8)gF^VIhK3g^aB-MVxb?LV927<$w3u+jDlah$G|ZMQttSMZn8ko7l=;*EovR`Wx` zY~clx_t=th9!sJ2ygpD5!MWoH2(-l~7E?Xk?>@IG7JF{5#2Ce=HL?C_9h|Eb#}twT zO&uui+t*1FYU_j{1>($QT;cOsBh=M8(lA`hB>vtlDgbC|#HsW*FWt-2tjwF4)I|r4-CypKh`e3$GS)s+z3B&m8_<#aW5Y@<*&>_c1(I+a;vsCV-3*8j=n7Cu(J2u z3JcI+Vw}5MZlpZ8p^x&%k4P|ySM?Q*tdF5$!f4RngM<$;B2dE7u(?N|smO7?CA-EP zphqWuX==Ni#Ar=C8;HlO4t38c*lg4s!(?{f0na}fla62j>GFiP6w|TYAcRV^-N{d* z2Jd8en;6E_lYWid`h)jS$ivku#xDH}qnDXPD6WyUW&G+9>a-!vwj{L4@1~BQR-ynX zv86eIwlwO@?7bRhO?rPI8(_)qG$~gS7)Q1y-N&YXMTSFV{_$)>p%}54^*e?ga){Kc z^)rWFZ+@P9vC2#a{Wlo>H|l^8rScLK495-V5)D2Rh6lKtoY2mbUY%>ZHX9m}B^!)! zX5^6~R3o|&prSTf>V)W%TP)#lb-!NokhbO!0z=Ct3VIDxJIjz8iGB3rR{3*%cYl9l zu_wq*6v^(Kk76TWz@C$gCDKjC6SVgW4qh+#Ci$KVYu;QE&hgg_A9q(X`;<0R6)#)I zGl&?rBuoKrcII|yKny!vuKLcp=)*<-A9^VTak`of2-9S^KdjyP?5_i_&n@NvjsKsU z7Q;WA(-R>`3R!B)o%HkNcc}9>c0K*Md70M^<88Vkp{L`qQf-cG-|<#O4&fF1;Mi)h zaGaEsRI!Vl7h$xWv?YN-NM+3)eZ+SeDVTiy^rQ)0Pny=vcT8SULO5o9MY!74X~;Ar zZp;kJIyFikym*0)AvVF6A$YZJEiKuo4-@&6%UE>lhJQ_azzVOf2ASaFmfTF zq#tZ4qwv8>WU{8YchGH0*ALn84v^(dJX2h1#HH1wV23Dth!SZ4JbYnwf#N7TEpK)@ z-Vt-ZgUux=nEoawCLRd(uP#sN@bOV1p0@L8$fwj&9a>YC9q7M(b|P;O{hW|J zK#n6HsbXjJ18ifT2)Z$%#vhxkg8W>nw7hi2uw$%3%cNwl{&(5ya_JZ!p+&$JN;JYZ z7Cf;KDub@paAb#2xMR3!?n0aCrg+`gc$VQ(AYR6wg2H;uGnQ%LxQd<8U|7r>C1HLj zBoMfujS z4lce)l1nC(wFn~|_aZxv^Z8vc<*xkYqR{(BE%8!CU17#NJ&W3Fis5m6JiTDu(xZL) zEOAT}lCNWfx)shZ{>NzM^P^H4gJ259{RIvlqhbUwoCbb`KPN!g7|jDyHmNi7L7!!7 zGh)mpsY`k1EF1xoAm_JEOIY%&o5+r@(xCSH(D8QXr0MwxKqaafY`CWs|p!|qL6Fh zej24__g8Da6zuRGv3FTo0pLs!7wjx7B{}3Z2FmI9>M})vyphPOky!VFeh!LsQ!7FB zidujhMyt_vo_5WPxy1x@l;0NNB^;uu~OXZHqN)EDIE*__Sg7<)cMuiMd zh&S(x%(v2#B$NJ%dV#|W^wl*!i8t|DMWq~Q8e7iMk3SM1r__p}zRbUu1V$>XErHvM zy~Vc`CP=nuk|DIr2&yFyQ1^SX9rHyb!>p(Xmvksi?hqtUjoDjNYZ+`xUK;o^$t(hbt(DVSG)< z@2)=#8SJqNpKnrItl#)2mFS-61p1&2_@wgdYz~u-!8*ZA{KfW-3e9@! z&w14su4WB)&ZaH-xa>HogdS}Xz+g(up`LlXT}6yw7-nl6gX7NS;7#%Vcy&~6>TGp` zf^f<#-=x0=2j>O~e|uE>`kHt_onD77s6}moUFP`sf~;Nl4(OK%YHXXpe0`1mTuLva zrx*3n(Gi8mg^i|I22c53ff4#?e_Sz6%12Xwd z4z3Q4l$wY_&OCF-&Z2J`&iM_D?#BWz?`6S0nZ;5hZU+{5%q$M01t_2U)X?>#AFCHS zouN)QFV*Us>NRYZA3$kO)oYy=``IL@Xjn2pSJRtSDsVifml*2Xt++=7LZJw1+s{_5 zLQddEgu~q833DxV#%2?Eb!k*Ce8mAnfrF+N{qy-bk9UGLJ1o+)qr?>W{KF2kd$&#i ziIG?rwuuwUYmk2V7*5Vbh>~5L z-(m2z{t<^HX+$>ud_bvQDoZzaqrfv-=ufq(#S@=ZRDg%;j-z?&_5Zu){Qe0uyW*N% zhpcirEeTL#a)G?g=mflM&HiKQz&~?dpP=$|tr){Bg^h|uXpEtvc&<*yW033%@!@+O z3k3F%V)g8lh(-{vhpVc8EcK_Q1L3VPPe;9?@_hWsVA6Y|

    @yz#g}7fG9s#3KFEW z(rRhgx3q7t@kf`e81}m#o_Q3WEUR^;reFldDS5wm-SV6OlRX8m9<|CQwOpBy zqKw0y6rK@n>PoL5em58`-S3lut*DX~5j+-YZP5&6DReZ@ z2NQMqd_4YINTZlFq1uMl?fi%p_iR=5Wqg>-UZ|3jD*y#gSPezE#+?bedLs79IDJp6 zDfQh;d+1taKplOd)UT|(J6!quCRqx2b>AQ8me`*I7yCYp_*3to z{lE|9i{7``xFm7nxMeDmU4qT%R#~aCFm5ig_#b>(o@2{gp%)%B2?Abv{_A}G z-&dEB0C;GnzAoQx+5&py6vFMJU2al?1dg65PNn7#p+Lum0@{b|MtKvV(>bulA$|zR zg6&s}Q0jrDp}wf@wAw&g&Vi*^Sn}{cRlKB?AdvOS7?2SNo88bW1Rbpz>Q|W(VN^Gu z@}*7SuJ*x>A-l3~7V6YU0i1R_g9vN&pUhE&bIF|>c(9An#)!uQ`B&)XpaUrsjHep5 zLJyjKW8?kLN`Kd zww#1n{9-aXI7TJE?5|`(H1OdhYpaV(ZFb3Xh(3)$$j9ewAJGv`c5~ek4N3*H15J^Y zquw++R+mkV^5T1Iwp9I7R_&&r&~QKtk;^rK7!EUsxe9wdX>fr+uz&xX6)>1nR;@gG z&_AdssIvktS`eD9M@n?&4fSXkUy22zE$me-(qyzpoB%v1|M@77lw-7tjxG2$%21Wl z2h~O=z=AR;9ObXAgjCU)o{q(E12q^i?Xya9z9MA)MX7_UB4$jPAK74i+ai>3mCKd}!-Q3d!u4SrY$Qk`=k&&^ga<=)Nu#U^Gl$bIlh0=45wClq z7`7Hur31c*lh>J2MK1q=^^yKg^Ef`)f5;C1_e_ry{`0}@TOqPVeC!|H{*Z@#xX_K^ z--)C#qOSd-Q+GqUSg|`9&`)4&W~5cz4wX?ynDVKx_y;k{W488)ykZH#gqV*nfL1K~bwGA1bVsnsw* z>Y=Az=OdQcC%;I|y$MN?Nng)IBuqs8Qp(rv?-UC*g5Y}mXKZOs-4x}z%|>yb|t zK?owwzxDMtLChxXrCSC4bUB@N8nIcqr>Yg@6keY~-woO{TeT0ihJ8V3Nk%-|bqDt) zy|Ly>(ulk>dgry1Nxu1#R@?ruE^u0(j;kp_1@>M;6vwC?f^G%7NEmGwiPKy5VigZx zo>G+J?@Ki95657`d#h^U8t-?WjIFeC@G5h5N#%KIPA)rmSa|$Ul;u^d=e*00mSf%7 zaL7htVcqKvy#GI~!#MGOzOxALQ$uKl`)VXJzbfB4DTgZ)p71BPJ;VfO0ndBhzptEU zShMj}-j{2($Gxj``O1_gvN&%D6<53}^%PWjYxE7;Q-c8H0#vGJ>fz*lnI}bYl1vAI zIsGYXOZ4oCK}m~^NC!LM6^y$}{zG4g%%dN4DR^w~d&*_WRF#z?AZuZ)+`<%Ag~-~q zJ3sN__cUhw6zI%8izKoGyqS}LH&eA*0>SDUJlNxJTJIVX95*Ix;9Kf~tXKU|IZpJN zp@W4NWup;5Q;)Tir41c26+MEOsieu8i9SriSL7Ia?(haG@)afBSbNPZh-Ti ziL+?Yu%6Oq^|)p32Xvufrg}y)FMT89!D5YYrxKdD| zaHcSmSf8_w=XLpl?3+G+M#x|U(JR3kWfK{P21w(*Gf1f&XGY-%2R4K$_SXBAJqLeQ z3cb+MYU8#}nIiD4jW?{#pMtDo{_ooQ$EOkbw;bZ=N#<>2I3!FkBUF5Gnj>KkS@82f z&uIf8YSHM&vsd=$$_Oe>>TgZncQt?7u4-u@a0kfRO458tTg7HQ+kR5h8cH^0zn}6= z1B!#z<>S{j9!gSHztaWS!fCCK?HOrRUCs;swrqBM?J2F-*&o*jDIsv= zjOmJC3tF<5jVeOh2H8TJqjA=2L~LoIJ&~x`>H=VR~yv!O1^YBf?}? zs1_=f`6-?po%9&n-fzC*K0g79*Na+y+NX!UDaaY@IZEZzXz4bX^%k3h)z;?jF2u{05Nr&F-}D@hglHGbktM%(Zpv(^1TTnF<`Z!B zBz&y0<;2gUh|T9ZpdIT4=({?TF2b6tQcniWoduTBXjQRg>wDxuJe(ao~CpXpAg2cH0g*XdR@yBwhB0@{X%$j+TlaD0M|nM1u*&2q7c3B~+m ztwNbgg3*4>1}A-Q}yj5FHl(8h1!6$QbjG9jnz-kp!btX_cZkL;KE zVt;6ZjcQvLW>J}|7L$Fol-udlqx*#UND53WN-z+>N|kK3g|$SJt#)8n|FW%5&HWZ| zK|YTdz6~5!R1b<9szvS)$lrNY%Cx_dXb^Clt;MC)DPaN6J%DuyO zbFEMr$)a%ip9>`}3b!|QbM+c0*5Y%$A1pfNn_EGN$#K*;2L04Ei`2_`DJ$dc^zPzrus*xFdg5CEGW z$JnF`dR8wS#^{VeRkY+TNVthd^=H>=+O;px3JzQ|DG9GwjSwDg-SX|n9-^uwRjb~? zHpK6NUU^AHY4q|cmuG`lS3KL5MGcjP7yXvJ3iI+$IC`+Qc_XfB!tbt{EPmKdwBhJ zzE}kKB@h==066y1ieR${8=Nn++ua|io6KkZ*P3lA?(W7=|JPtH3i3~vN4yFET#%;% z-t-Ms4dx5J>a2T17~H%W9~J!0mxx_`t7(*q3B85Q^b-K28d#N;+e1qyD1`I4#7f9W zG=EWaO0`m#tDj<*^+4barI7*l2up$B>v6RJR|stLyFB#9tKZ~I2EO824&uKCa@AR5 z0Vey(S{3WW?tIHJ0EIx8d;agnX!Rf7e2L6B{4oqPV#k{$DRRyWa-^1ph@dty)J)I7 zN%4eqo7CP#)rzPwpU-y*DJiMbkd%(Y!9rF`#U6S#3uW%KmqSE5u+!^&wh_(>S3;Wi z)%gtuC^!=eFw5*&RtW4~)RL;gU)F9QL4iTrUBZve?J8=lI6Km9-vnq^hLWR8MyH0A zk#x^(9wQiLaru?<(X1YRtN{1H)CtXN{Abk3;u<=c3Q3RUxMOn`uiNL%!q79zWt+_Z9MAy)tNja$vRro5%uoO>3~wwS11u~s#8X6gxp<9HMp zkN!!h{qdLHz?WQ&UyQ2p_{(2Gw|diBt7>eZh&iFzQfYqe0>X`^CVU)V+*$yUv2D>b zwlpW$X^wlI&`u{y0orWU+lxR8fUJrf^1LJb&~c>oVkozYWCyZq!@oKsYzHu-;9 z0FK<6rPL(iu3kT^a6W{8;b+UK>nTH5Zo6f7zt~hLa_QV}R;bbmmItPwHP>gy3keE2 z;tT>@ISNMUe3eTp*P|)?cIOKmEgc<|vn2(MDFOREa3JD2p{GXxm>2Q6m<(xxr5(OJ zoGnypu~;rw>vgAJZ~fcwz{1s_<{BCvo-0?YC^ehT{`<%<8r<5-$?bB9*CD~1O*reQ z3o!7;#!l>Sq0nMyvS(%#rDbIc6BE($*bnyhNJcfXv$NC7XWK6cq$pte_x8~7@m!0U zh2@)s#E9XTqp84nenprEWfqz-8Vf31xxp)~f)SRfc-O4~o~z@@R*W_N=GfduJO z!jzm@J@%Q=-`56`(G1$`8d+OV9v!{27h4rBN`aN?sgId#lMvaYn(i0r(Xl_}rFhl% z7f-xuaOFdRjHnh$O7hhLpk`p%6<~_EP@@q5cbfW9@+OT&bs?}c8BwVVZ+=9UiiWZL9BBtD&zH~rd#IlIuja9j zvc#e<1+A~*R~?YVsm+)8tOO{n)C}c-bbbX+X#rbcNv~o2tf%o}lp0hE8X~;ZY30=I zM@rpZo3;Smtaq}(6+SWJPxp&MtnMm|$N?^1Ppw7#h!KV9>;t5fC~La=@WY2>{)+ z_*f3$qF=zbN9(Cj)TXR~Sn^HiehUGdDrIf~g6>wW24o{i%V)QZ-rI0dMMAcpR4B$2 z@D?&6E6$3*e2ADB@V*ECVl$o2rW?-S+2bU2R^`3lm$SI{2L8yfTK)iawW`60Sngg9UVNIt@Z)>KtKmc#hdkJd(xKotBuh8 zn2T@|0)XFDBMT96vYbe!3rxKMc1H3iiTHQZ8i5==5-npO)DjH|DOWwc^^l+qbuvzxi z0@Y^ALrPMj=C2d04AR+8#gaiBz0hz|+7+n^_AHEMT%FHM#n5kXJv0R5H?erg$U=yS zh^lQ4hy5D)&Z36y@c8Ww=}w+#X*1RA-3u!^9a(8>c}4Z1-rrt9R!`- z5U_(rMb0?2`2#Lo#senp=oFyLl=13%j&0L~rUN8LNFBx0`pw~aSY~VQ&v%!)jH;?C zqCb2WSVr!Iq72WqM)h@DEPRywMWBF?p_0PR7FCu}&!CM2jkpW*CKV3z*5YR8B{)5- z{Yd*^Dr@{ZN1#VH0hWcO;Y8jXkIS8{xI62N>2K3LMQenQNmK`9(8U#=I)qe--!xls zOYoQ!<$m|xUyl}l^e)r-dQ8b)8B=)lY4X2M)EPRa?@bxWHk?w&>Sxbp>I;Q4kLoO^T9Pn%ZV)wf zs{gJaiT`K9aLjB7;?$b~N6AW(&D{*2&p`1oH=8D1vo`I$1&n=lNxv&lXakj*|?+yixrI0ER8@=tC%qcO#uv)Pvt;CBZ@ z$m)4(Qubl9<9{MFN|pR}&_>f%M`sXkD@Mw#Av6m8L}?D(kUNENR{HxH)%qQr+QBt0 zw|f{0aajk_wo#%s(oiuPnkSpl=%O$9teDV9&8npasMqS`do&+|UWF6+!V7`9E z1lKfqCo|9~tgN8zq$ca=^76EJYpvVkDWPrDX^t}o)j`bW^hp64U6ka2c2&LvS6l*6 z#e4sH%y zFZ1?Ti5*#XDh9?`f`V0*aFD8<~a+^1dr`Us>>XFw0>KC;p#)ZClID{!&^0_#M~3LC>P& zi4H7yQiy+!1kM2YDM2r$n~-uRqfOQTXrgd>`&3GeI;Fa)jd;(*+D5x&AJ6OkmHmdN z#c`Ms_ImNcrT4!D_b{YU-s^mmm4@YZH=oZ})Dfs4eq91ZSm?J=(VIhAv=$SdMIRqr zIYj4RmSlBUyHOP!ETD8T=ENLC3Y8ZGJC)R^(MMl$0z%!|KtRKwr#oFC_3I2Fy6)#> z+O`-@0SlOl#ag4qo*gBm1x;;rKoeoS0r1?AK+6%CdcEC@EaVD-`M={zYQH{Spc*~Z z&f#*^@7&BMyi`{KX=ZvTS<ECR1HBKL__3*RG z?#r{7t-jf8`e@0KNx0YunGzgGC=pRq1*eTsm5 zA^g+sbfzq6NCukLySbRX^#0*q#>zqaLhx^U6raCG#K{{<$IQK1%GSLaf%F6+VuQu@ zP^jHGqyhIrxl~;d1E0Gmi-J{H&1S?uDhc1MU9}isegyif084pdB#4v^&B(q)J2aNf z($&LBdFZ3aMz?j+Nz{(4E5TFWy#%XpH1e@F8F5tF~E$CviNf(eW zL4B^J=|4gk+S<~GH$`jOG`ZlE#S@+cD-w&qSjaQNlr~!z90^~lQKPK6eLxu};zbIUJfFZ>>{LsbI+XyjmKR4Sr_JDI;<(;LVF;pne6zw;;XVaBEXUs{$E4j3pzmJqD@#jjA~@_Nug>ybolS$|ZK1;b4b8Ca``dcP?|MlZqf3gAm1;(0LQ})~n1w{hrAQLC!Uy z(x(~643qD5v~{p^-e-($Rj##Iz}u&2`s)5H%h>fkB#{5k!hb%!PXY05K6w>!A~=N# ze4Jz!h7{M6B0ViFiK;SZNz)4{i*X5jXeA$G&|I3P6<|>*0o783wp#T*fWwt`D%IsV z(E4(NO-h~!G+v4TwCe6$h$IiMCuc~mcv{0g8$qlO&1*2{rHPVu7AEJ57eye(Qaj>G zZs`-tS5F*`Sq?mvtv|Y~{=^*rZ45x9@)M$e`VU#Tp;(Ra*E4dUmt$-Z1k2=8QG>sb za{F2IxZRoFfPkGW?t#;sXq7cyJu;7vPa)~GG}+~=do$GXHvWX#3J+&FPpj5#BO$Qs za5@v0s41TBDUx*bFB)j?f;;J5ttzJ{o@g_g?_Q{wj0IioC_`Kr0G16=^xC|FF2(eV z1I$`x2Sz*ErYwytB=nQsO0^HU0Rp|km&yax1&=R6p}*JEjcr=2mZO*O!^vdb`Bk=a zOucS}={F9;?Mbe;TC9T7C87jfUd}o+9M{GWaM@=7RIk{MeuL?x1Nu%s$t}>6fo_}e zJmp@C(n!lnqpF*y%EkktPpdmkWUxTS(L@>uWjiP&1Ks+W|mtwB# zLbJu)HBLafwQ_HNxthzsQ37agGi_mn)6{^01Z@oJs zTnKn4SgP)}?&xd|Lxc|aJnurqL4!0w+Fs>*nLhYM&`$xVLm?O#7!vyNT3f*qh7jd> zcW?1B+8Vew>5i$uLRIqq=_2sxBmL9aQ1Y%vYYPsk?C&JG!nw4_S=W)C00y!B*AXHR zng;_4hZ3CkVF_PE8YVA^H)sp=w!UAXrE7l$prCk5>B$5yhUnN z-aKv|fY(%K^KyRzz@NU=ou8e_sH>|xp3IRUw>16S)V6wq)PVaP&`SmKnunHrd=FsO zW?Sv|2kS~{YHkx<3!pzOPCM#x4n4&BRJBGXGdMn{x14&#?C)nwD09*Np&K5zJZkEO zxAK{0OT(C&vE*c*WZ1;z7RXSl ztW%++ba|Itqhr@`*sW79!5LHz)z;BKVZQ|V3KP8}DT-3S{h&k3XWApwiH{$;HOPxS znKI16!)dV0$rddEuIpW(6|1*5DZfcd4+jr|Td(h>3s&*ur7oy6^BOR-(#bogff`Gr zENS;#e}MnSy@^gOY4t?gJGFlX&)9bOn=CPtyWm%XhM35ld%j72{x@VK2e1^|?7sLY zpFOSQj$r0rLsGK6u7`KxX@*I&x#MuxRA|dw5iW|<0;Q!FJ_XG{#EHr8x3fwT?xI`l z`iFrKxQx*P!}yNm+OSNf?h>&I5wfp{!>z5YxuP(y;qHgN5KwuXGc&bfn4ULxKf@m4 z%fN!h^AFEK>D)ywDDO%P8OvU4+n`}##NhJW?;cX4omD+vt54zc4w@7x-47j|2X6_X zzO{5Xt>qUz&=-|dJ;amA*qts=>AgT!Ue&GRBBAUuRAh^lb}iAYpP7XqOmmcm&2+c_9KtxBf1xW&O^V8A>ZC)~7X=&7l#kNXDJ@*Clb`YgGY`;!Mg zLl4XMWL_2&KAVH&LMtYU$yLbsjtRJYV3HD+gmzC0Ptijzn@Z#p!ijBeX{bJZrlq5w ziS{SZV5`&3p01KoKJ!ijU&t~2qs-*hXel^=F2!sCbHeU8kx&|KG6W)lt2ucwxuRe^ zI}E9oEPs9~i{}OimiqyyP(SRjq-p{n5&B2tz8zi_RBdDMg~nDBvFWB0AoBp zP1+o*IKb4r)#VZ36-6QzJ{WLWl%n7Nc(Hy1#D4y(P!nI1>Gi^&wmiO)&io^%{`}@A zt)hv?_yeR|qlW&7?Rc)v{lQWd5Oejr*FpdqyC@{AYMC;GPV?a8c46{yqr;)J$cQ_=OvH1DRPy zK>Ey@34#puN40hnY$eZ890>^t>23&V1F)|a@RVyhna&L2_p04wNhYPFy7>YSR>)>0 zCnE1~uX;ZDSB{Dyh`{6?h`49#g8HOs)GK36;NjsDZ8#Vh7@na?)VI66`iOk0aVg?6 zT&DnJYOi~}C#mavx4y@)E3cG$BW9sjzVv^No*?lL0cgtn1c=Lrf@_LnZbx>KY|#RR z5pAr+o-~N+_#3Ta%-OKtmp~4X=O`m7+D6?RYulNMdbK9RLQ zjKQ@|&KG{a8TkYwWnJ>YqQtF5)b6iX22?@WP^K@QhgMI%=tSx1{hl{3TXVd4gg6)I zg3x%I#u}f<$mYxV1`D$u79T_->5rO}Iaj`8>>B!^#nh~6T)9rF? zTB3*V98G^8M|)td4t&(bq)Wt=@xe0glv#FqMCktf@W2kLl-MRBXO}MT;`hR!n@D5n zz{PRvNzCul)uT!7{~h@Xxj!5=P}zfM20a^An1}Cw&?rsW#UB+FSLE{hk`!I}wGrD? z&DlxSwF`eQ1kRhHc7GId11>7Vcb2Fmhz(tz=2=xDpuDOmtEqW9vj{kblIFhxoPJYD zh-Xch+sL!lU&?8BhjthEB#0@AmPEFHJMq|jr>Nfl7=U(#)sdrq(`KF}iA!jd&8$5n zE$1VXm!0WGC`+^ojhJi#cnv!{0q2xq!`Xjb-{fl{iL z*vDx|6{qFs#m6EDl`Sy-0#!OwH=sJ9L$B5JZnl^FAf62gq*M{5KndRRdeV~uV(Y@o zpGaTowny1P!T#acDb}UoyMOHIu7?nMQ;w-viztQ33`;!1U}qT` zs%wW1_G7p?#6@BURqeg^pDkSf$8vb#5ScG2RiILy}ZCsLnP!^cXB>Dj0ai` z_d^rBrpc-6wzb_`L)JETp`PSQ1UXl`F^;K{4^Os}!qpr1|++R$7 zl5rIQS5A))ki*>ca~!`f#&%D%R~#%{>{pUR&|4HjPL++=8Lr))FW8z-HBU4?6vMZa zj0SUCC%Ke8es7fEEQ%$=!|3nGclU(xD(n;sl{4TBaPbK`@@K9L5<}2I%N_QxHLnV` z!QXcdU7w9HP3Yr$ILF;JkiQ^R?dmBC@JMw9IPIZtK=RjKO-Zrr&E$*0CDLl>A!k|d zbf0Jqrop)kEPoP=!^)g3dvMIKoQb+A4H*HWHwXWzt%do>3t2#gib3xPK%b?WMKyfK zt$v0y>TQMS2Yuw)BAmm~(=RoA4^k&(?p`@#dT%n}c+?KU5eM~|xS$&^q%GV0xKlce)n zryC2kYN;;}a6hcFziY1=Zn;`tq?!Z_7Bfg|>^hic!}k4(QIbj}7v@kFi;Ly6sHIr( zSUeAtG-mQdprm?B^fJUxXQLIy=2SPE?i#(yh7xT|saciO!Z>gERieV3h%F*H>2;dn zGYO5PB^?cQGQ5o9M>xetxf+3vRvNx7CRKSc^1>m$RuU(_iWNN$7^s0a=ecf6sqgZ(f2%^~xrR+eQ3v9ZU44xT`Yb*?t%n6z#8Uvfx9JW89p{QUevX;iQ@ zNoB(hQ-D;gycxEfn?*8aOKMXWJs5&DUmSB#1Egk-ztDc~*5zlw5gJ$hyTA^Vl7hs! zNTZqm^P-(4wyle)Uf8#`aT*-P*!HT?vKUD-u}j;8@+YSn>ynC5nuT_4=*Wa`&|S|sb0PYQNIZBrsH zrqee2rsm^WLq@y?9-Y%Us_`eT?ES3)BHgd8p(19@`v*y2!KDtSv1}D!*?dl7=BYHE zx66ZtE)vk^v;~=Y`OTkEYo;ZpH){p0PDjL^0N`QTc~QktU{RwK`pVukMo+Be8ZF95bb=Jd+ zHOCFXk~x*!mYCha@_8Ummy#BMJmlXRX!=Ns%(`~HS$Xn{uh^& z4QxcMN@(ZSU(??yJFM zl{93Zuc+yUZP|cex?Jx$JaU0-7Mzs+Q^qIwqUM9 zSvF@14%#g2NHXRJ%UX;whO3=1xZtg`Y6_E&8pcMNGqX0C`YdwGs1}%7dMQFu1*GXU z*a;jlvZAY?BuP4R$a{WIup1P8Ao*jG84Qn2Vsln1DH=prp3jVpnx(0)5>^dMYHkr` zV~^td@$qU~_9kNDiUNEqSM*YPq`h&D3F(Qtn@#^tlBMkVveIIxvu|U~H)JUp()$^V z*k4AO6u?0uVG2ujRF!?X?6_idzc;^79f%*-Bvpw67j2@Ivxori)9-R82B7-!d6Myl z1cRya=#7;Hx60B9!9v3Ms1xcyP?1z{fg!5eQfa?&rsgzIJW3hjFZP-XN%*6O*^`d8 zwYK_xSsM1M&Sdyz5xso{DC9cj{J2MDOX&a8fc5hKYrwLu?o?*Eyi9(3K5w;$v@d!x z%29HgEU{WVs-JT@LJS4+4NZU7obnc#KJGkwQrH>q@-eZd`D5!((zZv+Q+2~%hdea~ z&u+Y0wo~XVYTTHKRfL25t%WX)Ab4|tT*z*%)+fyGAaP&}0->T=s3zIAcWkzP`Lm`y zNP@S2GS`X|*gQeZTn(pCLJn4`hxX*{aC7V*~`nCU-xaWBmddbQL6TD3FK4Lh((L^hQ7x zl`lFr&Tz?60us_7q^yve3(iltnkRWo^~(g%Vf8bD;#16iYS4b(I0>P}2-KSCUoP1@ z1IO@Qb;k3N7vt5Ic3?kM0!6gEll|u8o2pEi-fMG8JrO>hDB_2Bj8~ z+pms76{!UiW>`OilcdCOO@1AVKIvP#wG0f~aw^_mD-nRMw52-OQ~USETh0e)lG z2z*weM#!X<^B)mLU8F^^z6C`6kh;{AmOP1hFFNq@AlPhA_EC zuK9STS*dR)=C6%pHWQM8OaK5Eunb%Sz+5R%1TfQ5BGC~f6r*QrS2s88ocMEh2dCx7 zA`F#iTjxu>qZRv|-Y*tRDYZzAroU4daykiN!sJOEl>EZ>3-D4n)VJqu^SW@y6lF<~ z;tq<)Pzq8hZVJ{=#|%l867G$|S&OpF{^yZtG#*2-@EKuZSd+0dkxGTSfERebS7}Q- zC>0KZ9lN5pd61Ouu-^|R<;BjCAY(?U*Y~(!3sCgG@ouU{Qv6TNmPYtR%yI57fl625 z@@N}iIOGz4v<>dYw>t5JI2-$!QmfE$NeKFtDR-;MNi70B(jf^@I_Jr_udb`uT(3zL zlY!|V6zlzESmx>`ox1Zc7WU{VqY{6mv)35Ck)Sh|o%nee2B%P~!gLROm7@@TZz9#q zWVng;AxJn*vysgHrCo@7Zv*JoY$Ymf(U|S-vxtV}Xv5_jr`A`-%j|kyYt|$CCp306 zGQ|d6qlM}jG!X&4Fq!3mnNEG|2;{8Dg635uGpNDrIe;lxB+7L58xfig?Cs~a$Y26i z>mNr^CWeWMm{ruRCGF*ZA5wqV5?|!8fnzq%H0&v&WcE1WIkvj<;1McleIw45mJKs} zd{{S{c=|XR^0-;Q4X|?_4@MXM!WvUss7CdRMDUQF+Er<+!l~f)`RxKRtv>(zXD1A< z%Xh(9e}d=zjxQJ$-wt0b@_M1a(;0GBVZz0DQ-;dR1o0pLN((ZZ$}d%F)zfZhcy_b4 zWrj_$xZdfb)T%GcoZ{_q0FL^i5wW{3Gf+^?_;?d#{fs| z#oi$m7Uin*4$)h)7Vd~1DnGG>0@yxD3+5@2AEN|szU&E&>CvHK_?$@bGG-JNPZHD9 zAkHQJ*HgvobMXrod1>o1#XSotBaFmI_SQZa!{n<73x_j_TMyj43-?a|v6i+gNoZ+7 zP;Wp{Jfi4jH6%-N2V_vSBAaihpD2WhQX83!kYpE>G?Dyi|ttd zQ?tp9e~e~ig!AVQ#uCW5!khr#LB6wVC)wHOtOVhhO`}IaQ&zGhr6CBy>vAV%=aK3j ziQo9*q8bma*hC@Y^V{9D&>qf+9U-4PX&pz>ZZe(DOuJL&9V~i}xz{K^dO(?!_P%$r zoJuFjYDNy1?Pxnot1#JrYqwefaI{&eBad!}J1W`CCNWJY<38K%ZvQ#{zfRr$UM3}G zj{36JXwHA>P_mzxP((ROc+C|nL)5_18M$N5Uu=i&nuK}HAa2`suW~{(3*Qd(4*ViV_7-x_EL4vcFv8OY;4@;2pWCnkQ5FOL}RB4yQoZ@<9sC(=%6NNBRiukN& z{vD#JC|@Yero+a-og{jvoiqw3FyzO;tfs|>R>fwl#~y(p5BO(yS&GKe@55{1rQ-#t zpsJVo{+RO=do6V$4dL8})_SXztKQ;kXor)B{&@@>&62zK>NA}{CK~<5rWvJ`+cYHQ zpPQ@Pc7u?lO5h!N1YhgrUC&VY^AolCgd$df@}M${Z-Im?$_*z9XbJaYq6DBJ7DsMz z5VaceEKk~n$1F&1|4{#kJ1ZttNxpd06UcnxqX#8gf}P;rm%OCSxBk1kuZH0?&Vl~u zju3?4O8N&To=T-hzKPECsl=>SUfQm<*{IZfs2L}T!$J>}06B(XwrPa#2M%f1g0ElXNC)RrP>g{H3 z)(Q|-4P$0z#`x^_?stF!Mk0BP&1pvjpptQ~W(uLHhM|FxP;&v)&SBGQ*Z%mc3 zk%k3}+W2#EVN<)#mh^P9KZ0PX&{%N3RF$D>Q2>g5wG3oKQZ5d`V$##D^OKHv0Tv_& z-PA$awf(Zf|Grg^A%MPc;yJyYbI;)n%hMukDCnm=5UKvVGVIawiI6`Tj1pZ2vtE+z zA#$qY^_e^x%9=HCex!hT+mFU0~i2`#by%b3LB&9Xdp#= zuy{`n#h6Z&x`#3#Zbf$0(q3Rvezma1GmxYCzWX+$RAy|$TD3*+H_#-@p4)T$t3gb` zPTDfM+BgRLR?#}QSJesbWHi=dBqhGB0+qo6yFJ#+2}&lX51bQ3pY0Io zHz$~&{vZS8z!0YpAL3Frr1w`l%0Ughk#Z zP^Hvg8~#b=3F=BH*`?G{H1mZE_KiSrl@to*#nOry@fRvCiZ9fkwK<}AlFB4nW1M7S zl1%)=XjCDu2|OHRz|-`rLal<_X6fbef+kFY)+`7Xz3}1&7{J^t>DXdpLJW)I{R)-W zE=~_e5tvS85?MR8c#YQ~*LXU3v~WxBPox$Fbh>y;=|R7zWmV~LAp3m;(5>~U4S>|t z#f7q98~9XGw@?{6rM@tu2tcq)8f18Ek>dVFKRjYFLeU}%9D!re3_(a+@}$ycwI1&c z-Ay3$dA8m}ri@Vxq8yO=M{^w>StLCT+Z3|6$u<>$7Y+*Li<&3Le^pzV6~jdazv3Jy7=L zn6%aiHO<$KQrRazz6jlh}ERv3)rfPgfJI}EWClBAhJagRi5Lv7++>_g^3fk*WCQ^r)C z8VEQH-|Xwea5|OY4^^&}G?PIzlf2cwAAnN$D~H;tn+p8+NWRR3+n#qQmL1%q9 zY#t7i9-3#?%zUk}EHUMe@vvOp>7c6e$`X^i4%xcUPFGk(?dz!8YO*$~6G(l86jc&4 zCaAT)Vp{~PevudN&1>ODJzspACALNekBi6+iJ1ae4*p^LPkj&wF`xpT(_ zuctcOx;A_&o{WYXt;X*Ie@ontS8Prhy7h+dl) zx?|HRY`RP7?nXemOIo@?x=TP>1*E&Xm5y(5&UwE(?)~Qv7zlf=wdQ3?W=-;ou;in+%GhD0pu67v4f z(~u6G6aS&EH2PXGvqyvra$&b0KNL9W41q)vv5MlfNhioeb|c()r2qV~OIZHkhPC-K zi)A7k>`nQ|tgAfbBa8O50C*yE6r|ArvJ_U?T@oR+0jKYdIRfvd@oxc%hFDW+Oxske~yaJ zEH40VMF#I!Qm#|Q(Tc>ks{DmR?*q99kF#cH@pDsjs&WRWCW6|BJ zyf*JmJM5+FtWtu+-5|@C_cIbX*qOB-$5zIB_oD@_~!=A;T@ z)@!AslHSQLmtLmNiC@Yq>{VVDw}ht2 zD8QVIR)tjop&caun@KLKL)PYIssEf#i>^UO{b2?PeuyihCvQf3!?AVy298dzhi(1& zoG_JRQe2p>?x!E{Pbu{dbx5f6%u{(7=q>6DCJae&=u>NdkqYIsiSJ661^_5v;3$H} z`zgVpp~-X1s|Rbk!emc#81t|B8#I!Y$ZXh%3^vnuu@=9TXgi^jg^RC3RxN zNAx+hnzaj`^0SnvGa2{qKTEXF;+cl+*|zk#u+pM_-XD<(cb;S{H_v9Zndfct#+~q5u>x$Jnf`+lv*5&ALM&gOsm8|BR}9_ z!<(pE>;VaNc`K;njA?t|LBr`~x5_O?+zRU@I+~YB#_}?+Y$&j+ip~u=2bwFq8 zCt%VF1e@jj2~1E`EI>1IU9hUEOdn0EQl?&+**e_ne)h(o(=RZY{feilnYKZ3Xc(^m za_Y4GOXU8L1;Er`bKMkXvh3nd@DEudlBH*l=s}x3V`iWAOf_P;R$Z8@j{`Y(d%q8L zs6LfsR8L`+PngTW>;S8})@p@j4N`5WtT|}M7F9~vs%}%Mb-(fRG<4nw8@-&k(d4YR zm0BI^*HI~ugDaoE>T}Tttm8T>>X~xZ|9h~?s69{issohe4R^2t)+suoX0&zHcECpQ zoU>uofl#j(%#`)XAxIN%LyZqf5#JR`(Ux}qZWR7L>b~E3d3o8t0Qi7d6ql>-=C{r7 z*$(H=lBZ(V*^VsS%Gai$vA@1=)b?E8{c36S*dukx(ufAQK>sHID9tM1F)872404F( zmRKw2u}wU*AvQV*pp5J#b_G^lFS#z3g1b>7DW0t?LaM3|s;RB7udit;;cpgO8@6qt zs3@YNE9qS&hhuA@dj5U=qD2eg$)E<Id*q*kw5&_H>)p#n9A1@L^FJ< z!c~(YB2d_gR$knk;hw?GoK#Zpn7bIY9kzA5{zJUzQ;w*&YmgEuY*|#=mgFC%3p=D- zFiKStE?4R08y7vfig*T+^3owX)hs5VJ^nvyKK$Ix`Qm|?i!DGJ)Z8-=&%jk4{TCO7*`GVSC9a{Lu z^PbT@zrTpc$=j0iB}$-6 zMJ$31%`C^Mn^Dgbv;+68Ca**3&0I7BRL$?s3exTqtpDj2VtD?+tPj^fk(_+*3m4%) zFVPszIIdRU(YEDGqTw~iMu82ma`Cdf*+)}YnlXG*16xd`ODQZ^so|&~lHBq>?m`~3 zOxdVK7#(0aM%zTEF5H77e5QoQqJ#OAxAke3=nmHkE*TZ$=R+M_0zAH|tsW#Ow0CB! zN@=Es=iAUY3BFeedRjB#N+QcfJP!DYtT;h=`~^Uz^MY5&A%N_*PYTUNYkI(-C+0m> zZis@5-5D+{cC?W8$4#z{D!JTEp5({2P2W5LO6us`++5I?1Y83R%dto`ZilfKiygcw zpd4>d4WQab?>b~|y8dNDRwa*STn=D6p-=Ezism;CcOw#o+A)*U6;cR*_$*_Z%K(qo z7x}N!FMbD0rb8jrm-f%#;TZaRj}d-WKn+(7f2v@vGZNa$T-5&T$Cb# zb00=h5q*7qrS-C9a4u-&q_LGFqTh-SuH3KxkbgEBdawZ}+7)a(09m#3Is%*09WeVF zNp%!~-XM@H?t`A!iMTY9@nWq!l}IXqU#;buv-u6L`>Dy(6`B=iTpxvs!-0eGQ)#qu zum@S=+>vLK$LrEg`kvB1SVk6dWycpfvX4#!;bpbeYKbP}Bf#i6950N9gpa<5D~gP4uJI8^WDtXW^VE$XO)8eTsel_34b5ry8Zqxd$048 ztCqT1$0cdd#QiM#HDG{B@BI!g?B?_N;X*jTG&B#_1gJjk8=tU9Mv1Xg@b@O;RX43=F!lw~YCpku>zh%)=o`q#_U&c4yLP$Uj{ecbe zkkq0#h_T!Ss1W>Sn#F$kV^Tomhr~;gDyz=ewu*eAsh8a#lr)A-R_zV#h^wEUNWqOb65EE)^_lolWQW%)$0Bumw zb`SLRTAtTJIWo4ltmw{r30+qB@Lh8`@Qbm0Cyt%V=PfHX4b|~T zzoj7l)B3x`6fAmIj-q$)OWxB=RN=-amfES?JuWAKsJM9!$fmpIk};EbXN+hqUWB2c zX+$YOrN%Y@v`?AbAKPdp;j-!Pvc$Y@Kl{P=$&z`&jm-nD)AIx>Nb2}cqsVCD5^33k~)HV}nTBj~( zn{ZbAsetlHj?{eu1HTsd{|vx+){n8(*V@e0m#5n;M*;;)_nIXaL;RLicU_w6yKLC2 zi07QKNmA$YSX({7vL*-Cc5!MP11gj{v-Z!3<3^Io=gT1i)O&DI`^{f5TEOC39}y>qA9@%wqz<7L%B7QRJ2Pi=P@!iMQqvx8jXkT81p4_N7}IRj zHJLiazn~nnahXI)^vp0_#Q7cuGkNbi%`^CKO3nC{f1m=@d9a{|T``Fnj zOl;LpfzCwj%G_55RytA*L5J*d6ysztQp=U|!!Xh}SO*e=(1dDLsYZjWM;>HI)AYbL zQ%GOTG4vZ&s2F7Cjai_`Amq1P2Cpj4_j?8*Yu|scY8M~n&kkkr^hL114FPKZU~gTV z8jVTA?oZM~4bFR$(j;6yoC+|DoKqa@W2vZBN8#$^mX;RSmC-MVID(n|1!!-j|1#E( z7vGDQHsdp>Nv*~t(LUvD%y3kJJh$&4Fv%q|wn!mG>bbA^Dfo7tu-=%vLiXd!J?T}w z3|?ml=CGtJI6@q-LKx-Vi2K{j%KuUsI?5s6bG-@ElqU zzcks^nz_8+Ia@4JYY+L~<5cxd+(js!ho0wWW}negWPV5pYN9(2FL$o=NxPj?pA??^ z3hXu+=Jc`#_)8GVLQ)S{0J~y%1!h6K-|im)qN!L%4##E{6xlRW0x&<{N6Dr?X~+`i{6*H^;c!p zhx=dN$b_)fXld1Lu)}{O&(?T_^}40PT776Z+z()Y5yC))n1~P)KwHaEm)sMoB2n%* z)9UUhw)Lopc|fIbzgndjd0XIAo#z+db=~InLRa$yR7Jg| zNAmHkog>ZZ$kke}88~aHNCgT-Ei4*vFU;lQ47zi^@v5)KUprR=YiZXsd5Nx(jD+Xj zR+egt7kZ3+y7M(!Xcs?iGJS8u6|YvE_0B z-J!&%-YHQYm@Q_G2fx9&O}?p>xq#cT!A$g4TO0nL=j4i^R6is4moPZ!2{_s}ECg^V z-$3ayvzpyVl3-=)CS(JG=hIkQw$Bo=J}-LgF_}AdgGJetX!RL3NQ-?#xm)2?RT{?^ z)wX?F^}4TAJ<=qA3=qwi>~oa%#ofy1az-6e*|7TLT=CsMq(6Hc$v++!jobai4kx7V z2vkNa)n3=C&n&L2tb{lplS#p&s(OOwDkb0R>Qp&47@7?BW@dT8Yh{n_TJ^`gqQL?w zo6th8MK>`Vi@Ja~zT|c4%szVtdg^!Pv`xP&^@V<=rXv8Pn^T&PPo=L?r%5@kMe2!G zvr}=g*s8U$9|eB6%l`DM0p8o;dpuUk>`yX^5Y)FFzyC!c4qiwb4&Fibl{7<3)M|5$ zHHp{3Vn}sqqFhy$0YQvUg$w%Mpu?aOW!jV9psgR!{1BBE$1bj{3AB58V}#|)=1(Q( z&9T83v<1#cqYL1ax?zMmr+!n;c^b)iONa!ajVSnbSWatTolh;~3af({5 zfq{St^O~)L119_mZmceTGfyda{5fdt@m!r{vP-iTJy6O$_Ieo{#?EFTWxAu{_9ug z=sCEm0iRD>+;W@MNgTsJjGny~Qph317hGt~0t@!O4Bi zkaAIx&}aHISW~Ur^M6!pizYN%(tDGBLK9qs`{mPHeDJifDxt>#+s z059W>#!^X|AgBS?t+GH(s|Ngm zQt+Q*yXb3B!;)-X{TUD&Rj9={5s*#V>VNGsKJVy}^ZsO$4s34sP#dSI>|H{yZ!3a_ zN}{x5sl;7kZV<(nmRiQGS-=!1{lof*3Lkirx4!B;h*Pezg3&hlU=siokFJ9gSWrae zfU|tcYtQrFBTe8U5vC`qnum;j&c(?3XJVS5el1sbr&GNoNxcO6NZG%X*5sjbvjXGw z)OG9jFngmEGoii@REC{sp-TIT@n#9IRaUFBQAb+3k(y1{ZhdA+E4=+)`TaOZ`xh2GX`iEy6zfqY`1oXT0JtAIt?Cahrb!4L)XQ zu#@p_1e1IV>t2{3woZfnf*ZKh+{VvvJzH_tX>6H_ad!t0oE+3z+i}%^jHFHusPPh& zGqv1ZNn0ZTl$uvJ()scM=S%I~YfTC0C&TgV4UM)vSANfXxy;qnZZhlL3uvFj9S(<& zO56vFC{*v;Cjh+asleT{J00^8BmOMOJjFj&P?7Svvq-r)liBnU7tWBDAov=f)@Zka z=Sr^y@7=4Jn@c)iiK z=N%9CEwK7E6K2)Bu~ed4yW>rnK#8D!M$&2%LW@do;R$tcR=l4|=#q3ib$!|Xaa@#x zy0C@{=|U!r9v@W#=5}Yih7ohz12uZrO>h0HVMN<#VQiYeZvolWZssB4Yq55yWdjS5 z4I=V}fIzF#kCreDzb~_;JRnz)kLb!>2XdKr(elLu(K|7S({_P(o*q4Qv-JdW@)z6qHEXeW5cJ&%G_MgrmkNdt($s@)ahYxB{U$F3E5 z+qkVvzagsC@(2YXon!fX(L?qfi9&5~T!gf9Wg+y=1)|{Syz&b8`HKJ4Uwk)lqzg6Z zECHH5-y6iEw|U5=i6BfelmH_hyhx6}s*K2B3Bl~V5!1>YLZ@Ht)X^AF z{^0a-qmd^iEq#91S9i~3Sq&jbKRLK67uKQS+Q^g7_-&=*wEnk`j4c~wz&o^r=N#{+ zd>LqB)vN@A;IHP7FMf(C>jrLrrrRe#$fefyVVmyZ;oB6hNhKSt+1UGLE8rZ9UHrOp ze41z;?YtLN#Aepsilc}=KYxFQ^j)4X>Tkw5lo4@Fogn#7dOPt)Q-fxWvx#l_e;V|z z*vKW)8MSJq-VdN8BuMaVYkrCtmJOeMkB7on@I7+@=F;fKgrYj=v2w?y{lSEFqHY}3 z|EJHiAG-R}clkPoUfi{j73z^;Vc*pCy{U84hz`0#Af7S}HVg1nqf)1Rk1*4>NV#7MTk7!NCDJCMC}o$XDeZdD z;kEEKlzhb6UlwR||B0AR7{>I{hUd)}Pot|Fm?+Va-T5;b`BX?b@$dl&RLJ|Iu1`3^ zs2~tNbT>WOC8%pnHE%`BKMk;7@;iF6cD@M&vR|FLQHwWgUCw{OHZcsDu*27q&kgTq zkDk9cQOZ1u;UKX{wEd{g*ZK@eOoL5aLVWpq#?VKzx@R|miOx=Pc`FXEgwqrH0Hj2H z;kLo@K+~{XLwzlE_@hs&Kb`y5cNXSgxVt^}Ju9`F$;bR7W2qM*-cLPcFm9y=$9UNp zande&v8P}Brh}`zz5e@mAJtg9r1Y_61A~ogB-%3UQ;~K>2#?us?HJr`8&-rS}1owuq-qA|CvAl$c+- zK-@}fAg{UxVhL2}9qe@y)}GEQPCNjTU3R1R5e$84oF1&D{x6H@KOFr?quWJ4Xn5lZ z-L|V>LT*p)p5TqUe}x58YC+I(T18(5QU!;+te~f<`zr_G9>t7!%B`oT*Pq5^n_9hv z@cm^6b%@usM&{A}G9^`gjmbc2!R^={gnnJ=b#Pn>fSe;-H35;D-VZcNT$|-7A47`l zQdm$W9-+w&7pmp`&u}bxVu3%j(4J;qOES^*k%?+!Q1XA_MDQ zl?_&0IEh89`(rFtjWg0L49jrLk-xr#shUt{4|;5FX1wM_92s3Xm}mZOsf@X#Uzh}G zBYT5wCk{gJ%?fn+y_jLxR`Dk`_dOy(?X|GUMaqE-Aohq-hdPET059J6vP4|F3&StN zT&3Hzs`Q#9PkEz6^jy8?Mo!R}6Y4%5AukWtQ2{{@B79JOU6RHp(W|g*qX8{5)fNV{ z_0fjPVqM82>Vp>v`OfM298?t~92O>im(9K?*MMvHxp&baWG}YXhrD_E+@;pYJZBtu zt*R`im}eg98~!2M=}mSg%SnI`-`cI(ER>XtjN#)ZX$)=h#&=f?LKp*qh+klxjqQkN zqf-CjnUQ?Ni!Z;M7~yLNVUMEo=LnZCjrmTtM{|onBn6-V4P4iDh~ZnG=_L%_*XA4h zel~`KaWVtxmf0ROfYTY)n5XM@wmD*Iknn!PGC);TT{xpQhD}B4*GHoHfb~XaFPSA!h)SyJ{n7@p zAE9S7CYmYwkSgHGS)t#-ob^ty^gJPCCto%>@0$V${rCm&S+@xQi@VuP_aY18u4cJm zohs@V+hcl_>9uKX$_`OVN}j2bS#1=&Q4T#4U!4Pfi`L_B=$~>_Uaef0Kfh|SI)VS_ zWyPZ^*eyO0c8GE;F!4z0w~x%tT>5;0B$0~BZFGQUTE8i!Mq^wq$BtPMsKNRBiZT3@8opLex2aN z2MgO2F(pA-BzTph_AD&U0{_3(M>&y9C7;Tug$S|syyG&?D?5R+ZOs`N?rT8q_ z&hbqGXqGJzZc9F^I+H{43H3_;fo zOXZ%&IQ30!e7}ElmgYaBd~;jVVfSZPKH>bl2MFbI7=^5lUzYrH5f)%CdzPsbDmZxA zE_eEGFM$-}rxR)|l74I1wdHfQ=J5~?aS5~y8Igpw&u_NS(Fqvovhq2tXL}r>Ri4Xu zPyPYgJEI<9_P4e!+s4?YZ=LExszXI9&pD2NbP~MZG~+9b&Z@^u3lv904oQ~tmY@1) zyVy$1=dvGTb?>k-D19|htJ4W=2I*LfmfUq-_n+vJN^>6Z_n26PQxuiQ84|C_o@ZRf z3G7nzZ!)4>D|z4%bJ=JuHL}~fSV?J#F-*&3z>YO^STz}1N#th8spEOq4HYl{=pO#1 zRnfV&WPBnA*bNpdy46qU_EFI`T#*m<{>UVRWxX+k2cF8j-Ondaa`mRZZh9FR8ReM> z4)tSN00Y??zB+csYL_7)d6B{C8+L`!Vy!F6=1KRNYEy2PeIm2?{)>-`PC|%WpUP&Z+x1~)DPO9{A{hz;Prl!o>4vX zUL01O&qj#fT=ZL2bGqTjz08u|6U!jCbN!pDhiQ)WpLDAnxedT>=TcTRdc=uONjn*qkQht&POhe5!{x;U4 zHBUO#-wt>AH`%WCT^9(Pq?fq&BKAj|dTHk~;-`d?b!^XmRjbi9wrFj7yBtWMwBOo4 zWeIiF_l3ik!B5!pxOKh$W6r48s`1|RC~2}*pQ~O22Bj6qd^0y_eDysZ1;q#9=$rem zVpCIHbj82y=iTH&K32(hcJxGnfq{M{h~(aXVwG&KxqkU0)I6p2%HO?a(LW!>cTLp1 zru@2m)cv{1paH_obYu9H^ry@cPEqOWf5!SDiZ6&~CwIM^H30$bA_GaU>jpV;vA;YW z29sA8wd(y2^UCgS>S(X>)c;;%3<;p=8E%Cm6qR;>tH3yzz z<*>AX+)e0KMtcpJzo9yHk!B3y-*D%EG!?y4q?AYwEp}EAq<{gez_jRjLECs-v8*a` zCRSX(GjNFm_#(vyDDg4o{%B#RGB-LN5h&;i`b8q&e+&59SY*DL)jc4tR|Bj*MY`F< zq=6*~`6>?=1j}Q8Da=X4Ziw2(0L6T=;JP%h@0R7TRUbj0q_;7_M?TvRvh2j(KL!B2 zCbRT~mfCmEq6SLPZQ@0JwqIfLP=UAzyyj!RvRgC@Cm}6^=WAFL^9p zuk=Argr11cF#opEPR2I@2fKrQh6Y%BGP<{fdgR-G-V~4z?r~q)q=ih|6c-Oq$kRFX zqU2WqUs}(X%#+a0dhMEZ+53NKRnjmtlB0Yf44M07*x}uy$dbaZrsgeB%VZ4JJP7d9 zaN_WJ2g&DsZFoPpu^;Lo-g58hhhK50?#aLeEu?)HU1hUmRJL#v{Ob9u25mvW6D7g4 z>JTts*3GX&5@Jj z>L2oF)&WtxxJcF{WU3zj4H^H+^Zz3SDFRh{)(z8p!bGX$QRFfjo`SSg+_;MH5Ia!TUmL0Wj%KPu_*nvKpQ`(FUhe8nr}! z75d6wMZ>f)@nZ!lh{_)E!x&;E>RIp`CnGMipNNKrE!8r4!kQQ#7jGRw7lbU{U+=NFyDK!?pN?cID=9q( zR*j9f*Imfukdh7{aT*_guC5L=G<5Q}6YBc@^cE4*D+s9q)LJS^vuW6}KuhPt&%q$> zG{~>`Eu$rSn`2iUDk;?apNBG~(7{Cwm)`MRHbd#xucn1U?jDX)U^ZR#{p~lEd<29@h z_ZLi6{X6Snn32~Z9ql>2nZ16|dJKlx(RM(WV8vUDk!+LBW|GEXsg}J86GaeW7cZUF zjUAsk#cl#v*lXFnJf?E|(ShUXxsgE*L+qWFmYwuyOg6e8}Aciw$xAMFzZtPoFN)7npX=pa{g|XeEFxOgLDpBu0NaVj9iAVFF z63TW2XxmfZ%&hj`#t7U0{jN)lIvAqlDU(FW`|hIy__W!YKR*hu$Koh+$We2~^+mPm zqK1-}CXgr4>l9?wc08ggD_OpEcT;y+ULLQ6Rk~8(`Fw$9n1X_$<4h7f=QDHh$H5@cpj56>To!CSEfDvKxV% zi-S2@YL}s87V>B>1FyT`<_dw*+InpdK;+{w@uy9iv|loIky6yV$(3Tms!Wsm|z;jnrnmdkdQc9+H92FllA^X2IW0-rlal9MJIT0|7`$Gw60z5I^ntCl#ad$_f_z_=n&p^rtAS zeEB`SqeEnSE!csOX)P6I84QBYsagVZPgdSno_RDJEq0#BdRAR6I|PdN=*APbtlNq0 zixr zrYaxQ0@seqgkD%!SlsRwLM_N2Eqa$L!EXh=^(86igNUW}#?Zmz0F7$7=WC5--R@BG zug~q@0(rDx|J&QhJ!+INo;>~dGJC7Pi85}N_*2Ii-W_k@(vx!H-pK93u&|{NTcG zXyPFVoz#UYId`+48bc$*KG#@Kiaw;k`k86iUT7Oo1_YL*xmFk*?Z{7id$qmdO*b8g zXAq(%sPwOYrYDN6NM7!?(_9NKs~&WmH##~xN;!dd(`u)i#6qRcyT3)IDnhOGIS-rs zmSdFg2D8vv*cp>6E_8Gmjh}x7-xMf{gI*It-d=4w^s$8CJdUUw423>hfX)d5o{aCM z&6K>o0o|KI!y1hw=;~w=m|OcAuTj`QnOnCjoU6gnJG@H?N7Fqg@UE2qzGJQHdVlNA zsPR=;PzD@U=#O;s2R+hL{h;93F1E@t&Fbu8MRM-js<;SO{fe9M)Us;X^;%c~d>zOQ=`S79^d8=+p3jz_O}qTDO>eAKZ14Nb zwY2dlp9Sqp;Gm_VTVp{+V^UJJ7T|ap%i|krH>Xkius(lk8$*~F>}&X-IMfE)(%^y@ z6V!`PA+*i)`MY{~pWD-+yP4oKS*B_5>)8^r7t6Ft&LG2t-jz15#qT1hOhedqLQ^JE zRPpUm*3u6iTs?{Wu^={eL{;nm>puBkxez~sS;Wibiylb8%p?tsHSKD(z0S5my{R5c z;q;|FCvEyf8N~lawa>p4#3&hopb|FT=w19;b}3n2#}CNx-zsh$EeL)y%px|W3yjC9 zHfI=IYl^&4GoK4$aJkXC1FbSW!$6>COficY*S>ymhrUac!9meag^6 zkWK^)NMzNDovwZX0b;3PWRGNe#e@I!mg2_Mp~96?heiYYMxq`qaYF;qlP=+qz@2x~ zxPg*T)Lt0iB;=ZH?SLS~NBe^8haR6%e@>&4j(6@5^6}{(POEWa>h%?}^ww5#66k~9 zo|c2lus@FYV6J9l)K~>`DTP+PD|pzYHMROQwP;PNh{(v$m$YAm0s5~jtC1KkyKl*MtAMh zZluKT81`l#|*E!UwTQNikeqUEMtQ_=qhJ?1Dkt-@ygB ziz4AtFf!2D>@Kuj4kspAFCZ5@K+U4uDUC4HA*izvgpbaG=I;H{1pGC$VN&&Ek%Z-Q zR`K8~U2F{syr%bPrWN}$WM40Drj(#^+jFw1Y_25rhb;(|@J(;ji!iqnA1zka9!00A zlkIRrI5_s`eKwU`cthHBZArLcm6PdabFOEb@|rz>|Ie0+?V;yRyU^>bs^JR4Psat; z%=gv2wGrCw-pnqf+Q!paLJfpSt(TbeNkyj9?+*uSK2$XlJIT|vR&@Ws-3vUmCoPV+ z^6@- z={%&d0@|}E;cQR;X30~4M_xrqDQ2JKIfCbYuzrinGY~y?jP0^0_6bA!599E?BRO-( zQWH*&;+tF5@ip;K1!a=57yfW-1b#=%v%>ecy|T(9LnY{CXodf9J^#-|G=>VO9~r7& zv9YY%f?k-10$vg4BlN4|D+gnr<9SDj;I#gDosDpzU|~sufk+<5J0e=q;x3ee`{3k8 zg^rRxIoPckPYl=_q{(bJ=6F&@q^)UbX&B08}VH9ZRA}c*IEii5-Tfxi{1Tdk!m}tUeUO!TqA@`)rL-o zx6JFj+he}bo1B9Izx%6?yi?OEXdk{B^`fPyr?8o>3yf16lMFzpf9?jytFO%`3d2g3 z@UkK;Y^gX@dfAcvD}Ckz!>5kEC=;_pjE8?dCyZ+NwKaWRCB}PMv zZ39G`C7T)CsQnV#=XcXwtj8Q-c-2tN`*sVvKFqOCPQ%~3ucEuf&gPcY)Gy?>dVH6f zy|Jb49Y18}Cg?Cf~U5YU|?IvO&Q)OUZC#X&ZA>}NnE8KRGm zkBhCjytq+G8;u;67}eKZ@_Fsw3;{-Z?l>=V1V6AnUiFV*R3Pn&ew1FfnLRz@tpX2k zo9)b75u#SNlbn4*4vXI?WV}vNFocpq`_@k)I0_(`a67r^*djn0!AGkW`AnVkV~q-D z+2zTz^?^hebp3cjB@%r2VqrgjGpj61-&@^P-sJDT{0cX@69p&azFD7R{B-}PgtKtk_l@z-@dud_bxF&fE0mOPTOcvp z?{MS13gl8KYm9mvKxWW*g0=8AemdZlxf|r#o9QxJN>*y>Dt={#aEeFBdF_=q$Pv8d z$%!ux#c*Yf9o6j%jR2FY{pb;7@=!OhwBZNw;5MX;ZjhifLe053TV|*E`GQ7wdxT!o zt}CHpj!{of+S^v=7cl)g3;aidi*Kz~!=!K_Gw!^MfI$sIESM$}MnwR;)s(;cY_k7V zaRv)?88@ zVcS!cS><@ziQ^au_p^4SOCt-+OAN9FZ3T0lLlKn4Dk>qJRP^?!;>`4pZGVj9TLr$j zqy2;hQwMbdfP@u@5P9B-#^DYqn9B#>sjrEcq(^E^GGdyojw36~lzCRi90?=&eLVyr z;ar-3*8#%-0)lLieQ;$eV@I)1y=WyrU;{ej4;In@1!8V5*!_dO)fmyN8NP&J9N*xsi>&zHwN+De|laVMp{giP&Yrk!uqx`9jtP0 z)t@!hek~u`*$*L!$~fygeh3v-=qh^fu%eA&EQu4P5Wrrx#p)|DFP6Jeic&mQycu?R z-e9>-%Pj3cc@ipX92d`;9XgpdOFnR0+d}%s0EPZ?>jv4r{@9BN1);gKc$hk@Qc_8-&ugmRJolPHI1fkx1M!* zt~_kFzB+3b=k2DFgaa`#I)+3`gpVEb2@KJA$~3Z37`f>~&3X zJRKLYrV-5)R%0ZarS@m)>gqrfFrQv#b*O`>W)_?G>;=0p=&|G{MX)pEfL504JHIJp z$q_p?S)5BY4tV1X0KVy9S7tR+O_O-Y#!y7cxb7Nx6bu}6x$b8@d7;7Cigvv$8lp;7 zaUr)V9;rCh=|SE&A7IKi6X>;G7xigdZ?-SN6>W2f|G; zP?}!h={2BC6m1y%64x302y<)3ha|=lxaiyjWtv*!m|m3 zqE-&;S@x7d0WEW4ZN|8m4>InclU-jkRDWo@8IehE@j$Ml$M4;;-79=~F|H^iEvu;D zlT7#STg4mdt7PNnkY^Du1zv#V6rsBvZU?M7ZP*bxh-U`E7tNvPrO)bvpY+w3S>H(^ z;5plI6KLCRtM=*gDg?+X@{1ua$L ze?YF7;(xBoc?Hg2!Cut}g)EB_S8<4}wmIJPj1iCc`?-#vVr;7h+uu5Nij+${h4+ri zX=FKIY|m@Lu6x)7WA{Qlp26kw$JDQ7ZFiHbDh=5cPqFlv8yf6d$X^{mnB{?&pvv zv~t0UH!Gmk+}Pa7I)~byD5wq8V~60IbS+YCtxjeHo9UE7Xxf%0>KRNV^9ykZ@tXj` z6j;~4*ZiE$3bh^Nb{TJFGteb0k&O%O6w8is+02LNwzv$m!{M{fNJ`0i;1Ndz+?Qe$ z61cF0%OIw{01rD_?mCNZGbCJs;f>bE6d3@whz3@Vm8s%BN}AnY%pGiam#qC|CZ~w@ zTph4iai?uylK6vMvA`S6gq*xGo!=IhFtkPn#P)Z6eEN zFR>5u7Hk~;$_=!G*e)XwUU;hHSzD_R+M5GnM%SeWb^5?I7fe^tZ%TQ>%(au!a2a<; zAbsz}e)O!dQ2HZ`<*+&Y`an*8hA;Bof(JAGSbV}E=mexquuX}X_SLz&uFCB5ILJqs zZ+y==Lf$5Tne@e!loURJs~01V85R!EX>u5#o+_?|PEfbuqAJ{a7mCs(*H-u(FT>3* zER2G^FX1{WN4zF5pY+n|dQMjf!|aMua`;ER%B_P06~bvRK;x< zKZyhzuA$p+ZGR3_u`)3N>F8fKF3lluTgMd0D>qS2)rkvTU&Ahf=?< z+I2_6lUgk+W=K0NO0lgY?$M)$pRVTQ{5&bc#__B+|BUCC-QFbARaw*v4jBoJ6nX&5 zrifBI+LB!FWPPjnrO(wOiaF`^aCgBL7W34yR;ISFXIom~Wwn?PFxkb|dwWLfc2A=+ zP&59UIeA7~d4OtcS-#h!sr?mXTevlDb%v!>qNf?`(O7CTMD1uY)7!lZ4UfM%x0tyG zfc zrrbERa7*K>4{!a;P*0)bo^O=gE0N6P-};8U#_TTRmD1aMFiKq?qUO5n19kMM3Tqe! z2`l{-BK0OWc!yPvoX#C6kbEzLhh4s3>J_QZ+}8Il`94GjAY%<))Cp*7(SGQt2vP75 z8aPkU|8JmV_~`?8gM6NFDF}h{Gd2ZZ%BczDvTUf~Bepv2I}nl}EnV zu&|;N-e>JZQ>l@r(Oe~ZBb8V>%!gP6U!83QgRK@iKS4R5Y2m@)ytoob=W(QY)a-`q z4RJf-%$ZLN9SJ=Zo;hE!RRS|nm-}Y`9ukw=Xeb`I-!8@s4=j1u*nD`8Yr%>;RWX5dc;ol*YUNH8|w%I>DFVpwu4?-N+ z5lf&~9xdDbB;Z5WuM!NsM}O{gl=O;aiu4dbqy)U2?)c*T-WYS`&f>N?<6y_kR}T)~ zr`3M%=KjPr6oQ2@6sD!b1HB(Oii^+yj!{@hSyADEn*kV5li!?t88C&nraxm+-nF}* z6_9?-k-Gs~XEtmVzFpIUxEUy0-xirD{#Jmr@6h1zZ5fxvthx4h(jg6gV^NQ|ctz}8 zTK1#T8Rn%sSi%LO8Sj?7<nLhvAvGa&Tz}c);4XF7s=;EkFU=o|fzzl!sp`(!r{OSr z>GRdttj8Zqre7@q=Bn0nmBl6g9Iw6tHkRjHQk*)DjQhOrobEx&VKqy_XI1dtK0U!# zZZ>Yuip4B+@0Q6}JIgm6Ygu_tq8k3eP4Em7yVXmmU)w^V@~kpYwnX>DyAj0cn>%_R z%yK+(fQ#rj!XWsHW(k`~tJaUlpAxk)O;hNsX^oo;cPnSCgd&~zWW_aONilAi*e(8L zXpck*@5A3IeQvMIH`BD6M{om@PQK<7CHRYOYmrdzirVxB^Klw;#Nj0_rW03M{dO-p z<-CyEwascE!#nltWh?q$7|7&f7)bs~rragSN?LT0z?fd|EXmT7#QbnW$zv9?BkzQU ziJ3G=Hk=h1Qd^y0xgg|UdJ$#T3#}6^SL&<_d>pTo26ll)DlX-IGtHJpa~MBOadjD0 zsQC2wK8AJ`{QzDHYSt=n5jZi)!o=HO=4J(?H&xb6E*cEPUd)xRBN#4xTeS-iW={3>S~u%KdD*64JirWu>N(8G*G<$wY9W9^ZUK=)XL#* zV03Vum_`p3<1G7AkSrQIk&VWHui;tmwot}d8q(5N{$+34!K~?P%_4***U0oTrP8cA}REnnW>py&ydKfFxw|{#K z2jq4BM}qtBZxWR`zIq^lNBa|4$Ht=|p943?f4jyzfCULLL!hx9kTFM~WxgsrSUdSX zaLH}MMWYeiIe-rC;Z=D2S3C5I&&|uaK$%8Umwe0~>)Ng=C?n+ywSed$5>H89r;J8!r z@!45-Pbemo_Trb9``*cRsW z|EdUr{G4Bbajv+PoK?|y^x|M?*>?*r^*3{nZJ)EWDQ3!+U0bLEtwMT?l{F=-*~z@v zDuJLjYH2Z~1)FQ5ShJ_o(>QCfKX!zPNi%h@0J>sP}GP?C_!a_8snezu*| zw~hO6=e>#OqFwt_P}gcXACBE7%S~q*rH6+4kje-&=lr z(1?Sg9tYYV69^VEsT93QEf82?ZlAPuE~NuodbV^t4NWO{d}#~^@JPTQKvc*)|2PP| zaTOK9)+8b%PJ2zCjLLA*uCzS+I-wEGz1essqRZ%upe0S2>8UC>Y*6B>-f+U9@YHcG zHyqq30iSS0d0OT;e*GVmMViyisMEIxilH)?U95*B{9uX$9*3SRZJ)TgtNSvw=60Az zez}g8N*M)*ZihFVr3*{Mhs1+~1>*d0RZ&muC-15Pt&2W`54zd=hVxOxom~g~pqTfC zJw5^ym)Py;P48CO^AxE->@;_IbT02tiRVP8Rc3uUt&#ubeC{b*XcgMU4ZWNgH&g6c zBDt?{bcmJ-E{OZL_p*8n*-EnIe;qRNlpO|Zc8pli%f}j?D>?&auXh22t`qER$X2Z{(OEL zSrrkxSqz3<_8{2J^SA&3q_7JBc1eZ)D5w&s72DtHbIvY#x8XN86ekUmvi)3nOL+KA z4_iQ|(GQuXV`*YPtX#K+09Z-hGVmVFoN4Tz>qZn!b0bpXq(25k=b)RqTrsA=nSR(2 z82(+V&H@7u=D^T`=OvNhO03&8BL;)(&z>2}hZ1#FI=!c2ExvK@TS7iit}d_~S1Z|T zMF28^cyM|B^HVV0tn_~J?lt+e(=XE9rPi1ON;)f_vYqR0a>`dmJt5Oo>Id@H99n70r7qgPi%RP8R8?zl-{lsS%%;ue)=eyW_RvsL zLStf%g_fj`_HbEj=LuWweac&J4h6R69orq3JU@7K9QC1c&`u8G!M}hG`*CSp50l|+=0r&vfjVNkgOb_v7!mVaR z8MR)w$p=@gjLmMd>F(i(;x0J8kxVEY;>3WOW@ESE^P+F()$teQ$E@Ju;n7O#siuuV zU)p6RvwSQ2d}j~%w!G%3n+ z5TxWZ&TV`@lJ6JJ6ebVRV8zKO<{db3`R;zPr~D_%qT(qp8gzOZR$A0r&iMh7eHBEp zbGYg97h+DW-N}tlbYID&ah|3WauA(FgbBD0Dc`scd`s<3+7Yirp7to=oKdh@VEp4h zySt0u;#Gu+9dTs#cD!Ftfaj4eG_+?dSyeI2LRE93$8Vmqf~MZ$=Y4|ysTT_cE5gQG_zuCcY9pz>*ZVFr@btPjQ8wL$5)KQt(k2TFZ%P}6hPowIXed6Z8 zZR5_@GcFPtFaTn&d+uC^bs7;&SpMKSy!hlfArmMRiT{SK`Pb3dmp}RIN*mbKW~%Ul z3u;as>IDq^*e{62Ln(8Y<6_2a0n`kRdPk_B+i9+>f)h2&fN(2TzNkzeJWYE*Y3BRW z{>27|6A*8^Z;mHsf5I(+JlI{s9n`wsNKA@C*g{bj^Q;TQMm35m1BDKH@uo~u$*OmB z6dkx+xbC^w1IX_D7z7|owg*XVe|Q~IoLfQ05C%bQGiV15d+y z{JknS*Xng=6jA}o5xSc4#k*Z$dD2*CS67J)p&%kQR%Q=q2NS#5rpO1sV{Si1Gm{ZN zt~TP#vk>YVyESd}ByyO!{3s<;IZ=%~*}a$_}*056DdH=QBYpQ=W^_E_e-HW6gbXd{?|z+$voCsEa8 zrY5Jm3h1Kj*4Cag&IypBXy+%FQ*fy~%jZ!7Ymp2!iO2HE(f7|pC9 zzsq4)3a7<}N#;vywA(-ZZxTnPQ955Gok3`5y@FY~0pqK*SNa#!mBjG3Z`4|m@;YIU z=)RU*ZB9ORGr5E2sd5mp8U*Oztinw!<}Tqr&zEJx&$v!CCGill*0u=@Fsz$h$(t^T z{uIO(k}BEaA`h1#5GoRTIGL;UL0827Ry{gBj+gd@>QeTLk8I_+8Uk7DY`N3qR70%V zrn;p-^np^eoF}^)cq=+LK(@a4we2}*i-jRCB(v{-M45N6dUYtZlra5yXS0ZR;TO9! zn@7>rz?l}S zrRl|Q7H%muIa~d9*HE=6kz9#wN4DLVa@rW&4PKV2k-*Ag&XT1(F2bM~54FIu0WQxP zObeX!TI)YyfaLWHz{3)g7WiCH2tv!di4V`zJ|~WC+P4KjC}E$VT=Oj+-1Wiw#WCNz zIIXZQq~}-C{22t&EMml`8l5brO0}-VoyX!A@QbpM1ImcXMDL8qyNEYcDmK{)OX@bZ0-gPID2Sb)-qO|OFN z^WFT+fd`-69y|G+VG|HPQkTP*a%p)ow^dap`=48p=~V~FI_CIt8J|6!~6}=OoeZEsH^G-_{?ZWmUKdLGop<7q99}iN{yIW)4R-E z#FkgzB3wx##Y~(L6%GjN@i7ONOxhoIE3;E(e--G~CKFV9B$dYId}=21ROm~Qqz+Uh zuX15GZ1;O*pWPW6Rg>{5GBS$kVjYrzZTTT1m$52UBY>X~Z*>3{`Pa3`uvJ5Z%ziGS zPP01#W>iiWtqWTd_s^ZrT!uX;p>S=)(q#_#IZ~wfC_62Efv=c5-Z&`0wGuWF`5f@Q zd_*aPYj4GF7D&Iln#J{q3DbVssrb?vZ4ruWTS()h=L8SSj}LU=Ykp%@Qx4@kNFhCigLm`CLR=aA5(C=0JDs(jzT?%-lk;7b z-Tp&{u;(+E-|eoQtbZjTU)U){Q)v5eso?!vPmECdAzM<{Tyv$on}+TiK+DIZ+lT`? z_-sF1aFpTN4Tk`eC4X#c*F3iUzI=UBxiBuVKbh!f)*5H6mPW)E?1=~6ac=x8lw323 z@_Uw7w+Z3DvQtjNFvdW}EJ_RgI>8`OHZ~F?q)E|uau&KsL7xvACwubK50i578GEE1F`?Ytt=}B`s0Ab%;~q@dJ(pr&J(Mxh#I<%tOc$alrVKw zwcm&{%{_cxC*Alfta`jZZK_*ALO2p6h zav846cDzGm)}MdY>z<`#Q3yFI$3P6pb|p#BkZh`O>_w?VnZ^HP0pNzuh*F(oRuT7y zq9jJE20$rT70P2gf5~wV>^2&{VbT_q#vPYFirs3yA(Ik%HT@tEAoa1*NczAW5ucmj zW%{zT4n!YH;KIthWtSDO%zM;&S9XpPUVfko)qkU@b>gQUUT3UHyP!CS7*hyUPWN!%X z|)?i`BO_K4%m- zdENRz=9gPx5!>4Y=VpS{JWr0jpF3w8HGrwRTe-P)UkfK z4mB!%PM5qV2;E4vX^jD_cKcKJFm9r&74R-9^L~TS7b)k>Qj z2FC&`E^zL)@GR$5pPkH4vw@X}W#bRhgoZ)bz`uO&H~Dtws5+nDO~cnBBk)wF20DUf zg98Mxnj1d$2yLC=lCMfD2ni(;nMGSw8qHOiBrZ!q)a+ZdlK-RU;DLX7C5j2lCe@%i zd^`-UQw-GPDttSUTM48&K?}T|6X1zf3c^s_;S~>Kzg=I$Gi|+hBf<_XP3#JIbhG5C ztL|sdlAXEch98Q*;43He=uocl-y_XqJng4uT;{{xcj>jheHY==Xt^_UPQ0sY#Sqot zK>1nZ`tDceY`se;#dDB$FO5Yfl3^Lxz10`|h7nP<%wUk4RIC6|rM0!qp;&e=Cq#PkwydPDl17Zoe8Y8b!$}!16z|Vp#8_ z-ApTfZmqZ=Cu(OAt+=!{dy#k)kR3`v(pdb@rsvE3zfn!2ktCuvwBJ&d<3HC89-e^4 zH_uueUM4v4{v{Y$n#;irry?w-bYSRxkYmjt9^2C~-y7o0U_sIL-ix>s54YMNV zc#mRQKF+@g3g`=c{~&U@Xneub3)^eIk`ICwqIuLHE`%J#x_>DRRa-O%%f@D3m$Xc!1X4?1U5RPA?>IZemYfVB&4NQ1NtL0?RUyjqqK445- z&M0H0muZ89-t>I11!--hoJizwM~z<4@kqy0!3NCIsAEVB!|jIDYb`aOm>dQ!rOhyp zcRiAuxd1toW<%X3hd|8Gf{6%&a|w!GcdMoScpaD5_ry_&h47bpt|`2thZXgFBlHK) zK=Z_nV!Im@VL-q2;(PD&Eahrp#RnB!R{_;De%U@HpBe7=*8~+=<8wW6qPwNqbw*IW zH7J57eiyH z>`6vBrTDA~u&b^a&ySWrUD6}Hw+odo$3lk{^K9j$;Ke`OkiNJ72nvNE&%bp^FvN6o z3f@@K@Ws^kj(!5Ic1C~JW<*;4$9%`_4{klpP<~tnzROI1heR9XsA|h3f$AUeyt>?) zqFCuX-?OC{jYW8$j~lv>|M~SmY=NTbaEm9_dLG$E+1E}@M?})qvyC(;LKC0(rQ{Y< zA;++#CR6yVLrU&$?ZChZ#j5slG|6;8LOC4aM)JIylhhI#LIoQY-?1+du)G2CNRxxGd%9e-7i?_FwEUW}l>8x@# zb>dDmg`CVbd{;mx8+h3itAv%TXRD){ZGy{k67TQ|E~X>0EHlB`*u-S{f2& zeYTHi%R1*v>0+2-S@0BgEI<0{c+#y-W;8s1Ys;}?vlxaL=aeO5_JibzIxxQ#W6qhg zFKDQTffoQ$F{!tCmBOC)&#Go3yECv;Q4<}E=|c{OgDJ#_d^&k8(0G2=<2d3WbIWxn z4+f)ZdcyKW`(MPr>Mz8exNdI#ZVBBHN1z*dCC!PV?zR?<_lBue`(hsWQH)*Df;v{< z!%y-8me|ti$+=5+5ZlKfc7prswPL{iN|n8|;P(qy{EZ5Tw(O)XIIPcO{z|MJfeu4c zJU&3+BS(GEAz_@l;C}7o)#gZF0sU47D;%3)9xsoaMFhZNU^=%kgg5%Hne=$53?GEj zmW1?o0b-}E*@z`1Zt^6-d94e{e=oYm%AlUJ=)GNujDbR-@(A^$OpM zvtc4>T^%hao{1mKgYziHaS|{qZZ<)?ldAF6cW3b)@I+;h(3oh$2u*YT-gyY$-n#{NaNL#Lw6Xi@@g`JI?BF0j3j-8 zS9r^0K5ind^l5*9L}}!Ey$uaj2YF6ESj%UcNL$A#Xa~k}>8~9p+pM|9>Cx4Z#*aT)&kWf{skyK~(*K*(MKyFm- z6>N~ORUdfJS=_X%?oCDw!1GFGNSC`EL+|jW3TJ28`whX&Jl?v^K{v6<&*O4>9*@le z%3_-$lmd-hJfg67uKfb*bc;l6JpCe~nCT%4*C*>VtjVe9I>*j*)b#1Zf}S8*@bylM z%?R*wk?@td9cMD>zx;*J-I)pTMi!O%xCLcds!4PKMykRdMojj_!h#yLmEyezFBYb%gNz` zI9;R^^!w)>&LEkb7z{C7O1_V=vTH+BY~DjIu0f$^ z-ntjzM5LrLX6R)~i+OfzP+MXmzGMW^k3W}zA0>TP79#*^&qg?Z7zlULT#3wo3|F8Mb#uWcGhHx;K_6HE+{nVbdG8_l=1@Vx>ofeSTP9G z$R~--;%rXFkwNkMv!|-PPNUO1rT_1^Wh^|Vl1jD^icQyHL&YK6E?`cTtSC-5U~K8_F`Y?D0afqJp1o^^gDTL)~UB9 zc~sEN8|GW*cWvJ-`;L#=OUL##Y7Ehw$#iY%30Q5tc-iC{rKht~8Q$Gd^xLB0_+c5xX~;-9+Yl!Ksu4x(lnd zK^ngc+EZu9Y8XLnUmAUOEU%-^`uq}g;`rSiO(knio3&rvhkte7w~3e>m(=0(8vJYj zeF+Qat;w4HdacXsUXiQ<+KTfpw-4P?KRuahd-is&MYn$TLj2c8rNN`Zf3dj>=dSLDKWid$|MPSB7)3mh zhaaX`Gr%QsP&wRXP#L`1{FdbDBb;$#oz`C~lv@W)+S3`{@kaf4s*4Y5d-vNL9L~X! z7J7I0!Nxq^hqMDY!@jFt#Iw_VPfkT!wnf`@+E{)GrYCKf_Swe_>Gd}4nsAzr)+VrQL zcfA;7{h5{u9T zR3C#Nl*nc)Pw{*JkM*~*N+eXO3L(?Fg{%&m+NtH}ae`80ibLHPqt2GM4m}khDpWjYUvtZOCwRh% zTm9YKvSZnlu7R|&R7SAENYd3LT&|XMkG~BUD%P6_JUS+b z#ljm(YB(ibEo?BC<{f7K-7nM1nF|xbs1#;8wAhOEYy$NKoK?lD2$nx+14+DUZ_qEW-FlO4o%1 z4xF^rZf9S{V!!DvEDq@-a=Wm~XGou~ZL#B@F{l+8{qM;|aPsc8T%!VZ5u%c*UOC*Y zQ`s)ofqkH)qFH#DwzMEj|8VxQFN41HEyo8JV(?uZ0b0YTiS^-dFqNAHMl;DR*>bkS zhqk0)VX7d%yhc^xq}k~qC9vXp2DIJEWtpqo9HXY|jZ0DW;h#Q!>pFQa*UyCs;I3~` zsVG0q1Rn2i*x~eq18<0Z+5`9LQ0w}lX|eazZl7_=aC*`+22#dglRqu!R2~#^5yLl; z;6Iv=m7(_IAD;VLfIu^I%M8V1L2xRiH*KCBe;P6^f(#A*YQX7_XkT>_@aEDxl|K%N=z~^U3!-r|l{oVOK6H8tiiwRCF{{OU+(^aq0VlNfq;qk~ zaiG`^O)-<<&-A{f2gUc_)4f+Yg(lFhehef;q*Id#c@hBag%2h9vZ9$6sBKoZ`{>Z; zMnB$0M^NDp$J(l8nDEn>a0_TFZ>xPCAFcr$wUOkP8DF1;SmWLsJXhQ$)>K#cQ7$y> zH$l%&sVE?8p83ysR*ZQ zOfT{X$x@=~uRNRal05K)U?#KWwfA|WIwkhjw~|0(aXR>)XeNSgfiQBAjM1Ia_gQb<;VXXlm80YkS|w&&`t!MZRKAQw#3ru{(63nX9j z38NBoCqJ4hP;MqmQ9VLbK%Z334C<2ij>ZtDb3y-MTufIOE6i4O%ZCYllRa4i6j34Y zE~*MSB%ETu$9HIqWKI|AJ-}xmK>aP42V$SoubYHDWIbFVc7MmT%NDR%YD=!(jkEYo z5l7RIZbj{Xw?Mfw2p##y3^sAGdMS)7<#Gr$WTU$&*iOi??fKMN@{)*TX4(^(|4VU6 zbu8D8vQ4AHfJK#^Z;O8Bq3YRS?a;d+4`nwK@hbC7>R=!1jiE1p7QaIB&wk4rwDAqojNZ@q7zfwj&R4^DAgimZWcSFNqB_QM&5 z4j-09kknjYkGrb84e=+_f0MXfW{89O50r zL-L6OwXC}aFe{g5z#ojVuawZogCyYyBCtp`5L7PzT`e1qeHWR)gl&)*ZeBe}@<0jv z4n-WgT;R-lRo8_?E9`ut%0!`)3Gn)msyIf?30paSiozhbodCkCB`uPHq3aA zik^=1Y!|^zU$g(#bqGi|{A_=v?mnqinFE}$5B|NUnoO`yZdTeIgScgMaO0uKTIY;_ z0Hur8toq{g%CyA#r5KmpYG{TEa9;l`FtR`1P`l@0O6a~(z@1L59>-%1{CMcsHUU#9;wP@OfOup*)&lgR3Bq8YDKL8Y^l=ul5iwbs5$ zlcH@?P2v%M++r|Z0U5mv_~pD?6BfZX)?Ej|iN?+_X9fh%uN%^&+D^T$*1x*9Q3%BXY);;=iDoDq#=mG_Z>8hPx{n^9<89qHoTJ=Mrr_1BcYblkE!)$bYJ#$P zck`v)!=_Gj8g|sc*dH1p=(5YzO4g!Wd$*u>Bzi>P1gIb$_w$0qca=CY1J5dvNljOO z@5(I$cr%OYlYDD`O>H!;a#2Yb*rvl(H1q$t;c{KGmF|pbq&!lvOyZKw}b1e17~U zb2%h|-4UM^(8q9~(V6UMp0U&7071ZtB_a7T>vdzr2~j_tr+iXR(pQC!5#Y#b-+}95W(Ljd zZgSnB2l|cml1^3NNbui}SIsO3aPT6eaap?~;Z^EWDopH% zkdqm6AgCm@FbF_54Ey@8RsUUqHUPkm44T8ndbZ>9xHCD$>G)dDZnChk`ma!!MlZtZ zmu)Jk_M-j+q&8zdT@|lqeLhtpsNnw%NX$pTi#&pRL`mSjE`2kz8!O;OM4faz8vVwO z3W7bm!-}TL%d`C!|Ma5g9~z2a2(k-0QIR5>3aQa?|7X{u0)!K{jtOm#;j$Zz)(e-; zE*3+8pibKpY%vK(852iQ8Q#WSI}G%GNp+^)U{kkD5B18gC7SplLkX3q+0Rj$MvvhL z-H%P1Jyx3JTGdh{2*V!=V`gF?`uTJ7tH0~OrG2Hiw2u7YpZ@w)*T)AhX@Q%yaF!CH zvDkC_jdRYpw|HOemN{JL>KAKEbA0ZIx4YpDvILUHn#4GDdB#+%Hu_>to1(CWz7ZbD z_bBqDK^-V_6f%YIQpxPOUr5R8#_lr2^}Wi(DJvZ_!NkCqvt5-zoDavqRqfh&9OpOH zx|4{<$G55#D7B>1a)w5#@Gykhg;%&NXW zELbTvhzlmt@>aBerYbp&eCbx-RzrHGUfQqLlz5z}u%O6^vWyySCj&X?#w5 z-$96o{DW%29v#jLbaLUCfV<-mC`YLZ$-gGr#XJwuX{GpeBE;wbg>9pl9%&Vrq2pAZoYLs34bJfxjy+bj7uTTG z@6_e%9)}J~b~b81w+?%bD7Ftf)YiZ!2BSmKhM3DT+K2{A1WIL~2XlN<4C7{3H^8gi zOIuPV`x~PWbIFcKRP$+3tv?N)qt8PjI>^*>p$xK+#Gm(otsmI70eVdW0oR}EoHqmS z1|IL0A0cQuO)P3fah_zm@1?tG`J$t%rNTcrHUi&-9&uQ~ zQ*u!dDzVJAEbL#6Nya8jQ6i08(EVIgLr{<11YCW3pt}LQUk_ z(0KH^yYx*W1D0!bLFO;zucMwQ*a^b^KaTFx9gmpWW%2fNVbrhje;CT!r%Vdu`jg($yXe4`4zJcU z@9Xtm^zNwEKv`>hx5tX&dgr$&^xb)T{PC}M%sCNupUN8D8s3v+Iu`xtDBKcPqa(uq zR}5Dz$`*x_P}y>_54N<7&4EORHvgWm&B2V5M~I@s(d*4fId#l*Yw+4xYxC~*WnVx& zkd5!apZ>u4zLViVe3tm?BpbEm3h|ukQfAMWqYy@X4EC_OypGWZq?ZS+p2?WULCbwm zv8V$34PPxChi* z#I&{YXc21D+e!YJoYvHyk7R+m3;It{qQWKn@P`GaBb@e8jf;?~yyt%fUNvx4IsnX#tS{tf3HxrA!zaShBbynHl*e9&~oBN{~5Jd*pCuIGjeje64|5jUk0{hV{fVS0q8(O<63`@R_qU^uFzsG6l@5*^k zll-JCx-f&;X;qt2_I`%EiYaLK!Ydj*Chsjk2_1IifTcqJQrrZ;78)94WxRtyVB|jc zN=17K^eU1J?+K%9m}2anV8eH%5H3#ZMGXd?@tc!@__wf?jtV-Y(dLgjbT45_(Alg> ziqLl84~{kBjU<1PkY_5vE(uC0n9YI@On5l3x(fb^`=N(QcrI5MO5F@$2`5Z~{sZY- zi|L~HuBbZ=&o7~Zq!iG1hHlNlHD8dl6M$2&>+JG;7X<$Lvl{mFjh+FLpCP{FdX9R8 zF)`C&O?vvKKIvqs`~^BJm1nf*Q=1S_`Y>jGUK>0jySxKxqD_}>(8l05o+Cw^=J(&5 zkP`i-MIzn{YlR1%S~I%qgAlQ285CaifA(Kw;1gG}gYNFhb#U^QEIeYtS#BU(DwP4x zzJ7F*a{ae{XXGSeUkVy>kJHq&E!wk>B^s1xTZ4(C@2+cq{M_zfce~rX0fw3Q4u!2% ze+FJ{-ezt#1o;#?M!_>4jQnnQ3!k^mH41}Q5NkCCb>eg)PhUuAtcHZ+>lPp458Mj9 zWroDYg!i$05?WJl{wP`-L8P-4dn4)w}q6W}P zFbYw)bdareHKelX?o!RY+NE;F95j%O|Fw%j@GuBpa(h>gg3#!wjPFY4fUYJf_4f_d zC;?ugjQ0KF9CqF?=0bj*Jb0$}9E>3n!iuJ}=Nxgxq85P3qrcgG8-9r7^f(OBqNQCi z`2T9L1-MrvlsWzK~xfa&%Mdv{*9_{LC50==^rdYe!*ziY^ zV!O`ZtRN-;n`w&nU6i2{H4){%^jFGOR7d$s)-?0k;%$SHd?38FtUWvHVNYJ^Ca<=; zb&7I@FWlGi*4u%iqm)luhrd9jX7pPRKF^topejzy>EG~hcx8eC73^Gta@T>vF5;K6 z3;_+O$A!7&7|+Wn&EejZ<-?}Bj#z_o4Bl|Ge#}FEK7vnRZV8Yvdj;S9DG0ooWSTVA z`t><@nE8m%q`}cK;3?6>6#av29J{Eglu4&@F%2XzAs1c88W~u6wmEFeOE><=mIsgs zS$y_v9fVei^i7#r02!9cdP0_Nu~JSD$wy_~uyr|DDYQDPQUOfq;~>L5cBHQB2Y?vt zqe6G34-NXn3_3kQlvlzP;0n>MQ0`1DRdHw=g5mJ7EO=^Z#)CKB9JtYpXAX#<+)1)w z^tLaOL*p56J32aY`LNEEj4if?lEE`Uur`2+J@!~4m|`ZD5{2^#pAIwppthxJO2wROdCMMDo;xnM=g0m zY?TheN29*{1vN@(%Q4s8Cv?+e(6Y6v-2c_hp+D*7#^@F+%v@dk!D^f1#eA^5xv-| zwf#Gr>E{6<5p@F3cTeMX4}5vI;9S4E5YgC?>O5L!G;RqKzfsMQCcw&y5+AO>d?RwtEY5#B8zUk^bjEV)uK+=aH+90GfJ0BUR0av* zM#b&AJK}xI9V2|CCJix{@hk!xN%4No1h1C3F%yGj%<)u?E^f7j>I^T#Lzo4x!GL7R zviEDvf0Iz-=Yn;2fS9i^86|5V>6ULPERngWtwft~o*+i}IxH z2q~I;MjH)IDt=#HBH*v;O+nQkYGDJxQDO?UXa!oduvEFAenSFwz74PYlU^2G6KEZXl^R?kqQ8DPP7%OShc?w|X(5;`mXY1_2WAaV9^Q&N5b zWwmUcgh*;#Q68@#>M@X~1g}EL?i=5l9*lskpT@j@@v+hgX37Dt$%0c_r=W=#s;i1< z_!y%hX@H)i!2yaI4Ft`n-VjcV;d~j&9CQ8IMo45mn80LD?spW*vM)hJ7N0TeeOX$u zJKVb+v%$)jZr;;Mrx|@Ge9suo0i}b$5aa(>GyW7$j(2MuhlpM4xvoSQY0%9=-mV?6 zu7nreLLXWb`Ha-?9P4cVkS_@^HRI z$R;fCIqo-}1mopd!%=YSuA|B9Qd>D`R*TPThF|e%!pnQ|_^#_`rV^&<|LUn_tMTLkN&S`4J$^wNJ`>#9}pWxF8DuVnm+8>gnV-kiyR(YUq@eZ zPVeE>{Gk5YxRgAgLw&6)^!To?wlDK-PHfG}wCYt;=>AXVeQ81llaqjvM(YjhEF}W4yIdw> zeCe1#Am@3b(OLQR_N>p@kMlzIY`5XE0>#{fEFBkK9y?}4U7%AM{y{{nHfLGW;IIvT z8EktM{{h3pFDU*U7^I6`X3MQDYQrDJ>5BE0L)R!kN4%NVwJS(vm5t;C0Nl*?4I(px z3_+=Iy-BUyX>%Xkxy~vxU?Iplq!#`sYHh_=gC8 z??Lm{NSK8D_{-0hvv3C0&eacJ@G^9hg2G4rb( z>^|-8e4vpiyE6g$$E8{}@`Vrg8q7#vP+0KEx$KS%@0UoJwfGov?f%N$4<|Eund_j1 z3OP1zIQ-g%vSTGzd;cEkzDiMaHQkK$l$20d6YIc?k1t_uxhbL7{-N}rn)lwPn)jh| zWHH^ELw@X!HbGg+g^L7dPT1^)e~w@+;&d`!(E!j|na3Nw_Ej+|y{!rzzcZ4cGGXMD zdBV+v6D-D;!`KNBmVwBn#t$|J?e&CFb+t2*c5OC218oY>(Z0ZSezW`^L{pK}`#<$3 zaeU3<1dif42#exe@tG`q{Bori6LU>i#zV;#&KL7M>8D>^@5%S{qs4#OSrtpxw*bjA zB@zQp6LTLN?OiT781n5t2gY6lWxVEvK0q)(1Kl-{8~3*p{cC;^2=-E!p?(If=vdU2jqH{H^J`l8f38X8@~*evvzj zQ4Xr9tx5Zxo`Jjx;kSF9>hQAN1q&`8HL2lCwa$vM*Ti*hXRjPWw&DmZXzw2&{naf6 zf~w7$D~t^AJKe|>cb(!D-TxTno~LNNX_nj~;&b z9I)1@0#qkrJ)EOaX^og~xk$xQRQoF|l|OE#?Wg=H(fWa5>jo!_nXc11UgZ0K;uax@ zPf|blp10gbYyL}ZgA&cH=@#E5S|0J@hK(FjKU`v^_mw?@zFCMKulx>Mr61UUtLB!qTJ~XOevzJHesHf(W-f_w!# z8}dDJ-@tRMfA%0w*m^uWaC(@W%Zzjc4n7C^z<9m4j&JN+$}CRf$N&evP>I>?^6y;b zFiK2c{m!AmeMvg61N`EY{?HAmaMPN0e0uiMCQ6_0cj#VQomjil%29E#3yI!V&(QyE zKzW8XhQVO2q#6EN`$+xmxAx2#hp$?^lDOa-k0a?%I3?=|in7t|*c1WTVpuaM3a3{bwR9*0VQ8_RM-#*PEOV7&?!-x_hTQ z0F@mWwx6+3xlQr}7<1X4jwLj9R5UX475x`)TlHj78ktZj%J|bYnH zy*P%#2&$y&q(&@vhjN|Uo|eOtjdz_$WhNH(_>IP@xMiavUDeeRL4$7BYQF`GO;FIc zA!{_^fQZwaLM;!H)M!wzX>)LW_$cdIO5XMpy{()m;wOTO5gP*^?z~TC>nR#GFvVDBs;u?$0vNSaFMP)aQ z;JHVrJSYy0tC$6BPsHi`eaScxxAix`h-+Rz{&NOU6;XS#nfl(#lg;D{Ex9#)9jqSK zBr=eN5zOBSP8I5mX%;;U?3=NSn}PUaKw1Ts&~3w?1PcC~ijEHcXuGooFIL&SI}q}7 zh6d$`^yI6<%c70r8UWE1D3`K*39I;*0l154CRh`rd$TZHe@h89&qIw5kj8R;zgqS^ zG0{-i;p2N|O34X+D`O|<_M`dAq7(`xCJqh+*ZS`2$cB^hWG3DA+rx#;oNLghA8c7{ zza>}JZMOW^6aJf+{ldjP^JjYUTW7Dz=BO3|23W;6N)CG{SyyC8FCNNWNp5fsQsX>) zkN-}gs>DC33Kw4kYw8qu#t$X2c9zbU5f%TRh(NC=j)(C6HbMi~Yi~6d_ zn!bdZZ;KQQwMx(w!KLOuo{vmzx*#tKjZ6SUUZU?J712eq3f9(rN{QND6HdugcqU_1 zWOcJ;!}Dq#4*?#2Kx@iI8Q%i!K9cw`sN@W~<_mBcB%6avS_?ve6MbHdik_loJ%u;= zHCNnwayKxJ__~E*61(@FH!Mr#pEp%**n03W`&=KKkQw@TX!s3^f{3AxZ4a{xonaqe zP+5-GX+Pcf+msqS(lc4>9y+{HBP35nC7Ydb}w4SuRdZ2Fp_>T!?PwBT4(KduBD;ux(mtF>$21iyE%BO9Jn$PBR zb>dDn5dHjJZd)#Ecuq{{dWFFpK=!V>&Soh z9y2YMI)E;y?H#O>{E3!(=`~W~GLqv+f8wM<9(tn#?()AGUHX=k9ZRraL4Q#XIF7_k z+r{nC1{+HF=Rg8n@S8&SoGDwe%lw3#PbgAi;sp*A#L5`$MIX(G$^Q8Bje7>N@0-K;vlMh{%HjE&v<_7Xi!k8Y4(j(wbhE2rD$%2#)-c4 znG-RTprD{NkTa{_6_WPDtKw`{ON=t3EmOEs0JguZz|DYK}OGVC#e4`)bAKgsbL5XR?x-X zi%m5ye;6yLtTeG_VcUA^nMPV`YOT*I3oO2>8t+=GPz<+!#yeGn+1<-XFfpj&^rd66 zOVv13;LZO|>vwR`ziT}^#+S!fZ>`X6F$4o+z1@0{uEWdP73i2GoRm&N1Z8rz%zF#S z*-!ab(qG(P`cyYO$3+$XyGYblPHJpxpwF`h)VHgDWb)K3woCcGGBu4-MktQm?-tV< zcst{tamjXMU{~z%%IscrqY3)(!uj`V1&&V+zd2VI6@_(IF`Mh_ z&D6-Xi5f+t{v>#&WrzD3)@{#5(&8B+5I}=s^#@nS=Re+@G^FQ6f%gBBM`{kY^#98K z%ia5bQ%9wqCdIYgcz?RG?B2TYjp>+w#93N%%_q5z%ad`ETYkpi&$F_AV2q6s6B7ff z^mwv!a#pvugTE27&&PfG;Hl(lnaO=R%N)~ycCV~a&XW^=*7%82K?CCRO)-i+#GQz~ zIC_E~wUL|rn<(%4r|4U|d0NH=SDXj$i~{YqBu!C|#s%Ja=X2w=dMwTH#nr26e9wA@ zX20!=e14-vfy~GC)&?75F;mhGBu>VzK|p8HkbZRn9Edb0UCYbl0DP%MYO?mnUpX$^ zfAf*?y<6;jK-w8C8Mgy6vksC-yUxSTcv+YnYJJ4c>XUx#pOUa6ust4n|F&9~odl(H*_Sb76P-XTHMVG9plX(GrL)ZqH)V$-C~) z+V&4MNJrr)^~Ip~Zh+GxW7lbE+u*VaR!iJOF{{>lS8>QCdAQnJCQTkarGoSec<*~@shp>0WEdo;7R zv%ps1w80NS;i%653)wAyesj4Op+=asiBK$i@>DK3hRFk?S%o|Xd6@ARx3POn9q~Y$ znu<-04%hN}A05?sufw}sDK?gc=HPt!^w%&ikOB|`EaHjXeX5;1(lQEK%+(+gm}od< zE^h>kSb2(02M&-wfbC_P3K3XI9k`7U%eKG2wi`<2R(KIdr$9#7I#M%!TkdpUVz(Ym z90EWIF(C4LY%*QlNxrr=OoFDoer5Qa5x>FnK%GGADw)22OHjBZj0<(J9y&!TQ?ZU0JC0J%#?UgHL!0 zZDT@eP-ad0?ujgC?TDUhPJV8$kL0TH0)k787WW#} z-YgfgPJ=n~^VubOfm(7ivIXgfi`lKi*q6ke?CC;YuUDnZ)_r?9#pE1}2Es?RqeRWg z&lqSSd{&DsNXZT)nAd_aH!T9M=1T0fo3ETLsvz{8!;62vl65I$_JsVjkd9%FxXAhd z<-1-DCO^JD`G3fItEei#wOy2Mq)WOx6_9S}PLWPQI+T*`?(Poh?gl}S?gr^DY1kiY z{Oj!hjD6!JjydO7?~}&)idiHn91FOMJqw1O7k@)Q$1yRLUbMGi7|_!)e`)@>Sal|O zQM*Ggn#$oH$=u-iF|uTNwK5wIyl-{+q7-N5yb^p{>K_vh;cBz{poG_IYQ4OSp{%2lp9qlZ~(W|%5E zm!|Fgl=Mq~!pie|>#y-Ecz?Hd<>EP3Ts-f=*Zh`Be|!%@L=xdUN8r3^@d}l6>^dk5 z^NotS-iQHnN=9bF%B22qRT&_ zlb?vz9QQYRgFd+5;fuy)4%CWI)Ep1Bt7OaOu`0J@Z!0PG(u@rbeZAaBVB(^JPskp| zt?>DS`q2HZWO1GAnr&38hOkkzCr002>WfBEQwbH#n7xlZ0O~Kafj&K&M*eNwzRooj zUSg$Q#0r8Du%O~4gS;N~tnk~VdXp!^Fn9XdKVt09e;JJeUd5$PH$1mw=5~oC^c;fn=G8+%QZ|uOThAnjG6bt+Vveb?b5k3d-de#Qn{yNpLed*SzzHSgi*a0 zETVdBinG`2x`RykDfsf}m?l^R3>*rIGFG1ndeZxPz_U~Ny%#Q6DUMtKxouQetflpM z1ZcO>&4w(UhVTb#&&qTzYqzuX59g($gX|s`aOdqPkLO!KzmlfLLO$n}R{XDl>i-MD z{`q=c$*tGfEYYK6RN;p7qu4K=X}B$bUoV> zgi|02euz@UpWGhN5%#ac4@c=Z;NkSf+|B;+vhcy39e?KcZf~$B>XuH#3p4j})A`3d z9YNE}m7%?A`7vu&R#p^zHer9tTKQ(+_*mYz0BAyEH!`Tge3kf+shOpIMukW?0o}up zm?F)tX2=tR2WcPddH{_N!~v3?v#&`cal<(jygYV;9X+MTf=-B8PgQEGhDTjyTxGj$ zJ*CZ76KdnRvABcP`cCDhSoVviA|X?30NOC(HSB>oYmMIc2{7!|PNF5x5@b$RBf= zjphO#9sG$Ww4A+lJ!4I12(vdwL;-mUb}DzHr-DQ{*g=W3qt2772pv%xKg@#J zF4*@b8juz_zgcn0&WY&Ni~mo0#Q!=lT;g7bYs9Ng9L%M|#w<~{HEb=eeahdbhc51pd2!`}A24?1v(&)$^V zMv#ZTbWmNWOGsDa3WxlVqmc7-`}X|~J=d{rs6<*}XnM8l4^>m)T{!)0DU$kN^6v26 zC@?RI6|==7wbv8K$X2!p^UpZkPT{fY#rI(`@F-6VX#gthGlt{y6}|ppP~|;uN+6$tLl#Py#f^ub0uzjfp}~^MU+(ni0qfcl)-Y|K zDgn#PVBzXQHIU9uj2_+ZQEWI^a(7r3qL-^`Vv_&*cxBs#O()%}*VRlSqd2G-$$TGz zSGsFOJ7C*jU3J*rxayHqsoS?L`GApd+ri|!HOkO`n3IRU_4%#4gVopoV1?iI+*^Z;@>g{ zP+9ndU6gU4Mh6~?XjhnXJ@B3{On>>Vq4@3%=~}TeWjw3?D$No2h%`+)G}X73AD70= zDb5avOVmDy(BUFQ=KNRl{lALvV4$W-{+k6LOujw|esR_GlL$A=2;6ovvRj1@0Azv) zye=W74R9+X&raHN+MlGBOXHHq6^pWVvpKBB>i%p``It`rm3pK4cx!(?r=KFXr|RZ_ zL7Y-$c1hMh^{Z+lZE2#%*!Y_%_%ZvdM?E=RE+RBa(!QN*xyX;|iyQ4SZz!NN2Lqo$?@jlfAt!yHbRN8F&uShf!eu!R+ ziI#82p|5${#$R8e_oh!Sg-kA9=THZW!EZ`64Y~}vj|d)*qaH*8j$)+>DnBN1{|7&J z^3JC8X9gTjL-w||#o$Xd&ZQ=?_jd8@` zQW*>>DX5NUJOMkb=?6^>gJQ3FQl3h&tSjW==0eb$8!1ng4UiATVN%aqNM&W^U(w{y zk55nX@kcF1^CynGMj`(3zl*Fo%D(>l5am;x~~d067D-`(?Sl$>fzqWmv=Z5h3Ke5RIAr7ukxeNQOzIo0(OQa%c^PV572 zE}8Fp1=D-rIFrg;Zs&>y)x4mZ+ug-w{jjn#u(7ex!h4H{kmw!)>t&v%vcX2}Ev#ef z+%f}+md<^hUT$2q+gro{D%_U1SEidQ6a4dW75DW_9Cs+M18~yEtka}lA0zHs5JB3{ zu{Nq&xjh}sdtX(lc~oGZeuER%;pY%gc38S)%P85+_elm|4)1Gjx7GY zWh|hO2%S3hVpu%Z{>Hi@Lm%{OL||PvzsR)kn9Q+FQ~tGK zyPB3jeu!&xFX3PQ*{rzC9Mh|={DHbOk;U*s(VI<4VYG>TP|5wO{a||U{_jXCW3jyh z`L&~K<|jT!fo7#%ORye5BV7I+rKk~{Ut9%u`c2~`iM)Q}Am|RZx6yuz>ATI1c69ws zE!ZtAm*>^Zy>4*SyYm zaobAubn5NhaV6>2zou?b(@>?}%rKUK8#bp+fLLByku3E~SaceZ->zBv-mm z%3d(MY6@8VMLlf~=@07vhJ;6UbrXKYIRBg3rcXijajW!rqDk95>_*6SpUgA!1(hO8 zFwsH))18vyC)A_L?jR88Fzsj}lBq8i&uX-L2!`*O>mtMf3=9r3s1JeP z?FwOkvOo+-lCD=*=`R-^KJhu+^A3zHGb}ypX5B^`0-^FGi%DAV)`=q57A=3t#DXrQ z$S-PkyreCpNj;HoOOqC-&?tq{$n#x)pzO*K`wO)Ng@gsIA0^DUp?pf3ZAJNZJmej| zuev!#PntGd_@`gQ;F0t*5Jm`K4`mS?zfFp8oVi*8rl`#>AL!V1F=xo{X%H}Qa59_H zpg2(QS!>O4*7^*JNYI&hI~3O7F?*-bN5fnf$tR??k@CzE;bM0elqjZ~u;f-|>T|FVU@NF(bdB`;pD+OLe0_ zjUxyj+g>I!9ta1)+O-T>CE^bim7v~_%r#`=C&6s$MmdNl$Rm9|gu#$~T*vQ}D34OA z+=Rz1d!XV)0+@bZB6Jyw1*zaK@8YvkEzZ$z*JxtoHV$1pM0cn!$0C5SCoWL@h!GJvJX%B)B10KqcowQInhYnTeBV&iaEk0#)r+Hz zPYU`P=I>rsJH7Oc{xthn$2j${&8NROcrjV1OdWn0_+gWt^^`>+fwT|=L|!$;?|+Y- z1Deb3=0tblp<}j2o|>5 zZ&s&fJS3bZVbfI2-H7hQncM`DV6?s>a-(SWRY}yfxn9cPuouB~2M!Z@%>s+DKd@jd zMe}i4fb>?tP$t^Ox>F~FEpp-Ia(rNhBK)4&y>4G-`s78ts*UNE(ZK#j`+S{mLxm>{ zNjs|!BqqxL=J_JO=24%UXZ+U|VrT;e+9pD`!-L>8vsapDb+bt}YNFcUu%OumIO)x9 z6jI+NKoiBq4+QLVwZivx_68Of-VB9q2o#a06q(jn8Sma~?)LkgUs)V2_fS@&v1GCa zV(WdXzMr!Rgn!wx&l?FcSbjj9(!LB9w*c+W_q35`|?#pP8sgW)x|5`izq=A zW`+*A&@=3?5x_J~w0^f8SN89W0nVFVkoAgeYy0~WKGE*tue8U%1#^R7wSTM$(7U5) zc%(-M9#CW;tQIbWCuAhmzCzAO6@X&OrE2tc%vHdnMnmlR+9H0nKLrmcEA4Ol!N=R% z)QBpt-Mrwhn-ZuUvzv~zdcAlueX0?!xONQhw-*>vp_0a7)guBa&k_-DIhSpgO_5kN zc?QQ^5}7`A(Lo(8b-qM8e{fwn-fH2#nfnsHZou%^nm2vzY@B!UT1iPo^s2@2rxrj- z`Y9ByMjvaqz^c35<2_;Oj1TYgUvWEn>@Og04}U#DkCz&afui3*>0<(sz($o8M{n1P ztCQ|3Rm_=pMPe;8e;n1jA0UpN>wZBMSm-}&#^&JC8*${Cj-FKY<#*r`(TS-nhan`m z#<|i+4#MdhJP5ICHJley@DiZcd)NVYh&Unzgc#LrL2aAQfHXwH;9k2N8lIb0Q(X0v z$>S7&AL(fY+!jGDkeia@3IBKTVo}f8Hkb{X_Dbhf;WEw20T6_h&#epy+!Yyt(c9Vq zHZ~M^dMK*Q4y94j zniOKm3tz_J)V!6=-a?js&}${2X%i7pt`+s13fW9wX?COqU4WFG50|@Rq?O!X)2&$B z3q&WmM4Tp>@)Q-Y$F3s(&F1L!=84*`54_!g!Duy*rw!(%UR?M4n7FHYLwT~f8{lWR zaWB0)m z*4GV`eB#nL02e(jAp-=0zsCy83T{0o-O{o0!*XG3vh{aUH8`Upxlk;g&Njpv-&l;I z$^}M6DQ2t3?1Fv8v=C|DwjQIfsg>_3pVY0?+I1y8VCK~FOEp1fJA%|V^DC8|YS#e6(p;xan4*M8g( z2ci&AEQyF>Mf?!ioC2K`_Mob10S$PnU|;%iVDP!D@Zr!%9z{JdC3ACgseHg7b@GW# z9w7dDZ}Z}W+rJ%a^E4QJF~F3^*QExU1BWrcP-QS)B$U>V2!ZxESJvynJB)QlQC}?S zk02ijuD7Za5gFL$hJR7r{&>2cbMPQ-B#Bg+Yr0lBedTm^MF6V$d|dX0oxtkmj zRmzSRB+v5%Z;!_ED+XREB5iK{$)fmvvV6WObTMT7*l|F6#8bTxU8uA1_O@wXI6S4} z6GjF|$@$J^@#otzLPt>Jg;}B@J@a<1c`ry=FyU}x8r;P(EFm&)SD;@4I;3Cnl5P)s zsO%*`Q*{rsS9y~zbhC9Z+WISnSwMTHQDHwyhD7aZ-AJ}Vr}=7a8PPE?AYiB;RpEB8 zpn%q7>VMF8tEY7K;~3f?cjkXQhK;l?2?QF)PV3CrqggF#bME}@CDU748{6J;_-b*&)hWIKw5<9k5)eT0?sI@Sh1{=H-OFl|ODC}w?~%Lp z2so{dji2CHP-6G8+tfjCK?$brmnWKpDhM;-&Jxj~CWy-Cw_#xKgZFA&8Tz2G^$|hr ztCF;E6yUB%rqlVx_PY$k`9tHW9z8S|kegfG-r|fm-=zz(73|D<4QLA=Y-OocahB#SrJMKp=mKFZ67k=^s zqMYsl{~{>Jet*%TBucfzggKuBdC=dI(1%~4{tiVKc$(S2&N77=s|GUX8567XGO86r z@aC#YC!Hp=^TN~#ZV`XjEr^-!w>IN$8Q;17VnZ(+0%hgH3kwxjfU6^13hGmlg| z$SuXokSaMbv7%~>7RJHA@X3`3%LPVaI*p&n%W<17eq*ph4nL8lRZ}k_?FkO%T`}ex zZL?59~B_dqRxuZU9-fc!s90vV(wivh77py9$Zd`Mw44Sny&sB>^K1dwC!QcF9v=M z&-4aRyJvOO{7oX0OMocNkHxfE|*XjQ{UfQNCLCbeMiW_<{h#o%!&z+Sk- zYx=A(0;1<%RaeiyB^^|Ion)M6HDK^@FMi}COKksK6lIsEr%aX{*8l)79Ek3NZGYTT z{`uXeOPf8PQ|$9s4SrQ?lPkpWRo;=qwV%=jYRFLle~PO2J?UUL_iC zKa&1K+wVIf06?hd;2%s0le%73*mZrMH0Eb{%x^$+>p82CCCEJI-GsPOm9y=PtV&$) z%JAD)I(j`h;;uc~CRklf8ycy$8sOeQ3D;|ILz!0)^&i!5*HkUfNzdU1<6BIG3wjLv}`wS>||A=2-dp8=C9p=7$nFD?GvdVXbGU!=0t8GKl3R6XjMS-aGj3lJ|F467=DZI|5L~GN~ zs!QnptjS!OW&=AB-^4w1)0Cp&?YN6x3e6V>W)<~6Opjt7y`SQ$e?lj@BV_OcgFYak z%-m5Ym?{gNBrP7_{|5PdP*6VM62j;itDEhvr>?0jE#9Hg*Fn&R=>qJdi zQATFKFk#Ufi0YrjD%IcHpPNSjCqkBq&8QLXcFmy>BSQGmcP}3m=b^)lVFHn36iiqf z* zcd2K87$@$N<_t=|q}DWnH8S2eCDYh}csp}=QNmd+jY>Uv zmqw+p`SBk_}ox3$&0yk#&Z8DPiU(>w}5HI#N6F zj@zJ9cmou5n>vDlUwfVNeT)S6?^nN`kANK1Ed7BUa{fWKE7_UdUrL`a2GgG@3&KvR zRnNsTdtjiTAeV$+cWA@ZuM#gz8l%U{?feKwcl(|=>-UB`36pBMSb8QvG&PJIj)yGY z**h@%SMSQS1=C*cjoxY{$@Y!hT}(Wy*+qAlKOHY|i=Q5K5z4Giy3P<^2~0DM%Q@b2 zB(bWL>|$v1?%$8vtv7gg8biL@7JEf;h^-BVw5-cHhjJOeDAV}rfm7$jk;fjMhb>^w zKM7zf=Bnc2FkT`9VO5j_dOFo{*jb=OCI`UfX}V7q;rwn-3NY=eOhTxpGOj&Gq>!p1 z3d!jUKy3hD=yJYk@i?^f&d6neZN0Y-?y-RT>+HzS1Ek!zPgM)*`AxUtmNQ-$68(M3 zVfWi#tb{H*8;_CjVHuzcnCQ%!+3SCzI=aQ`lU%R#Aek6yN+-)W^}uX{d{G{_#BkK; zt`PUMl+XjL%*3!b!Wrhtfb^Q=BB>!Pp2xe$RAkIPnHK-VP(AAKwuiR3jpz&zo0oYg1X)T!c+!Z>K1Bdd7$7!=BVtrs;Gs z*b>gL<2yg#m|xEA%vU43({nustQOxRD5&&-7u$rQ(33b>yY~pR${Q|QPY^ujL9T~n zUY3Qq4@*Ts=jYetd-uL6-!IIRF3R#YJwQjB4C9tQg#q=^?rK1yX4t01V#=g00}6Vl>kHf@mgmV9@^yz9jIk`>~5#` zZX%DN5tj3-!MG59=msZPpROJ%Ay+sEh3v1ic!r#S?#!E)Drw#^D%RP1RgUf)s)6_A zR;gy{{?;Q%@g{F8^47WnjEMzu>`v;mSyN8Lp1*5%C=R1TF6Lyy@{%+nL1>g6ASt%B zd`+N;CN$>T!p}Je*vVYdKiG>#_)`?vo)nc>2YaynlhpS=z`n#((dHEu;_QBgOJJJYh_gpQ`sWW z&z`WNk);LZi@zjGp2J%50I7X!U~e#udw`G(ia1L@IC;>`F29MSx?SGv)%ISlY}K426)<}t(|DA%g?9rqIXEBU#1e3neIyREHl z`o6WI=A!E;s`*K>{)c?O;_zn=Bp4W|*(h%^-fFJU2*vE#Cof}U@#^VZ5zz457<1j2 zjuSr)_C6dD5F>4~;wufYQ++Ni3+=}6ntIw}yg0r=YM@rUS-x;JBW?4Hlk3Kiq-UZr z-DMn(<90W6h`2Ilt?$FDWoH{V8%tpf4fzc>(OYzLXZ^j=>dN$iwAgBbVbM42>qiO$ zqzP$tU5L1_d7O2rz;1Lnq$aF{{-dP{N?zF@rovzso@|mH#tJ>m>AeCYAw;e`zNl zBe3?4>ht*Ja2t#}jC(t#^Ud5d?E6qx5dL$F+<^nk)i?6v@W+DcDZJLp>d$V#5AvwW z-O_R@Y-lnxfW&}i+#R^JbLHn)!lLRQ$jI2 zQI}{PVLIf-!f$;RX%OJa$PH>s3pW~ZY=YQtX6wId zDhPV#o&YI*T}uTtxgn@suCrO;t$^@zNI*+CI2@1BZg& zHRy771c^fsj}|ImHw6c62zl(o0nhG>yjq!q2OCtarvb$*=chd)b~sFORim=;gaA zX&Z$!6NtNAwcxpuo;?l1G4yO_{IB{YKNuiGq3Mi4ew7Zu$x?X`Ua;u#M=TxE_OP-f z+$7!w1k&~s8j`q@zAqAbys^k4pVu?Ltq?j^PJY;PuB8eg6eS&}2qMZ!A9=1fS9hQ_ zkues|fDIY!hTuKRtZUT}E)^TPy+(yj30J1{Se-I_!96Qix-XuSH#4kR{2HK!7OXmNYS`=Pv@3I#w?75?K>VCLqaW=z?-nmT^D6|Kdm(4+Z#pg zW=7LNwi8V{J-@(jsXzZwx^o{980O+*;cm+o9NCN~CN?F8dee*3wjD_yK;hf%$I8pg zuXIGRb#)^*qx&b>_on@tM;QzOzY@lZAR6ZthKpyZ^Y{+xwM8`6 z4#L6!l!aTCDq!Y7$S^D6LD9oniiL!PT>TxwQuqZ)fGExo9d>4e*rN_2+3skJqq(y0 zVJLP?lI!E5gaWwMlnu#$LAdQB5UQIC>22E9DvWSvmn!>wN^B3hT!+z z5`PiOI%pJz%*^m7WDVZz==+t2k^xD=FK=~bbJn2s)j58H)E|=`#DQhtYM2bS{MQWb zT?erhq?B)5M@LkTHVjqM=GKNrwf4mDVI-Ys#Z(mxo1on)Dl{9Zo{m$^xF z96$#y`rYnS@o8s>rgT3$yE)(98$yGF3#itU94ykGg z{Wo~&k808ce@X>*t$e+0@}j2{nsqS$xo)UA)*4?I=9~yiKq__z-Zkv z(nTb=8cHb}HoEbWmf~p*4gcD@7peWXep!E^Fz=AC5D;LjkRd?^?lEi!F; z`k}6Qb(67xC7T(>6ckGMvmHMbi4ZXygBULhqU`UR4TXSDuDt*_2otA2_0{)c& zChYJ>{OG&8y9K*Zc~)L21v0*k8;I71981$k-8S5hLY2Hdd6A^^@r?J$ago8sO9qf2&gk_jw8vKu1ILroPQwZb4Ef)A=7uQ>SvB zB`tQlfB(&R$H$ztDDjzNGZhJ{@kAg;`ts(CkhW+|_x3zDF6JYF9<#3sQI zb4~G8?A0z9mH*gRYYzh^YmPD4u>DU?Oq`XU6FW%8`v$&Jh1xPB>{0O31KmKvI@Jc$ zHF~sV9f1@dS=?fa&re8Gl8`X)SmI+oY)MSw%givYxPM;<}59&As*!lH07qG*HJYxYL$WppEIiwUyPwMiW)(3Dp z!^q0J{T=EYI2>x1_>Yuc*Z9m&Mh_e` zbvTMTuN(O%5o8c8Z@cGHpde{u-~$V*=42MGqKY}nXV(W)%k$ir;qECX+=p?Uwszsh@LL>; zUVgtHBH-!jrl47RBBMx^!Mo1;H_EVIUafKf<@T<6Gi@FbW)o4b@0`LagK@l^Zf(Es^ER`nu7z$*GSFI|ji;GKv9;Ze z-U=*}Ds~Yb?bUH7Nb2fbL+>``Hzfag(2GQx`Hkis1?{sHw$@aR={y<#7Bgm1-8Vcfs>M6;` zqNLHY&j!8*a@B-JI<3VjsxB!Zp*g&fVpVyWVDb}B>~@|;I}D1I{!s?{g_cqC7$)Ei zH9+KBr6C!bwJ|Q8Sq~=#fJBf02`%Gx&;v*$q(fZD6QtnquBef(*gN8*QB1p(3waICJ45Cz4| z_WW6BTxC%sj%%Z^nk+)CmP~>y`;f7Hr+I}?`S6W}-V8m09j;zhN#zsS zx1K@WGC@KH^6msL>C8qb1@36J7oTldPTp5N-HbTvI*(>k1uIyPDIY=UjmN9tDg9I; zNJiIFU{8j96a6Zf$QM01o z%yE_&qL7404_$F^nL-jBASivp-bqG~dpU-%UUZBsF<~?M^EgHyr1YDum6ED@7YXUp z_g;$qr(uaN1Ie>yMcYysTPc%YG2N5wjD#Q-P@8_jR%uxLbsjPSb_kTz*(rIaHIdoW*}H`^8$ z7q>UZEHqj2`^EipBhz%pGv4&k59c{(fTQCGX5hsw*s|)V^knb-JpGOpcVzb)(otOH zo0=N7{n3hx0{scwBT{e*-Tq4?{}OIcQAXhYGzEC_IxoSc-d5hv~S08V`NdKnVQ=-n&jhjK^mxoI8RxX{T4FY7ynOeJ? zK&eh?V$QnuY30fTa=T18ci1`=b#OC|hgZ)Nj*fb_@I0g|Y$vPMQG;`w6lKilo}ujP)y=L?BZnya-&sh;amN;tYZ5Jhn61(TX}R| za%?g)GaZ~tnC&+}Uy8@7a)-r+Pl+`GEKPK;k{O+AN$FED?l!=lVsT2wqrjN>SYruDhz3^UE9jJ3Lp*brZ}GkHcz(>3_Eil#OnC-FCeQ(yadLNEdTP}3rIp` z8&lVx&hHvzImwf-A{VQf6v43&Pa^nx6t9sYhMqwcAO6xdh|RK@(h8&4V9Y19G+Ovu zZ3dP}nPe)Vqh5ay`4FgcT-nJ}!pvtdu|NvecHh4>!(wV zU-}e3yKL`zOhuOLn?66plsCY{S7%2Dd~eJVI8=s*wv2AjRxJ4a4%S6tBx>~FL@VK*sVMf?pHUvZ8T!c-M%w+~LX;={wC(v`V&ALe(I@!%XsvvI zJwR>)^dSnBEaJHvp7aZ|g48YNz5Sg*137Kx+Rf;;RsU5dnBdDBqo(XJ8Y$4*_9XVUDz~r z;W7H7KijuqQH-`Yt9q?PkYfuon*y1DNhn`;I9VtJQMjTHfeu$$OO+@olm*H)G~{4> zvuw#|bO!?>Z07@++9T*fe-v;BFtIi_Vc9?ucJo<IcWv12KcY>$vE z>f|fVG|gDW2r*2&vsfS*M906=LiFU99T~sES&m&wMCSL#NKe2NnP7>2-#N52FwIc| z3XxHB)+1dRwc#C#Ul_0tdNMtAS0oHsVL*owuxWC!5sm-*?AB<);HZlfrYPd1BGKEl zwcmMM^njLIEV>VIu+UKWM{2H8KS}=^GuT)uJ_&1Gzv*z_SLMNYE;Bn*(*!GMtgi1X zwe=rHO=VQkskg#{`q zkT*UGiXBaJ;t07Tk>X$ts>$G~GC7*x?hmT({kCyFDsdyw#-lS9AxD;BEi3$9K;hFt zklI;p_D_INQ{aIlFC+d19 z1A{wf-RW}QqUGr*Jw3zrpD>r;7jahozTvpGVm*2nKpkk=%wsX=#lOaIggbVM>Xs-?UU?U`;ryrn11Hh_lV5;drn z)z+CnO!OfHI}3u0ukA+=hJH@wPB#}J&ehHAXEF7pSsx#tct*`qcZGJ0mVOW$=DKWZQ`-;R8f!#&5bbw3PfKnWnax2sT3i9(u2!GmQ zmCc*J(NX@31kPD~iuNy<8QHOZqmFjvaCl}~emwS@Ff$dpX>rrQmLh48zaEByIpTu^13v}&&t#lRJx(2*;V^^fI_pH z$hgcof1Zg}|%e@q|Q(3Fy+%hY7yZw%*-g_MnAMFR%JK z)C?)>Ks#{Frsq8?xeU@GXxPaXmN-?i3QmrjD&_l@3F51L5`j<~ma`TMDTXq_-4StK z0A&^>W~1?(X-^L~j*3_+T4L5;M#YQ3rG=?1SsP{1J39SOjBxz{yHrPsy0;ct^z9DY zSx2Nj$E*kSP4+&RmK$Hb^a>wnP8RuRk?~3g2P^t`ACs?-i_-#e_6HZl0Leb3_EhS< zK|^M%O4xVReFK;oWWRZUAnrMD@Az0A773fJb(;3nHfQg)5wq@-PKn+`#fZR1?HS72 zcWjgtxEd^yM20Xw2_24nn}?P5`RT=}fARRLM1PP@BLb2%YJ=NT>< zlWbn=CMdeP;LvS%=q7pmMq7D_`z_-gzJEwY<$*A7iHLq zp?F<}MN7(Hg+-<6)2~khl|_!-Hz~1l=xI{cWwkZ)xq&83 zBgdn5*uO;w$jq505RIXw(ze{uPl>er+2-`9#3!p;3AguP9TIm+r7r^HcwJJ^$>O6FR2H4xW=9 zs$BSK1lilEN`3in-R!yO&D=e=Z=m(7%#WH$oANM9d z@-jO1JDjMU{misg#$tE!mH0YlL_&g_=s*6)`F|VvhH6{>W|zXI{+3aV9`Sg7+UXqd~LxQxyrakCjVi-r}AsmWF}?z-!wjW=FF=Sr%xUSB3)X! zaVo>cZlOjqjoDvdp^!Z0jq#!jz#WoE@x}c-f*RBf1Y&*@>#iacpA(+@ht5{fH(7BE za4yxcOo527e9Cw--hGGx5!T9OhtQqM=d_rDj4xYb?etzGtnd;&kCw)UxWigXzwYx5 zUEi)0#oUGPXvrCeD7dhK&Q}gnLahUa+l?^vma}zMf_)WNOO+g*6u!dV!oZ2d6uYUQ6q}+QNg@xPr8>Inpf|?NzxP zmwm;=&Q9t5#lxa%K!so_roInZSNV-0yC6ATST2M;-l6+XLl2ypxd z3wH&$m%^bsRmw}*s|kU}lhtxl%>QBQEu*r~y0%g2Zj^4MI|Ze?yBkEhJEbJ0ySrOD zB&DT6y1P@5K8xplpYeU;obzw*?Qo;_z1Es@Uh|5@CcE+cgJGz%oDw)mof?22P5}hH zS@`WtsVX;Oz%pe^2Gmfmavs*89AH=OgcoU22zCI(Js0?FOaR+44-eTpiUzz*^EgF& z6lt|Vv@MfnY>~E7TV5_~5{rE{>k`0lK^(_1u`5yX@UX}x@x0iCF_L>zKsfu@P-tUG z0NJEoseR#S-dYep6@rg5$gViqL&&--tETsb;85p}tn+Z_Gd+e2$3qI&GFYV5$BGX2 zrZI6@3-w9O=KNVHlnKAAre2TqaY#_hXn8x)a^LimZ3T>d_ssr2M%C@9PS{v^v8D7G; z%`a~@o<2n=!*t&Fi^?^z3{}VP_w+(t|J%{al6Xf2tv6H5_I5P0N4_5kR$rS{2vuvK z(PlPG%i)6AVmW~Lnu97};P!e@OyIAcaVvI{yIvOKo#b-TY}cfg6d+L7p#1$x+C=VrtA9U+C}e zdBMIUye0>tmx;nF4vTw6x~#1c>}p1+yHQC4Z*K6wOtyP0T%| zD_(h;5}`pOS$PN38lxBvYt{@=Xe+NA_A$y%I^+?^I4e)7omZNfwM2YC*cpZx9prAi zJFHdaP4X^svB-=pt8w&beK@~mm>~%dPLWKax>vjMdg4tjdg2cV5E;818IvF5C|a_n z(^vfHe*~6VtuD6+rY>%U(TCx+{&Y!;QQ(3L9XNxO_fKKK0_;E+cOof-PdDQG_id)& z+Id#@bHfNX|23nkcz%+$U=ensbBSyHLh_q8@g;%8|0Ta{{X#@%xK6dVsQfNtWyoOQ zOo01eFMv?_d(X73l?V?Eg~|u!Je*pDnTz~kjsgk20Vtsv@HA4xOBUeRh*JOF@jdT` z(|rpsT|im|4GoPh37bYR=7N#Aeu;K^8W0%BQw!&`J)co5?F~^2PbXAcTl<(rSKp70 zShKMOu&}-GQc^Ub9UP__LH!I%1N<;$6jVa_KdK#=yUBw{a`bXmy z7FCmknHeMgbpIEDm_%^UFAh7D0DM`~ytX3lK2!~OJ>V7IqPXGqIxOI2Zc4shr!U{hN}EQcVL>X@7$`8; zI5Fpns4;u%;%IaIe1dwsCH4LG0F@|m9Ea>S^wHCiR7t-%UC?E*D zP7Y#(NUyuEkoUZ-7}WTWPvZA`n|gGZ`!eNbMUoVY>Q|<=7T^SIyvJ^XJfA}Bte=Yb zURwASpFq4g)iLffDfZhDJuTkRgz*99v&j}XubpHyw5AMI#&r`%r=hVCr~oYAb0c^E zWN(x!q~r0sBWm+Mi>y<$Dvp+eZ!@=%HU%-DzmxIO7Je#{^k*l-K(-}CK?>~L1)Zvv zZur2s{zUmN%HhLZ;f_@v26+Wa!zXxL8Rx}z5BBUBzhg#U993%hiq52@M06}+-1Xj^ z3|JV>|Aps%J*Y$m0+rOyWn>l|{zV4maz0nwQuEM~s&H0|lSWa(qf9G^Llz1th zJhreXb_sgR1n0Nj7aK48-B;M}k&OvZE#X}9tgBLQ#~_mbyzue%3@3oe8c{_tEx^dy zPYyk0-)gm;CwqU+`f(t3gX4>@o3yO-defs1($udGnFZWoG4_%V+)xmpnt`yd?T?Zd zQObFu2qEOAQ0lVMCf)R4gypbZ{!lk?u8hSO8D7Y_^t@9{x^09ip5jAc$S^rHsi$p1 zhWShQy{>k?1ZWq01T=PMY}bQOobRXj=+tS@DpMA}`DA}gjsG4>0Y=f!Y>=1-Vg!>2 zVu7AmyH4)|H4vIpr#QfZDM$pLl+!S(9q8OLmZ2P>xTphCULCaL!@G-6V!~BNafKmz zdMj6fdaFeOqM0tx>vM^W&@B9xC=Ko3{mH;D<5{EeAxC^M{`MpUP7i;N2G$gq_1gzW zkS^cq<^>w&LIc9*-(04kzw`wtn;3%zaXkr^nJ7duV5~@^#HM9ZjjFkW5oNaiA^1ee zPSt^}zO)wrgJcet+TJ#Zr%y5CU?33zX==MCVQ%O&#v3Gu2%H*0?4QBnX;7{NY0v^t zyEk_HjG68+KIO`0@+JZq{=jTy&U&C%*_RSKE%r%Tiu-EOnaN|5R(tk_Ct&g<3~;qw zX)6QYyTZrcn-7K{DZBk;9rw-(GjpO}(*!cN3LLbI=FrPc!ts`R;+*n|eYm7+5}yNU zAKBMpL@D~KSg`8`-Ci?KWwEyrLp`pt*yB3-{g61XYNG9JRKCh_zY$VFm z7^=i#c9_Qr3u`Fdd!C1QH>&C-g2b(mhn0MDB@-%8zeQQ5NN!3-$(eDF@%e>>&qLm} zYD~4I!qW3_K69Cnt8en@;O_JT~(KRWcLGQnaZGep}4vZSf(T6x;!udjm$ zA(a2w?&&6z?G+176T#l!O*&>sye)J;%y+MMNUcNl4P)B;P;)3bw!~|n7*`dbu>261 zlLPJK;F#|z#^!c~*bigY<2uAtxuW-}1Wfj($$!dB0{MOJ&7V+~eT<{* z1R+4nih-Yu^=^SO`A<|yRM3|B>ZB^ZCdEiykbcXVt)$goh!3*_8z$X%C_WajZbSJpY#f9FP3 zq~-Xq1NLUY9&5lX#UZcq(5L;u=i#IYSb^C-GlhDTL&6~1o2tY(_V)DrBYcw$geUlDkV&}^>4X)^M@PpJ=MPLOz7gM>D#QK58{WeyeQy$viR4m@d- z-5OO4n_sTzGYeHmBS)cC7pe(<^RGaij{SY}sB~wNoq#y()!W=07{?6OFO~!W>(HDe zG#dUB(3ePm25S!RO(8vAJ5vteMNlPMqy;Nt6WRgYK(ij~(%jW0I7<1Uky<`Oaykpp zF%V-hp61rpvWbBhAy~G!+_tEZWD#K;LRVt*&QF`L183bBostZFKj$!rA2knaQDs)J z3R(+cO@~y@n3#}&NKi51zP0KHIg!kLbcb^xoAkfj4EsBp|IC7X(1c0*l#dBIt6D;a zl3$R7<*je))6?tzKwp@ks&naUpHl>r2str+*rZvBMor4^)TlvmgCH{)To1HwRC(wO z3E4+^K-=wGR#F9cn&#(NWajmM7$MtqpSjiE<7E$sPo@~OV(}Zb{Jfmnh8Rh`kM!N~ zTNpu|iVA#D%33C_cWpcsLn}qzj4v7IKHBfkga-=`T%TK0_R5$p?hmd}nq{AgI2{O@ zL;Yd10L~NOof~Ac;O5^$kw*()t%j27m6dSJyHJh_IDa85E==8hRYrr*FHts?4TiYC z4c|bkAt9Hr(l}DayV{r-!$Q$mD4H?BCRc?__i##hT-QF~D+zY!?oz$qTtI+qLxYR6J;NTiqvn)(u@poAEF1TMr@Y>WnYv8hOON~H!A3{TjyuZC4>-aeKA#Rk z$QEaRv2ZC4tFg?)qy7&?l1GuJZZ{X-zmg559y|HZL#oV?m)%Ir=2YvvPV%4T5?-z@ zyEjxFVir&zKyYjPn@2|LE z7Y(8tSMc@Po}ozHYJF*##ZmZa2~ zd|pXj^QP?hzEc(J3uUoS%qqW_+|et9~r zAh3hjqKMSPhwifOC_gJpn~aTicx2xPLyo@O(w&*fL)DNP5&~y4Un69OshB6s4-bdq zsN9A}U|`l`XD6tHJFIlR{U#0x-i9>8?-?^H%f;kh+VCOXfS9>*=Zyp8N0@VAWY>x!1M|vQXWd6`XE?qH zq&>IHA0Jd!$o23_2h}9@uYYF55RSWAoh7)% zk2Fy*!xriH{T07Oj?(>kzZ6BU*+}&1f%-h{#6HMegmc#9Fy{Tc5e;kW^5c!*l+)W=}YHg+yev!ttwWR8xj^kqj&VOly}EFuG>O`u}D8%|PCBH?o{I&Z$D zOT~Ho47f}#IO8{=V_3aFat6p`u4r0*w(u2fyyd{;0&!8J8Vt zv{9ryT_ZyHk++t97GNEU$^OI0$Xk#qIQ$$JzdZJPCGkJ{eC*@|vsala)rq{7&BZUy zrT;0ft3bSW5_4!###Gtne9*HpG;bRb5vM~BNqWw`&l{~@yVj|WLO)T$k5buClBWv^ z;qM0lEyxQw1P4uM6MUx^gD%L?C9bbWh4EJK!-Uzt%z$t#Nm$Z8Lu86rdyXwKhW+_em8RXa|s%j=N+^cJ`O+~5NObG}~f7ky)Ff{>^Q9+}F z9=ngoo;f?ZC$dqbJt@gXb{V)?oKYPn1#o$>$;mhaiW&w}$-#$wKriQ@1S>^^Y{Dq= zMV27%ESITO_v}DgWX@%lEj7K-690pI^>YL(eZO7W(^|^wO-UJtD3!kFk2wr&^`|VQ zyChwS=fBA4O^5I36^v7U%3>+1Q06VVCNY_s{g$1O{SW(AZr|&Zpl&MJ|&Hv8yi9078!z%@6-bCGpsT{{#>; zyGOT7w&iV#kAF-jQA0HsAJbBXgLhZzlGl9HR4-Nj_CTsG=Q`Hv;C!>l>N_+vlq&9; zDqk45s)myHR}($6zQ=e{to_J5cJ2!<*8%r{D4j+si=sq z;lh?uLj>bT0Kve&u4e;+RJ}KbrVhKG?u$i|uRa$=-3QU1Xr~p0+O51!kv*t(5^4%6 zi-nw?qAX6G5pw8?a`YNgYpox{X!L?f(Vsp=@jn}ry{Dv7*-KKhP)tT+r!UE41H zi|$ZIW?lTvv(L8x#D5d53w-ro+U?f(PdjIo-3%2Kd7ZrykExiVCCJLQ*EFrN4t!Yo z58gi!v}PJ(nYgQya;ps*WGhmsLB->NRiI=mP_gNC5M|>^h?v#q&ucLMYiM76zNT=} z**mNlTTG@XAq4m(BCa^Gu`wSLf2_hycVt?TQGi4LcRX(+EK2HGJ+d^TlM6yR zY3-u`P-&uca$|-lM$e3w>*JX~)|cYwa(gkfX&EZxc`Qg7?#u$Qqo}yHbfn7cX_ex` zBEDIz($;mo+*!Xq?7nyK0#zDO{eQ#${$%*~2ywZmczL~MnH}lvVvn^(xArJ%%M4h5ikH>A zSd~V_cdlk*|6ElgTT*eT!07&Zf)+9SB0kZa38%u^zS&rmd*yJD{_ijSiSX;i%#2%* zA8QQs&4luFVEWS#cj(Br%0gr*$L3y+n(Jn}#O?+vydh)>JoR zyeU@DRwqWy%ntrcvpdRrd_tURyCW!0O$b=xA1v(wau7kVN{iVF3Npw)dv_ZJO( z;pzKHmw(QU6hL$nlfGvU*>2N6&&^a@WQ7ik#e9wd!oM8CVw|6y*U1|vCMs&bppC9j zlahNNv!KiHpRJZP_iB4$$X$T=P|saGzm;NLSCx(Atf!}^V z0jq?|{9bJQ_#dTjQfKLA{68%-yy(wr+w~c^u7xI94IPTefll=Ms2vX2?|rQ zGs`DBOJE#9p49YX+Df}eeA`P`bBPA?$@q`b6mTa?Ll=DiNhlbnlvjKn-aVD}@SbXo z6GFMjY_{JJ*4L77wTL{!YaUYS36{SPl+QrQMJ$Bep9{-OkW?-(+JcwWY0rM1_sVMj z^kXUs?9(J}R52SPHRP}j{&QYv!n=TrOmi|+^Gv0BCB9Isem0Vz7&}fF!hc^Wzzuo_ zi3%j^+VSs>3s0=MsdaxoUdY&02^zf%N5vmPpOlhvGt8Zw+jsnBgm>`cj=$hdb=;+g zIMFXE@o9nErV*xV-HhdZy&{Ihk2=pBVqL<|U~AW6w_a;9P}X9$_%i-ZyTL1puvjA^ zR`p@!%VkRk6j+pmMBR9;h*#xVHIXQ-;J-J&Jbz~ad9wO59r)4pz%eUoI@Ajx&t&8r zLK_^-H{>M%$V*vrQJ$AnGXtqZVU`|xGQ)1Mh}QjeiPC_QoV9P)|Ib(Tw5d9$p(KE<;5ndlGbJyBd5V~u~3VIb*{=pZJBqA(|K+cRPlX^!1|>=ek?Br z>2uQ7gZO6*eG=jHBk*I9o=09kIjL+!=*lq8XzfhQ+a$RtQPix|s3+ht9db8b{As>B z#k-lJ3j??cDKbaVGrHPayC81`e%rAG%EQzm7Mjr*k@y9 zwiedgn5^HCXccv+m@dtI5F%i8otfY1jWeiIM7+~A7Wm?12j ztu*R}G+p6fb6o70NA;hwBxnT9VgR?JM>BqVyr9C0`J<-~d+7f<4mkmSB3PNI8O`Eb zU#$1+86I;f)EF|$n)3rpb-!H%GLzZxX7iEHysi$+ox#a_B==0Y zfMz?>v(Ky5{*3$Na!hx(eVGHZ<~H+IR#!u)3w_c85Qdx|ZfvSPzs5@@rrdlTeBgAd zg?EEUq*KcSg@;z%0`xM-(&lODsZtqMoC9tM5vc5{ zkA;7Vh%4OsgHL+-cTeIwN%$zqVRe_fr4}bMKnAX`Wsz$JBm*ixQ);26(@se=;sHgW zU0z~a@`eSF_r4Fv9;l^RH|xkPq+d4`eyG2%FIV_%v(ReL<1v109T6@9o4^GOg{pAaa>XGg1qW)!N$>{n7JH zHOeQf#z@AJ_DwC)m)AAb2;YhP*BFT^{h!i`jHUWNd3`RW%B(~get*wg3+%~W-Z|JF z+!b>QwgGGm?)-4yyIrVO5@IQ>RA}$a!@kJyt$qJ%jLZq~cxnYI*+Y|V6ePQ-&!iu6 zt9xp!3crk|g{ysC9TsuEI>n+>6Kx}4?_4wVOzhJ3JI3DsLdJY9STvMCRq z)s@g`Foa?UBWS4~Th@_=vqqcUkjiP=6R{!B?1@4c^4pBYOT_y~)1F+kU3h_pVfJk= zIB)U&MkzQneZTl>RZy5*TGEfHaV1l-&e?76Ju@k-IxjXyx~(Q;Sj^ z!BDfcLg4V0fHB|l*5UGKSIYTu%HYv6i9!C_q_c24n>sZ6=)q}cSEO|W-t*L!N%yC! zfu=+B>-kP6sX>5LZ?-oYjumMeVf5;|1RinN#KdDj2#P`=4k9?!3(Ee=9ssu?-EG;Feq zz24sOu_%49OKu*_xAd#u=5Vpa|JPagpO=I*hX0kWfDm&-+!ie78my8ED25jW3JXER zH(f~Vu&oHi*RNTHRd#5pRmk4JmUJAtEEIR@5*X z8#ss|YwT8$O1Qf9O}$0XVtzdl#&b6|tL{McPTjnx*aJN&g@AoR5lOjAygOZ;G|eX- z?Az(6l*U?==%dPjWy$I@1hH8EKcU`ih^&-Fx^4v!eknO*(}C(q_=}yBx_LZ8!Lz$g#=&AGw548NCQU(7GRI0*2rC!` zeI7p7avq))Y?3_BDNRj6%x-i!_R8guDR5>DH#H%kk-@%m4Bxi0wgvWSrmwGdk(7fY zft-S(iK`fB$+?yZF;>+9ds-h$6!iZVp)RpPie zoERYirjDnHYc^4cw?D*4veYW zNGT`ZKj;2mZ>?XsY;-~|E6>SAmH1=>R{G`a4dpK_-nRSe0IWrf4$ZBwo^y36;voIa z0(l=QA2SSh2dUof?l7FjX_DB3oXqbd^2z&O46%T(tgWprhwHJXqnm~;>#fDMjI^Yz zu|=D`M#O;*R%|OLX-@Qhkls?A**H}U5|Pi`O^H(e2UzL))c;IJJ_QhB$x(sH4m&AE zKMz&fip~}#`q*;cm5tAW_Cy&-cTyD*mqw;Q1p~;OH-%rJ2^1+7zE31zqa_Ii&|7>R z|E|XS^XZsS#pzyyDKOg%gYHbjpj^hP`J~N_`P(lJ5Cja_qCcz5C0xIO^xUl9DEStD z(zMkQJ#&EjjK8QxVsd+|zm9X2v4TCLIW_>)=V^cw9)O`3IvBbIXF*iqP+O zROl=?5F(<*pc23ONGq&jfR&;NA=h9ctx7Qe!!f&)kvx{&Wsi?@*66g5nn%xi^wcsA z4k4WX{6A>Rl^5=CW6|k%D8tk99g&2^mw841ucPq#uOg)D=c3e-7Pn7dhzzjrjU=5c zHOmu3IJ`YI*m!tSeeVybK^O3?@3hMp%Xt5+?P!}E&ONaJbd?F?(+_Vf z=SeL6Rxii9BdgxUh(x5j&nLLc?~IED$1=K9Ju&Zk)Q`$C#d@tHGm1oDObtdlnW=JK zUb;cH;vY2DZ+CeRr(q6Ali^9uCpXj1FGiFaK2~cLad{RIzbdf&$*^8>B97e}q+0?w zTvW?Op~WU+fYlx1+f!Yw~`6Tzpp{?;kejn0|A?>+yX zme6Raqo9gAztPJ_ok_Y2Mn{38a829c#llJwI`TFw2{Yr_5?i8wUa&c*xN~1BT zC@7#jdk{Ciclk-d+x-N!9;u zws24AvYV3i_uSkZxtn3^#fJ06g)iP|PR5xsS+(P(hbKw9=TpGgr)9CQp^H!S$qc8e ztOi(@_Om}z2;aRE2Mm&B6x?P1(#1zPto@mtz(-77^53}aQQnT02G~lG)I^|D4+^@- zPQS_6iud`3^iO#UO}bo6x7nV|ZDn_3Lr5$fTdB?Lj5-B`WBsKw%6kn!ohehJ7c2iR z#VK1Qga8$e3#@Vc|9%{wc!E>HRwd>NFMx{4CjPFyNoga z;#5=~42EMe8jQE+BVq)*H<+g>E)b^ME-?k;k8aW7IKv_%#jZv{=F}7>ytXwdI2YKr zbg{Y_8G|9QwyGy!>frFXcb3mNzqr`=Og;NhL<}duCmU^|qI>qm2w$4>X(_hcNnItL z^`p1@v0BmYbu^(slNSVCdQ+7;CpC0jKWuC0j+cUM0jJm3*y)EKy=bZ}GjRVM?c^7| z!hVlZ_dm0sL*yJY;0?Zj<_z$d4C1r1Np^Gw*+bJ0+T{`T#%;$)Q`?dFF;8 zc`u9o>k_yb?J-uVdA%nmQ;0T0YF z$Sr3}AS3Vdaz_(Zw2@llO0h;?p5UoJ*rd&+U)DJUusp&00h2RIGe*&3ki$3$0x{ho zU&Sqwl2Z2+4@$UM00SJ z{HJ9PYuzO4seU=qSak8OB>WrF(W%t!-QV&I+G;$md{jZ_aBmss;X)z6eQX5A<(JOu z`siCw5^UD$Jp$A>$bjh-SFTN~_Of)FJ@E?CZs*)6AW-@#01M@dGmB2e9_v zej3F9#%H|5(qG5M#y9}CL9EjXq)g_&?wN`;xXN^?ShfiCx6R+ew*%>~XSAMNH#doL zOWbq4`&6;I5|a`<7XCB#M}?%yo1*Ub0x{y64WHaE19m&(SBpLq@lw^jwI%Pbp|b%ro=77$X!C ztw;QD>`3~t_zv`?WTEsSQ(HgdzEmy-?XOZG{Fz=hN{^J5K?M3kOw9hT`eMr@=3gmCFreAG(SfkYnbNB87U zYpZq#Ffq!fkccYLW50za!06GHpGs#K`h?KLot3~`8qhM7N_ggx#1if(rL-xyeY`Eg zAJ$L1=C`d1eCjwnm{zzVQL~q54YGNhq3+xZTUF!j%z4m7sTp(jTdvm~kZtBYl0wW^ zngi0uWmy%SM|*kZ>@@DU?@e1 z@cKm^&WvZ5v9zpg(JIku{_4r9Q^|eA@c&QO8|DA%$K=}a!Vw_Y^GeBaa63<|H;9G7W~^5|rqn@`S++Zc(oy9y#+ z!g>^as%|zS8_X;dwXQetD%V6Y&lDZh>DstPV{#@(`_{uYzJwyU-w+Ep9eDCS5mEJ zw%Zteo*BIeKsbvB*5W-!u>hD6dmCvDkpX!7%EH2siOdFBye`T~-@(0XoPe6@{63!$ z&ZE<=!eAW4UBZj-KU*ty`NO*tIi$s+wy~)zp zJM>_A_$Ou=z*>`JQmO^NJ$J+DgD?v+m6{~6R z|7clCtDs5vJViG)XnjUPKOb|!!^?xEzgoL5KodU{sAn5?UK}0hp8NcFG0wycyJL?9 ziV=$nHPm*H_4WGQ751$(Kpl#}Y){{6rGuVnU9Uo;X-$JF^jeJGO4qGC2+*_{78xqp zFqI-)xlW$;b|+A~p+Y)Pm$&^0GE9g1CFuVb*hPzS9=O^MYNh?Sr00=8YUL|6J*fiy z8S&%UHJ5g+Q&EQ4x6N4#$>JekS}E1>Wz7G5RV6@qIevlP{L1qKcJoE=%|miWfNNc| zj1@pMdZUbUq{mYp>Y>H)UEN%yM7sE$_t63KHn}(k7$|q(bqPAUIuSStD|&t7wEFX% z|KB>4*b$h8LCLKP^AU)S9Y-@LTDHFGO#%;WTe=A?_M&w!k9&7KroQsK#_olMzLoA+ z`;5b0Q)eKVe&J&Ps@Yj}!ti4IzGtBE#~d6cWrv1=Q33+S7Pi9N0D^F|A2{%;i`XEZ zZh4T~81#V=hjZ0-H1rMzq=p!fd7THr3TsZ5`*?GimiKEy3!qDA2|+<$|Aqj4i~-zf zJeSJqlAfafup@XiFD(jB|n_#^y)=3Dw+9oM9|NGSgd@zsL#&Z5MUq$VNR}}QLo0CccNwVyztZDdOQ@^pcTq;}mzH>}$ zWKwxd^h|6| z9qG#MJTi39I_87$5v4vhOzgL~i3;qvZ)t0zuezoK`zrLNajzrSjNF<~8brkNT&_s^ zGBroQ<-P{8>ry?ZU%&#@Q!$oY47;)w4D^V_a#Tq%F`|IEQ3w`ms?}el?j7a_xU|re z%m+kB*^UYO#MdaQ+Y4#Es)Pg13npO%;d#G4x3~K%wxaBuTQB@j0*}fNpv9dNo6&>D zzk4?P%kLB+1E0j!*uKT#uP2L~_-voyK?5EFB99Sv#Jwtj3!rwBtxZjH@TT2jeb_4w zI$N#*oPG()3gH5fO&|Q}R zwLf&C@q4-d931W@!q$#k&gpW^T5`8AMm;#%BY>b?uXjVq=l%Kwr@HU$n=AYAa1uuE)7>JDXK61oi17UuJ>OhJ;*U2~=m7VR8ca_4uyb4B^@?)2r zPH3>~bAOb+v^I9QoUwkZ>yvfcB`(8PQN=DIc|BWCtbKYi=*_@Q4Av(*@As!h1Cy#} z3n@?Xs=pKydq)k<8cg{x{8~gi+WC(yN&5a@PX|;_SNd6RI_#gxsgh>j4Zo4R$3h<=hAF zW_WU@qWg|Ow1z{XU<`zy`wy62JJOlhSL>kJ@ydqZq&%ak>>=y6x37L;bn^DSOC3I5 z`GYw3n_vHJfU`!-?=1;Nts;=JTi$&R6xCSEU2b4sk5xWntKE%|@E{^0!ec>BO4_xn z%4_rUQQ8f{>c|eDUPTC4RQBZ5gQCI6XMf!F&ojo-&C+m)d@M}Zu{$(e_=@kd98BP~ zTQC3I{H$g?29E9}AUx;s-OPsT>9Q1nZ1Vxnd=hK~5z6HMRnlnvpwtLHTI&=@q=|iY zGG82~t3rUDf_i<`A}oJLxLdA#W&IiIBSyquCM)7KBZt<0LN^-Ld7RrBrB^5n(PFVv zf9-(d0QfFxb+#Gbbq2(rN-RdN*Gk(XX?dO;Sk!z9zO=y36#xCOJ2k=P(diX!JIRgY zHT2g)xsK1pa>jt4dzX~wj`QaifhwA?h=?5|`u=T;BO{;{cG6ZnESI-lKRG^5;Q5B$ z@`VNxc(~wc!^4e$g%M#zcY`WUxQHV0<=>l}(W_2UZgE%iB0|WikD`53Fy`I2M8B%2 z`6j+#Pg>k8);gcwJLqFlQ=8qJ{!;HL3c9XOz8JR{4zJ!n?Irj`z`d(%q8M^k>Ro&I z&MZQzZk(Fi$Z@mUUAw5YF=@34eL?Lz3UXrzxFta}dXR=4Fu!y)E9t~&NDZo-ha@O1 z`QY~{c(oL0PpT<_VELZ~aAXd(YlzPp5{L;u9w}NX~$NI`2i}@O^amg>5P( z<&k0v6XZZZLfVoR34}$-0|Hz7zo>Y+^n5iid073Hr00Tog@A>}@%7moPi~og8*y0C zd)aQL1j9}sM%MS?gqz2QP?O+7w9@`~8@Oz|x~WF>a<6%dcw83v3YZ5s^gHP=cTS~f z>wG2n=*j>hJX3#s86B324L;NXrYEiL7vv2V)4@4i1qmB%>T|E9l>3?}xB7e8+5-^P zP2Blht+=gST>3Y|74kfyuH&XxUhRwwAcWdyMy`b|HQ8OQ#Q}Q`ZG%6XS%O_f=wL8O z-0eZ_{mQ&r2w*NA2MLDX<$n1)_DlXU*K2?6PlZvcl&5-KNemQ@p0!D-QEt%_Q`(xY z>m?i0JvXQPuPG17#;MZV;3D_gGQ(jL8b2&qxiJIBH@K05`l_V}?LVjF*#u7Zcl}VU z1jJGc%X%fYDBbOxu-bTgP6Iqlb@T?v^hMTMloOKgy`zrs2)YW$ynI!A{?-%lN;t~fJ=FF5jvEhLJq zYwdSluP+kwOa{E=jVZP}^uF2&j1OhC{O1$BEQEb+d-&-lKmFlgXa&o0Id8Tzr{eoD zw2bH9fGVQQ_iH0N2L6!J#bZOOFLvyGpIRB##y51nR}ssrDfFQTxyNnfg2pVXUx|wq7u7m4Z+O zDt~j?NANmM0UWc$ac5YyLX)-Ox-!x|PyK@{l^u|kB$D3PD3hsZRHF^JAo%>=x4abIyU3|`mNlYw{g3qxYMD;1&-Xh!zxC$3m z>NF{XgrZUq(YY9hIR_2N6$VOYDaX>G&C3*^)C{GzJkNiq=Dyn-DRtB!`~nL(;`Tcy z(5!JpOM$p14(N}D_YNFtuI9BDYhE;X4$97Eav@GS8Z<`{5T4zR4AO2M;CXCHc!Xw$ zu+v-jnHSs7Aa}0*7E~fVq+re(^`d4qhuN<92%o7j37b3=F;Vg`km9Lc6oTe(Wx`Cg zK8D%F#Cxf>ZxWx|@ine#lXK1XX*^Cv#vH1^cO7hovK0RaL3Oq*jN`-@TvFXYIwQkIHZ4^5Cy zlhD7##lPgaVw(QBf?|Nn2Yc)n>taZ&rD&ySFeQ z89N?rEC)+4`Hk$!3|i9(Vx>r|%oGP9@1bbO*-SCH%5_}vpBJ#{h)ZM*n%%FkM>YAR znQZr{0!x(sa$UBt!hsr#EPZI?O5851kgxB^9YLb5vWAySteim`CJ3hEeO-GRwQ6)x zrw)tq&r{@#^kQ$MzZym7(Xezjk?MX{B&hWuKOMtf=$4~=Fi3$Ss|lb*VKfXEvDavF zl~+I)%KK`5gGX*L}bde z6qE2txLm2m$c&rSU2p(gF%wAv%vbMm>fKoDSI_>>b9h0uVVF1MOlh#PHnh|*$qPO@ z1k4mnq4I3b;zX2nwZ5Ocs*JR>1p26`jSix_GHZ%w_KsXW<)3v}i*v3qW;~~c6SBPp zUD+_?s&dpD5cDFBMzssF=mw~RjfH4}5LC!r;bf!QG3lRY!7xc)=_V)I%U;6v{ zCV{MwhL$$02BnbP&dzQl4aMU-3_3L!y6AL?$rOx@^GIXKNT`PGT6W%Tu#Ic#69y_{ zEWamXKE%bvr8$Iiv)>JsmyBra8pS+};QxDZfpG`ay>gsZtImPuW!6tyCESOQIu{og zMdJW55nJH@nYpDeX})JS-}z2)K=_ z0DCf&;}r5NnlS4FmG{-P29?dddYb@P`Q`2jBKW?QGI#tFRx5yPDFFq+5<}4|JS;31 znDE9`S2Gjwd8%GBj_G{=32tgc{lo1U&eL?ehtq75<#vC6qsxSgyWcyOM28QMf;hM(hzS87Bjr1oqg$k2);BUov(~>8!`dTDg9e2XJFt?^h zN9EQvJAHjv)f4YGxfvK_Gnn_?n*VT!>Gw8l18WOmpAyj4-BW$HU5ei?SVL*)&iVjHEtf`#3T38CZyE!y3JXm+DOH_( zb$My=tIj}$vh4+Tq8w&4)iYbaVwvH4SCiFknA%bURQmy=I)=Sy%KZgJ84}Oev&H)L zzJTcbC>kWLCgMfGHJeu2@}fo@XzC@Ef-w_&DhCG{pZJr zBnv1AsLLH7BJJ|2RkG6;Pf1_IfA+Bk zigLTM3<;Qx5pMLO98y3)K)mXm)7VVINe6I8RI;K5i!GA>>jj{db`DF1!1Rj+FM7x` zM)4ZzX0B340ytP;d}c9(b`%9I5yCdC;VcM{Ag7afT{Kr3bY_8plU9;Z(M3oeG-T#U|>v)hCkiM>E^szK7m_WUkMLGqN)8eL8Pnl~3cv<)^6|!cK%{K*>wzE79 z7!>00!3p1+5OF$Ztu4hyfl)!*CxOk6KtgpQk6~5@1mA6LhzV2_FO+W13 z%LCDaJSg9V=`PZT@|hJq)gBc*BkG0`FZPjL4R3sq)N+BXYKg-=fqmy4NUg%Whq9ev zfbjzAdEbBFV-WiY&~;(bQGe`^Bc#ya4>&-MrvQ>Y>69CK_DYkS2;0fZf~K# z__l{VFM1>eXpSBfi1JlLG=P-jW1-&WI@co}FsD?N0tygm3qu|*im)e6aPfVO^Ry$RQ;8>@b3Si>MO&djJma@Q)v*8 z?iP@g4uN6l?iT58>F)0C?hfe&>28qj?l=$c`<-)L-%o!zGwi+gTKBp`=Z%hUs$BhB zfY0FrS5@7mnd`r~*lxD?%*gM&tcUD@-wcsS=PsTwS%QxQ=bSlred;#2bNJ_^ zrX%%Wn6pI!e2u=IN}gTq=N`uvbV9N6l$o3hl;we0W0&6Syd$2SKWue(`&fq)C4}y5 zPX&CpnoG|{H3f{M1b7m9bW z24gI~S-SH>i|H$7YiS?Gz~KeQ*PLwSfmQ}3yNUwP-mR^13!$cIb`DmyrE)5(GBTeE zS4MqAH8_(n-BZTQ8Hgi3qo=_#cNHf}_+VU8l9L0gPha>Df=7tnly|9ghTAFEOV5t& z@`@h^JRxwf3SVEIkoqX1xPS4sakUfX(XzhN!1NRr0z3s4XVjgQ;h%z>Pv0R3PROe1S*gB^JwO+#AoN zDr)ZhLnyA`A1p1q>CFr1*a8lx@Kzz~LlYGQqgha_$lMhw@D*uMfW#bHcf3~+`GK6l8X`)Ebw>MsAk3vX$b zN&5l|4za8FV3uD2a5FKo_kx!=sHlsuvh`T?+WL0lZB z>q-4Q^iYBIdY2I%yzG8qVWvll0A~&6v4p++QUG(%5ys4kKwB?YxEaJLQ4`~aIv2o{ zCbfKwdmeTv+4(}mZU3vcv>ky%jr3iOx36Yc}NQ2_UaYFTFn>pC42XzCE z1os+McPpY|APA*zJ=kvkT7Ube9`sN8N3D(P^KB#l+~XP|T$}>8%aiuQbq@U9#l8v} zGO@9fsql zo=Q(M5Onr;e_yB=`DZCEy^Ly<_NuyXrqAZfx6{{cqb1!O{9GmbE#oYcVMYP_Iwst; z#nV$e(igRlPN#yc_y~W@sXB4T+kAFMghca#sI-Bni4Zz_fp>mypDgGzg>(*G&CfpVc41^6+B9#fLNjWG%h{}ZY~&aq z#msv3LtO8|GBp`Cf^>RoC({4BQ0ssI{3-hm{-shOnz) zxUBZz3G@@)e44P_Nuz zDmWrodMl~^rmM4_E1`D zw>ekGNIkE)?4@sUR_SB0`*I#{tORN)2TZ*?e2Z>{6F39;zoLZ7@~qfe+ex+z1~&zy zHX!~^Yz$BPvJcc1nN8RmuKguiIrNUvG!-iTAE_VWuM}~G6DrX9xX8Ix2e{a9pg@jr0>g89?75U*{sDdxr#lAb&BO5GLkk7j=3n^55#S9vW}G z`On?_B4y8w_PcW<_pP6RAG^Y5_14a|)`95tzD9j5$q9DwCuV0rTG71b^zg=+pjDNTKfdAvTf9gi;zTezyWVQTzb&mVhJ{Fpx#o4GwGz-E z6bRJRG73(bdLt)ueR&*=R{yuJ*;;5muYMeK>0*dbD}P2axkjK99{>q%aUT9fWPVgDUxH=!(z-&>=tS=I>xOX9!}<~(5MN=RoE)&O zta>Bbo6SHRr-om*%>>SxVCqrTw0TloZ+~rIX!6y}O|vTt%C#EP)i8k*H!7<#im}$C z%Z4ZarsUoUcQ+=F4}%2#6TPnoU;`CX#j;!_~^!AT$w7^Q;ae!t@^?9Qlo>u}WSCDk>G-rdV$lg5aM^gZPo%v3Qw6DIvhsJOfrM@8mGT^N%)TNwFTcsI zjJ_xdU&F90*Lyb_b7m}IPwO%|A{5Nk>{6C-8}6)~PG4C~F-1W{|DdtK`|Def(ULND zqIlo1ED436eM+^V&*K+fy|Hvh6&#;9;9$&UrbuU{>>wgx{8~|gfeT&ZfTz5A`@32_u=3wPnqjvW%04iTC9Hi1f&aKqK zsDdBn?@qkKe?F?hlgy-YT-rGpZtE+c?ArDdrm#Dl+io{(Y<^xPJx`gk^y0heOi}mH z$WHsKpOdU?lq`9dpL@OOtVpXM;r+kJoGavGK5(zEA~Xzvh{c5~Utqf*IY1ZUsX;NsZx_nrvW#}xVS`DR|{2*}SfkIT!m8ed?C zhbXUs}zf2{S?M$oY6uYJwu7PnX7a_?f z07%2r$z2dA-Nf5j0zH^Vu>P$1jlFOmfg`f_sJx#fD`9s7TIt9JqIGRJ3 z_@tyFN+<*niBgRj_YEhzb-S;9$kik!+F#T!5Q!JP+4 zGG9VSor)#V^@>HY3P$O=(@=!%qmv38I*$JiBnz`dFtc~Lnb_ew|MAphUt@`I=Y;Em!@a(ubDi^; zm-qPN;i^ZKQb7+XIyE~j}vR=t^AVO00$gqO-}in%CT#fgfZ#QmM- zBav`V&9CZdRy#$HH6`q%Ud9lP1mMxWES!PcLa_kH>dBbV0xa(nIG=-Amm|^3K31Zf ziW5B3zv?Dae`@bhmfB9fu@x!ADtmH|ou15Hus&T~1uB4CbE>qMh=X>)k8`Ixow}Ox z-~RluhomUPirYCjm150&^no;6dRt6gb(k&<&6HhcE*h|QeICP^ahJp&ZuM5EFPW*x ze>pNPe!j%^JQRZFz3E4kvv6Td)I8HmyA&dj+bG3=m212W2US?B$rlr!UtcTPKd-PI z&K5H>{I(M*p2=px4!JTvo1~k`R^oj$z0HkB^8qdHhK+vFpYOsKwh=t+jF}2Yj-p`hv1k6^!!6q z6s=X0l#&Wf#(cWYWV$!A_33YO*~8Udr~!|+jNRTC#i$$RC!Q{uGECA>Z`ac@*T9Yx zpIpx$RiEOEkV{#w(6LWM(!K~dijp@UC0(UM9bP6dklLUM=%qs z)ro-5)bU(no0JqLio$yx^$VvpIY*4p#V-IESLhfNy1+(MeZ<06-uw}UHLgblsXOT_ zwwKO&ySwNNkiI8;+zpa)yx0;ooy_rHFczi4`}Qjeuh2+X(j#mfX!=nxsDqU3kw@`D zYsE70Fl`M6uvd}=ev}=P(O2bQ02zOaNhUU*QcQC_0sz;~N#Gj>2TS|e7}9^n=XjUJ zC|8p>qKJX>Sh&Z+7ytV{u!P^}ZMI+*%EWsBMy6hW_d?`}7#v=J|X>+&j0 z-NaQF^wtV8$7z!&P08gXc#-2vSxvKbk0%SMhx)XC4O=;yJ+jd%+Ae+jO+P7EZyqgV z_0}MMg8gy<_5l09{vKklbpX()I!8XP;kUm(^>hz z?>5F&l4NV;G=}4O2+Ma7YY<<}f zQTPN!!HTB8Zl$fR@zdmAQ!Z89PySS9!pe>nj10Vzi5bxr7IUCfZCY-nrHaIlOnT6H zISL3+V|@~$g)swd^l?qQOiV|6bLLdTzK9tN#*rcVl(+MR`uiF}ko0p@Z)2aiyeYyVYCC zT4LT|uFI%#W~dmqC0s)$q+{^&G^6aXa@IG+LwK;WU4D}>$YDL^1hIHuvBo(uekvmK zBhDT^W_rUhw}x^F#SsY_wH$O-N*Bg-3*gSD@UEm=9gHS(03u`=C%HLmzPcNsxZiwg zO7TVdPQ55lZ!ldT1v{(TjVAKR?@|Hyhr{k`miiSC?25aMf`Bl|tOma8ghJU*`DT&> zU{Tq!7T&EqeY^z6Jq$_c8~O+XrgLS*UsxVT{3xO;fMbYG#APdT2RtB$Nh{=WnMo@- zt#@l~4!|G2Tu|-D$8}D46{#XI0L&Vtm`>+$k7VD#9d7oA(X-u`_XUyLQ1Ijyj7ykxA`2cI(YUWzXNR#r8@*ANp^(E~UKGFX? zH6NzrRhsyFnteL7S{-TB`j|B5XiCIoi{-n4tWA+e0DQw&Qjnj*di6(QkAq5mJY_~I zRJeV|(5HllimWH58UXrL=yShfA$VurHexv>wD`zlvq5dixI5O`EQA@7$A*`!(AIb; zBPg!d^6ECD5Hua4SLbCnp@0luh zC-q%^w61rl#t0xu_zFhL>!nzt>Kb!1H!MBfy~`WBZ)OL7og$wsJ$-&$&sPM&Q1op~ zsM^{&=wk&s!Zz$k%e~(#Iq)`GuooF>RETthz--N5|B*)Y*8RNQvXbYP=L8enD#RFl zU~Vy4Jp7}$I;&P>3VZLW9F;KiQyjWX?870Yti5**Jf=S_XhP-I%N0YBD-M;AkP$O@ zp)d8`2*kco-AHkxqp3FS~=C`az$Tq^jTq7?$AdkuEXZv51$zHID-Yz>|Y!+ znOWD6@i=yRKp1MB{egUmRx3MTu8{;DCP$lOC)dsSyy||pYO2m)%3!Kbh7WX73MtIk zDrtOd@_nR%WnKS0;FvT?C|~77!%BhWF9I98fk{|6K4tynJDCQom|a!nbLVW}fTnIt93Sok;#=wZC8lCqh- zc$B%|*{@_7TEl#Sb|avAS}ESk%PV^f{w5WzRi2knn@=Wg`le}?!(u2UJK*1*MT|U; zA;QBxll-@@WeNUR%T&H7s#T7t2C%qCkopqq~3lR1jkV!O&*MCMNIuwy>7jaMMQ`< z5Xmz#JO~JT0rMcW-T7~tjtqMX9Msza{W{F4JK>`^)p%ph%lULcPm9U(r=6m`awy{l zTpqEv`m_0*_^O`h^nurJ)W)xom8$iJ6HBydD7==kYgXSqs)l>tQJ;Ud`wDY3f@(_Q zFrY844BBA|VY8tm@@SQLE!ZKw-#@{Ib}abK#Ly}yD1^X^ixy5nyxFRjo!&QF7vHOK zpC^KfU5-awSB~g$z0xMqwPf6HZ8;rW1$n?b!!{EM(QCBf%yIY(PNI|r~L`au{C zH_gF05(SQzkXoQRlx-Dg??M(>oj&aSu?^Sj&TY=)rlbx&!kS_|C>K}Hwaq=cuC$R7%^u&@0wN{0q| z(QQILN|l;gX+Ky>I3!CnxMvY0+KsNHS~XZhZLwwr%`h=OvG)(OkH zu&N->kjcRvUlnz_Ywh#SmzWkH=3q*-y^q8Zoalg?n+E3C(oP=U=Z~iAZGkOTk>P18 z=IYN^^oXDMDMpn_)x-OFzlL*vJ2OJL>i#iHt29)cg2bg_x9jw`#Qt$uFXiqd$AwVE7y_#sr+gE`<4G;9)79Ui0lPgkR&))499RRIJY4SRYnuUd)%QGWkHEi$K;s+>wAVSfsE+ue?`Dhio>nZvbxA@hBaKIUR(%B;57qx0LK+xNZmE8sj zgDtr^l`lDyBZ@ScfJLV~0W1nfe?FWvuKQggFTIX1b-?>4cjIP7^jX&Lx1P0Qm*%c@6Y?|-Ha!62`0Ohi+gNQ)P0*0*_R&!(c6t44!yFQfiAn39 zry0%6;&6_d>=~>MfDg}G-3szml%>??NXUrD{z8VC=jKtyXzc(-PU=Ky&^E zZ>rFV8e%dBU1TRHh4cbD%Sg?Gt6?N6^sW`F9I|4?icjCwzlJQ?4bonNVq3XBo$luEhfrm@opd!|V+)0d({?_+IuL5iZnWb1TOa2|xJ-f{{x+q>UdIN@b^SNfE| zPChw=E);N~w%B~=UK`>0@m1E8h}0_rr*>~|?{a~ezLMumZv|&hOQ7SVj72!N;5Xf; zziu}WEq^pvUw$=fL7+MBmS+NQzS&Hzg2Cf{_$F4BzfrBk@?t&n+DFJmhsDmd7-SuG zx1sj>N8@o9eNiLk$@{+VC<+5}Hzyiib(CmP@No#xbzo-{BEB>rse&r#3$WY6=}m8H z;@lkjRFN0(iGX$m%bPw=M*4#{G{!bD2C*~(Fka*WV<`q*^#ni{R+@rZTl{Lm4{VbK z0|Z>%b1>G5x$XCfH3huSulsOxWP1WWiY6Z%96&qi5tDD=Krj|u2TNO1?CtD`0(4IH zq$)Utmhl4YH#}7Xc&0CMCK7N~hn-{MiW`(W$FX^QZ zG?bGeAXCGBAOw)Ai_o{ax%D~MY-EQ2t`Oz`#t$ZyPyJpc};T07Xu)A>$@z|3H@zvNk4D;?UKeitI z&8z^aQ=dz56A4U2C+r^b%Di_QB5Age{7fTBBL(CR8i4JyOqw-u@w2#a`D;fOY7^j3 zcOQ^}^UB%+je(th)*71PdZdT?w>etka}rKQ3Li$%^ehEEF_-V*!C0?thogaMC9VoW zzd!STtyHpLw{T^eEnZeyZ5;SvevH8FDxPs7>5*4(b9L9|8OU~p z3E@&tmuyNnVe{M9oFU*gkOVGsTuL(OYs{3(76a3~VuP*IO1y*%+e+JDDYs1fMP{TJ z@$9XoxV9yS3b*PsHBcB}%R{1XpLeK1(j%l+({7B-BVi;Avt)yRiq1fyrzwY2tT-Ny?>~Bl_9Ps~jCZ0;(Z@ z*Yua8OtFK(;;)zghGvW3@Uw7oEm+xDqe;;#CU*9x?HM+?`^ZJFFp@BoO1JunsI~Sl z{a@lzVyMHE^lBy;2;eSlo_HIN>|N|BoSE_|3s*8DyNum1H22VDeRXV;4B-I zP=cx2SQ=iscuxhjc+Pkn!MtK}uJE73)-|+>=}N(+-`!Z4$76$CGQgvyU1vuP(XRmF zjqFtKThg#yxQf{g(sri$%$if^3yS`UME3RnrzPx)Xm8rvc5D%UeY)z{zbmI28dy%B z^W@7YxzdNnC5Hr7Re@d!2v*dey^sAQEKqp`1B)pjFSu;KsQ6GU)F^IxO9hIl+ zIh7}&@3S6@6Dqmx4Ay5X{1A1mwGbm1Rz26GPVb;at7BdrYFdHE>`LJQp>CtVCTemJa#VwZ$FuCQb=vR z4AQ)e=Q{R^w*ioE4KG55O8<52aHg2-7I3hl3;IFbqxk)LWPp88T=rFuBUC6D*#Hh2 z$5Dy8pLw@)B2o4;9gpQ2fRLqdSY;4zdyPVNBlUB&)?gT5m1rHug!8>U+tsU<-!Mu4 zscTrB<5PP4IBeL*rSc&}UM0;yR-u~p(c~;rK#TC79`j2@6v;l~g!eS-Fw`zku{! zGJLmxJ3^h)wL+X<7D*r78%<94M8TL+n&k)8#P35-i94@|a-%+TXoVqDj31UKt4_vi zan2~(w4z_)Yqed6wyZb*quV}HA@(gMr0@UX;qi%px7}-NO9p|?8h^6=q*jq`qWzQ! zQRU+J;f$L8+3Mze6)I5f?UszXD&Q7N1t=7WKE5;oDl~CBFuk+)luL2RJ0A3zs#BXL z+ET8}Ks`5p-MqH)`u(08;6o(!h)U%D|d-p2qHKrj#k65_dF(6 zk_!&^=zn;e(gz>^`Ap7~MZ3>RnHR^7{^Nw!|2O^Xz80o}rk{5vwf`y;nIg2az@qSA zwa+t1sN-b=!E|=U71Cj7u4x8`Qi2EpD`oVQoS*4zOe{GZ3QFW%w0pDdgw$PBgG9Z0Vii{rY~ZK!paI{ zv2ml*(Bk4+iHR({R9@Q3=i#x??4lwlOxK4hb+M-HqgBhQ7Aun;K%0sIH}uS0N)K0t zc(WPrM?oO16_i!Drb-f`#dAe+xKxp?q%?8C3i_@7E%uU80h7rmNGg0!^gBXaO!BCs z0Avl(!&phkA}@1C{gn@ajP~=!-ht$+c01<2R_5U&`{oPx^NRdz3HpDCUxxY^@Y~g< zcq_@>62SZ2pT@}%9PA1d%bmy~J z#DSq23j>0>5S+=U^}7;!5h`}pA8HLk;Dgr_Ta(2*TTdJ=2R)8&1D!r;54GwuE@Mho z3N#yL(UOMK~G3l$n^(OBX^PVGvv;S339C=^x;k zf~IJlbqMKC#q-pYk1kFn8wBCK1F!(7qKdKURf*wQKwTJA(wK;h)I=tNzdl4GB1*}Y zySzT^nE-GS$hNj4X(N-(4BKL@O${~Gi(v3WC!&ok`@vj!YLOWLyoW9bX~G-h3u?b3gP|qB%G{*`czXMp_E($gcx5eHULf;#r+2GX8wc>@q}KQ?+mPb9H3X& z7wa`xIfP@lflD5rb2YJOEeAHt_)F?(b}&g^U!dYy(^zFgmTHY7iCMRPi+O$!E6RCS zegO`~xhT0^?3g84*nWJ1Uf-oqo6SF3dp84^nV@q|laOatIdSi%I=qt~@sjs3j6b`9 zpd>#)+6+E|u1F*vJAhMo-yDPxnAuF)uBw>-!x?sNn)A*JIHDROD&pQ$hjVfyF4%Mp zyuz9{0G<^rnD<&V9s32%`CMX11r<~+|IDGI*JN!`8j z4B;z6w@n*&6qwC777tjR+>q2Z9u+0jcuB7lEe|Jn%HIizj;9SdGV2SzbulnZnBFG*w5~;w#cz&w>UZkWel&Ecm`|`6Bb0-pG*XuH~AjO|Y zXO6;CEn~bqnze|Cv5cpL?uwVv6q7kaayFA^ua>i-@bja$$>iGgqFtiT_nfQqZuGH- z^{Xhrk1KsqG9cMjBll2MhQ~yPuR41$Qp)9abCBB-L0w#e6}TUlo+_bYFn;ya<-zUw zf_^6_F+?5=?hiH6m-u3Z^UPj88@0r?FA)x@j?_n0 zZiGk1t@mXkmyS;{an{QYo@g>05S!0(C&s&WxvNvs%{8kyWiLUnnO7+Pl*Xed=(}t+ z+4`7RFw{m&Dh%daGOt56>iukI>{K8NzS4CnU|TVbX>{}2braYu5YJK26QR9*%`dIK zzMrvr_)YA3b0GN5TsAOdRdcJ__?&{x22vGh5~e}pN{?R-O!+2&JUQj5b8VxtY&k1*5_ znWR|NYJGoV=>ucfF6cuk?>9f&$GbBocxh|@o8#JXL@Xi-ra@KNbuP!_ zq#~*eEWxS|lZYWovUJtEyl~@bokAp?ULU+1y*@+2d@f4OlIn+C`cQ*oF7USiFhJ^$ zN#-e_;Gd}!p!FVMi&DP!ZRXVYFK4B^Nmviz1zG897)%%!kYvL^kz^Ja^W&lK3;MBl z5gaMuBO>(2rZgxtrB>0jhd3C=d3L^BBuC~=@rXrEpVmQ)Af)x4qU+Ub?q!QU6K`FymGaGS z5ViJzK{NoEi2xyC-wm{Tz{8JghFxL7c1Sr?5wHSKYk00kS$)W%HT)553NFpK25ycF zKQgLUK83hnW5DMT7YY@Bdit!;iD0VUi%A25dKqSQiF$Z+e|%ZD!+^s$lW>S z#1D4d~0qhfyNeBDb?67Cy(PT6{k@sB*@`2{s4AZ2!!mez~E6Mt!OJki=h{_%{c zCs5^JU>e?PuZTP`9aG9QGCo&V<6xwfEMv^s>d`Ur$0Ja=-RE8+w3N>Z9M0K)jZ{#P za>#r>t%Pp$UUklY4y^o$fwGJb{Q3SNlm-l|5AMP7M^!yPh~BZ2l{ycryN~F*hX}ua z?}CY>+OmC{q}3Gd9h=<&qWjIUx{XRxrS?af~1KXEjjD_uxx^!+fFylI&2yXag%N8 z?CuP)in!~aWLq_m`5{ZA+06}&C?+yqtg4WpDuBS34Exi`69U`PQ|I`$Qa#V9W+eWS zddsOC8eYB)QB(EObl1_9{*v&(UjUZ7{cnddqcQ5%Dj|CQ;tD&%*C-;wgW$q~PSNFlU#xLX>3p72F|87#QW{&+>LjGU>|fxh(S z(ML<9+krrH;O(qSTUmvbgGr3u13s9m=L}wjnl!o+zqz?Py;y;*4M!1KUXPEiaA&9R z>N`s->y9Yujp^LZktcr1-m+p-Od~cYFa={*U#1g|e<1Q~43}t}{ngh!Ia3v3^#~!1?TMT zmDv>yF;Mh7aKWIiZ0o1dzJvMc1isnv$_@*<(@yMR?fW^nMxRHE$)!&(t{Vgpr-KtJ zkBsBGNGEJ;mby@9rJujcKFjq-v|AV~?;ciWt}VGPPC3RITV(p2E3VkVIP^Kg6iXlw zUv8j?2vcX7AaoKLlAvM>%71q_jQfnEeyI+ES|&CVT11$~e5uxBtW5CetVcZIU~d@R zX|#`^yQzV}DW7-GN8Z697mRwDpw#?lLpVf?R#XHJ%-Q}K>EEu5xox?_)S+P}E?s1T zP@1u86jOeg1xrUgth?)xS@75TAI^OF%SaQ(x&yIM@n`xTf=Ni%#A+ThS?!`u1#E~} zG~CoOm#4jIeAUx&8*HTU{bB#&0TaXR_{S}+z1Z8%am)d_XXlgu=^1yx5_{D%#McmG zAAxQ=#}bef-aV2AG503PHH5Z=-kBHvdHtX&w+D9|ddzf@WfscK;Zg{dM_sg*Z06B3 zWOZip?T-IGbfa2e<+G$GAdYX>cB(P!lfDh8Pg|Upvwh?P4KMr<{vN(%v$VWn)Td~m z{OlOiO^RM*DNI{!$*3)HHTcd+r{lXw^;m1gN;_ly4U538kt9hm@KMRWYcT{;=1kf_4R_D%A7} zAB#W4KU@c%)cFMOSlue|999U}FAi2fOCOvlOA)9<$bF5_=7e?O?y2Vs|6QF*Rw zx)#lcDo*!sUPwxf;!i$XMtM2;nm$`jRBfz&5TT-0blhK%<1f}~xUh_hN)no^4p#Kf zY=nGKO`lj+n6Ay1_g-yV`j(P^Hi9%sld0q-6L4C=RhCHTRtCzPPy%J)iXV8@}8-;5N;W?RFY2TB*7Ml`?e|7 zAyv?z;39-Vl~kjbb=I$hYrg1ijLSb4c>g5htr&RbmkAvgkchBtM5U}w|*JykXgY8+nk zuzkN^T(JH2g@~BiM`x?Me+cGgYwIJX8lW0&rm@%fTLPr)ELV>@)5*`Jr+pM^GHMAx z2_Xgm8=WcaY_I!S4Lv~5a3=EgjOE^8$ z^eRklMk9>KYRPW}>l#80D`^)1747vteuU_*Br4Ik%M?KjJ-t)>)$R#l$(=sLUP;x3 zj48c8@>rZ9)r^WHjw5IK#&4v~c3$kbpgb{Y@asHLJeBkSW8Z7e?^P4L**zn~BE3JY zQQitHk>}f+(x&U@*oW)8g32LP zWxs!h?ALkx#jEx~sPG=v+>o#>YxAyICg_V}WVExe)Ai3HhNd71h}6#H^gAUgk~-=0 zRw}70KuLqoA37_eJVi{oJA|@#Yy26Hjr0PBz0RWx0rWSqt1rJ8Oz_#QhxeopFKUs z?p3#M{`Pq7(Z*w!!&63a?PHaGLw~pv6z~JHu&|hnjK$hh(0d>Ku$R2zo@q6+)6O<= z#cEt?!pGkU-^V0$iuPD+Ov+>el(ho{D~OiP6eto0t*WagIN|%u-m}I_8X{!jN;N2# z^%&U}q1@X#)3H9u`}Q%x(v5W%lHmjbzXR0Gcv^c$2S-U|P&g>JXRAV>tut23yuxPl zH8@NMEAS1gK0#;%OXDsU+wWguFS+UA(5bNh6BYZDw3255;pMB%B`7kE&CRTE*`n|$Nwq1MzOI* zGcm4?9*^AyA-}+Q($~qU004(&6H4ih-nT68mju}R3Q!2@0?`WZ0!O4q-&V(yoQgJU z5m?gT&@V9P5CiXeH*9NOH=2VQ}Svh3zoikmOH*HFWf(W5TR^77eVrL#`kdhaPy}=^(Ul+3n{lC>D&BE z=0yGdx00O~@<=7B+84QG(kq8#%;>5x^%)ECI|st?7b}k~hHF=zQaYp1YDA&D++gW! zz3A(`2v$t&*d7ck?!%ex&{MpL_>$TYgm;;3U6vu=JA1h8nFjzIsa~91S!4GCkZ`{4 zN*3K$A1roZ#$MB{uDO1_F|}zFjfY;rSW`Vu{ji(L673<>uv>XU8``HB* zNkKIMFP4$b)cJC|^45%kyra=#6{JnC<}t9*eXaPRa%DBAU?27js+asVXIndMpTfiVf)fP(BROgQ) z_$Hmdg=`*CuL-X}eCR6naC#J0>Yk~-G!0gNu z%9!k(FW$#|?7#GMKZ|rsQ{ZT5Z`&$h5lxLBB>yPwPTop{uR=_(c;tW z1@5>-Je?C#WLI6;6=v)+nbhZdMht~RoCx*>ovLFU#Gf!Rb5ds0NV!j3ar=H3uFuy- z>Mwf;prHUH5`x()=|YLzGyNS_Ctug(rDzcj{I%%8%cjnA66zLN=5(m5Qes!i#KrxKngA1?`1i0AGUdk4eM< zKB#ei8x4+qV!4vXi=u7}bSlG%yJh*jVvMz=s5+0)Pd&j}6^gf2o0%B)&&O<+H#UWP z8T)3~5hIgS2LtIyMc(U}%7;~DJWjhSwul|Km8P{a<1X8qdP^T0QYN+nBBV2ciJjDO zOa#8c;XAhkavD!c!v(@6Yiq7p4|{yn5yxD% zqPjEH4!n*o`#ZA0cKxGryi&Sw@J^CA}ug2cSTeE zI`L-OY3=$bd*y(O8Gp?hOJ6#5embsBj!CK^0Q_*mo=*<9j3Q#Os6+7_by$BITKA(_wsP!e?qZrUfX z$$-Uk7W_nlIOX(B887Zkw3Q?P`N@yuWZYwxyic>b^THJsj6bO~B~$mI`!I?ay}ry< z*Spw`or$$)*UneSuBH%2$0I{X6F~3iLX|f5{KJup-2LNc#$czE!TFDh+m$j6;iusB zg$5Y`$lj-Av|0h@*T`vTT-5I9KRGeMSC?i483@03 zBt1rKV^1p3Wb^W^_iB7R^Enc)#x@&y32njN5I4P<5af`oT0!xJ>z1UiA1jJGn|%K5 zmI4f0QbAF=_)QI%gN@(t!KX4<2bx-D@|CA{gMEE9N3yj661k^CR5FrAfqeWa*7-fk|b zn`F3C_U-vN;~a^ahu0(Ip${5#?kH!-2iw#$Jj8%efDdn};YyXnYLt$DqHp3euvjBu z$asiq2xE!=zpL}!IKq*7S+y6cVd0#{XaGJoF3fRIrHYhbNzGvqjzN&3?7NMN@IW`# z)H2vnNR|DL*>t)sB0v+1e~7Bh_19juu6DUnEcPF+ZgXEIvGhD5$G`an-e(%mx>&Xj ztd*GB#Y;ysdbBY2PV#patAnFqxqslkMn_YE6`gMKwRv~X1IHk5kTg!zW6#zt{v{oE znUj3tTl-t=HlJ*Io7VQZ1)Vps;L}UYkVpZ=jRk;;wdGN46r{$b3E2R=iypeG6<*$A z?R0?)>zvHDXaAAoDdxnLm6gH90+h-UCys{=$ed|!_opJYtKFFfcQ`9!)^ETA@J`tsxqL0-?n_=KqYL_3VU+HdoQ$Z*nyl&H@FGNBQdR>anu*5QhB z{_OeFr~hH=Ed#2IzI9)^I~74fX$c8|MR#{eD?WhoTF1ejEloD6=X!V*&bVb<-ft^8)D+n70- zrSimb@`{@`!SB84iw0Sdy>wK^HG5LJDuMedMTZZ3)#p{~ZA0nsEy<@qOiVXkh1PcM zL4eTgFIytJS#A#H*kDBN_?+A2l&{O*mKi2S05{1jqohJBr zg=+;ZLBG43scTX}eWC+!7;oCgM(zWZGCGu&`MGj%jVR0~=3<8REHA1vi+`WgDkO6~ z3HIba2S%#`fzadv|bJh|Bjjl%_NZ*B$sk0bWKE~}ys z|CGO=j7ZbH-C%k)lSYGPuJstbJ)d_TRs))6aep6O3D@6Y&~f~0m-_BMW+W`parh9Q ziuC*2CkmR6^S-jEw3o)y@7T12nQYOT?@5O`FXU9kY%c8bn~2u+d)d_7+xT^Su3R z-k=ys2iiWD(;V>bqo-5lh`Y#Q*J5V>lLeG&>qMQK&9=)6WkNfHo5Rre)XD@9tgKqg z-OFCI-*l$2%Gf+%az1I1tc9HGoeE(0pkXHWi0Wbl;tznFLOWGsi`Y^*>>9_yZ~sij z^6VY_E3@+$;iF~P8d?q^9mnJ(bnmfOg$`tt-K3$lZhTcI!zJ0t5v0Eswr!=38SNtU1c;w|PZjY!ac;7PV zTPS6M3w3yRUPf76^-(yQSI>LMHIAPb&#<<43NX{S+sM?%j&}xU%rQIgQ<9QKSj)e0 zitg6%s@0cZIvsF8+N_hc-%lAYrgA;tC=_nRJ)VPMA*xMtbfDFQTw@D$4^7F1+= ze(ZfaP93`G61RANO&mrSKHZ(LTxm>P)GjXLDo|Q&Lu{V8J-AL1fv8Ag&-9(AT(~J* zGvu4DewoRn4DBzpur%qXg(XSBv@~ZUZ2}4UCaoUc!1Grm$G5JtSySNxL!%})2csYd zm-3;CdbI$Wk9{?*t#3*TQ?%scfRSA6m6KVDFC5+i+?Fuqs1@jR*~g+nuXZPp**4PI zm2J^{UBX8Wqa@%tmb8i)x;|CDM^%fb;PwMsbfQJP)ctb$P+C$K(0Q_uo`-{uz@R!` z4cLBO?&wIm<*ndTvWi8)XwXE!vSjFWHrj2jU1Y_sLmm5B8LazPqHvyM(fjgN z?3vf?bDn=%kawe3XNi1)_D4Og0radK@abvm{(G_dl8UMF1Mu^JjzcJ#U5KA78SPZX zblgItm(sl1EE*$tP*kE2O-lphx(2ry-TxUB#*y9CS;k!NsbIQ>o9sVps&pqcft_2wfo#~czOwnCzs7xNxXo(uG9XqKG2uOp-~r9>E?aiWJM9#kYM^kAj)*0gH^|`| z7FapIW8-`mmyn)wyELJ?yx_Mr_`ZO~4@%5{*AvUH+=PrGcDB=ps*MG1j~+I$ds%5W z{#oHg4`p%~z*Wn@lEhCwU(p5Gv+57*`kaQ|$-XGRqz<8gVk@Oza)P$-{Y1p>m44fv z6b`zuaD)u}_h|BC6sDAVtrW7Uum>Szt%iYx1u`vQ-19@IS}#9d|98a!!=L_ybI00& zIAXYi)8=1;HtMtP8X5$^?N2jRWWt#IUW~tqsYI$Mz zT{R>rbMyoSx;Iaa33QX~Nm4f8L0FAidRR*^r{G4W31<&X4X5A&$ti!ZIq!X2FyOZ9 zanuKYM{sM=rl&Og@O9sdCUkk#?0m}=07ne$^|i{byWieDsOM&7bp?mD|JQu9RObe1 z)4ks?rTW*KyNJ4nM`tZ2_X!OGw{-kpGLJzE+~38-7t*xEYZG7rH`XhT|uftgva*N$DxzpOs&wFfzOn=voPuw#s z(|UcRR6zTOWacwpAlYq)Ooh_9;L=>vFImI>66hdT%Sl>a&a?@TEO&7WM}!2rQg%3S z$BNN#%#YBQ7B=q}xp}!*sj#Vh5IeuTta9}R?zjc7RGtE#x|rvA#|(t0HVbTFe^6~| z0oMPh+y0|k`buQg{9{;=c|2Ya^Y1-4J-Gc`N}2Hz%1E$X;@&B18YGtlFSPoAD(kTD z|M2m-jr;!oJpElpriw(f$)DE~99dGJ0WIWKHf{TQ<{wtSoh+Z3_*S;{=64yLrOFGA z)v`l)%ZUu2S;f)sb=_TPKuJrhF!e*e$e|*=M-k0R8>S4taO0Q`5sv<1Ryq?^A4;L{ z^ILT(|2IQ7>^D@I8l_sAp1h;q7L#xPlzb1g$p7@%hazO#iQNZC4XosvhhF6ph2e)> z?mmcw5QKs=R4r(UZf}JOA5U}!#b4I?-+7JF&!!Euz#b737lSROaQ+-Ho#ge=2+Vm! z_mO>f?@&!-g)`+XembftDE!7s^JCv#s@gAD09Q&%it%GBd&-~&;Ov!cE9lNEVfIHO zWqbSfe-Hp>Wa6gHb@Nq}OSc76g@e+Zhs`HNXhUSiev9Cr89^`cg;l1R`h*P^<-lEA z0UXUg(3SK%Omrxy0no#pTQl(YW!dSJWU_twq|_80z^|^X?1CSKZf^p=d)#+*Fjy*u zO=j0m(HHy6qzTuuoiv3@4W1_J71B1-fQi522Zl z(81Sz&#=9aQ?=qR=%iNI6MNkegoCB`2C2W41{t&KPX3I97(5i!?8Bx~Dq-$D6>k_c zvm0zO3R;|OS-wvFYOu}BD$#WF;Ld)#yo&EMFsy8n1J%1+ijHE)3bJgp1Xj~~wMtZg z)NPf@mQ-MD^B9<>M?Nv2iqz>=NxBV|ZCC9=?zJU^iB>T&_m=;G!LbhTgZCoJFrQKl*;=HnwaL)ZzV{`Oc^v8loz z^RA=~4*`k6^QBmwS>(g}E*=_npt@9`HhJXLuM)a_pA@XpL6)IIX+l=`N-e#QHO8E`;LWdf{j|36rMg@$n3G__7d+_|A4g9s43k47@*4N;=@C~U?bsJxJ<8Q^3q@n zQ4uHOwTPaOI$Ul~qj^WeK*g>?O8{`6I?^$I=k6)DNR7M}w2(Pws4B9{z$;WZeLYOx zU;1Z#A9<6gN8mLU{dTQF-`X2_S>dF}Z6wrY0iDnlt^7SD3G=FBYrvjD~e-(pZ}zHAX)t z;CV`=`Atn@(fKGmg`j_}fdQ~#^aT=?CZiX}3vz>mgpgBZ)d z{&Z1>KUqE=^O==b)sS=qHYB*k$>m7;jVZL0uj1v8Qx?A-|KHY7|ln9MnO zQlwta9hsl;;B#wOmkEZ1!;K*3fl zIy}3JL?&*C{SlU^>m8MD>~f~uZkVNTd%cwn-nDU9SGhY<)N%aZvhRhlYQe`ht@5+i#B%$2*p0c2BDsn@Gha$w!oZC-;gg*>S% z&pm)&P(}m7CKvcz_J~Au{T}qSi{+D$s)EJT={X*y879$R43U0m9Y={m1Ab?f?+W37 zGV&^Sh7e--B}Y9P-TK_lTd>QJdGItNXa;=QuKDicB{sn9MN!;}2Z+ZFc)sXc9rFs0vQWk&$>)fiF!cmKmxa zFyERNw_B;6W?sK{z!yc#fRYH=Pq0t~GKdzkhQLvSTSFW9e&@J(VwA*H27W3%c6O>lxNv==Y4#7TGqhoWZ-OKy08U@LrqX9c?s4^mP+*|&VN zl-pdPOSqwPpk{%D94rwEs<+ijBAcl+J9q!_mvseGgaJ@fOT6;`>>JQ zNc%5IFVO2?}}twHdIB~C3v9ii)y{r0-c-Q_aXaq>iEN`f@)Aqzp<+!a?x?77RFZu!XYN6?EwR+Ql%8>PLXPNu>DgrdL174jA=$@3w_HpQ-L=zpr3(IVR#loWx$R5%$q3RzHm1={(Mvv6cVL@ zfpXsgoPj8*`IQ+tFw+u78PRkJ42=3q>2PK!1K#9nYJRxxrAITPZlHif(F))fDGn|! z-u7#lnEAtyXvvEK)I0?Z&|w}KRxV8;2@Va5$fsCr2$3~yGEx00CZ%>A*eYGX%U6Am zLPuH`3yGHF>JY{+lSV95wo)Uo;IR*}Rpzjde|G2{Iy1Hn^p*wHam(*@8NM~(Dl zv$c6joet)Y41Tk6Y2B1B4-5|jo$KE&5E6%Jthu%R9?WdBc-vyv%xF|Y3k!lSTF0zQ zAVJtD>ZK&gp+KrjzrCD=QTqFC*_DSuF-KiOG@30Q0pn(<$uZ zIkE=XA_etS(VWj;uekm#Tr!ZIJ$3qwUvmPp#N>lNbD1VAkq99r3rR3A><<97clbst zNJ_^}EwYYlD~-Q)O_4|YxUOTU(?mtX{TzIQRaCfcsF{_K$adKE_HgIC5#jqAJtV&q zG~U&$OZQU53S_c06}nh>={847{x?c2cOHE%7a0tMSFs8IZ)3&(E*#|h{oAvC_xr+; z1OtM}hkErESjf{CwKOih@+|;n7o(X(dL_da?9!q4_mc|q1mmY`UJAbFY^zeqT4{!& zm?P$hi}M=TpLLhpT$!S(kdW|WftYno48qsh+AnJfo6J)UPRG0vk58bY2i*{*#d&UI zAp(z<)^XUnpya=JbS&r*`<&OH>-zB%JHGeYsqDRtG=rHbKmKn$C(Auae_QK!?}zHW zkpT6@7%YbVZCVc80_N6Yz;Zpq*z3c7EK>tJAWfiC$_VRSzcC1PBK(CsGm!g$T9?yM z9#%ocgIo-RYq5eg;QcM7zklH~i$tMUf5mvXWLm%;+p1SUD%UCL;=i_s_cSKQBE&gB z)k?L$R9{*@U6cHla?NBT7UphB4o!Exaw<>t?6_#KQZrr65G1mHa?hs+LB8Z?p_jc1 zI6aQ{uu#d=A~{P!mm`5M;`_=$O8B$Z@RgjnZDrfB_=?dvkIlSW;{NP;_Q@`3w3EVB zU+^3HSnaFUf#LGn;`jS!oAkojMKDC3>r$1DMIjSSw8g=3^pIWHc+8E)G~a4!d9i#U z&)*WYqJ-)TDX}g-kn7ftSad-|usW2kTapo2T!6NLsrk``E+LXa=hRjuRRVxNcI~y+ zKQPdu<_S|EyXQgGqpk}V{C=lSI08)(3U==Wc@9WblkR}_<{_~HriRT!-33a|==~Yb zng3Mg<6=OG%)!sEy@qVep#M4H)5M+}jQif@2fIPoi0Ot5@&5cw63Wtyqjk5qV8LS6@{#6rf`GpuELCU&vDRyZB>C1Fc zB}=w(%*$}m*optAgoXtG6zLIp|DFJJ++2FY=SXd`ZK(WX=UR3vi%&AokLvRtD_T%s z;E(*;-K63_bw3`+rzexwRN;euaV@=)jDm-8Tf)DQQn0rblXnWtH0=8 zGTUo?qc(0gZZYj|@RZSTI%%5{CdK6CBNAI0rvgDNHZLssw($*9@&-jB_AV|2GE_aw z%NIT!eAi9y8(ln-!a8v%9ZG9!f7qC>xOks5pw|={7;4I4J~#V!d1h{0phYhqiy37T zf!%E;m_V;HWuM%UCZIP;wvOY)d3o_GFB?Wf<#wB@xe{+9r@5RIkh-J1t|G~&T_Bp{ zvYDErEFO2c#6(_95R^-d9MZSZ+8M>AmHFDw$&%D7R?r?1atX6;^}m{gP%*GGpY58q4W z@!a%$HnF^QfNdwDcr=KRaM%>wjb5Us-hY$thO2C&+-p5uQ?@|B^2L))%c}XJB?cwR z1_M^-aU#I<0mO1%+u)ohyGhDZ%Ifps-tp&T37w98QPX@tw7pFXNA2~*dM>2}E;cuP z5r~=*8N!Z-is;~Z`_%CmHg9YsnMKH7QmB~8+R)fT$yL7pkxwA4u ztj)@DACKOg^<{^o1u0)}b?MGJl`!`2%R-8~Nmq7s%n2D)4 z>xy3~6q1V)=%_c*t|&^sRM~pnKVG9~uc2$hc8^KpIRtQ8D%G&eOS9CW9-=NG7@$4f z+&ie8XyIcHhZfroUU-?>&i3(poH8PCs@;U7?s)MX?aY?%xALOA@W{URYUa-HZcThp zU!nD_qwRuQJwsJPCFYk0Wn{=)9@@8vsu`6b1KtyNGJ@B}#q#+(xw<0tzE<$l2sRe* zx1UuS%huV4RJf&I{V+jig28+o?d@Z*ajie#I<;HDtAEH^aV+>;y4SEHu&d~(np8xh z!ZJRGzyzC`qe0oBUKF^9sbrwd^_2pwrCl z-=dC%*n99tq&xWc-z)_lf44WfZkM;;;xnp`4(V(T27U>hVdv*n%#yeq7TpX?OepjJ zkR3;d$ll^|(}uV4d!FLa(xqn)z#|rU+#D~Mr1HaCo^|CI4-F3&hL;KKPq2hbZ_Z!H z2$09|mgN#>T7xKCgFM^XJ{vEkegB^9%MqC+?|m>m?DgAxD@W;3)VfN_1De1E>>(NBOrCiYQW$#U;C80Jcozf<$w8$x_n^-5y3 zVuk1}nS`O?+tBnMOfb{i&_ljhnOA1}-Pw*o*O&>*pOGP^iqJ2C1mne>I3P|NbWnbN zKD3L{rDhY4g7a$YCtlrxlOg*NGkJPTc$0RzI|EFNh(heMClj1YnMB0U#ix z)W`cOoelDVH#b*9u~yy9Q6k>VD1L-(jWDrpekj}c`QWts+hU>V0gE_(v4+WBQ1Vt2 zs;KSwrUxpH@0{2lac=n=4FKmn^%KA|n*E0O)Lja_a7|A)5- z6@FlX)S<3^7#b6WN+}Npa~*x#?lslYMv@^QA(7%V|BFYhhdqA}KxYa7DCQ)<8?;<% zvZt8GvYO}rg*@e<^b*K%nQn}0GS>X8ulJ4d-eKy-^oC#&q1AK!6VvLmF@??AgakEwM%`__4hl;#5g0_$p8Pd{WudJv;yGIUB${MW@muc^HLnAHT$*T-|jTajoYy|@x5wUgrHc5i!d|{ zMA&6fd3izHM}ZO$e^?OK8>wu49b2WYeq5Yz4QJEE#d$}I&Mywm&Q_DJ^#|XzaL&*B zz1WuR6aC#+np0w}e-A@SV?PknB9HykN5)F}B#R8WT+BaHlmDsY8y&s&T|EECLxW9x zT!N%YUj)68@M>T1C%3oR#9^WG4Da5BK~lN5`rSwpv^d?rN=728d)jRmzZo zU^=y9h(}m;Jb6SU4(L~h_F;g(UB@?q=lmvUhh-Fhz}QII82(^v7U9ZB(JCB!SJ7&H zsn(e%ealA4>jjfEe$|o4$e*724+Ko>Anh+A-}e8C#Mz1oe&A+_qjq+&mFAD*Z^Xu{ zk@>g68iuNhzP8a_{_TVFs=oy*e=0d>7p|6X!*0ZB#X*2hNBm{#%bDTo6ebt46|V zvSVOpgiEHbY+CND1LO?df6-uoIpL?$OBufyda)bP)~?d562*IknZzu0Gc#hFd8hd2 z#YD0Co|DCU5Sj3}e}NKN&r|XFlX%OlE;58`rfA2T1ul1R?~cp)v$p#~N-20GPpOmy zrb5EN#IzyQVs(D~sdB7C9P*tLGtB)FaJ=eSoA~E!?CRR|Ju+Y4WnpApW*yuwvJ21% z2W-QL0gp@i!(3Cn<#0Iu9>5msbiLd1@%F5xbjp&D1&)6d@RChr(JRn&U}9j%0?<$p z*49G~4$}hwuLA`*{;){?V_=+Ah?Ke)Fw#gXB~khAckMhzOGl^nLAu(0t7^cc@6`$C za^nw5OrEWf8-|s_M7f|0b@J{xM_VO63{1>DH0iTWo-P%#KV7717FOSbf3OY30X{sm zw6q$dL&YkA-8ZkI%|v|3w)>(5r=5Q|Q;WXd2OC)$Pvua!!cwtVaQO5r90H^V; z4fiE?TF*oR^^A>j>1)-aoG5)j%J2w3-TA($nPMEB=O9J+kdzxYtB89a4xn$vINRH; zN$f)(SdTodl@I4%n23pbLc4W=;P3b#$4j2)@EHNza=u^6$VVI3a3ZYOYsS5GWG>$c zffTpo1k`Xs8AH)u{!n%A>#4b&mSSdqX=!WwiGdl$z(^b6=_PQtNh&uJZO0Kk?z-H5 z^2%RH2!X?%G*=L0d66guZhQ}`i#Sr%&=-a1SbmjR#hafj$%-qpBrB^Tf+6*PwzNb> zRJuAOsy7dLl3QqWVM`z(ose)gcdb(IUdppKX8+zluA!>RsO?HoRGW$0-0}(QAs})# zeMz5*;ls{EzE)YzTK_1I+iS0X`%)68H?L+vkI&5^IpDc{YuZC&e6Wflzg|wH5OHGg z@;K>D59^P=DWlY`Tyhq`53|D>l6Q9KxpnuE0%gM3j#HU+nU99a6Ee3V@ z3-BJM()*x#2BZd8`xj>sq6bnjAj2ztz8t#ZDpIccSX+FsIDnwy0lB*6S^BCXkGmoe zX`Aw~EkbUxA^7P&3R1TzB5>!{I{8SC&0i0k3u0ovIy3AbAz^Kv} zJrU|X0l1v9I(<*fHi2cn5A>xTFiZKm_b{TdlAa4;Jr=o7yA^{|rKN2Pr5GUY-?8_* zEa;%@T&)&*2L}hs#T~`1jeRLnBM!-zPQ}lmZDGxvJZbQS0?t%{!6=RoS=8|^9LHSS zTU+W9NqEkmnrqza-z5bzY6<=e@UJAlO`I^}s_2?i2co!5Xp-WN>p;P3$dJ^9yfcFQ zEa+zCi!48G27Y#rurNQBDkF?5Lo=cMLtaVvxe1jzKC|BmKe@afZUKPUsrSW@zPZW% zir5;`DJ3TrU78iQW6K+tD$gTyT@I5^bG(rRALWxpm!Gxgi~{ikjbv`v%Lnyyr}-;? z4!g3FKXH$m{$0v1&ri*x1~LHq3b*}w!Mf%RqI(M9_@RQ_@^rnpH^Q@>V5b@dUl*N{ zB3t?Uo@v=7v*r`Y{*~yVo{X|`q3q-GKa)8B^R?y7#6-G8WE2#u11IcHR`i@l*=5QC zXGrZY&ktKCC(1lac8IvI_7nh4Ope}<_JqDj3hVKiT&aj0z-fk{;tHUYLY2zgD_$LS z{@#@{S(&=C!`Xo}KesO|3f@deQ%*eIUG!c!!p5ne0%vqUHVa(Y8P2l?E+J1~trS7a zL&&diY{A`~lIn89_SCmNkM>#)sAletkw8Q|xy7hHkdV#mNN~JPfTNiNfr#H7dSYUt zlSE?hS$gsc4h;?Is8Xv2l%L+5{Jm)^G~9mupEzXRamgIatS}SA9KQ%$YFi(mcFhFi z=v2GuJkJFD&!@*WQ%7Z?awg-l?v33OkX{O%#YS7R>AF$iiL+r{bR=1PCI1m?`;3Ps z4iXIi{2Sqw1h2aHM3`NHcWlvQB?M%*D%IwxG;H(MeXC#XrrTd4y$C{c&g6zuU_%j#zqLWG@GjSI(

    @CYtdst@bF! z1e0%Rk@cv8u#eUFm4^CDH4==7siqX+lF4c5lc0xwtb-hb82&CjJw_SFRQt&@!Sby6 z?ks^#W$;Panl9Kw3OKuiN5w`v?As-gA%ELx6mQf;3kyD3w&ih)6iytfT$!L+I}&E_ z6)}p_zNh8(pO!5xj@C@L2t61`X?1uMcuiGTlA9&s(FFgAuMV%Refc^Jz%4ULSajbX25rmSa+@Apqd>Tt@Ldba zj~1H*cK&AK?tU3{8$N=r9P_oAoc^{q_n`(RZ1N4;-9zl6fKbKBqEpi|w-Y+^#)F>B zV-$PZ51Sb7#J%kzCaZ!zX~lID`|PbKpZ(}Rg1>Jr&AWHaJ}b&~j$c~=OD#DW89d)Z z!2OFc3DTlpFVb4~FW%Ve9Q&M+f{5b8+G#-0SxnnRiuG?&GZa>&^8QM$dq0*OgojHL zG5DLBCC7h#O&w}34k|7hpcPTCuO~LvrkD1?(Yrx0V!oKlRuJ@t9vxVA?|9rT4fQg) zG*%FCAErZ5Fj?S12F|?+m(x+q40I!j$?7!LC0RPq`?xU0*S!vZ&Ba5N_LfBW+%^Jc zvQ_o;q@(*Pc6~k_O8n`sDrk@bV=db4))wOk=o^9AyF1~7C5Zi%ZliM%UuQFix|(7n zww_zq&G#$@2L-9|5a!bYHR()_d3hBQjO&9HQTYNm`Z{xfP$eTup29$gF;)(XyV!g;1!;A`7 zM2dwU`RoH+Y)y5;I{vo@hD=(e3G~B1HAN1CkrduIbO>$^ZccgIKoU7$YQ{K@J{q{x zMq|&$?_kl|mi`^&eaU`z_N#gVBhL#PLY&cVjDi_=H($K7iG+&Gu5mc={x^?rmjI4U zd|!4L=&Y8A2sR8C)#`e3F#IBh3dzC(BQhi9*I*3a79C&isB7;7l#VhAoi+a-Jdk>Q3Ob!Ysw_cV(w0`4xy5q06+ zimwnMaJ`B9V3L^}mOoDH*51L0sY74$8QN|H1(pJg>ELK-ixb~4!b#xizPe)O(OUe# zQo43Q_srU4?=_r?Q;XQ2NL+Yd#o4dY#fNk%WB%1`I>(rD^QwR>+^mhW@QYpocax1S?hbPlwsI=GMmF^<#f5#Mi)QnoIm_ zZ8Qmt`U(`l+!d zTO$Qs1x$7)4=OuYSK`q4^4Q)}taPBgYfM0`r_O#0zJ#g5OLLFJ1o)FjI!rlkY;G#5 zO$QSU$X;{)X6&1_|W{$8y zmio614L>%|lR15a(*5~%SsIl+T6Y%^VMuzf-IWpqb{(!i)je(}85IBy7Lwl)%y^mu zT-e1PMB9JB#TmEWn^%Fo#n9=K^YYdkM@ zS??crZE^^m_i|b#T~UDUfYa8`WMBlOI{V@F90%0R+dG_Wzb8tbVI|o?>ph7>mj$oZ z0L8-N<%-$S#%l>18(M!QmxJ`hFG7vAt{y7SMq4*4*N zRH03#h2IbCHiGezwE3D_C%x2+bXU1Uu>!eSP|-xJpgg{=z^@w4*vw*kbk^3)I>B|v z&V2>zM3v>!;R=5ml56TGd^M7j;#ezppF}2F9q;JQEGX4uf#7G%vbhMTPIx;P3{k0+ zf7cTYJ&Zv{H@f#Xl^p{S>p=s4;;`WSMFz&%#1nXCheHjqn-Mbc6vjeE|LrIsgbd2c z#;2x+X0zl}sl4+2OGp0QcYBQ&G5vkI8P;}6DwB@i2eyu1H!j3X$iCJkLjNIXV=?It26gRsdoRer<{cJcP6E~Vc=|5fiX%bq>{l|g)9X)ddiWiA$zYgZzcFAl_mBRga zC?ia110~2(H$Dem2Oj)q^ex_GMSQU;DJ{)zM>X7x#x3lrfPm&Feyvy5m;m?&F<`Qc z3z0Z=9Os{e<^8vRL0ugK^whjhWi#)Q`cIgf?&rc#wASe(FCm|~X05YYw3Hj$?8JP; zaB(TibBi)GBLa7|exum+Q{oFps9f*Foz`z82)U?UZ)xR2q^zTNaTepRLdYc5NV%Qu zTuF@5W@%tYV$T)|I_XWe-$X)l{3M?%44mgF3A!I&(8n411T$xjhUrMa8z4c|Xb--T z-%c3iW|m$iS?Z$RLbLWv+}k@Iq=?r0x8A*ip&mR*q6b|+#k7b{iLz_LaeR5L{p8*X z#PniRzbf18r3&)*7Zg`d$Z6m2vq7A)#HnsUAS!gPp6bvSdxO0AswH##`e{|-Jg3o^**UD?N*#yrhgz^42zy3~#?0<`b_j7#leq zJ5g z@LaW!wVL-0(YcIeldJhuR117){>?2yFK}RUmp(i%PYUjG;!ez3Lj8qq2^cejCnOaY zzoEs1s<{3Z7?)8jK?cPl1ENECsZ+W*%ko8+vN3qAEx2%E>g*h%Yi_#w)w!Ar7NFz} zzLrPNt^a7XIHh4^G#nXzr=en}c8Eb-Sd`87(RI9cs_a99mzx`j&9XhtcMP0V887$c zX)2%mEOowM#j-N7?at7p(Gp~tENHlO1wKW^$Lenk+=5Xp(nzYates#EI;oF9+QAgs z=HfOK&TNStY6>&%JR^We#UpL$v;7@P+AiY9ymylRKyp_LUHgRIK)l_>0y47 zI|(P3v(PyFr;^?$+b#CcW(0R}v)Yn-eKVY~z;+7e$ZGLyKJ*(iGc#_N7P-5r3r;wT z(63lR1r`#+1k*f7;OiK0kr{A|^_Z5wLN0)nT=#7SC{e~bP1LR#qz?4&G#Lw-G_Ouqj;a3=9x z&{NbwbiwRin0&0oYzFC_LEj`ny1-H^w2GRIyY+V#OL7XKN^NNxrpbAX7X0fk%%F_ve1;swvz_dW8X z^1(sX&G1AkxrLw1>Aohdd3g%FHa6e6vdiYpL=2BdH4L?wK!lL=bW=zFI`@ykG;eyc z1-e=pI_NckoHWn8QUw8&Z%#5UPic{Efx8UKK)J#xP_M2AUk>1tv0LCl?S3TLXta;$orOdt45DhZEBFy%eE^ZWM zgMVp}bXgq{Y-@d6e0YW;55|dXYN9Fc)dgYf8H>TVd!70R2baNPfOe#oO0?3IZj{gM z?88>gq!IefUDgTR_$pwLv$S0^$r`+&Ys`Vzk)HIf@n!21%2_0-f}=&bfGPR?`H3KW&D;1y=6P!AHw_Bk&eP*#>P3mo@+%% zvz3%i4I(m1#rif^n6BvAu&e?5Zfu@$z_)cNj{zW><}(V6W6vU;Vo?>yv$e--ov*?A z(7j3sh^_$NurL*=Ig~F1D6hrgk!CS3eCYKkXx+9qmMhvYX$q)}szB6(vN8nJE0m7#Rw9Y*-g(K*?n8y}`Am`usr&|f zl}fhQXJf3te}tM=0E>7M4B7i@sDZhXPPK3(swB%FI0nuk1YzVzTr=i;FLl>Q<2miP z^mK*BW#=@81;7wX+#eN}1WHX!t=qa+TR->fWk8gE(%tcp`H1NlTlh}R&(OKvvfF@54?Czkmi1$_gcpZW`LJt!W{J$ z=lUiK*Qjk~tvOv`QM%(uR|emE7j*LGBs$F{{hHEKytPShHiZb^JJ8v1Pr}lY>1gh$ zZ6*f3P?RY6Pwj?MY=azfpncu9Rpy26cM2$7W-vM+q<^*N<0mU=>aHWC9@!!PZx(>l z^-O>p6d3}(h%bn$)hMqXcZ$s+6m$5B)I>|jq!B1dTX!Jxfwo|)3b8v7prR;=!VBey z>kd|1HRW0Z5(&`6KwwgT`U!aM?(2o?p<)CnehK*te%t=qiz7waOG~Y?LiDi>b@I90 zgUDa$om2#YQg}{Ht;*X63OHof(;|%n^Y9}0zp8(jxSNrCm*$6Z-EmpqQu*v@5W?T)*q0C&oZkkryeFx~5G{ zUqdP`Dcjp5;^!^h}b8I=-gg%yFF?LXcMh*RSz9bb$T!<$htAO(J0yBrR7Nj^&F zzRM}%=Nd+cdOpF;&LF|x5SL$R)|{G7P56qs*bJn$@A^qX)B06|~OevJ$?Fb~J6Yo<;k+?sXN(u1ATz3({4%;diD)V(&_$K89SjY+CB9vU0yUCF5rq|0S9t2 zp0gSLZ7Qe0W=gF2CN%W2;47lXYh=h;`rMyLK`_BXO)l|TV1;{Cp4@jOK}n930vwH8 zbnG!iXRd?{_w1l|5r&Cxw)I9MWqCamH7DXDmt%Z1fm>F4Of5^@+?KI?9wHmKEdm$P zMuB0m4`Yq*8r2d9o*)>wtZaGNRjE>PJ|g+^*i3Kl$IpN+#7PSZkYvRiMsze^-x zomUWktW8QMKZGeqepuV1!$rvav0IU$JgoNx+dp8{`$Ewl(Ll%xwCoC1#7=y=+s(Y( zJ?Y^29Ar~fZYyRupw6@G^76#x-Gy{SW0DnDm(BPGaK=cp==_O{Gyq=`Hg^Mf)%^2~ z?D=5EP>gl7P3lY7m4?j8i#>(1?|hhz z)Qc||m$KlW*fl*zdJPQy$EXB~HMIeLcp)xAQ^HvYKLcvM&sF&y14s?`BkqXGU7#?1 z$Geioeb}c>z!Fr4*>ALNP1`09AXyv_2z z!t$P1FbDjOxJ(ayre|!1+s^CuUZg+pulBofg^6xT>hSkj;nYr9yPy2OA^P*%Mq|o0{FYV!d(ddNC%9k z3wkub;a0B#DLb=RcJ|xgJ()<~Y_|5>jW8r$0$;K}inOxsC-7Q`mlhw^4T7uE05!3e zq4oHB(g_X^=`+!F&ja2f;wZvr7r@^eO5w6Zz0%$Cuuv>r;yIq@R1lO)2;ze zIg(zK&v)bExahv}Hv&_Z(+CvP|DH~F5- z-9h|}N0uAE-lB|&qr>78em*{K)34xx;?_H+kFVG6MEX%Ba zh1Olze2htGkHVq26Y+o<9_pjpZ0W0FfXe4d#8GqA15%-fZ@90k-@~}R_xz%xw6^IS zNf+sTIX!c|e>SompKDSR3_efep!r1?vCkYY5^q}iQ%SCD3be?83^5)vMK{A_h;qWc zvB`pg@rTbsM^zIW``M_F|9J4b^~C%q@I$ADS?N)=VyjKXHg6Tw$S6W&NmQiAe!=a} zb0;$&T!-S;`n`D$T*?AkiGYR&Pw2q+A|&vF<9k2DJJ3f-a+Qaeo{ zaTd_gcXe!_DE@RS27x~@eK^>Kuy?2tKb81jEp!hMv%M8uE{>~5W2Pof2*k%ikePIr z^R8cy9<}{$gWQN&N_T1mw)(1MG_;N{Zs(~#G})>a9R2$Uw7|iDU>Wbn1Y0Qfw-_}b zb*fWu`ZD|gt?{~?a7nwJugX~!`mM)6W+>C=x;uPT=lpK6|4SXJxw3A)ulrW&%-}QpGx&sM>{fbv5E%P|QcziHH zv{1ePc5D@6$t%)uC?qOvTpOTP-qcs8I z#N_ z>A4xf2$IcGr{aH~BNbV9dxe*M^4YH@t4ZVX4(Zk{OcYNJH@@E~VWdn7*cT-^ag!cW zw){ZZwPDmy8)_6}K`7v`h!kUhz;F7os+^7g>^!v8;wgh`h^1`Zy@424pg3Rk=@BTs zRMz?$kxzbvTd{q++N_<*MB@p8mo5k*pLD5k-C>6l8%pEH9GK|T;`J|YGN9sVf`5{P zx$KYPG{`tdLzL0sb|GWWBJ`WJJv;}s+CO+Rm6QdOoWH>HKCPM*0vYe0ODcX*C+O_q z<4H=^9N&5rq(_KF$c7w-rASsLk#X2vK2pro06#w19n7eM9*TPvSz9?v% z`Ds}@w?G_+3f`obnjHK;JbeXQRPPt<4Ba8!pdcj((nEu^z|hhl-QC^NN+~fgbVzqd zceiwRhcpWJ`1{{`Kf#4$Qft*f4AA>nqh};B zMD#9+aHtm_H6ISwJu3TR#b8{W#>AR|?X~-Wi^#-}vY(9`?hL9P8 z0ZFn2t%4SjKi>9HIH2j0@&Bx36GQCKZ#0($QPXQkf%tGm|5Sb~AOH0U`T47^MY&(T z?~Z@+;+p!S{GRb~n`9G+6MW-!H7V08Tv|~YsOl*EZ@Fd}RTY&H&reLp=TNRMzhK?> zn@>G{I1Kz)S3Sa5f#WkSmP$=YneA0aQMaEIgt-!6`PG{wZW2Nifmj#5TiX0XpD;a9 zM=}HwMeer=vd=@0kU^O4nSvhdf6oG*6RW2IB^ZcNEVL1qm|d?(-@>vh0iNh;56uhN z#7{~A@rkeCoW1_(p)%LQY3N6kL%!$zBj3+$EfPA-b(3M}Jjh_a)TcgyZbYNr?l=lc zY6GU@+zCtL-+u{{NZx496%$)l5JSZoxhh63WBd2(lrnMYrZ;6JwB z=%HNsBtagrr^ier0BZKvW?il8s4?fK$E@*F0xxb7SBVg_w+aCTRg_ysJ>s~SHqiRr zG5BV?-L1-lgD!@~>_X*qbEfo-Q?3Hz)gDT4OHRC%i&)_|Q5|vg;wJ7|LsDVaU*Np6 zjG-U$R-K|A(C=qzrVa#f2Ld-TT`}MnHy?-5vqexHEu;Pkei0xCTm{VV>7?T6#+P;r zUt@NJ4s8Z$D42O$6G4?-e;|`Yi7%OjYO_rn+Nu?X*-a-70{-X=b>j5+3nLyZJhG>J znA+CTaJByQq7@S;^#Gkyt-nR-{PR%s!`H%=-PA8)h0JP&7(DUc2`q}>MM>V*Xbdtw zx{H0ezye-l2PW5*IJ+gLL0>pxF*yRAINFYOvbB+5JM%%)%C*ITl9eCMSeEm}W6aZg-{AV3=u zlRXpiAc`fMs#{6XVEZVE*~7W+^T?&}-ZR(YEM0<&U8J4T4tu=*V7X7f%JuTY>!udS zROxU8?~0g3q$)5<&%#F_AMEUm=!+HMa};9~3`xAdXXG!vKKhA$F89(kno-0>3B7*rW}#s@juNGWfBfA#10D`Q7RLJI71XB=2wfk@ z)<;!VQ`G@W&bo8EJCa7gb*W52(}1vwK;`|5p%fj<_E5eJGD-fIo9GHn^;6U}Iwy1tJiV%vB@q6AReiE1rSLibe9A&qK z-mKBS6oq#`qRGT-WSbN+-xgcaZu&l&KRx$10d*dV> zh_ncS;#&G{$~>^jiP_A z5(Du#LMA>=-7003_2T@IS!>>%*Ba6~)hhn0SoJoJ0Txm9h_jT2Nw~2HDo0#LEjmRG zf17O|GLatm(NWY7SoY#dhMR|EYJMp2U6(z7uVTA-^WL`D0%hlYLOD^vMJ9OJgCL{| z4IhmkZJc#oW}YJZ1nwV-1h*JX<3Wf*Kx1B6`^50v>4H1UFJ^r-S3y!@KLwF&2Zqeq zww@^9UPX&uiH=NAqvNyfLbX(B_fPwe>Vxb636VeE)KotSd^4|=4TFW@T`e;a$3?1?Dxyh#s$?7&uFPK+r zhe~WZ+mYePkFS&&qgvU7C>1#x^-!_M-y?^)T0VnWSVjuJ=JtOH5Sza1UF=Hd585)_ z+g8!@eMcqqRY7W}N69wFUw@@os{w@42KDmbc%ZYx*>`;>k~q}ZJw849wQIZtwl9cH z@WE*Nh)<>(ysp1 zd+OuHN_@6$-vqb7-FWR0&1y5y*wFSTz3aEJaraTGQ@qw%H%E`O$anEGV-2(KrFZQi z&c`Mv8$P$qi4wtM0+u=iu3ds;=v2BzHbf*u(?E#C{N%CNH$_pt2Ls)C^AARWQu5Z2 zvUx&9#>C;`g0Uz{yTkqF=Mgd?L^?HX6~WJ0-M1$`xT93H2BcC1rg87aQBo*`6O2B=GRUh`i`;2GGa7kcbd(wXRe06az^)YIgGm zS^(7U#0Sx*yI0uMLLUPS5)P?B>42~o4G+AIk#Ve=kZ~QTvoRC#N`Fy#5v6^~y;;f6 zN?5&)9#mX)Nmw5wNqxv3n6T<|CLG+XzJXbYZlgSNXhoPtjwW9|E^7VNV4KKf10+f222@wbYwz|1sgaWv z1ARrj?!EmlQd6~2SFocr(Q=82w-o{MY<69n2@J!b^2(~NzK!-RErm?>IAuBIPt;-@ zMJ59B)FE;^?Uz1l!&M3WR(ngyD#eUU3A0`i9Kr_@7HXG0fFx@?d~(_E?s^Q7`J)5pLOM`Q`P2vCQr@l zJO&_UwlcB3tvs`|k$`wS{r|gjkX(Z$Bq-;kzC64WikiTYO##8`?H1%0v{SkLr=(to z5xhX=xJ1P}-c!)5^iG38oiMhQH)94{i!E*)U-@L}`iTT9p(#4`Pb)C-(xr#L4|o3S zyxT;eSqKS5Hqfn&N^#(qHYhE>*<-x5%o@Z|xicqz$IO;>iNUi0^Y-Au8f68;N=l+E zr+=Z=<{uvOWdNn|d%+6RoaTZg_fzkbpdsI?CK!w&cC(Yvy?zK*C?97s#)~fZ2PfuH zuH^_0b{UwnfV;QVi7|$vv%Fz3&i@Q?UYCb-5tuonnjK%{ZXa(-sExVrT4DZ*L7&-i z{(Z?T9D+LHs7o*KBZ0q!#j8a}<$Wb;14#xGKhu{iimedZ5oDPB@DC<-I)x0WIjA9i zxc+GpFl8EhhyAUAXg~DE+}xb_WVt?BZ$-%RGHDPS^Vf(#EjF*csBHR8zKT_HEZX>N zZ(+=7lUH+25QP5^@aKP)a^Kbs>W*Na;1ZD>7A3Rb@FFLAE?S% zf9$}Nh6?aB6n=m_$bIu^_Ei8v(WEHZh<%XhUs{89EzJF>xgv84kU8bZF-pz#k&#`e-J@ zS7!QM8bkjzo6P63Xi||W#Q{vO!`DpyF%#&VKj>XyFFNEPC^G@;>N1vPeduFuOmzT` zjoE_8p_{U@vcOl8&SinqI!uRXY{7&<2gv3^|1Gc1{(^AF%>R*~oNP#vZ`CswdJp;o zCZb}N8}knRve@iq6XG_-dzV~4y*I;$65i1n_lQyn{qW%fq6XmsDH^O;sfDh9>XIewlC6fN;1BF{fab! zb&k{Zo<$KTiUwg65{jFCks{cjcr+&_hC%j+)yTd~7Z*E3-PX$?)*Q4KSS-mh4`C;2 z{K4_|68F#M*QaZDy4YqczBF(pR1Nl`KO5;}1J;Aqs%#Tq-vH1x+NeVl(4Ol5$Kx$4 z`NEW8)~ZiuY-3h`?8)r%e7MBeTN+DKw4IS+fCsC)l|nHb`}_vPuSotZpv+p>Z*4tf zwRgH#{8M7`9r&1Ct5j3BEkdIJ1LITgMQPc|-E!M{lvwvz2y2EhMR|}5}hl5S>?8izu!S`0r<}Wa ze{WM`DOLgJWmh?(dYaCvpW0n1Wa=1?T@BbE4?)#LfDjQePLvC~)ZK@M>kV!2Is6)9 zO&u6nQ~>mYMJwDB0a#Kut%ibYDFlq40^oV3icGFaCZ;nK+lMc5*i>x*QTZiP&-Tn4 zh@+5PD8{8z!z)8&)YBN(y!~!eQfem?Pt&OS52LAgZe_=m*;j`{Za?82W6LwswGrRr zcX7JOA=Ct{DB`ZWkZ&#Nz>40WCO83YA<5daa>med1be@=s{;|p*@~4@5)uGnCc3&ZfyynHFFoSzGW02E~8~KRsjt5m00*&tk$z#lRy_4mnEL z8THbWXz?}Qy%_c-$xD=a61=4+19quYHwaom+zyy`FYbGJTifX75g$8opY zP=z=`ANH=Euk~Ty>_N#;zd#4BO&s`17c7|TmWVkw>k1|#Wir z0-{#C4ioi_z>NtgvRi4~#)k5|R+S*vPFWn8b`||6O?}fasY@<@Hz(`J&DOO;|Q+RpgfAv|_W!k^$ zjK@7LcG#31iX?^yr!}GleeF9upK?mh3~Aa?^?s5sXQd^|OYt?oT5;KgLjv%saSknH zq3*#Y)Gs7D{qA?ZNqFG|#$Ls0*I5%O!XO%EL7fk_{# zUAf9(rC+tuDDl_eC-HZ428S~YDIqg4ge}2gid6=A6IpbkgvI$a($lbE)&Q>ic zMUmE25Dq@x?_6RhcHY?s*|6^;AvB_~C>>}ocT~O#z){>gJj|I>3BqCPA?_bIt&O2P zJl>L96Du3=P1JF^-{T^$Hs477moiFdNXQ(wTr30iGUio=Sm1`hP zj@9DzhTqRtq%VwfOwC`!3m4x09ooQ1l*7;}GU-D}1RUNSl+|1SUfEGT*o{#HL<;$P z-trz27pQD=5xF9qM}$r)dcI}M&r|W$d!te9y=a>9E@HWi;l=8sMk#SZT8i9JJym|K zkWI-(R8p3%F!s@ra;uN~j)}mqJM}@zhA$Ejp$c??p#^t`vFXJgbKz&NR(@+0E)^nI z|CZl}xXWrdN*#^a$V|2Jk)EyFW?6jlF2>VD-q2W|dHQRj*q@@u4m{m)Mu=WRfL6Lx zynPLm&l)>K~nvr{uptwjM@XgW>4O;Tcv{Du=EvK!5Whk8L^n1HL% z-s>|jUmC!MBCzc7PNOsdu_60i_r)R8p}0|N%< z6syLcfQdn+bm~i5=CC}EHWuPsVtk8mh)kWHMKL@nbeeF0?n91i(o6Cb=so4BWYG(Y z-+>ttr#(04SL=9jaXpGCnyw{#og}=JUG!8YUhmVvVwq)S(u!(pDgfgtmnFQN0$gSp zW@Bs)b`$912&=7kVTnR)Y$0)!%WE$Wtw5L_#{%*cH}onhVba2ET@6w8t^OF@73_)+ z;bYo6%QN>Fklu_jxM1eJc*cG-u44Z1>nmZ8slz1eARJq;A3RYYFtHpR9(A!1L4$!z zJ0tAZ7;pv{H7)TBWIZ0^_+#71#Sqa?CWvdYp|d zdJWc4=0m^$CC-T+#R>?U?Tt_v@DSZOYGIrl+koRLKVIZPwifv+jBbVv@9<-m-wl~! z2ER%RsTc`u2(b1X?K>u2Io=dr_BW6)9aiF$X=&+z3<4hB7#53;^8uh-)~RC*Rs9KA zF?M)o*|iYFNdIda>niFU-w_nd8%Fv$k^1e?xp>wP!QgGbdr1sG@AW4^Q{>09^8qcu zTwQ(^8N(a~oG4R3&2$(_?A`qa-9``vD`=-T8kVqTk9gQa(%QIHG5i1>*T7 z;BqZ8Z1>+aFFq~QfzSSH3r9J;th5>Qc^)w}DT?7X?0}kdg{z6I$?c~;ps<;p)tG)z z^l)K}>YF^=A(E3sh2?%q{Lk|QKQONK%B95gIc2e+ygZ!Owk<+X zGWE^o_Q7O&IJQ^z;Kmy;B-WB}O%o#fLaPT~!5eS5EmLkVn=OK0U0)76K9u2*WJ9_^ zkL#E11bQqzy*fP$3k#X!5S|jCr+9+-5ylpXnY>}-i+6Uu{J4;7z%Q2JMGri77+L@_ zv?Q)UyYI4|6q&o~>ziuVOy_ohDY78@odVq6SK~uvwa+owv(?Mdc7@jDD{JZHnY>Z; zw#!WP%ISy@D~K&?n1s5fQh8C2+s(E0yBCVpc$5wTP4`^~8wVRD$d^|daKPVz49j~g zXle%zF7|d@#qlzN{3z3UAAJPwrDNVF>U_<~<_CQms31$X2M5(Mh9h)Y9 zKN=%3^Cb7x`S)fTV7s+@?%A27NJT0tUw+$_bUlkYzIvMx7DNc^TH#_;EKoRukudZo*Y@TBRSwAn17V6t9TUde>11RC0Kb8$l$l{P>z zpNIQV&%B$#!t0P!03C+`2%5NB*Px2miJI^e4Y+6*f%DW17ryMZ*Y=+F4yo{l=IzLfV5PO?o- zTjWV)0$lr&@@AeKueiQVxePN`d|!Gh%_KfO>$!~+4s+yJqoH}*in!+*|F7a&V_#9s zs`~W0YxeKMhQ0)k+U?KT|LGBc#?%E|*vAVx58Vf&S)riq`YNF1hI8nl?V%9G1w8-N zVxk5IMny;4oVSH;qGim1qLea)m?b2^1A+P=jO+c$+#oM3fg?9Q?*D)(k}(qw>^?Uv zYVbYpL~UN+O%2B&4W!fO;jl%Lq7JGpesMLej0wWBR*_Grs(y)!?kd5= zc&EAjqP%yYW}-E(o}risq_qx=U-SO6$_9RE|l_}phlQ0K&p zK7BkhZzKUtgNl6ajAPLd2*csrp%!$UoGNGj55z3Os4l}{CM*I0{};2&q-1LWdx^n! zg4;&I*LWcYsLszwnOdRaNO~b2JnO?(SFR7+iOTyysMHf4WTxN1_F)M`zMY^ipvEDF zWrDW0w$1N>Ng&pP*9?2VQ~|87uOi&1UjU6`214?}FiE2DLZ%?sT6JPR4S;iu=QoC2 z^@KjZ`bdI@0|```tLB)a1&9SHDH_RTTcV!K^fBuA77u(2a7;jpMmFKJ1f$?+dkU((wS3Xbxc9XsS{OSPi<>i68o@(DKH$fFuZxPnLc?=n2&&qb- zwW_ssM*&YrX27i>WGrq~+)rYEN2VZ>wD0-p`kN&jhpJqZa+))RJxtQ*qMazPf04+_iMGa6YoR_ayu?gbfzPGsWC`x(x z=fv{(oO-54Mr*M@p)*xTwG!x4GDU?LUtZA*a@yJnqRV)IB3?^$)r6f%aSCF&+%>&1 zXF%>yx{KeFx91+eM}dfg(d5OTNNGcK2D=wNYKm0T2+;v6N`}vEYW(?6I<9ZMd>|ep z1e{efDJU?d{%5(uaZAD@Xp(V=q0v6PVN*3f-yz#%GP(}%+v#;>pV?22HC(?(8UkBr+E_e9t;dq>xDA3pIlw&5p0|k4evI&~nkx36(IP$`X{(z!s)%8sn!_+hT z+V9u_lNySWt(`yyH6mkjh66Yv=f>5;Sc9EUe*5cpK-5QoD6lu z{UkJI#I`(Q`%7IcTRUrWRCA;r)q5@|W0uSfW1=8#uoz4_E9wHE&RVZJoudR`gLa)p z-u^cXv2Pthynx=q2&a{+3>l)$`Jvg*@XgM)R>F@OY~*{$@$llrCiAL*E| z#8=R8tc!9$+Y^cuyta7ty@Dh|N@$pF|Jd#?{%BOQjhVSUlktDB?Fuo&k1Y%l)$?5m zDE9-EZEbCl90R{N?!N8-Q+RQ_2_#|aftP`iQ&(G)>&Sria3@VPpbae-l^=^1SwibT zVahE?aon*7Yk8?p6=2U(YAdQPDQenZJZ_SPpBDC+k#bw&Q9)ck+qeI9f7nfTAg>_y zdj_pQwdR1!9WNC6B<^Q;;8*yM1^YJjyJ5zS$9*)7w008VKu3@i2yhrGiz7Gsyrxa-Rv9{}q}W6dOJ-DW3(@d=-o; z^K>1By;PU{e>lg&A`jR186W)XhTf9V?9n1##a3O7JCXhB%&GwwC+)qTx78cxd2siY|AI+XsVkh5qoZb zPd+eL>1H@1VAkLtiB$$Ur_vtFID2A*mM?kJ!iYda1LMO%(zrOenJe9+2(SdzcSZ2V zU>F^x|JbBvXilJOP<#7ZugdB`RVG{*BANvlxmr(?$d^ zWun;zfG?1W2q!M?80(7Ykn*84OxojhL^SijThoETOe$(DLhok2gTU^#f8#`sD|Sr; z2?R=S=58hH!|%lWfUVh+W;6Ywz5i`tw9M_{cg9bjirKIs;-ly#N~+|-mv%UG6ttt{ ze}z5ogonliXzL~Co3lgqs&i{Mip+9TYbQ|Z>NF%SG*oM@Zw_T7QG}=0ErIA4Kf93D z;~ufUhV04JrGk)+ipE10e^HXDi^NDhK9>Abd~rg2Vx~nDw4f68N%UK91946&kxW?u zJqH)pO1gmZmIa+T%&@k~38U6Cr>0A}VLH*Q5xO(gsozXIg;7*z`Fa(uM2I}4o^S>3 z`1SBlW&agx>4pq&H62a54XPmS$G$3VxE3C46TeMHTM_Oe24V&CVRNbW9vX1^#!lMr z3p%2eExU$(N@RVX{&K#BU}dDF2gXnMh(bN;fU%GC1;@=Jn|%0WJ#CR(({6O@R*_uN zA555M1VS1^ycMQB;i9z1XJ(qnIzBn>g;OzYcs91YSdSh(4#X5b8X%%Dpba=pf8?SP z`6~Yk)h+wha3Zwgi=Z38aMp}8?Et6;BzGzP@GytUNl^*R|C;);+U9x?8dQ4xzSew} zB|1>a@+>!i9?q#arz=zGWU zAMM|Ezl%bKz=Vy9YoXC5wWZnHM)2&eRXb6+LYOOA~)*?&j4BmGU+8Xvq2x69ShxJZ0nq(|PTt zLS?$Ir;uTeh9eop7yW}P9%!@Cdl)P(z(y~e-4x7z73Zgvw(?yOE>-RRI4Xs}$8-Nl z#r^5l8j7~Y*6B(pvxY+8$7fZ@1QHt;$ zO|VF=PE4lizhY1R-`GbtOQ$Qz5r(E8f#*&luhVGCWw%74EdE4&b?Tx;CUlZ$^|;@l zkICG1#?!jAvrS!q7_s`R{Al6tYYfcU7};O!+#ErZi==gBa+=(x>uRO-Q)?a6WeqH_ z?gOfwN^ez!KxDihPO2D?9ik*v^aw?yZNiy_Tqj=dy?8Ess$rsNrWB|YV`ZBu0B7#* zIHP|(G2=^I2(#Jwhj0{WD5OJBdDl2wY@X#FeEhJIXC2W%?6&%D=&hU<r?huutgYMKLx!4SR!^8qXu_4vcS3D93m+FhcpEP{gMQhRuEo3Y=OEu zKJ2byj~~%nU6o@`SI%AOf#zmDFut%b_@b!CnXJC#J? zC!8Pf=|IPabE*5dFbIy?!w^jkQ_my5b1kNW?~kJ27HkthfkDOU2;x)PKko9N3(Rk5 z6-1pJDyvG$&={rSdvCzbbM+@g>$Cm{7G2Y7H->f-Ivf}FKvyPqqg+wN^< z8l(UjZs{_MXvk5iEB#G5Ca($Cnvh>`cY*0`UA}msq>Ko>^w@6^;REHfK!l4r?`A{P zQQVS;P^Y!C*{1**85(loO{!D0*|p+aVOdtiq?CM@kFABZGMG3SfMaqkAJ!kLI>8)& zR`UWUKTc`Y$o?B$cJA9OqIWw}U~2u?lec@5iL#y^3g{q02!bN;eybvcFxh}I0j1t! z%Dp+#YxJt@;ny-3K86M&29SbmL)R4cKjs1qJBuoc`i$X)*32apeAvgk;+&I{K(RdF zSyrnAC`5Xltfy-*r6PS>^#%qz<5Uw8rTp?z&iGm*&lF~8CbWf9WezjthTvOORuT-= zfI#AKfh}feSY8nUcIY_)<&DUV6HFXDG~q$g_{8YL${CogJdSoMc|HXQzc8KRGmXs( z0g(&~QyD>2e|c#oZ>0wZ?k`8W87*M)3Rvk@B;|i;Be(`T>v-U3&+t>3C&yy}c_Q|k z?{#XNa8Kg4L`a|Bd{n`X{d|?LtrY`*_f%fG`$Q@l{$f&7EA8B`Gr(8(TvvN=f`zq_ zn2;ElWeej%kIgw)k3%@VIu*Vc7@Sn%w*=wq)Fh&;QHP?2ckcq=Sbr=mrQCksuut%~ zpMPZe@tGr}URA4vcwd&di~OnxVcADTJ3^#{|1 z)0aELVN>YvklI>qFzo?i41k4S8?sNn1aQcBcRW5OYZ*+2Cm_B@;Gqg7M#_6JdhOYP z9f*s!h_QywyzvyMMjc|DU#a!cBM>YAI^V$3+J_u%ZEl7}5!@j1(=q>)y&wEb&dPct zDbROrq*ny%Fb6T?Jt;Q_Or8QkDwr#d-Pobx9%7*DD02tb#@Pj_1{Gw5yrrxsqhJ+NN&dY=kTmpuPp^Mhf9Ax=VihnU;(w;9QJKWnF24uakB7}zF zxp~U0FTR|4IkSV;#=*cK^^*@A$%|vh@*lSH8>-nNK5kM)hu)S|h1!78>HsnGCKX$; zU3WONMW@mT{BO`fncp8>Z@xo_|Y9(U#o`cgt<73mgfQzISHb?doNe%n_;^BtmgFbWf4PCfyp;KVG6{{J&cu{g zy}t4qN)iAaAcrcQuy6?RU*ACMGTSDC(#${0f|IpX`U2ZV9+FZjwd_HHRdcq{*?A+* zD%!DVu6gO^F*(otr;R21c>|$r&0hf+cu+?nN={0Q`0?uc@bK+WXp|OD{8);DYEfx) zWA)MNSE#lu`?Yrc1`j4K_4}p07d{>r@O1VgM(Nl?-n^Y5`UWAGq$9A({Phr%`-@6L zJ2!z9K^LZ-f@<M5$BiexK#p z+2kDQ*bNSmK|-o;z1X>s!|Bm)(%46=mmH|B@=Bwo{8gcy^tw3l)(ZIyy)QvwSrrXv z*Gz6GWL$`uvr$p<+2zo|b#?6SQ0>x8O%)X>eF{c5jl;8K`n($H`~uAXeq9N_ul!YJ1hbj{b2@gd zzf39rUT?+dyT+rXVe`Zv-RjP6wxmO19*j&m5@bY!&53M^Y3Fsvf{GrZA1usI^1Ee3 zXV`JjGe5NYtd_$Zjw1U)gql5Ia{667Uy9Xv0RdiAY`^90#(QU{dJ;s3yua9g*&iPr zHs~|^&XoNIf!iM5pCZh7P;q@>8ap}sM~Py-H{4vG#~_q;)&ol|#qV)g2Lfyc9U1!4 z$`ipEbCvsq&u$*1#JI_2HAa#4<#UCofIbuEV6vIQi0Y!B^;=|b5vGR7<_tDCl51B- zC4wfSIAEZ|JL|D-b4c(aEZ18C4R3jdiX5_nU{DWcRud`4w)TvSFQ`9it3M?$M=3#5 zmzx>Gc~9l?q7J<``@;)j59C80JV-W_-h)T)LaNKwen#+84df)q>-uZWs(e-!Ug3Qe zHEYlSnB`0MP5bm`9Z7)K)}n;m(zovc!JGR*>3xwOUW{enW30wOHD|fb*`i^MmV7a! zJW)hf&Q^chu!dMmY56Q{$VIu6`lO}OkRjq_Esfu2<87=5JI7Lbn{BkMIF%_`IVR$L zz0{kg>cz=$GiA4+MQC3Sd#x*e@w#IreTxDkJo#s;YpE$7HT?|vY{eoyRqkpB${`l=Hdsd$#b@M5rw{-T0b7XF~I zl?OE6epq%9?%*yOFlCS$6$q_M*qD0b8Y&$+4+vV~pttEjNFIm??*+~}7!-MYY#ZE+ zc8O5>l7WdQdl?#YaeGU$wDpKAyBYnq4PK6Qg`nAakwnj6CM6P>4g$7eHSMO^y!QgD z8U=t5pu@q{SRigDF=JQ)VEZSCvnltBOlbP)Tzsa# z##PF{8Kam$AshjX?*}UlppK9J8z`fw(*AixE4HjWfVqxM808r6-(k#?4=7mD)Bu66 z!*Mf$<$JIH$ya2t#<`7#|5>kzoI-m3q&u|9P5A-a=B!)kDBB-T1+$4mA++TrnP!3U z&GI@4f0O@6xE(zlRgm-}=edjv-%bRA5kqnD&rp*rE;f5}e4*4K!Gyo~l%|^M81_x0 z{yB_6aDIy$wt}-B8-R43^r+JryRk(3VG;5`BbZ#P) z0Z24<+jb7aPwr^!@da!|FJNPFsgk{ zLbc(DEcqA)q+e1%&bDnE6DjgY6(Ol+G5G9KhVPJFyIO0Fe=F%!wFJUmLcbU1wgK|5 z4Bo)y>w&E!!OZ3AuACPS4uAb2gj!v%kUk6PeYS7@!~}~J_U;T;!o2hG43-&?amURQ z@E8vH4j8AFf%69h?J>s&ZBd~rMN{EZ55qbo^0=u7L%v!(s4{p^ZV3D~-|L#c*wS_@ z4tBnb@|=bM3Azcbt+4q~^@-xj%Fq4}KB)QIZkryKRM%)uaPd-eye_(EN`7U1IV-=N zsRpdI3`$LdJC73)O=XiaUPc-Z)-=v0n*7%pdB2fegD-Ij0|E3o(g0EhR7F$`vQ}oy z>>PsH=cu!hvnn>ua3*ojR+aQ6(+jPqZ!AoIA}!(esQIeKIS%LxG_`k90iKX0cOf~m zoz#~!sHktr`5yEb8#+h4b|y`-RK0g^YSOC~kKlz~aB)$Smb%4pjkYk@~kkuCE@4LbkUKJ`A@*f1y+ZdCO3)EWy9=ee{!Y-3RJ}=V(Xn6B7p1 zu9J;)SUK)xa~c)N{X3s_D%XfXu1yVRu28Be;@r~qvhc2 zM|)c&Hc)FWgsCAw#J!Fih)~~$e~UXA#S2Fn;dKplrtWRG?#0NyjCzP_GiBLN8r^Vu zHt_fy*P|76%Qf?o=KDg{!wnYCkIWq10>vfVi3j0>p28s(u4P`w^1??89J*Qubty)T zNx54CP7598{ZM;IajufHr-hmM6sM+3*;aj38&cuBjxJ*rmF=Us)XmLACSDO30x9j` z9AmJ!4r3zlh1ca5nAI#AzDw4eIR~K$Vw^h>j!S!f#B1o?S!;b5`fLWm$R-5#WOrKE zVu_f2bEN1_@s{zNgo_y025BITkxZbo)boj6P%$`){wnIkzgqpRHJmD?Uk)VJcOYAW zJA|N5ftL(ZlDNX}X#)OK87P2gL4k<*B=j9DXj}PjJMUZVmUy#$5zKI#Ol@24fA3-P zGqJKNbjX58p_(A3JKpR!P!jDy`5>tLi=o=%&HN|eXWsjcyt+BCWCF3x>y8m$7V+#y&*+oh4% z7+|oI4fx$brRTxaj-VNw=6kl<8rFq(sz3i5i0sQorS=p{$DT>WYsNBOuY7wdC}!2D z3|82n_P*#PvFeKnmmroO%NF=2tm!Lf{nc>Ah3LlhGr+*_S|gjK!K5E2dun%YkI@!G za1$b?{lke#@jwD(dc4-&UO!GQuMobjzr? zbTq=wP?8X2UK9u{<}va`qNe3Yq$W*|JIdcks1b$^`50?rmj^ zH&ox1z3$V-TtkPy^ZRpl3z9Qd+b=8%?uTMXkO_AGzPwyG$mmZ-=DXa35{60j{^690 zAj_aci1ucw1si`i4)? z7N%}M`HJUNgiKi0_T!Jthuf>dw$?67K?P#Nq9~=p7uChzC2g87&+qJwFJFBJ&UbS( z)-wwjhP4L|U|XjRRYAwZE$ozxI9&VuEe!*5GT)lzubF*$GO38C6;K}gcIveV{^avyz}q-$X4DK zWv)mg*4h$*-k)Cej!Y*jEmO*-KJ!l0#{x{#d(n2mD7{RuNiQCZbh{2cR4)?DmyNyA z($YCR%|EaB=52*pJMk%OlR5(_(&uvvjOWwiuQ9>vt#B47GDk@(G z3go6`42ZbwkShzWc^8o7ESuy+C_nHGrR0i1O>!=I(rBpw)?evmh{7aK-#p=SN0iuX za!2c|oGV`W>-7_*!y<8PtPhupp4_R`mA1=~YEZ;k2K}l71YDV{L2k!plOfycU1wd-$hyu$&JI$;rT=4PXPdI4;GAryXSAI5DJhh zjEN<|;n;=qCatFfB$(;g$eI~A>c;W0^Sl#?nhBeZFJ`B=UL5MTpdu?U+M`kQSzw43 zCv%@k6ZYXnKtSMD)VTf@klsWI80@f}ghvzU%CJvtV%tXJSA1J}9)3*DO?YuTAXaeY zb+YSHmc$+H`2==0=-M|)g%+9=Qmpo`Q$)y2EBdazG5Gm0^xc*PP53YVP!dE7UtGlk zj<(UnG?(>vWp>|~_Z;?{l~UjBlm?h*`rT-Q2{?_qal}g*0>;R=O)SG3suU<1dZC}t z4kKv%@xI^)igGqC{OVD1JNE9SgOIImoXOb}NG4(#R_T&7C^Hu~;r#J3n}}==sAxW9 zvC{}2D!I?FVa-9VTYO{Krq`kG`^?(;qxt%hXY*_ckQ$K4$;oY|U3jqD^cWEc3SS5~ z`Mr}vUQ9;-mS}Vn4YM+JyZvqxLmutQavHTaNAk-)mZ*j^;K`_< z)13?v7Dt(@J*SHYtAn~rp5YNOih%3u@EQJ z+XsAxq&Z93P*VAF`X4`f>zYTx2}FB-E8sovC#R&S6m?NvfcC}T*R~s)l%wCplFmZGH(N&mz1zb6-BSZ#k@UTET`cp0latOUyVgA(hi1h z1;^=6O5p@8la8`lD1=nUc0>;J=n!W4Y10Ux8ixh)W8oHx`voKUF6X@vX0cCp_B1FEn+!X+9M3y zia35eQDchb3Zx#soUp3*MA_LC5eeaNo!IiWwr*nPJL3RihZI^~aTrmCX|~C<*rQ`c z-~`ByGx$8)L=m+$4&NL|R{;t_FtW(-&!=89oKPE0JZ z5&3iJjbK2*9a;REV&hX{e|!UmD+>dyU`iLO%O*72o*gI3E)kxX!>#CEi#idxrt$j{ z(+UBpliH0CrvaXzNBG0oMkCrDZ18!X=+`bsqi_6cRkWMZjF;YXl~QY&K}98{9p;B0 zPGjH%5stBu!#y0%8V2X^C$Pq7EWJ2m?2Dm*SwM1;azZ>dPdVm27Hg8sI;%U)*m;W%g{XCsgZJ};-UAkxx>%$Vhu zPp#medgP94Y$`@YmAQdR?G*x_8`|$8YODJi9M&e8GsG3!@J+$RVvJgWS)G@cNI;Cp zR2v#L6A*3r&5_py#}82nzvL6mY;&u#+o>+i?{kZS*)ArK_N^b~B9V|zEl!B6@TQhj z*lYU@mrbsHX|o>B_0qk2sOct_ByYyy)}y(d{q0wp^wFg)e3^YzI4?=7W@SRa^77nJ zZE1G#w}ZdF0AZ@C^(fD@7Da3IKjyrL5E1ucqT%q9I+vp zHObvRXsCbyyF;d96~vWdNWOxLzx|+J8u_JA`(h?~Cb^58bH9g*eUw0SAb&E(KC^Xi z`#Ah?5v7rTk#1tGa1^F(2ZlW@J0f|7nW14{b|p>XkEWODe}gA#VuWC(ph2uq+U~$* zNd{LgXx5vLU%0{dC zDZ-KuiofRd8lpq$zPvv};ek(=hsb?@UX@pQ0)jE5Z}GM^sz$IOocsN6|1c^s;Kis1 zFL$xbPnMog2gNmdUP3tq^VqW9`cIlL#+Ughy->&T{&m^a(Y7();-eJ-g3Vt<=(DR@ z?cOwQ2%nFsO<%9#I-5MI{U55{!Y|6O+Zq;0N$KuRMY_9X=x&g1q`SM6MjB-3?(UEp zq`SMj;k$jFbAIpn{sCg<+O^i&d#$fja&BrM&TMaf`}B^}we@J|Ajtl;Ka`mE>XyOe)xM-{ayK-I=v+nePEq~$3Gc_% zuNby<^^R17a|#DqzkzdY)B&Jpfzh=aqL$_4RjB*4>|S_kDsZmb#Ag^-PM_IL%&jjM zA7nOH4EN)qu0b2pONSxMywH=j^5w9ZAgYW36MH7iFu$E2j4)JX(>Oqyq@Qwb{T&2K z!eXH2(36x7u6$SfhQNEkS;@)P5A935v5SKSGu^X+8W zW|3~{kb~?K+V}(m(T#~^R!h}SL87vd;T(Qrur}miL&I-y96DmN^<+u|`3=W9*2)e@ zJi`kDwrfAciR8%O9lpG46v3|OaCrA95qxkW_qR@BY|R*vywgz%d2=e#G7#mns|5lD zaF#)JzlC;HH-On|tB-BwZ=Cs*hwHPT8ONOd?hiz1pf^#5D)v{+@nP)p$LDKc4e#4n z<3`T-8iOcpyPZjfZIKC-NUKacW?7$$BUJB;5mbsT+p6OPp7R@Jnou1qJ1`*-5GTto z$g;93!5DfiO-6t3B}!Czup6-=hGUGNn}y#xzDJN`<&+#Z52@;B#n|$Z&X6$BIbcvc zjRVXiO9m2P=^-Ev^A{P>hdxwP=$xs870Y_=&j2bT7m>KDK8J2*8T#ZH%1ZC^2hBr< zV}2AA&F@XkK2IhN+p3)5%z>@7*et*1NjHrC%*H=1PI>F-2!rtzyu&cJ-M^%ff#*nD zYdmHaT|V$y{{Gx#2<-C`_hXKae3CSEiK*I?63ax$6~+7|ZXuzda7Po5nT>}g>~P*VV-91j?FV?kHqFJ+U@fw`A+obo#%PtWj+Fa8`}Ltmf~%foKxU5Xv1jD6 z>UVJ*J|_=i+#A&Pe68_T{9eId4Xa`4>B^ImsB>dRt-omHpfU^NSNuebSb1i)b4m(P z_3hYA+BzavSN=@W=J72rFQ;?04?k9G*2I2d2q|wTL~)&)%qgc0d~WZHzNEpI*R^Ec zcfS5cV0OHTr}I1AiHP6GX+3}m=T`vlh6xUus%gWu_Vh(^OyBi&my z>hC3@rVa-LhL`dBbVkpH%Xc93I~LaDb|k;6LM^5HBoZKsny>Qp33C9hRum0z-1!Cc zY&z+3xR7Oa>{0X@54Hd2?F%!IVaEfZuo&^4$Thwfu_o%D`O)WZ;m6Q3y1d0Pk{=#4 z7!1?@wTkke`j?M)+1*|h@>Gud-}GXm2Jh5Hmy}#nF`XCdTu8>esRqepS;o8Z*d9^x zzf{ZpJ#C+`hN34VgpVeE!FiGP#Ao)Z__D3EA8nBd!58q-Rp^4Kd7@R~I2%75VjrRA zGV&BM8L&&)>y2@J>2nV3;X4-&PHB9FpxbC@%El7)hN2BlV$$E=Iq;yELCUO4d65qf zLUjt`e&a85GUxo*9V{s>?vLc)tL92J-k4qN3Z=|CLq_jWu?!9U2$5e~)X(eIs@9q055}vmzxj$}aqr&dtuWOdy zt!R&Uxe&2Z)HlYEEh{~oITHuV(xuzpxnJ$ z)+ZFsX0f>L-F70eH$BW%RTEdepEK%wj?|qjTn6rC5~^+QSZ!8QL@R(xH0O0Rcg?4G z)14{)S~u!BwLeyDJDwdmFlww2{Oct<9o%$z(`XFtpANUb_$UE)3anilWHk}@*`2N} zpmu(n?o91P(Xf6@)X-Lr{#k>V{t=s8_t!O#{nd*;h-@I2lJ)%hvM}ED#=&_%@FU{O z&hhp%%>pmmK9(3ch1>|k#_4YO>V_|zOJYufh)3V}?K|F!RwzF2JzN|N@9CuFR|2u3 zDnc?BqzzSF3ZCMbJdL=|{T~Gek}O8Mvvpnv&z_zuQ;C*<8MW?Bli^h7Jc?T>t7s7O z_S?_@R|A9@AgkC#6yO!dZ#7l+Yl6{4JQ#d#r|cu!KW{6L8V#(b&@1xz!>)e&!(@&Y zc&oWE7p^_wUq<~IZl}SR5Hf6|T>Tm*Xg`yRzYxTr3e@{(2 z8O}9uQ)@D#t&NUp`QwYxMzp$X@|S{wir$NaH7je<`8TNhr%Z2gy3HlYU#7KCh|sEF^Laf0O*fLDW{8R70^Twg z!qr))xLS~_-CAtf=vU{nRpMAgXrzb_A1FwkBvSMvQFQXEUo1ZG@Xqu$%MlNtp>%Cm z2Mq}ftkP8_CbXVLqH}H0lMu1{M}h1coFP1VdK{A)66~uWmK* z2e2)@GqK^4V&vOcYB0o*)JLsxegJx{l36!)Q(NJJWph^;gst*qyx-M;vMdWJ4w>Gw z?OlM&bgvoze26fY;+}^ZwsLs57ZxBEYJ3%&KkxfFhK6-I&paMl67^Ih_78fA;Zbnk zH}6hkw8%e)uv9J!1s!-_a#AMU5PeSm+8m7n?pgLsgVic&c${QdpM;#7B{AClsP_vr zl4%H;QJgpTytB3khC^3T7#K?9Wig2^k>oxufZ_QqF~qO>UgRg?)YKyDR-lSSYDv%RDXN0yBNPFlt4bewIFn`l;&zy_@Aorp9ZR{Sve=( z@Sd@?4<-^r7kJ0N>R#)g1b#Y;6+XL;^KlkGc#tAy|y(KuJ&yqDRK&S z$HA>eVtSoS93@VbVeuQEo4f652k9hhGhJ~REu*8Mvk=m zC6*S~H{uDobRHNIF)CT~z8*p^P(hFoEB@I-dFeBdtLm%WfnJS|%I*L+ucfioGfCd` z?NU(;aK!kTFm-;^vL!uST3&L}3LsA*-k?5)fODJP-Rowq;z*~cva)+zU;@}O z7dvjKckG)lYDDn8olbdavz42ek2a~!wvJn?oM3{1uXlb@M&h+({9;5-Pyay+KZf^! zaexIMIcrHtI;VPjzs~7Ju#ZL~YVEzkp66(rFVTB?n9TG(HcX*=Z)&HJOmS(IFTeY- z6DWs`z^&<M44EdAB@(Ptf@Ln{f!8gXk?c#T#G z`o-adI!H}K`EhYz2`DhmTs&E%yjihD`mFcerhF0L$&G|QVaG<*|;6HB->cb(`6s;YGpft6Qx%DIam$<;gH>aaaq2hx?X z08bkOU*5&t_FiQJOm55?^OrEy|82GY1_ zpCxXQ73*KTR8sPtv^BUhqufHRb@;%0h8&zlvu%-20C0=nawPE+CNLcaaFNWB#>!81 zVd6hkInRkYpFKvJCEVIuS`xS(`^vVG5IBAClv*y-n``W10TCtV714o?8yZdnS@ZSt zwT5G6=O6!M#Mw^5>x1J32MLxeOrvBlae8Of{iI5@A@sDJSQ6COs&O1WL!dR?17T4f@Q0s;X)aKNYE4K=O4=}Xa$fYi=WZYxrIyY?^wqyj zyvHG5T$T8J#kN@i+#pt?LE z{Z;?B?QFQw1p4Pioz@C10G790L)j1DW72$Sh3fq(8^O2mste$^l$eEI$u&7J!{W>p z@djkC_jhA9f;^R^Kf(XrlyA`ZIWlNq-I&+(2NcW%2tIq`eK+=JnTzB(Z-8Ny;>tr$ z37U5X{Xkb!mwfTya8o$J2to48PoUs*2zUj@bSlsC2GY3SmLdg!HrgLICQt_BvlP$> zoxQyHG`9R|S(JP#PZ!(}I?EJ91&$3RDyH(`AGZD!LvIL!s>|im%`5D3bMZKh#jQ&% zQ+BBKzpaiG0-a@aM&0~Q9C@0=r`4ho^yA26$%L3kG$I5yx0f#Tps=UgQ)*h5PsjwE z?+F`;yH8CPC}6z98?jmNaAi(|H%cC3Ffe|e-(33S423#Sfai*pR#<(VJ2~6K+^`@K zep8jrZ{M9pN}$DZYb)kRjn!vDlG6TXdjG#+RV)mzAq*ThYU={lL7N^nD*KTR+LmZH z8%lXyrPOtuP@RxQ#0UGj1V!MI*kO2gFcIqF{K@1GXSU#i!|+kRt1JBo0!DOjK(u73 zjawrL9>*qZNgy3tlne^6W>2oFz8itV!+meqGmnO)1IBTF^cfR5=8hM3U5j-;^}Py_ zo`%iz_VK}9vb2EL`2j943;*c8v61Ug01;mn@p=B%_D-Y4ZO`@)P9mLD3fQ*A=J7kS z5&`$J2Q`N4%JLuRpO6}P&o(TwEnc2=4vmcTnobpV|5+rcrrosUv}9=sd&Eq{HFJO~ z-o@i@N>RvB8NGmkLPZ?$W<;Fg=Kg7x;MrigE*}WOaQ?e&Zu?RS=Nu`R9>^tV>x}BX zcW{7!a{a|~{_kI9@hM?gKOsU{)5JJ)J+^mGd+8_K>8HZEM+A`m7Pzf2=l+?LXiin# z)=$g=(47fh+TM~$_h}WSEQ3dJx-eW^8n#)pOaQlLNzU`)tq>~&Le)Ee%3HYhp%+uB z+tshy{!)L<(|X(SDLzvp5NY6gpfpa`Xn4$4Wl8YS6_fLXms+<}(4+f&{cwr#ETpve zH%#A=TN0nQGkHyoo8=Ol<1_6x1=#OgErDLp2*3MwguQfs=?N4JqcBsU^QWh~_e@1| z!eWw$ky=ssK*tQ~rxJv(CADj z2vn6l!0*H%Ic34xofDbahOO8GhK3NL3jgd*(!+39=1RK!#}ae`3}jqfbG4g&zT(+H z5G|&{R&nFoN41Xo&Jh$|B-;qR&=p57^nJ@^TTKH(z7b9i0Nm~vV_qMI&AHR0b#km$ zqekmAsQFc@ya{n>bvrG18X^4w4ZqPfNktSj@!L1hQ~dV4r|Adu{)+HQ_0ONm)tlM9 z{`CSj`|XB?_CnG4lLjsxKDNNXBpIx$ zSQ-Jv>{ZCBe$zvwQJx*g&dtsNUCvZGbG7?Mey5>4n}v|qj@h-whA&tFty%v6_Im#R zQ^lO&V1dvdC>{Ou+9+591DsUFW6dEH)OHAxFyb=fS5v|ct1hP0G4+Mw;-sAVxSyC- zk0}-FlV_{{ZH)&3F{215Nbx^gvewh67xk>x>1=nu*D31g#QHTZ%VkQ$Ko76#LtCgn zBt%MSAn~@}EO^#z3&9AHYCK=~xiQX~AI1V-i}xDtWS1243K}-{4jmnxsG&b;v#5s^ zJWJE}N0@F9k#g6{N)QR56Xv8=)}8@bdCbd*9(ABlX}(3V?JgRSG8I~sAn`6RI`Pd9U>yC4J>wkWS#OpN|^U+EF_ z>7xnE&e08B1aKOjTW>Ud88rX*FseQt2TX%4Cz8yi)g zNbMd;jDFw5?b02x^+;e-$O^Gsd!0j&jzxFT)LJk+-2QZMB!;iPzIU?nq%Z8~m`6vs ztS{9mA{@KJb9_2~10(x}RB3L$7bvrZd*rhVe3I=?J8;h}kMDrXuGNQEUMPnZuIAq% zGPXP`-p_3=ytFAfm)yw2STbkt?2JEV5@%uT<45;qhq@xE!xkOQmTK8C?#?gf+%t-RaenTv@E3N?&9%;e*f5-b z=BQkGHPrvh%CplG+4-2&)fD_~${tx{-FB7ZftL01j1Sn_9YqL=B>J69kO#P9z@ zhIkl0mYM_Amv1Rgqe)0qG%(#6E-C;OQO|i@nn;$hycl`}?!^<9jETx$Uo|F-*|2=^ zJ#Y03SCB_6d)59y#c?vZ2zb}+_p3{N&Q$xvTQoGQB_0PIGO3_j=IbBr>iYT|Ot*1$ zbv3|`Q=o>2O2d?+#qCUTgNfkmLx%z;M#6!9g|KGh$M=^u2%B}-wN2&RG{t|W(|{?z z?7gb$?BA}MOa$u^sx+9)#6-&$J(T_Fx?qnvSDU5K<1!{)_hIVoZPmDTc!hvY1?ZL* zlaC?9mzV6v6?vBOf+2}K4pbz~52ozr(g}^_C9IamA0Nv#NR4GwQcl-yQF$}RnR6jn z2JsCpO;5XysUGDmT6Zl3Lp!&*UFqfQd~dy-;wGH{J?6xd__-d4m%$3!4{)-$=`IJG zlAj?ILAb4L6ogA|g1MQ7>!-5`%lf#A_&1$vaA<~wnKw5a?C6^$o;c)Te6pvf?Vqi< z8*5FiDsxnn96$K1;U-+d@Ma3e*pUakXPD!cySP$Ua%BCPpt7;QE!$kSaeDX;PL|z6 zyJXTy&RPVV%t3GeBPXGTv&9QQ<@xcClfydE`&T_Lt-j(-sx(Hbe>w^T0q2^u)zDfs z2Ks3y3o;SKC2w>0{P;F3(nJ!-BIRV!F|g=#>ExS*I8cBe1(V5sC#v%X;30o&qY$Kv z0)v8xMV|$hJT+|}oDhjSU2m+MZ?1YF8DhSuIEbDwZKIzprNG{pcLc2 z$xeY&@zZUL{|EyIpPkY>5+wV-OwE6LJtP|Dt$XN&L%s|Hi9zXPCC5-L$ zq@(^XbLQm*lR59mB*2IkLBGJRQXea7r=3eH0utQjFZ^|%CFp$&6H;568>^w?eB+8u ze!(pswfOJB|Gi{@`nj;Ei78yTtON`{E^Y|~z@C%n&?Ol(*yu$J`LN=&$lU1w2wbmU zy=v!f^w1@@x9re5u1=@Bk;u zkt2{<=XFhh?=)|AIx(DAS66gj>&+$T23Xan+RLe|^YFZB`tZ+9<0!dCQU6DXEg#7g zC+%m$_dGJ=GXUOMCk5D07s2q|Wp;I^s(PL_3={=)_4LDK2Hj+-& z4E6Emdf5mkEfmc^Eh~=vnBRXNWd?$Q& zs^1zl-0wx=MuJ}^vy?+e=s^#>S0EsRfrV@-S85>weeh`GsoywOvZ)Q`G3WbUq+vr~*&QeGyG5YNT_27Ra;E4J%RL7% z*CHdxCsy59D^kC7*~}@ZQ(;>~y-yzwJtaAY_}M+J9wA7w&l^mn`Clcs?8n^BmdQKz(r5oZa_B*Z`z3hl;yNhL?*!2(3eCTfb8(@q^!ADu zNso^#nV2Z##==QbJV}O+D|0|oj>S!_iqMXIPy7_Z_LM2Af`M#Yq2%MiR{b4EeCJ7Y=KR#;P%(KfKjrUDVJQLZ{ z5%WHi*L=T?Wa57?x*kfggs@L5{#x1cw~L*DMn5_pP0@=9yGm zP0=ULaEBjB6=tQn9PKP#%`lx`&6c>dR-HJImjJG;L854=N0vc+_Uy=F7JshRfPR?O zQ7iRsQ(0+-PeFcA5n3?|Ces;PL*LFm2Xn17ZQ)^F9+b4OozY|Snph3}5rb%clj`aU zsmf%{vEUm{ce>e8biprA>9;KA^>!TCwZ=#BWo@1HgZKUZ^-83`{q_?xjMu{Y-kmu`corDoV>AQb0~yU~6z@IOjKcfr zxm)ExVx;}XfQ0NyNB5?>I;^MLW1ut%vhHvK#Qpyr{iPuXOv^B69nXxnM6x(+MedNv z&0g zJRTwUCzb8oN&56TMXThuO{bSi=Qrwe9#c3F%K^-_4RKLUnz-1mqd2`{My~|Ze<-t< z<8L4cmUAB$&cYP!<*Ovcm_zR_x{_4jdwO3+%WMUWveH_dPCzz-;w%{I-vrbRB1&oQ z`ht~whJFmFD`Bn>{~tbn(jf#0tag*c6oA%=Ywyriy?kk98|MR_(_KHcDV9_2yBqd+ zJ!M@OAtBOWWoea_zMy5VJL2G0+Gc*GRB0|;sYm9!kGT{sSNPg?6FHcCPgT8Wd@rA| z?*EE0i@X!tW4Wsrg2j1j!UmLDq0r65%sTA2`f#EL4V|^tHWzZ+inm-mNg*)qn{#pI z`1~khIVx&uRRW$BjSm@Q|W9iuP+X=@n`K!l7^DuQ@l) zIZTWQ6YOjRW~*i^&~J0S?(#&-UZaCl6LQ_r%5v=>qe( zD*k`8t>-a;%-DUDQI1&5^xsu#Y_Id7&;B@jeFjr?Y7pQC~w zvaa2DL+45&7?bWvI6tkip7l!-p>b~4JUu-*!1Tx@K2c5L3{Ksly0nk&PER(0Ix(}$ z`Dh>D;ww-Gk^yx1D|t?PASi}Rg`9^bVT7|8T+KGJF7l&?SpEAfk+z@!rAkxT0v|Wx zJBFb(aDM@nyswWRVAmi%*?#d(r0FTPAiQInbhS01)9K%H4+ft5L5=0qA`Gu1X_sB# zCR$%%R77xs)LmZ0$OhcR4-xQ<^(@f#=g2yN|L=Pb7%v5w$nl=~H^&`8@49c8&7qn` z2ZOcf;^R2PtZ7Kji|uYuk&#;=;tdV4@12$ZmIjzToi4MH{{CIO_`EgPrU0D}^GQ?H zs_QmWA*YOArLH6@laMC%Etr=f(TiakaOLo;NB6Vg7G zF1Dl%0{K;HOT?0qr1HCzq*E;N18-K=C^_U7ftQ-*xA8D3yvTt5zUWq#3H;Mm-o;DT zMw7D&^MbMcqT@%x+bl^f3E~XPn#CF1>;ChAjgm9954?Z^^e)S4#0e<*e;66g(NIX7 z<|!f_XnPLiqY0!zG4fDmaCx!W(p{G~@FM7p2+`+cTp`lIa8LNp<0|9E1*4O}R%PD#iOR$wKc_7_~B-}w1;;%UNa=fjEX2hY0} z`F}%F)jrnkv6oWof+9D!ZQ)g&Lu!wbmiUc*?)?1=L6$}! zXuuu}d>gFO@gf^rZ!0oYhT1Ir8Mclqb*u=;77wi&%G#kZOQ;*?7tWrzIT3-Bl%MTX zsYGfIX<{4PAjL&>7<0={_%|Y|TZNXr=a9^Vm^Cbb`B*2SH)MfCh-vuNJ2U>Qu?~^j z^pA&I2H9nWKK`NJcaCML;gA0Ry>8c%|Ge&yt)@eMcQuVQfADDsf`-9-&G$PdqT37& zOBM*=`xU0BmBzSuw!IzZsC34VB`0Bu+Hm6jHCS~0tG@nAk=P}E|MQ-;l&vi+r)_&m z&lO~!nfjB$d+N|E{Hn=07NDD3EN6eBra`eAzim2e`0!X_HNy$akNM^bP0hdH_BbGh z(gG-&^YNLT)qT*k8Pl$smWNxQnqGJ`S%?hfkjHZ=VDb$$F@Mj5d`uh+K2ze7F5Iot zo*Gux(hAXgQdl1v)@^ECq_mQz3vdl!J{2QzAECahK6$c;XDKBz3aG$iD+4>ED@}2Z zQ~u*JirY1!cdX-bRFs_%SKnN54dv`Cx3f5lQur!b-9Ilc>;B4Mc{6*iwM0|56Ozx& zaIK=as~Z_`!>vjvk6gp3DEr?BO;mBZhh|P*#dtui5~iM$_p|5e?>lv+4?M6x^K*Fo zaG52zzXyOkJfuGQoXekB-W)nY_w-gw135ZeK{NFJpC2al)0)DDeIC%Y*Hrx3&(sV=0;D6JbspK`pGj6C?h>-<>-iiiGULmv5@gOk&QMsWEZq; zQer{ORF+qvw{W~_>gvYk*Yq|qATdi*fwq-$$Gz(x=SPPtqDZL4^}bIsL{W^wD9z)+ zPYXul<+w$jXFGoYYOjx+0$&?nfSbr(z{3T})`bg*6%CY^krQ-h<07Mz-QIa8GC~w} z_JiCCs*Gx_de4$O(3{boa)3}>EQWe2$FQ~;bfP=*%Nv6M)fxUO~ zgW;>ob@K<6?~7mJRM#`;AnAflyC_?NE6dB?Ws4377x+8u^nY-E7&SE6SS6>c`ytfjAyii?Zud>~rOt6DIdsH9XrO7S06 z)_*9DEg_HALOvvthRTezW zg@#uIk+W7bJzC8(5Q!N)@%C`yw~V)GR0Hi07}UQtf<36HZwqaGDT9Z#A)+Yd_ultc zDE*Gewc1gG$dbWSKnZF}G$q6``tkFR6KvZO(OnmknlT|ON zS+g`m{?}b6n-#)OqOC9YubEgFGu&i$Y36yIYP;_`^Go?!1prv)b^HMI=u$-A zc3qVA*BEnd;HT-eR8@mxj^=ttmI(9nrlDgcB>J}n(*ma+u=i)ih@l^E6oXU#veh{3 zyVJCjP8*K`$Ow#LhD1_?2zZ6hEz&FTY-T+1H0urXAQ^=Dwddlsc8-7_(0t-2N*&t( zy(^kSANtlzOj=6-Pn1Ee-9c?J;;Z#pIwIC~ODQBj*u{wu5o{fK|JW5up`OnmS zGhO1G_8`ESs~NqsViipz9nxeoN%UCSku4S@2t4~7W<-m}b#2U+*lg<3QfoSPQ}oMA zL~%9{wP|lh6iQE^4}CE4`oV$B8-L%K!*@3zx7F|b$;_-lX1TcI>y>+?;lJ_{|N0d< zkKRe#%8EhWu2P&%pwvL#r=Uv%)Ylg(SKN6c;WQ$hmdX&dVbZU4{wgrzN z|F-E%_Wvmte}MRq(fY-C;$b>TP)5P$Y>#FR;M$yH7WLLDKFON*wb_j(bGcM7k1bnC zQib?7n-S4Inl0AkEp9%)WdvOXjUh4#&n>8fIZ$Yp>N?K?-rFXKNewzS=6pv=y#_h( zN2kn}nP({YX*!PVyDGM9k2lRo-6AS=M~^r_qqHoetEkHYH|Y8IU2!wUgtW9gs1Wp9 za*+D_jPbof!&g{yyS~EAZzk=q@@1Jk3}NH6ec%ep6_(4bl2MTPDa4E z7-61JO?TP|vGok#EDWMP1yt=Gm9ju{6B#8}_0!CfrXU5K$|9ynL}E{UxZ>2BEzTo2j-CxI8>`R=os+wheP$ET`^Rx^F6GJtzwml`$=W(H4m#HGIb?PzbjPnCUZy0<;)ssLOp`}++xjAk4V zp7MV0Pbi*v-5ySGaCYwF#*Qt0x;?>s3=Ih(r=ZyUDq*W@?68L=pGYE;5KjxTSxD)A zBJnzAEz@mQKH65*)C@Q9d%*_0EImqN$0xl0*%Z5V@CNO`qYzMa@yOG>5^vsqGc}zQ z(2PX%Y&7Dcv46jf-+_qm8KsSs&y~sbcs?MF&3v=8vg0q2>=#`GeT?s z6EczSUJs%z+LCpP!Gt0{h7Vb0f4{ zzM{&P28+G1M6dkqRrtQeMdhDFMQ@|r&;O!roa8;{ylnkxpRb{f)pg3aSTy-##9s%W zf&`>c5Bv8(%eG26ovWoP(gns0y*V>$ki^2mA|q!3jc#)3{5Goy=ku3$#Ql7=F`?rE zfW(#8R>I&H)?)X&D3r~v$|-x%#;#;&_;TABK=>OkPb`{05RINGQgWCJ)mwI~OJ#}$ z$kcIiMW&4oa)pSjm|b|Of16THN z9(<$RC2#Ir*m$MyZvS{~96Hd4Fj*IO4>x5W@U&bAL{2zfho@lCa%Kp&nVk6O z0dB3NxExwxegn8AVB8R4=pA=TP8UZ@py`CTBQhZ9!_*vMX8+3>_=pRDl(sfUgw^#m zzLU=`i*wl~2I3DoB_YAH3+#23@PNfo`5tee`@xTtHaP5_k(Ut{3yXrBBm42$3LX6v znw0eBj02e$Fz_MdXSHK*Z><9-CaZ-Lk~}MOYYA|hVAwVHc+Zs!9FQ$pH6sGe3Laub zMKShSGcp@%SE|&D+wwbv2^M`_{=#J&<2%c zPF`c9qkW0=Dpf{eCI*RilYp~03~;-z!(vaKYXwH=kTfP=aGCG4DL(}1ryY;ji1h)9 z`~61PFmaiqQ6VmWlY_ zS#4xbT@0FImYr}atw$JNcDQ-);-(OHGDw)#x|Gd>XS6SCKc!B>D_B7)?sV>7OaoX# zwb$AElfrjztbjr@``;r3b`X)8J;Eds%F1m%8$HwR8}AucMX1%MNe}zr#_3(q>mJ{r zA%2Io_h}#jBzl3_ag49`ovkRd^lqFC=cpp@EVW?qrfKomDhbQ!YvNf-tzt>&qJH z`0%BHpe1D91r>7(FPY>0-f=-Yte?gYn>=RiW+xIs+Tn8d*dHSNEPk=u3?-c366642yP}R%0vV_?{7vr>y-hb! z%=909j_CH?e;wVt?D2VZcV_6u3=s=QBYsKd=RB{CaM8QVU#624z$^+Zq{0FKxaA?~ zLnkSr&>WY(AaZ+eD?ypc3FuXSOu$Eb6r!QCo3WSQtxvLTtuVo`3y}^4?J8b;B?iKm zje4}Tj|I>A?N#S1r*Z1gUu9UH#99++MTgIlvFfK+)pf4ly^>YlR)0-T41(O&+FbbL zsW>L?@~N{*ZB(9Ag@M%>lN zp2fMGf;JLUwHt>+WBF`&((?=IeHebg&)$72hHD+Lep!yVJ}7D}M}wLrG6T*A+x5Dw z`G8wXXGV~fKd<+74Y@nrJzb7``Ze7@VcLByF#2@Xyhp;MT_=6LaPA5!{0f4U-~Ue0 z?+JvBF%dnc`kkon_u?`sXh)lU)#h;pLp0Or=jViV#Gw0rVS%2R8AV1$#{FuH>ta9O zoMxSIrgWhKh8h2_L~(Q>=;3aeER{znt#Duj$kT;S>7p&661I*f*M1KLC|x7u(c|F41ROC%>RErsB~-ct3$~ z;Yu${MM~9QTu;}6M7);)uI=_`eCQbbYONKm@$pUX9Y1tha^1wW5eB?&{i9|e!qE3R z4MGJjSx(O9{q9a#Uy>skZ*%ILm`Q#$q>?-zlq-@0P?!iF^e?%Q(!OJE9UW_K9=cB5 zImP*ex#S7qk1OC9Bfkg|TUphUBtYd#`U1?!J)(7_oOT!A_GniAcnf0gatxkeFp0;Be$Z9*+YnWroEE z3d@d9?z;`S*Zl)~O8PS>s}(7ALk(~1OYBDL@$=&oegB~PHG5{G0f16DUVdT!^M_4B zpO4QRfR3R;>RoJ(;N(`M_i<4{BDXVf*zL&p_`vx_FD#_baQ6A(iq-wx5Gs=`xZZRG z2VFZ(>|iWQK&Q9`X9-0_?%zl?wfWAeS4+hZxcai0wF6y6NwCJ4oH#lu_mU5p*2)V*Zrv2O1M;5#uU5YQ$vzUGdE)~6JXxTeee+d8Fl-au4Z8pLXG5Qj(9 zbLS`52`+SUuEMm74W-iY+UFyZ6$B-aofbOa$irW|o_L@F08JjZm>e?ift=|M{(SY3 z%n4g#+yogi@pV&2*`AM%n3zDrsXIav`$Le53BqX{^0*ZX-BTEFd+7zYQ=Wi6-vq-M z<7dfhWmBEqjPC)fE+~d*$i{usM+hGi?wb=6Vp+Sa~xa7s7vFmHbH4;NUGU# z*Wj*9#xAJkBP0MC9_GIA7SGl;A$>wkXA#U3*D#&Co8Bl9){r=pk^Rj0AAm0LpHlzl z7c*dOhZE&a@C_JZ+<`6W&n{o>0sfswE%iOGW9c3BbC63|%4GEXlcDt%IfzQo2oTEq z&1Fpw8BKKe=KluGaP-5V34Hy;phi(yYRx4c6q($EmJzGgKj&F%3->a` zw-YuX>-)yHwNf!Fqu7L!B9O4>E5&H%u2(vCac6>^jx#@rmrpbI^4;0E@Py(`Ci|{M zepxR&?EG*Tv5$MtCcMFSiEH9qIhi_r z=GpJvCKd>S+p(;^+8_Qp>bnbH?XwX`_}N>y5yyj_@C)MRMStkta!M5m@YEA}w1q4` z8{EdXk`uVq_lY9?dH1yt|}eGJUZy-?{x zNz&x!qVL-$JHcoTwyR}`&c;n!^VKHOAdsY%R@ysFIAlDsp275K1f($=5F|nb4fT6} z1t*by3KTW!-T+9N$ZD>)SFDNSAjqHHw6~|{BRV?yG~g&EYGcEstfB&}a3NQpWV)?( zLvqFXous7^LMYsX0l@C~`1rHkGWUf>_B1^`J-6>OcSc5rxZt=6a7ESq!gv|z*YZ3o zN}I0|JGM#|3SpQXwR~F6U01#s_}nBxcm)MXJUl!^F-b{Ds{~huekdCJogSI{GcqDa z6jq7b%eK<$hRY(+HaZvgwzIdFUr-<(9Odldf=#~ocyk2i7J+EK1YlJ()X&MUJ?t+H z*v3XhVUk-QAtH92xVpJXDJp)v7o4JU^)caM2{jWH6T8vZczv$j!fWkR08G4smH1`s zZoEk?P9r7+pO6LKzWYrt$EBNCyq4G|&cBaE)Pg z=-D;0V@xq?Yw55FT%fwJ*%@GGIVARsx8?{zZLIZ(ei9Q`=#B69Q_H#% z`XJ*+#`3Ch;s}rC38UvjKzdwH=z?>2#2BylO!&N9?l96ch~ld4$a=6}K$-QY|G(S* zUp{*{!|3}R%Le_%sEJ$OBPQ)yyCNZQNBW%F$aY#*1nN83taRK9e%S6Yd!l z?r}H^NLAg~z5sA7e602MP78-6JB((q1pdE7kCSvJqBX;dsB`TX`30!^N(52M-5|l^ zea!tm5E(*9VO0aa;Sxp2kzq`W8^3(COERdBJ&631nkGj!nMJiH4?e@E@ms6?J|qTMIBPuy6#(?c5Tn|Yl+H2d4Co zioROF$AIy!&=Ynb6KLs|fM&8;F9!ijfWgvXq6|YRLr4JRAs7nZOTHVgsQ&NQn>19X z*7xU=U|D$ZTCf=a`=gx4MmD$%1WCiSZAW1?9T%~)Q&N8_O1J6h|Aouwdam6L?9&x0 z`$P)mc&@;)L<|lCPy_LB9?k$j0aZ8z9>ri=TU-0`Alo<3_v4*N@8%=R^PH0*jLB|O zphB-p_mmCYNwd?7v#MnuN8z`Bx%Xvt+dWU@b6CGEM$FmM?KgqCZAhVRslWK z$Md!ekwu>Q@Gbg~KCHo(hjq^>!10qgUq|d~(gH=<;nQ(>wc$dQjv*)lw@R%Mx*o7q z>Jdz<$p$?Z$(IfL_2%{#Tv!%9+a62M^x*f@YHY<$Fvg|w6TZR1-j{XtWp0kCQL zJ0&H|!6C+2oIIqfKL#JE(=`MEmu$7vZDY1n2?DLp#-IWQOd(BKLBYQ`%LA7GYm7tW4DI4363DT;b4vb^H8E%LSCZzO0&Q*DacL#^FO} zGrVN<>6)QMnIY$tRZR_C^$j%1$rAq!K&0dWfQqz2Il6mxf%-{0ayBFq9R}*;j6fktKr?>i>-4f^o+BY?a8>9IB$;$-viU&Ayoe0OzV+J!!iF71L3R! zjtwvx#biL~emV;ksWmDxPgYO-EgIo8T1mSWQ-w9%e(ey-9MWh<8d{kAekxu@z2%Cc z)uU}Q^x>Q(QRVza4n#?S9MA2==Hz6Mz9F=Fg>{zFlHU1M&YvsfI>JYU9iM%b;h8Z1 zzV)lqu~$tEs+bTq-s_>4BwH4v?>Z>2wf#(rI2!4H#(~ik_}#0D zkUU$+;kL!%Sb)cSLKI^`Y$?Yvzngu;W#_|ZRq`?j!RNymkOHGI2 z1txf`Ht{-~o0g0W_WPOxJAK5iIAJAcz4MK-mYRf3b&UMGVQpluMhxx^B5yiHn>I`J zj&>C}S>?adN4@&cZz2YjZxrnfoytn-!U+VO#1Eq=yIJmkI%#AYXk_FNOYjoc-DJmM8LhZ$i^RBhM>h}%6JysDy0mMixku5{RWoqJ<7krgRbqOK1 zJUK@RSA=gs6dN+*S78oB0szud#PSszUoHg88zJC$RM^dEn01|n&CuW5d(Sb~I%!Sj8!2O+!SWIXKi zBiaarUt{!R@c<+vwy!uHJRa6uuYfh4(6Nyqydljj3%Au>?>)e9Il0g?BaJHn+3W!8 z@Gu3?EJro%a2O4P?0#*VbRxosn;)CWq&MTq;?T-GX{RoC_7k(dZOn49oUu5`HGC%T z)*e3{{x{Q9q6Ewe9y?h$SM8l3ZH>t-%*2phzO}i!lpvE|GrNP+}Kh-MsaJE z?wum$&&|0Enq25_g)5SNR(#RVMd`@RMdWtjmudWiZJ1D5$xIgIosFld3969BbZ}t{ z29QxQdg6@3$()Y1L&A43ra2ojO7%A6gCoikD2g=Pjfm(9d}owdJyHD~#bu4V=a%qX zb-6WFQpdA~fU(DhZ9A&2i}xVNHM)Sv9c1jDXcqIJYnS{o7iyabk3^xdAl!~hU9EPE zfguS?0TBYcZ00#LpVZtpYJJ#4wDeA>k$bNY{c0%l&B^k&?Do`&oW`oy9t`gw>d*<1 zk64PnWk4o?;#c39J`}#o_0}y;3!E%rvS2D#F(|+#C2~^r_6EyJccrAuMq45bVnBB? zd?@Nd^a@O=SAT5Qk&{jHzirPBKWoEVI0oiMK%VcjN_7@!E?;`r51BJ}1pO0`ms}fv zHub&n{x{f%%l-D$p?`YVWL`5+Ey`_q5)r5N%--_-xBQP2@vT#R;8C=bBZkHl_b%1K zoww>fJ>>QM4S6GE`GR@(biO~3rh)GhlHm?X zAj|BPRZ5c*Fh_^3%!p<_8k8fF?JL+&Yl~$Uj zssfl!k|V=?af>W&&73_0Gbra(1qLvvAo?IaZ0 zxR4np6S*WU*BAua-EMie0vO)VOmZ`Ppk~@X8zuN72I+sfkBbUN0Fa8gHO_IGdj3{M zX>?lM>9~-nuw1+}{=}h;(-yS`EGxPM6y0N&2Jn4p{1{#Iw^WuOLq)~d?*H0_i6CgtFSD7COXU|*W>a@YTdWjw%HDJp z&F7jI=X#d0E`+SisUoCjr)wwwH&12pS+wIMEyt3RGUGS#wS%)ZIJ?>PldL)fzo)iY zbNq9sUIru}aYqPGRnIP4RBUyOxkc}4{Jk9fiRCV{Dr89KEF24r7Elz5$$q1v`Ig@ zk{~(XZIMxzfo%_)19mfLvTPv|o4FXfzt(qVHWEt+%Y)JR9o~0JWt$WltqPsQRjo569izaF0TFc$peedD^BcroTpmugoB~;~&EEjX+VTeB zlppEc=Qk&a--ZDoy!923GtcYLIW;5t_ZXA@cFsEK`PbZIJ|ZM@OhOu$EA$(Sr{#nk zT?&xQ0AJ!dcb8V()s3>{tKL=ii~5KJ&Az39XGy7G`^bJWUGHOOGC|pbi5PAsFjI`w z^{%$t_I4b1xH7Y0hMk)0{ps`mnR>7|L#c|<`)3bWVGHRVOgnxk2vQ}NpT7zt0|F$^ zE)C_@fMTfOmvG@edo%Vm%Vw7|O>KuEQXzbop_bD+<_Qm+A3~T;)6he?RUQvEUZDmh zMVwyuy-57fn_)M$D~&Q7n`Xn;8*LYzT0xaYrrCi|NEW@3z+}&=uJ;hIhxvl zL&aYk-eTFzuibE9TA$0o{93W#4G)Nw;&UCuz34f6WQC4YPIyZ6klD{}-xX z>UqOtVpCHS=D~ob9dNpTHnKPmG;3$=qvIMt$Fkk811rixS5@Zi(`r}p+Q#+2h87rw zmILWM5)%Jw&ACnOXtXX8&!wMTm0>dd7$?)>VD96i7gmVjkBE~^PhuJx_-}QxU`*yz zCWn^0gK>sQ-$$^ZEwh%V$*QFCO)UA;ES4AAJH{5~K23C*cMjwGvSQ=Y)1dSJ>TA`{ z`dXzd=_$pq<0;oRHZkk}=oioUty&sy?zH!F)!el&c3(lg)~t&SvPcetO_8}9G}%~4jnl-)hc|%CugaIwdXOx zUYQ1V@)|?cs#<};0l7jMCCTu{OP8RR3z7CWGGsGku_}OX!G{a9Tb5Onl4FJ~BI0y9 z5WKO5ah-Is;}Xvv(|b}q+E#yjH*xsc zweO&;Oioa}YpTujNCuCXnX^>!Pz_YQ!-n>_Pi=Ph%=~_tUVuQ#KVW9d(B3)`fvX9Y z=L10Dn^=Dfp|lwSq;Lx5nAVMX#}&z$=y|}g#l$LQgfxV97I*EH(A@M~Eut@jJ%l%d z0kc5URhAf!?%ebrCF%`DL!bap9Ln>he-b*4=l69}YudVJBX3S7^wu7BIMBpE3*>t`^zI>6cZvCIf>zS`rFahK$Q=?;Nmva-}Gt13H*Dz1CC^!Jh2j=IXu4i9p*ILS&7^VY=U^m zX22D#0f@{VI(pJ#{SC$Zg=D@=qOaQ*IxKd9W`6T)tQUavYq)?JB7*g3mbEx^0EqTm zcebomW+P07fA>?Z(q000Ff`x*|K8S7ql6mzp3d!BrAm)L^0zyh%qjC{G2>+TS`wDP z(D=B`*R<#nN3+`Zw-;hyV-oxgje3oMQl+{tAgT}OFt|*ezuNj|x=&9(>T)MEzfr4J zd7(c+j)3W7xZ9er``WH zRe+VNs0$1-@6F8|Ai`H%ZFP(pMH290-nUcYLewsvhL}uz{2^Jlf=U(yIgO1H`GwOB zY_~0nGkjw$)Nd+UKn*SVpseL&CZ$n$6TedUpVB7dr)YpQA@iy4L;0`AyNfqF`?`Va zK1b0(MrLKt_3}-P!@@lk7RO%5zdm?mO~qB67>>Ga>O4;N25Ri|w}-rVY^pZkS8g-Z z^2C{0iOs)gzPVbST&#TH)D0pmRB25MiS-MTkY8D~_0KYI8>W4q#BFePFR`U4lHIge zTv1V21k*b%NDGqR@NvpgO*<`n_jy~Pvs!cpU7y9i22ikas+lEb;Gr6hEqb5oIk-rU z^NNDW|78hm!x!YMx}8XUzmf_yU#Q~MhRR~p)YFS@eP()AlMb{QfW96Uaqapcee8lWMNemn zbyz-jQ`&6XgEKU+^IA^ZRDi)Jbg|_G;+ZEY!YW#}6N`6SLk1=CC0)-B7Lb(AvsU{t zC=wLMHmIa;@aBNsZS$c+Xn#CzMP()djY>)4I-f_5T5xr`gAZ>gLTyP zG0(3;xtW{B-_A$RM^`U=4wg>nS$>{_`2U@xQ=xQKC9k~fcKjxOdP(WQY=! zYEH$+jvCR5sywjD5p{>lXId?vhIX&~M_)_wXX%1g} z^r57akobm=w-9#~4S`a^b|R0tJsFxs7s05Re?+%Eg}?7pXTo%$v>-x+>q3Nb^T)2Y z$IW`?Hzo4oXO(Ml`8MeYDe%k8#fk?>kB_AUDzphnCCK;h$>)QOq@XB7Ig(DT_6Ll- zK4+qo*0iM}xuc0xCT1lymq~u|4iE3ltq-Sv)WK8fLs9oek~B)EQTe)By{*G#`F-Ef zc0**+*-+3N^aeAbAD7R5}iIL0nLlM8{;jmhSU0M+|_}vyTAZMeA6oY_4QXqytfrD5q z))*K>Q@XTwbci-H#b(jzw30x}aq#;cg`p6;{f%;Yz=aJ6Ywfg@oN?0L!sSD$LnJ)Y zmv;$(P3@_tjA=jIIa#QN2p^r8h{-9zKt>jTM&WSWmsa?W7TZ&D2AY3+%upspIhFw?wLxNjW z#!-wVZySQ?Fn7MWx+h&i_Wji^D`jVAnPevGHRqCtU@|6Lb`HHQ@h>U$-iVf;UWNxF zBZZ$DucNlwyX(;~#PtIL@)OM{ZK<2R&sLu`PTpF(V9U$)~u?fjFv~D3tkH?S=>F%dp2Q@z8&9~u6KM+P!f9^b3;}2 z8?3t%$)hO)ubXH1c{``n#uLvi9QY6=DJqlVmAe^sH5ogy1LI>dJ_Pu+RG8j*Ma_=P zP=dOGKl~o0XIjr|o<`G2)k(>i)6L%%x8k3scI3?BZ3rGw4=i9?om1YZECNQhGv@L*6NW{6$Ifr+xK+2Q2HPq_S}Hg7D-@vIUGM` z5RG4VE&h7e?dCb56|7Ve7nsSYvUaVdUG~KWZ$VR)Yod?Ih;G2?+4m=1rmzc2ut(0n zQ4X~MO0vn@Tqwin=;PY<8I9lDCGNi=I&RceMf6@Pfq_DodmT)9#=*uXSYMOiIJ~JsR`*^ztS5k-bJ$(-G$ zE^L}0OdJ5j?#PPXMUE$k@z>hTFzCMoHdN_pTLvs;vknpW2M8#K$sVrEVGbOw=6k#itzph#!-=K6ieR1i3HZ=ma*DG_@RaJ-o>%x$V+NV^j9|s^`<{u>LRDZ~ z@P@utQ_YpfPhhnXdIKGllCPOGG{n9iNTk@vF`YhdVt9CXiENe3*(Y6xPb2KoJ@uw} z!I^G!st6N>Fp_|Kpg@{S<#gPK-ouQoI6XTn)oF7v=&3Td7{H1qrdbGrL4DL1iT9Iw zL^j@fiREUP~nHU@QVF$BkmwiI|v~nKj$p7y+K6o4%2wjIo>OYtkb6=b-#t^D=9(?CjA{XZ9!U+c2;t6?Lp5Y2ciqYM&Ih7NUEjoKcPH&Yj{I`tsSEb(L%0WgGO|pk&=G?g~tCU%s%t<}q&Q@{^ z9Y`1siz$pPEP6;t*Rl|5kPm*X5VzhP!@DzSa8oQsfd{dY{Vf-Q^6zbNC0azy}RBR&9^ui~z{50Zu zk-@N%3N{h)5_oh9d4V8m%k}ma0z89uhrQuG6kfi`00_7epuuMFszO>X5_(urf<`zY zFz7dy&94^}tyCa-i>HywT##^!fL&&{0%fj$xtpw>_g1W&LP&&(dV!D#JwdmD0ZPD* zaPyT$>wub&3)t8_hODyO53~j|VbF*wxwv2GUQY3}h_~OS(9(57Ue#x+#zT|ZUbworY^MT;0#zWFAW%wjnWrst z+^7BJH9mwEmR7CO&B#Kx!!yOhhQCev!v?E@N43q#(L%fBkHbbRTc?Ll-n(O3xC+%} zx%@rTY}*oeM;k@BY^yNrT+kOXP!btK+0xSr=W2mMa5DB+ch;t+F7mfce&np}$kbHL z%Q6kzcd79$j=T>djjx&8BLL9E-|0{y6l)`}{lmN*S8#l0;8!okS#i_(#NYrw1Zq}Y zB+M_7@X;sHXb3snkTQ3(i#UV*e^h6B74erR$@~eR0_WbNurTMl8APZ_;|f~Tq~xL# zmv1ib{f@O-g@7)l`mAoMkV5Edlm>XtBG?{YluBRxlDXms7voA!#hMv%O4mDl<%V}~ z(&DJuLXT_(4eHy*dh_Y)nfS%w85B~MHY=uqdfxU(i15wP;NIDA9%@?^QSvNy(#v8> zlQWBsr<>-tz~Pf!oAOi+;}j8cn)&V<5Dd5O)A-^%-xn)%RJ3hBy!QkV@->0i{XHqn zsxIWA2Gb z?g+UcJ$4DSh8IQ&C#I6AU+AX&^c*5`x;}YcoEhf&1H)Vm;S!^v7gdvY$pb?pP;gO9 z4ugX%Uhh|qAXkopL?uS#)1|V#W8Y|IXTq`(0v=ns={I5M6JI2w+06SK1tl%ddMv4) z*^|(_Z5_={tgyeENmr)7$)7?CWMXru7n8o^rC#{YN~tf@1NIBK+wPX;?SVO z;jjy>Qm!gf@SWdBJLpI;Jr1eU<1RwjB$P*3@Go=_kxFFLpQnkUD)3kyJVxZ5^e|24KFNCPQC^T0`*xNKUB{eM;=Vn98wU zrb4ZV91R%Zr5|*gDKHv0Xs)W7J3$_CecZE`n+4#2ij|rTA{xk{v2qQs>m8mDjf1$~ z^H7IRxNIo$hXj5cFa0TA=Ch>A=ZJp!2ebuHW`~jMkPpVwSS`V8EpPV6l&f`lMOF8T zsbCHPPK|PjJfTUNAu$(UZuDS3tUHU2+VW%QHA-ruv&g=wlUC0kZ=JJ z=pT7XB8P{J*0gelG`mCa73zz8)x?~I^Sb*wF*b({WrtrgvpQR(P*v}gpOw43xXaXX ze>vdeuE^5ldz-RT#N54@GVM*MDz@1Ez*TeaG2xSIr>L;+lT~iBg6peV=OdkxaHYVs z=8(GRaw6*O`pKcV~Zh{iVI(n-6cUq&So2%f})%FC~rB z`RbKz)4@laIOFrql;5qd^jn8*2h&6$T|Kg5toaAKzeAciLA5N>HcYxeU0uCFa8O-E zY;5uaYuB#Q!AZ->gn9R_4w3oWD4WAxBUkM9BNm!x&MFv0wt-H$!kT0*z30v%>ypySG&uJqoBS z@#8j`*g!NW0S0`ICWsbKZ;ViZZ)r=Wj)_B<5gtazHzQ3CEOxqZdqQD7oWqLMMLT!P z86-gO>!5=izhtK-_F&{GKV3eqJwzgYrcx~=TkC5kyl>R)+18sW{$X1a{}K}nKU8Q# zbi~``i&&Fhc0OkI#r1<3QEm>#AeF^mewjWWKnubXIW!dyaN;SRDuW%!lS&n7p&Z*G!=; zVK=4{Jar4{6-fkhR0?KD#ehLAfWor?GjaIG9ng(&ySc;MS%p0 z>V;q%5gS7A3+Q4?#Cj{=0Cbovfw}rA6(;(GL`ZOhGgg#wU)a3f=A#4 z;jxJX6+jO@QtTIwD67RRL6qg+0Ei=?y+;hFATr3TcY144gl`f22!zhYTF=`-J&v%d zh!y`YsinY=GuIL~_7Q^$Jxu&nV#_2$L;%D=ksv=3F43T9ycxWXN0GD8iUa%$J(hMY z)-t83NAi|DDW}%AU81)(^OCQ77;_$qb6*iY{d#>XCa;gT_%~)|)?!~z+oIih7lUi8 z;ql-3kw@LZW0J0rP4CGLCbQmodK&&A2og@yU!=i9SKu|pW1aw^aS^sOo*Wx+PLp^( z6u`*5^US|FJAZEow&AlY+%N{5EK8GC`Nwz;2bzzH0) zHP1>^v>PN!CwU#nhDOIu3Ws-+2Og+PN>a37Ez~gY&!6GuGZiko+h*m`3Aix0AhsJ{ z7^g^kwAH8u0=djT943(CW}v3b$M?6KcCMZ_)}%zFLe3A6TXQAV@H<Zq4kykehAYPonq8P&pTAn)FnaL|5GanI&$(=MUZIVcKxnXQ64gV;J}# z#VL9e)upIjP4R)tZ<|LaVtDIrY@7T(vX4*999$|m(i)D;Qgy;uN~+R-B+}yJZ*Bw) z5gl<<5=4qih%i0(=onEdkx-}gnL}A~rh}RKba(1V!v)ZgCxZ{`oqFv(-u5u5XHZnA zGSVcIDUgf$jEbwLTDYa1U?oK~GJO zrTL}q3Ph}!p|>gXR+E`yC}xL)mBD1Y)Qn0fW_eV zb3B7`w#JY6I7IO7a4KJ%#eQ1^1rK=uc!fgWTcdEVEt!H+Xw<5{@w#0KAg>CH3lH*b z7BEC%{;=fZ;~Tc~>Fn&BZL-w^xK%KFgK>cZ@jY09tKdS(#l_?Z_ z-#p`xnxnpR|As`u8{Oc=0CkOmpef^fvmw$HA-8^2s?R@C0t!Ix;=KR2oKf(#cQt3l z;raEBTZJkqlkC1czc_8DDh&unbgP__QZ6#eE;&jaxyod|tb3jGU8Bb@wno=?4;Ta5 zil%wtI%-ri2-ZeD!+5u|cUg8L8Nz;J$8AX!3Co9TV-P2@Gw(A<9N_8bm8tWZ7k2{# zu%uIzHexD3MhN-PY^9Ly9%P8cYj(v;ws=i6L87suBGF3bke3V%25s3vN;S_nQfYIs7K9XD+uz?000(2m58@SW(`j|e#b*Bbnj3)D>D}ie(rhlN z^lc)MG65?Y5&=gZaMHos$o=NWv!#Zsb*2-tpA*!;hEVD>)a;1sZpqBNnJS&O646Ku za8TsgU%;OU55<$_)9ZDH`DAf9OIihP<|E}L(CuIp5nH$bEglI13xLKv1rQC1f7n3l zh1G(>zh24F6hVYPxCVZ^wH6?(SC`rsSXw&wNB6Ti)tMMg_1Dt{|cqDlVl#U|Gz>h z=E&&adrqLVe65^;%*uf)fmEctYAs%RctaUrpj<8{Sj8QZz!LlFj zrUsr+Gd4j<4DXgBtC}h~$BXF=lBzmUJ(MfwR zzZWXGr*gg%*X(s8q%~Xzxz#qh%W_L*_m7)#S0d%j4EB9L94kHm4K$XpHh=G4Gq_YA zCn^3T?GmtMvU?aENnw=s?Y|I4k0{Au3c$g!f*($dg`qXN5MV9E@H@4C9=1GrqzGYG;3mVRUwN+g8vc|8gwkjdob z3JViEh$v$a_96Qq9DqQx?WKMz9%86^y52*L;2i`*r;_Q6!spiwA_wE7q@ehrlr+3U zF!7DU&JXz+WPHzi`>3 zffp|ZKzhi-g1{6qM(%z*iG&aUFQQS2(rPun_JYaWQ85K8JHtx_pth2nd)17(g%vhj#Bh4}jJIoO7M)#X8J3 zuFz_wZinV0QaqOO2yz+X@lRM*vRMQO>TjVIY7G{1A|BHPxf{Y9$p@pU!rA=37zf}# zoZsKyx49INGo9`v8swJsOOINGc$XiGm$O=}t&b=BCk{7}GwY6@Nn#{`hQL?otdd~fWWImelme;?Rpe2u&^~(M;l5WcUBDJA@$GAjjrl!%6((~C%^2dGlhKN=S zt-|?7^}8KaQgy^9V1TTPWN?qB+G56Qq4FojP|c^G5kMKy*ugHJGr6BGcRN)S;Utsy zC(t?Vcap4F0h0v$tB9KSn&OlMFI|WOnk29Mss9+xRIH?=D3~xdPT&<76?r9O#hyw9 zzq!DGS zBkE&FLT-QSj0{Ew@%QQ3kd_|ASu?L){Qc4Q{lAkf`r3a8O0i|5{$j6E2&*%d{>~mf zsA`1p830&IdRa)qWeqkNnlq8IJ4zHL_O?5#JmlzO=S#?)W^gCw4lRwx2W@}$tyQ8~)!NPh=}b)V;r9wQ4R2{=#9E zj2g-ljXWGqqV6Go473kOQvjIZbFF!$x^tuQq@WU~$8+TYi@~v01U!Okr9N-Z0jJ;% z__4dn;ru1S>CY-FJ&lb!7@Zld-Wk-M59jDRcSkeixzZuELihld4uM^z+ufH0TVdM6eUy8ZOVIi?>y^K zOTa5wK#9`K^lYxZ1n)fFi%R8^duo+E1xBdykhw^c1s8YC)1VNSn|*oRYS4UK$@BkL z3xJNAvQpU0>RBT_WU7)JQ=~Oq!Z2;xo0fBtlllgCx_|x8iTObIBs%gMPITV3`|L)J z*un+-#oAFMceNOUl(hSw;&2;hn7nopuD&sbz+JR|s7*yhWQnuWJT@7zmHkwo53AfR zvL86_J>x=QpRY!#DK+&y$)CE-Nwmzz83q{a)?11FIeO5){Jshax+6zx;&N0Yu;eqL zvkSIhwd(17VX`_!vx*^=A&N6t=!TO`7kVWdr$1kNXw`42T>E=iO72G0PjhW#)_m+Jt@7l+^o91^@0xUIL$dqBDRMihp8- z0~BNq`hweco>hs|k;F2%+Vo^;vCd*({v87s(*>fT8Vy7!m1)!Qb*2tsV%BvfA_OkT z(8d8lp~d8Vq*E~9c(Gp#f?5uTmvEF+scUIb3%djtg;)uK zK!a25Dl`lZ=fS^5M2Q6PSTEP3Y|SAno}RC@8O*vXul#EiN~L5{slD72c=d}%d37-~ zVJOiPog+GKD`NSkhCGFK7JwYWU-CjQswq-bs_Sf$a}0q_HyeH?YjtE0ye}#1vHUbbKBUTE(SKP8EHbyFkda6Qa+@yADM-;rF1LOKXbb zzr}2Pl-HJ@Z%)Y^=Y0^9djJ_M zOk@Lg_JRe1nssOR4(<%7#br&;`~SROwyJR(TVzN%=WC#!cBaG_mt zibnjymEsLct;to}@|C|O7u)|{cs|n37d^tXF#H>ppynxo-R`RJ^X`4-bU|aTWFe7m zA1XKAkg6fhrvoz&`l!Gs6x^@orTr@QP$oeajnYN(^as+NqzKth2r1DcXHkjA7cz~x zPC=TL^n}#`yoiQVs4ZrlaAjA*>>xmvh^Lm(#r$a=4ji2s>cyDFME|f%50JHeWe($# zEs^B+eM1vX6m65MR%o3oKXRkW z$glW(pd*bLm|MVdA(BG$3LF|*L-4FF`+OKU#Bk|AE|sDTX~qNIiTiQH zM&q%F>gjFv{~Y#?=WCCV$HP#9VsU-Z@t#B9FVs4fU(oQ`Px#-v9#bxo;%A~Ez7Hmn z8y5aAv;~EIfu@Dlfr`fv+9BNi&C)9_5S_ji6L%~BA7((p^66K8i+fG={{bf@k%Cjw zUD`-JeGIOzOppBtZ+K-f33_vJnm&4c907Brmrn@>GrZmilDbBo2p7A zM|!+$SMV7EK|$Z#o!yn5&vyt{B{O?8f|kO<5b(Z#6w~rba>uMeEi_@L)fAx)`k zKIikgDp=LXC^4_>fhpRsUz@7RXPX&=vfTss)kYGbX^gQvgODTmIda%N99(EvyBD-F zi-hm18UXY3HnbTZPcynlB`(Z~%P=$xo=d4-TdbB2JKHXcB5W`7Y@iRF(f*nkBD+D{ zuF23yi9Vh=-_0{Sp2>dE71vhW^oTc{s^NMa)>Mt@oUW?2q+YA-tMJgD`>zNgbblsp zWF_ecxYdHyRQQKZo8h$1+siqbt!i^aW`M?qxA5ob1Nb7O= zvYS>6J9RXl_npQqy;Q3*aBrT|hIsx+*}UdNEiP8*z#o?PfcFSPYhRTGAnlC!hRI6-UdK_A8DJG1FZj zM%T-*?l(HXEAZ5QDz$u`3RGhcvJFQ~pN`lPPl}Ksul^7(e+UlAGt>Elq(Y@hFZPVG zCaLG^CYk@+MNw6W%a`?7+qT3(Zvz*=N^C_;ycvT5(5VltE<}qS`ujE%odS{MsKe{$ zznzn#e|?XDhM=L9%6v}7xKUyyNDZDd308=w4Od>=m{)-OE z>G}?C*YkLLwler##n%u_o}!V!xDw_AlBAYpre$biV|}K(CD)=59d<6)#aT$GOOVL- z^qTJW`prn8IyjZ|ZN-&ZgIuD@k-Ju*8|dagzpn&&z50gjPcIf1ORT9W5xp1Nel{nN z5%y~kd@3a2d$)<7+DdVK6vhyod-uK7BDbN7RjSN>m#C7CV8i?`dAkb=&&X=0VAN1_ z97chdwqJG2WYAPW$mD8tr7crsGwnHSJ}m@V$^F4L80QK{pv6L~i~kPw`#eqh;3BBvF^Fk;aa@YB!$>uA zCF~byD7|&@uh0YPbPGLKf+Tu%sg~-CU!^?et&!w%-<3#<*CM%2+MeAsvQI*7gNE3H zetfZ+HiY{0nBOxHO3{O;{zc+7Hg&eYJ^t#&=RtWZS7NMIpcd_K5)AnC zo*9p>rZ2D2i7Ds|fUjX8oOJ(3*4mp(fg)zvSAg-Fr}hzqC1ZVZ@|eV3eKfaHofw7g zV39ac>DW-6@{h$sqeRnuOz}g0hKlfIy>p}Vp+XF8P_nEhN_NRy`hb+H2}KJ6ud78S zLG@0RO)b~X+Ct6K&p|I6m-#n-{#mGv0u{IAv#mL%x9WHLljr9#tsn%UgJDN?ODOlQ zuhcs<;j*=$e*{^X+^$+u^6Uw8>l$e1HFU=Qnsy3X}#M!ri^SfyeWe)EMo+cw*Lay*a9B=U_;~ryspn zAIkX8Oa+LLJwMwUrv{Z)=9{xRPWQ)-k|&7)Y~vPEKa2XY6TGLld+Q$-*C;~eaXAj~ zpHHs@Hv{Z!XFRQuSGo={5wsS6uloYrDGFP0OZ~1Ommg=T*(M2scsfBnJgbZVi`#Vg{n|XKK39mEm zoX1D55cX$*Zzwy9v09<6hmihWW9031Cbx-l(n(})tS2Q#G?9p_mTBZH||&3HI4@2WR+O> z8fXTk@7zAg(;SPF;}E$P#mk68ftU;C$F0Z^LVfUHs;&joC0m`Kltvg0K57|CB zh{D2I+y}obghX!&BV*QBn)!GSCi`ihD=~az@D*bK;nWCONcJBdF3`Jdw%K0Yaew!it8R%cqpKa&Q(vd?1NGG?yyaF$BI%;nK z!qR)^v@vmNi9_$1X5B4X^`f05kvqELJeS#^{H31jHc-v9m+7+Xw6;sl7P)|v8a|i+wRNM`BUS{Gj-OR^W>`(-yW`gb^ zlyg!->vZE!jXsg>zaM2)P^@Xfit@>ZL&-&THBb2`Fo>}jS@MVpnol_wr{x^Xa@_~y5}A|`l$s0b1XtNnoVA{(UE01iZsq8F zkt%003k9*`6%yyZB_w@XjiQ5NY=8Q4jy$ATEYnyj@xf;Yc zibJ5-Mx6>vLK#F~rd=6AnS=98lzUS9OK-$hn@PB^_cvQSpXCE>q(l#uX*y4?<*<95 zls!oQ&;S0P;eK{bet<_ofBL_`zd}Dkf4?m-#2t{|S?k56&C{Sk>56m>0qE<@BLtMF zsMbNFX)qyoCH31Aa|)Mi*)V-)fN1I}Hr!0kJagYYvs#s_iK~guTm*aG{+uVp1A_TZ z--|eli-uWP7}Ha^La~Ri(QMy)g6niba(1cHmbBRSX9h7rST*-932t}LLx{a?I|l0F zdiZ22S0hl6tm4AsGfcY_Sv?EgV(2>yB|@^aIA~no2eE6IxZYNI^8i9|5c$Tzwa*;Qy7eGy)%ghO-U+4gp2|T0`gZmo4h** zR5jm=4rlGpPmv`AChNDXau(k;bIDPI2gIumFD6T=%vT<7BmT-)+txbig^>Ibk*rud z@exPEZ!6gko??h2Jp1!cs|f@IpA*zn4YGCvCzimS74$wl!`rIwv+46I;?<$*cVnGPNu zGnBQL%14Ev0hvApD?TTqC*ztcO-w#k9!>8?{*khF#%uu9rJkJo)zI-aCev4h_~#j_ zOn`h~P4o0N1_;HqKq?e0Z%ZB5)bJ2tZqxQW^-?P%fuAMzLcLMhZoP@DXB`7OD`Zm= zzzIi}{95zB{%>nAIJcVtvo&9k`HE8>v#mez_c}Y-b8eqs8`24sRjHD6Hs%J}o&~5X=6(1Ct&3DT@@6|s zp4=A1g-<7QnUT!q%Mx~htfmdqwJ4EIHRQ19Rw4%43AZB{Q=nO~_O<XFl+Pj#CQXNGLY=F#3q(UZlwrvCfP?dpnUQN%g3z`mgx05{@X6kn*tWw zO*$s_26o)48g%`xjSTZyG*46mDxvfIPM=jf| zSWa)uV6SWA4>=WK+#&P!u4B?ryIaCBVZ{?4Xv;8ihdKNg0uaFc0CYqu!Ik zkS=6wHeH6*Yu*i3Razb|$~2clIqg_J5R2*GJRFUijFoJmle#E!`y@4w|A-RRkCI*$ z94KLdyxD{sW5)oR(n{Pd0gF*VpNe=-iTY_{%DS%Vr4-IxOU!)Z!e#prLTc7L(F8zK zK)I6p`p1njzRFB$-mL0x#drGE`Gh1tJ)i4M#2BNck1lBpIQN-)^GB5{yhSKhtKrunO%beLBYCe* z{h)^*6avTf6|RGzi!Pa}KvFp>)^-14po>aU?K;s+C%lkQv%!Qi1Cx`J2cF@jh6y5! zQg^-s-r)V9B(HGc?KBy3Nk zBAAr8U?j@8G1I25;BK-~pUYc`jOGD_(W9u{LjOYEp=!aAT-%X~9hm=cm1tNaAX`Aa zU+}s+0x5mO5*SwhuZa)@Co_On1ic=hVlo57WDUmg)pOr+2G*bcQGs)ba?YMJ-^ZU{ zNASkwCHZCw8q*}-t-}`bJt+%V0fZ!eG7Gs}hCRY0ZX#@GJjc$wpVK!8l%U>v%PsWG zDO22_Ss^vw-5W+%^Vvo<;oXXVX;S(fOSIo9$3?XFA!zcN z#=kuolGq(;CPpwpXHLJQ;TJqFNdN8kd{J~e>fdqIAOTUN{LK=>J0aD`f&ct?N=bB# z?e_4?U+;e1ZTI}f9ZvmTq6zXC*0{O5E&kq;SEE9u*mFdBTYxnN18+UxrbxMuCy{pD z(}bhLp%0UNO8HCG)yeiPm**e)k>9svB$*KEN9?)tx=gs~($O!u!Ky)r_lshf7auHj zO;-{~sy>NiC;t|IS{n7bvtb)R;5O99QG_x+}LCmGZo}sZ~m`ja@1VmJX!8k z8BI21@Rb3OOU|9t0v92sdN5_;(27 z+L;?K?$=Qy4{jP1C|N=5DX4IGvheqbFR-8p=D1fi<5~@Mul#C zp2>)lY({LGO?V*a0&lo;TinXK#8KL%xiO0WQ_W9{P$=-fDtd$TOUv_P6 z4f1O-Jt~#SMs`Bh`2Uy6xBe@_xN%PIV`Oh;?Ir;xni1?hlTO%7%}N zd@9f&BwTAEw1}W~>s{0jx4zGzA35=`VM~>5hZq1>69@EdmU*2jo^?nurnu^36?G>8He?rTeR}4{sd|&} z{u?im7e()tV4z5oLU3}3XtrJdAmHwo>x^=n{Z;=}fu9q5rm|A>9(iwNcAv{~4eH_? zCC^o=VPXf$eYf8OTUFZnY*6vn-Oiqr*67ZJTo4n|ADUK5atKA|)^8lzJN|EFzR|8Q z+;%pI?}PF-KB3Mxg&BypUS20B{kq{%r0KrnAQTt>8BF(Y=aWKPJ)b(EzeqgpEy9Lp ziFk_z%#>B%5v(=9&UHD3Tmg896pY8|!p}`Br802&LHNMVpv8r&XJKxTzqzfA)Udre z_$WCw6>qi6FEeYZXfhftfrpM0j0_bJ5b*HwEC{S5QoGn0rngb=`{Ez1O*@Nyb9uS# znculExTJ@vKf6JN_8-3eUsbsNhJ1Eo%;tB)kBg`enh<$$_p=ht7Q)R|#p{Ppg(bUR z$0{^h&RbB(8As0Q3vjW>5%8NQlsFRsP+4J2I3AtgWbM7?NNy8fwIE+=tl?t0kVyMB z<;YH!#@kUc78afS;mz#jZW`}QZ@1LRRq!m-)v^DJy0jYJaiU#VCsaUrww|B=ubx&l zDEjI|QKR(%3E3xsQBRCt~vu4@80)}@yW}$etGq*%47wik}Zjq zLYy99FF+%BB784K-aGfg*z%<{E@;#LF`D^VZGZDlFfWZj0zY&>PkQPd@fUsGp z+AQXNba#us+fCun`TUe9&Ez;9<(Os(Vt{mgxOU_buVs<=S^f$!bW1{{ZXQlj-M1_* z7oUapaiF5m0!8qBzL~ZHZIM46zXRd!{mJ=dEjdCovam_`?$8tzUhjYRgtb`uz2s+h zrRrjVYyF_=FS_QZcf%+}&DIiE zj1Yf{SXYP3xttOd-&&;PV-jQm`Fblz0t6NX3d#>N+hD!3fY;X&3-&%f-QCu;{9IIi zJCUc?5#&}d4M<-P0itrSy>^LWIzpYqR+HWQ(CTV7hxw4$V!cH)S5Gqm;8>%ik}Fbd zIuNA>P2 zPoUH!`7k1Xw!b|XceOo;c5%InUR%`&;joYw?{;-{RcEeAbwEZ&Mgk(Y2Qsc|i{t8* zdTAB-e-VZQNn59+x0iYgH)M$g<$4bh)ybIEWrapdI%B~hnIn$EfoVk~ zg=Jlo-he~ST!KoFX(ICiTh@#`y7;>s14!XKc!9#g^=P@9vZqSkMwSh~nDq@zak)gB z+{Vkjqp2hSTI48(l;5D@)k6M^IO8uRq-HG_*T<}~-c;>6rJ5ddlT}){yxK6z#@--M z>-kid#5oh7>XUMn&tB2(FjMt6OFTGB59cx)VAmEv+&iuP8%&qYR~IRMpN^jW^WLcE zSltC}rqz<-XuM~6iW*u?{L^&zaaFumqNF5sLG@?B_~=3h&5FoNj*ht1{3gXOey@uf zA&Vi2AlY;qzA9y}KUv0A;*;L$C`cP;INz&Sysg3E_il%4E_#h-o%O8DmA6F<14ImO z*xrcXM6rScAB?AzaOlP_&mLG&TK`?uK!lICA$`?=-=wV z^v07WNR7I;?ymy?Ep0HrAp+RcMoseFDKuPg{9uR-k&)_Dr=yy!VGmzYK%sUnlf@Il zmxqV!nswj4_2)=%nymMFMGyPH<&OZ`lgZz^gCQ9jg-5XExt;K{WIra$5c|75zRpLl z{|#=No+_wQDpbp4V!(*-g}IV)ySTh8))~)DSYC{|8Xb@RZBIgP5h)E}F#~b%^3J#F z(&b}i0~|&LfQv>cBO}x9eym+Y3s%xz3$9QHH09tnS!C-3D02@hE31dI&G%YWx(Tsl zqAxOUl$*(_j}0NYe=@kK;NjtYZvGN9Ij&{^jGbEAdws)IImtxF}tx zE_V}6IEnK&v$%qJAx%SV`wrPRoMQZm1DWEl09!(TT%_Qss4v1rY>YdmUY!+EcE@Mk z#f2-qY|u+@yWsPCy2yretCCHhmV+fD$@}K{5u^UZScD9i z)ab;!jTZyuv|YgXnYp^)YL5`Ew=}EzMfz0N7j>hz_9F@T z=|;lyeAa!>U`UtLU6FE4%?6Xx{h;UoUyZdXnjfs*W9wfCe#hGDN!BH~jJAq>`^E~= zZ201Mv{)hp0A~o(v>|23T;&<+DD(kym@~Ea*h{W%>?EzC$ zr{}Iq;aQy!Qr~6-{WXlk>=ylu(^QwQ9YB3;rB!9+v{DAcXey4g3phNued~j1anPJ9 z!y5+A-7d((%e@<~eN3YZWc#g3Za1A5g23qHcvH&ZVt;S(?O))AdG?G1opxMCmx{q{ z)9Z*Nd7K{Lc%dg;O-p<22@REy_`_M1Wc7H-aF5JD&IJvTro0+wU{2 zF{8|pN@p_VbhN5`}7e6D;#{}leb6r}6k`%i5iAZIDx?c>~y2U3aC#%v%{WTa# zpn@0ne@fG9wwD9ESE89Zv=7OJ0ykjPWq>SU1I(69MmE!j1_q>xJP)Traaj!LBJB83 z6|kk|9$Q_ut*;Iy3sC9gZ0dnrgZ#Qd`>;|dgC-t3l0=JEtemYEQvJ1ldb!2v5GZ80 z0Dpy}^`4%RqXIVr%tz?q=0J2H%MsLNj~lyR1-{O7fA!-?fI)m=%~7$LeNh{!N}qz{N#pn=QS~VH z?z}hR5#D;>A^<&L*g|9B4ebw8m+GwK?h6H-XFdBp4u7@Rb^P+DQJX_C8-#xCOw9OA z$FWD&ldQ*z(uf>qG&QFL%|Q%TC?Co+r#SSyqVf=mG)8OV!DBzQ%q!vO;e{x><0Y%T zg^6{>Cyyf#k+Q5!E(swpg41FcLm*%qHh}e|pgs8_0_XW~dO^;)AmW{@auQy+qIr?K zJcWdKGBai04={#u7hWF>Da5NPDPtM{ue~``!lY&ehY*V-RuoK4|Grm=PiDFMCPY&t(%lyNa+O20#^Oa@_vkgSZ;O7V4oySeC}q4_=~= z3tq|5lunzH<<#@BT1z_V-i_#Zd7}14fsp^)Mqifq?8|)Q1etBZ?y7+P+WPPy45y}D zCUE4Y#tEY6PeeThFskzxE2@|YvRdJkplgvc+5d3L2WiBMp=ZqWjfQvn-$h!5?AMA; zhi0-Kjuqa{){Ali!oIrWUTk|A(cAL%p5FrG;Xmf3OJ#P5<)PSCkF8;5E zAX0r&d3n05#wQSfN~^f_>WYyd!Z4u60`&hHeA077%(v!rpw)4)J~V^N1`{M)A!zpN zTYjC{VAK{KtI>=uio`sPTw;ib-vd}G3L7H?gx(xrS*}_3aCHj8%8cv<0ZH1 zv!T~J;vh@FN<7WA*4Y?VpkbTSAE;yuj6#S}6SM}*!C}AghJR1}<9#S;^tn=7@^O<( z!&%VjkA*R?xp{%sho=V&>gF;(&;mC zVvWj2Ncw?J@=g``)UCJ~cS^!$ks|<5QCE9@ivxDb!D24AJ0OAIA|KR*q}Bv9!EQS5 zt_eGo(nbHgbD-1JK6Q;XrOEVtJY zfdCn$F}6?3pyN?GM4za4ACI&itv(8O$QpqrGPM=~$JCOrzJLhpa|-bYw0W6|BZUd> zY+Hy+e+UBTth@D9HC2lE^PV$T+7}H)OzRPZ3sTaT{EV;(!lMs|NQq=1i|$M6bb+tx zX_OyOjd*+?%V<$DGSeCQVuH93wnnT@kU(%XinMX1QtNXq;it zYk-LsmfP-)o%q4P=xx}nM7IhSAD5mdnYrPU@B=LZ-u z#USW$jh7mxL|&3VWYxKNK4FuBhf)UQY%ZH7s8XMG%JNSPCWa6Cth-EHuPlwO-t`6` zcU!$Xw>r4$qO2mp#V%?@E*K<}?CZuER*PxpCI$Ohaf8OmPD~MgV=k+8P31>ZI)4{K zwQ@DtqCII*Ht;EMDHF3E{5;It9WF3V?VzOH?rbUl^ES{cHVC>5RH;juNp8Jqba7MY zx%09O+^Raw^wohHrsUt#kX#g3#0zX(E}<_kYUwrYCcU>#UTcBbJ0b?=rdUHfwQKFs zk0Qh^`vchb{8>vd+UI@GIb6?FfmVUrcsXaB$4U$ttnc5B3M^A4{}ZKV&R_eai-w?d z-49UEME;pBCtDw(qCp~Ue)9*A=5g3o(Lp}}-2PVIJL@Wy+;Lbh)Ss6KOuhTi{-n^T zY)k^ZoQOFT&NU1BDoMyBCOtrR3};TA$;4pO^_KE#*P;CJnPD${!KWxAgfXk z7RUlF-5_TYfa6>bfqqKIex3y!j2D-Ri?E+k`Ob4I(!~va{kxO_f&FM1?HKD7(xZk| zI}vb_XurH3|Jzm=j3cI?MMcpn(4>#a{^5i=Q>du4Jr6y%pz0QksqWV6C{9DaeX8m# z<5g9mSzh3h}BVh7{s^L96D!Q!VceBA*wPN|?7rL#(L;9H<_2MWWF!WupsMc>c z3W3cmnUAv<0elsk@5Wr*=M%#I1`tboH&z}lV(vpf;3o5^;^HRz=Nq_ z#Q1r&2GP%7Bif8GJnZXeHZ#f}(lynh+uY7*9iASoV=7s&nQ!>Y@8Gnnqg9=zd$#a#b*;{{y+@61aqj#XkKz94!;xZFy5T@u z#zc55F6dm{1oO#~&sT|kuCXUtk|CRJ&l}=o>2wjp<_1>HUz9UmKv*E_@BW^oMFhOs zN_`9til2(iaH_rtrK2To1017|n>5H;^x2XU7Gelu`3-Q3&tKVwX&_Hd-=2fURsTSV6Cx>-y9N^9C7gDgWv5af^2(>)ch=hxkT9s# z?is!@IJRIJF}r>FCVl67M}!ve7rO+)$;Kp|yiML~e&<^JmVaITAt~(*sw4?_n1ICO zwZ_wCCaG2t{}?CAum>1{!Cc*OvxIEJS)hL?@|KS;o;To~hXU$*ujR=}ZsCtOL&0;G z4QpTg_cW}?o3qxZRH$RpJ97K9MX{+L;~yuAwnQ2~)ii3R_uRTT(KB0Oh~b|@6m5H- zFBrAuY=l3>s6zGeEiHZ$O1$V&P_I*kLefw7HbFXZ$-}N3Zf|0um6cHC4{lEf5eb?3OBy`Bd=`fo5c0{8<9kaS+^&ey2Q6_xKHQo_m1u16xbS z#KMkxDGomI?Yl7hJsi>G{B!x8aqhBr*aDV?Wfj5tLl_aU5Ppu*sq*!)l zRSyUw8&qwlz*T`QY3Q#f!>7?AomfYLJI8X7d1Nj)CV;6MH&E&jwgu$Q@*DSn zoSdAXgJ_)AI(IhEMK$-Y<=mhN-fx!W3BCU!rYgb5OGhDoJv29`UB+Eb1w}t%ITRlR zDBMBH^?-P1sxKT3&{!q|yhp0oKWXe=k#HEOn*n|CXml9!A)wfj7_L-I=h)Jq4tqTP zlgSG-9#lE3a`Hp)fU>v|aFPV5lbLkU?Y)dDks;Pkw@>$<^FLf)+YM!5Ige!VbMgxa zMEP$TWC?n*0R1_qA|rXpJFBO*j*iQM$5V4hN9AEY1FGI^pru!!QxByts{hI{o+C_! zhK7a;b*}_zsH>v@(1c#EGR^@n`d<$N(Rij=*k&>D{%xQ4apeu@&~lwg#P5+_mX?<_ z0K0%>7#??r4SekR91%Zx^7x;k2BUm&8)RoPWPSQx>u&sLee=aO#`?eW1mKHV<1HtQ z-S*ra9u`?QXcm6m4+NrHCT3JpCsVv1zAL|i3#e6if`YK**j*|)$HC=pePe`N& zM5Q{~Q*^GU$Hrp6$FUE113N%L#Yf2n{)R59kUlH7k%~F079b7>-30+i1Hliz;Ef5p zzuHxw^K8BztMU8u;L3qaZ{@8iYlt_ z{QgU>=^vZp3yv*MW9qFm130K5-5SFTyxZ@B1c^r+&wn*Yq$3}_Q$4u2x!%_zrPx%p zWM8ZS2Yqlm-qb%5u#J{R(@}De_94LF-(Wo`JnOKK{Mo7S6hH>};B^7mGaR5p02^&TFvh|WC>-L0~X?7L%O zON1O)v3moJf*$$o-iXgrpg0QBF0jKM`3^7^Bn>$VDS*TN3ygD>5&BX>C3Vb|omfGL z?D-L&7lQf%89=V*Q=}DSM;}IrRdxFmUVRmbXU)qPi>qy2=mT_Lz9nb+PlnIFGLY zLV)w^nb>3tv9dUoFjzQ*!;7?@ZGYzj#?wH&9jAKZ#et{vV~tt$Vw;C{s1dW0pHg%a zyt#l)N_Ltmlqu{C6P)s-qT%v)nLjc`+1 z_S&KWKdKL#tgP%5u=3*R_nt64G#~F&!g@9Ha5{3yiL0%yb1j@-9U#w&``(SLL96hyT}exUJ%Rc(BI-K#6K{~3;hu1`Ao6aV;Zv1h3{C>~;$ckJ zM}mHc_PA$8##J=n+P>tBL{Kx%z$3;P+GUbsw8enH2fbTECJ}jz2iz8=-eBWzHX3jE z<^UxtozXTwl(}w}zBVxu{ysmuC%lQ&x#4}%C>5j~#JTTlyAqwB6Nu28PH7+vS~}0d zP~+2}6Z|Rwlc-JpYQsS%-iLmnsj;9j^YsBNa>E?qr)i~nuR-^^!QXq?fyPJ_7QzWD zo$Az8KwE_%T4l#wajlPnJ}zmXm;pucuDHWPat=A}yR{O-IpM^HtWLqVz-Dl%OiS(L zpj|lK(0XcO>kegVCkf9-vP|#EhJaA+nYR<`aaxj!e$b6bML)fGaTiBe_dp z7L>XK5YyjW0e7~xFyRt_eL|CwShQsTjLK!w3zN|7u*^fqWgU;8hTsbG0fr$Y;cJt_ zatabI6J2lrM}7Tiz)XlssJhR6_%f5I)E%|ycr6{nJ~O|y}y_NY?N*G$Fh4{ z(8xp(dQ}OvjYs1t-g7&xWmV|ZvNUUrRz8#*dzb0Ol{gr(z0WzIbq>N@1mvBtatdlK zwY97$fI$q8(;8DR<;ikO++2+zqpTOXYr#0%6#Vn}iStZqV%<@!mvjzy`p% z1s?IHOov{9OmTVcyBrxxx@n1R)!+zPA9Hfl+}=k%`kSoKM1}hWKm+}%O7 z%aCy#M1kfL#gIVF|1fX~72vZ6lZHDqzrObnaLU(sQMdm&M zXj8}*-)jnaFU{9gTu!4y(PE#hl9)nP&?G2wl!W=rw``1Li-z-7QWGWpwlZdmOvCDR z&)~SJ%b<*UZlyKcH34S2c(c#XO_@`UVc8`5Uhb`1*l6q-0$KesG`L-^P8C(Fxz4Lp%h#=>ysN!!#PGnD1+^$6!}v?#Z;J*GHm z4k`Fz=i~gEuUCMDZC;^W%^)FLK8IWY7eSIO;I1xg4_SVo7D(Qj2p~Ta3x$i4QPqQP zLu6o5^$EsA{)OofOT^2f=*21Z5xAMvdNkjKyw1pjRAK57B2d`SGsQN703<#RaEX#8 z-+H<`|M~_N0`prun~GNolzt*~?{?2q<;(qXyy5?YGdir~%2=u<&>OZls;)p`SOdka zX1%$bI8rY^hSqy+zSOa7LE58{4DL$Rir@zf1P=1cz`yUIlLVwniTPa-I557$VSHdv zKH!7|18OyptmWHcwIX^gGVuZeKIc60#X!wuPAg^coj%>5EI1CONK6_S6BLKC4Aahb zkADo146(7`2W$kZ9|ZCS3r5ZM>52WJZs@S=IrFu~cnm7!pMnJk=Q0~lo0Rn(M|Z#`d*i#D1KqKZcZXW;xo({8j@EB3)bup@7#KZ=M_DdVi*J3Jy= zQs*`tWqj&yiav$OTIMr$5f)oSrl0|WE=9YA)b@qw8rIlZe(OdWR zIBY+|d@Pllz29|4aG}<3@0_WZb|P#lDs<&yz_SypwhgHg(<1%09h0lro6*?LE9~Ay zE5r_XW&s93iC+yDl2HIxeS6@Kfe{Hl7qqH&yUQ!HVOL*eg0Vja3^m;*>=~JaoGEmk zOl5pzW7ujcVlkRKiS4j?nf6mS&K$QOlO_ORO19!FinRJZOo(>r*#72)=t$|%?sy*K=^D&@bRd98>lz zB^Hr36|%jc>)2HMB@vNf2yMb5-)UAouRQ-BFMuC;onIrJvU&=A!xWi_N?N8E%_Ken zXeKe}wt!sV)LZu2fA|S*m}6_T(|!r72s1sw+29CVE4bw6NBpEE3*Q&AKl%fWLsZ< z`3Yc~V%%sZiA$j|ET`&TmyN^xK_=!Ica%ZUM{}nSv7R>%*Y?25XK4A`Bd#hvZI0K1 zvLVNWIWk(AzELnHW-8QK(IK1Benn@aw*$$+!*xhsHrhGn`cgDq3Mz2l~r%22nwVD+JQ8&bA%CA z)nozxr&d)4i42BGM3ajdf|0Gx_ zd4KC>{BQ_N^f7nX(2K-yD~x-(e<0ZW_)uX%huk{>csi$HI~35>P!E&4kxLv?sI0cT zhh6}SU&Fz<=)X|YHn7Z@%iJk;2qQFq_yfQ7>KMq7f2aTOPqGP~!GN6C9T;COwK%aJ z1&?58`T}*QR3?uD=1%_g@jS9St(5mb8+P)gs#pPWocBtQ=sM&q#RAu(giN9(&Vp6DTAJ0KThGqQ+FlFLgN zoMw97oyfYTI#aZnvwD|`wyrKNrhC40O)pkM5fAJIF#tJ1& zNiluZ@~@&H4{;;mzrg$cbfq)>GTYS5<-*1!wEQ0@xoKQTBKyk70QTZxS~4!G4L|SU zidQZoTTqo$V5l-uTaI8zbYHN0DX?6LB>HhgD**|Sd9_cqd~4E4TEj$;l=YECzliWi z$1!rZW1Zo$yuK7&gu8l;Wpz+lA>6;#*mUZ`sCQPnN*1G7bqzC3#JX7b>+>cf)D$LK z?kG&!J1jO2=$*9lJMTb)Y?3FCL{v_c&kn>U!rJk;AK4#M^pHVo)HhGBE%?I3WnfO! zojbuDqTOg9=b7#prY;K7$%t$$7(i-^ysFq-NLTcs5NL5vsvpXwg*Uo`+1xngO*}>= z(brC7p~@M5au!PGVsmq?^_UH{i%fQN_hllc7~1?JRkT!?iVxc#jmlYPn;Bh?xPm?A z_detjHTkgD?=;D$IGKb^#)=DO=-lLN=HJD78$mBk&BxXLO3oNdN*@@Xy$h2nQ6Zga zSKYxwKHpsgM<1g^puXJGy_pqX&|S1$qBEhAXIe>XFA)6#z1}XjxX8Bw`z&0EpH|t5EbDTFsF^%uDmM{ zijIIe-uf@BCI!`?kg-(`z&Fo^>VPDnXOR^=!XCu!U$_c46(YbxVagADAyb>T}K zfP+ae%T;Y#Lgx2?WzOnn62FIrODsk`quXXo!$Krzx%DxdAxt0DPk_N^`JMe}3bVR1 z7-a}2@wd)?Z`omPAQWOk47q62aDJ& z;C7IQL7c!Rvn~;w0ysy@ZDJz^(fJulh^u}2l&8235XuDZpK>%0z}u>}d`+WTQ!c}s zN10CD2{p3RNcoL!nzH7S%DOvQTmdg%zL5$G9F|YrM}c*BG9+1)5C>I_S=;A!#-vP( zLPE1r=qwq4Ww7y=l#@yA6XU3ep0cgrSQ(%QQX@-b*Qzx#j9Mupb5+q8tabz&5c1g5 z!G5Mrq^!ekqg0c_*S9^DbboCZneP1tPTDb&nh%6Ar)Hu$!8Xp92oLa|T6D6dpk zB~L3^XCWv0b`iVy5MgskjutEhH-R;I($RIKEJP zB-J(5@z@_Kf4GFK*dsY~nGpxmUadmiaA0Kxdsk{ngJHbWSbf}pKFUIQyqM|Po*#g@G<93h)7nIIB8#7}2p zD{z!q;{;G_&HD*;a5)h z6rL3WRH-ylIw!ppin*?58{&-)%i27}jR6n_7MpzRsGSYT6*t7}#O6{zpV*=sY_r>w zRSPUv+yXaOP1!OjM+s{De6G)kt4-$b&+&@?P_bjIb#df6Faj)mzna-D(8ju3fH6_4=hAsTW2vd25A= zV*g2=NV=~Bmtf@Ep4@!g3BBtOl%k9)s&uDFkx(3fpTGGN z|7LobHEhPWx?YK*8)LIK2FsQ0!IjH%hiAvl=W#UqDvtbGSbeFTj3?%2SLyuj@SY`Z z8jo`*<+N3izsDCi;2HmO0nR934pP;3S|!H>i=*~_!POPyA_IPS&`EowkQ%fdp*lY` z(ez&|!jC%g#^J9k!G^RtcD)4C4m`O3?(|-!QWAUx5jW1R^5lf>T4E!XaR7>Ip0_c@=IBrYRq-fxq|uC{hTM1|wohGM-7r!%9WEwH z!U7|QdPF-FWk`|*7hN(LKS+&&m!fHTW~J?O@kE$u7SQ+s-I>TWzr_f4TbJjlQTmed zDqHcflmp!?A!1CKkoN_+Bs)nE@r4pPsEH^aT89^}7VV*b|ipesWI^%%w6|YsBz5 z83ag5+SG_#?j7bB?O}8G{xO!-{~W~6TYhiR5f=(qfd0Ve<@*KSP3|4qSS-&%hdsG{ z!$F_(r|+!NtY7RFmF|c%U$9k-h5NBieJ^Zv^%p87&<8} z3B(BvoUP`~Mv|^U&}j#MyoJR7j-fd*lq|jO7RN`nP=(F(hsI`8M^<^%#%lpVQq=28W&@Gz6w4dlYbeTwlYsoyX(L8mbHEkmXOOb6QPQedMHvlTX8qtC@J9BjGjQER;V@2=sZl94Kx*)~1z4uU{h!X4e=`E5lI=_x zMLOQc3;O z@)+Tx9c*YCOMd57Kv?9ckIjRvc9-zjt0%Nc0!958m8aTo{H#0V))E(x12NViS2IYq zbTVH}G@hrrNBr;gMT9|gKNE1pE%g@9|@Z-zUnx( zy_(%GF(P`ZI)N?>MwWUN8oESc0GQ#knyAYONe zrMX3YXakw55y@v+KoWODZ6i}-;KF1|I4hM+pmfGPB$ew2+Z@i&<6c>e2jkdEDG{?Zf^W?Ib0z3Aw z^4LOF{OAN__i7tC3IpumKHvuelMgv|Vnk%Q2$@@XiRPotYYgaOnvzD>DO!li-z#f_{F>Pw9?4dZMc6!~~Piq7VC=v1=&Gb$1A z{S?hW%}3X^H#`?Tqg&C{;Ze@=bgfhGP=65w5Sz;T8RhQx0d$XE`TBf&Fr&Hmr)*_4 zdiKzEZ@$cNT05mqjX`}3x?FkXj+OlWT8F0!r=vW(dM+23=(kX@aX)#%p2YT)ydE3g-P1E>V$R zFlmA3Sb16?18nf84M_Dxe(Pz7v8Id5uqZVh-Hv9s>+4}aq!Cip+N)&2^J?i_QM-y{ z4moj|Cbgq_bVIk%r04)VpX9GzKhC#a;lGSBLXJ*?Rr&Dcj_|(AVFk@M83S7;fd=un zzOVY4m68vOHf*@G`7nRY60gFgVP^Md)cC-gP`HppWCrGDq_rTloVo$ZV(0m zN<|zjocp!)gz&JCvXj;Je4oQ>aL(nyrf~B7^dQ6L-=qqDE6@Dt=~5Lf4-XG@RbWx6 zaT&ORGy6$a_icOekjKmmfO_fuyTYKRre%{mH1G>m20XXseoM9DmxnYYeQN=H!1e0X zrEEi$dkN%mu&@dnSSzqmB?(PNfNdm64yk8>dSzhKlvRnV)JF7M;7JPXSWR*aZkS_w zu8NqR3FHhm_q(aaTrtcLLl`u8sUFkk8dY}pIM?5sO;_b7}ci^zlGLF zzSZeZy~7NXC zJxSvZ52erlLL^h>TqA%!b9MO(SP6NzrayQb2c<;taW+W zcHv7@y|&j2^FeB(b~NWgglc~NQiqhlHcj!L=i!)5WVcg5gKM7UYxE*>|ER=2V>m0z zAH1P`5FZQ}KbEx@cgwIov;X&0r7My%3CUl(tQTJh618=^nm~Vy3Di3%8t;_7!eAk` zTZ8G+7WVyH?!ywIWa|Qt=;mnHQ+45UqlxnX(&Yn({y%)ZbySvH^FFSGba!_nDc#)< zAR!E#Xzd7swG$BYYJMXGRA|h%J5gRu& z`EFZ$y3%gFi>(Xb0}I5H1n3eod3NXvuy{Ss2Du6w-LDF1)5^BA9X2P&=s0l^A%Infc4!{i5SdR8dUZydGG?b~56`=UpZI8o;vK&v%t`PuF4EOOx6j1lJ0KDJkISIu z;wm)-!i@zgOQu<{cTOE`K+E9W67Cxlf`9~!?@`y7=@ZN)(LH0#;SA1C9a%(r7QBoq zwQSZxOo%5e^CM~J1xZzpAz@-sz_kAp zlZ**{`+uex%~TQDx6JBQOJw7BvRm5{hrk)X@O^NlW5#nJ>`?mbK`76XcDLR@+t@RXo`YtKIhMoJEVe7cz|%!9hlYkwv(Po0%_r$$eP0DAK>{VP`0iM?!q?Y2 zwYD$GvGAtK^wQ7E7lu>$xWu)1g5NlxT8k%f(xuAT#REiI4OYP(X@`8$*%#~JbA5Y{ z7;;3>oIoHq5{C7P22>C^3ap;Dj<7F7il@P9-%;H}f}=)&Cz>XT-YH(b9xDdJW5E&kQ~6pt!Gy_VtEcv(|~rL#CUQpv~?WTd7qQOSIqrtXKbWC{keoAZx^K*%q4=MWiHSfxRL$M0lz&?Cp(E^osGHqXR7tf z)}=0G2Yz-ge!YcOL^8*>-XTnC<>b_tnlCU;NO1m+sPd$D=(y|_%mKfU;Wq|-iNbDM zC*qn3jF*449*|zxllYqU4>vIh%eZ_B z15EFLC9pu)*W^TH0@Y|*h4QCcf=D?;^MoP~#TH7k{Sx8!!_=g1z6u1;w#$ksv9E}pk*$3rBl5)-ajfIZ!mAL!fY@lvndp@eC=p6&a@NSY zI-UT87|bO`?OG-Ht}lBuW{tKAZSV9vp<_hle$cGy4W;np`h?yRa$4ew!*c;OYA8Tt z8GpAcC><>6fH|c2Hj#26Q`nEuP!cG|FvE3IjND6ai;HqPp@GCuOVzpE&hs(VMxDMp z1gfCxHhUqh)>1$l0*cgbAZ4A%>;3J>Nkx)ujlLKXuD%U~fQF~!P5(3jfvqzgocIM+ zVK7;om0Thfb19}gd`lUCguM}lG8)eot#BwPKND4YFu_K#mF{s02Ozv~?AKbt+zAd=rTn;W@zfAf-x9>S9iOiNJlpOG zkWDxM*~Hsrl0LE$@NN>$H0w+1Xb#wNU?|75UEjSJx9cPEX)RA5j;Z?BE%@-$b}?R{ zATmXCD>LpJ+lb&N2D#Z3>(784F%Y;365k>_Pp`sr=J!-; zQWY#tn&ohRxib3!0ipnn_rcW>g^_<4HZ_*cGE`OMQz6 zYQXe~c^}Ip^*W{$c$Ho!*6s^}`j}{>Hs%?y=A2}#PDyeydz&nGpTiKK^hzy&bQBf= zl0Foi*p-)(Fi%`;aK*r&mXE$#1F|pW$`1-9#0kuLOcItGuOzRGM4jo;Q)0q!c=Tdw zBcYstr_G>4cF@00bmRy!3vllcq`M8|q~E5LDW>dt2?dC`r4$|d3jyGuM+-`0JEvvO zb$bHIqxn8j=?;Rxdp{68(1DJn4+SM8K#Y>~Ee<=ogMsp`dW7B&eQ5c^N@-786`^(t zF;=I|7uPV2I-2^X2R=tZq0AnJlbrwZv$d2dPEDA|e5zh4$9TK%7`}QM91pMuv^;bR zZnGOyUPI@>$HI7rb1L+k^wi>U^u(p#wf0}d5Dsg4x+ zw%)e)hACIAt!ZMO(y^XYJNSpgE-wed1r3~zw2N3N>jr$>2((^5`Rz~lh!?lPuSPU(~@N|kEUR1Z1s-xhw*Zj$P7Ig!O+q^6I^ zNP9A_(p)Gqf!@$Kk(f_s1RQ+4^PCQwQ}U)%~B-UJ(J$Z`IJb`Nr&Zm`|Rur z^mMw<4J0dKh3TjsCq8RXIeV+@irUje zC8WI^`7|X}T)*>E%NmgiN2V&n_=|QvucBEn(w|uDj;=wT;#310MD*T5ZeLUI1rOn} zrn;#jIlqM%Mle=$$3BbYYmfWo47-vqi5xZ6nLz_1kIh;ONKIvpn~8A{k=TU!{pY?6 zN;8ddZd@uto)CKayP=Q4p{{$}n`xD#9}smR9V#at+5}W+TK)fIKs&N4+jaBTpH9kH z071t)&t)tVx!OF7rv*F=B?w7TWo6VYp&@vUv zP7@_%F}Trm%EHiWIiGoM>Zs&jCw7OH$RrMGhO5Bvl$!tP5pCPZ+V6$x(xsW!qDM~j zF2HuLcR=(=YC%G_X3V6|_yaEgCtg&Is*f@9N0Ns;;Nf&BAeSD?e0S*gCzM$~d!Z4A zrjP4*mVD>TmR5`vM_^yuABvts`hg&%`KYIGfI8hqw6XnFrd;HqlG@GoV~l1*u6T+# zL$f4xo{HpyM+LU}*C{-_S5ck+Y&vcD3!dHRN@SjW#6xuy&$($OjQ~QBI+g8y6klS^ zc5up4($TL>%iiL3Y?kXFMUI`_;@4KFctm7iGc$vT_}Ct@);HtK7RHX;O*-sQOJ0g5 zj8k800F4IpX}z1VnVra*_aRK8 z-BkaMT>(pOI+Vf|sQbOE)juypSXC7hNNvLa{t^ce33da7r@nd8&9J*!5@!0qKdUf? z2LQ1N{N)vj+sPb8cuL)S&J?Z!qFq(j=GXgo?rM@@3e8L%y>fR>uh(MWtxO3f*04epCU5sH|}3O0MZIws$; z`j!!rD|(WIJAME&C}{JCTbhk}QP*%u+zhud^aK>OM+%i^pFzs+rAb2r?+B1qnT@~N zSeh@>6FNT}M6N4n=~noYvO)n?5%=Qr7f!MEtP$&N`hei!$k+5nmQP)>@XQJo#XKx9 z-r9VHl~L(U3x3*d0(9(h#;V-k%kaVz9*2!_tF+wbe9~7B)&HFdzl27;mgL?g#2MnX zX1biIv?OJG&CL-A!6a7$=Q~Edbm9?sntN)n8P0T?(B36@!%b$p`{c=f;-}O=h8gI! z3TStpvx*DN#2ppYg0%|EtDm)wcg5j>gx^0qr9>9y?wS7EGF+iJ51wH9xzb~Yq2{qK z#hMmC-*6D4e(wSc1jNb7Nv-D{CnF=HQn3bW;_E=rcrUAXgPDnRa`Pq;e10{%RT!+i*bj; z)#; zl=yKGq}_CaC?Q~Pc-9~&TAnJPd0hO@0smu+cnZK0@f7&6)~y82nXzs7DO9rewHHWk z&baf|EH^oIYOUBxmxRi&GW<(0zIr(c7YR}@tgWq=8-uxO-Hw=f7)z#z=j`wczft5ZN2iKiGU3o~em*A~u1(;tH#gT6MVWbZgs> zq>=*`We&vYQ#+qQ3NSnTUt{8B#W zE%pj=XMbh3k#E@k;qLzaUCo5&>bi#&3IB;i`bje>lWriKe^RE-89aQX`qRjkCBFZB zRN2SWm6%_+K(xwMOOL_QZHf|~t0xN%3Hj2#Wj7dQ15>g#Ifv(p_U9AesmcwGB#LKl zgX9$|jFi1M1~>CZqV6u|s$HMqXt zC!NAF9MklD`pSDyJs8qZJRlm3k2uvoUDrRcZ8$R+`;6Ag{r&y&EgG=*uCo~HAC6*I znQm_#Ch|3UG(^GN|9g&-iqFuXDZtL|$6#N|DzK_@?`UI1@^&KB0au>H}wWCCemWOdT&Nw?Wiz;L_%Eh7QuRc8bp@O+{mc88Xxp|N-TZ;0~eK1w~`$L?Ey$=io} z0S|$vXnyC-@X~xG-es7J?3~yx=AD=6Ek6rL(7Psq+Gth5GJ}EIXX~fOVcmOs$BGXF zZdmX2h3Db^eUi*C?MKLHh%kKhPpUxq>O&NFZ7bhQ>3wOKAx4e%Uv+eK8L6gAZjTkp ze(#E%YCYnw`%JITNF-D$j|coO`sG(d>`4;AzGxsqxEB)u$l1RL^oGcwlB z-V`|-{VO162ocfQrKc!=BhG662zDV_E1PWhsliY#Tcgpsz*9ye#tWS#kc7)C2(ssv zb6^NR;z8*Y==;!d%>PI1l3@$6ks$`+{f`V~X107pWHo`Y|8pL?D<)r*c9iLoZ@yXq z3%2}@{)9v07joT zH;B)!9co;7zDo=Z`e?)?2CkCSFb1GnJ&E_fWoms#Na&SlC^20y3`R{!(CV%!!(f z%pJDcKln}FC&i&hBuL%d2EBj|kR2P-%9Emspnq*Wn$ z?Jh%#s#XaXg(Z=2bVUz0j{>FxpDxVo@#_O^8~IYtYRbM2OZAg|Ef2Uic-0m0pB)u$ zg(_V<`O=)1i)tEa7V*7Wj-Fc!IpUx6;BWOEeD^+iOt6k1sLq$d9+=l(76N7Qo zq=vGxa&TD0p!5bX<*VQuUU5lDgc~v;|6f-oC#~h3!{vIE97+sx6vihe>;k%Q{s`M$ zxMMt7N*2dCWbL{OLMr&pwp$YS;JYzymGAeT3}|YI^hyJ3B}$mK$2`i_8#5Vt>a4nt z7;Mk)kq4CqExaNRNl>QuO*bRBV+v!_&^qX&i}?r7laoV2JOneq(oBEhmS_Rk;)O?= zsiQ{%KxaJDSS4h?)pIS8JnR2v_{;xp3O{i8RKt?WQq+S2!GO>u$eoGCQg1@>dxeUs zkJ~E@Li}K~vpI|EA+S1ZK=_UFbe_fzY9t46g_X2#nyqtsDO~Jl{9yq}_z}Lu4?i!%6Yo9n4926;gt! za5*Tx5qC^$;Md+Q+T0Nby1Xx37YN7m))5akSLzgI9Nbc;@)re(KSx=7ngBi_@MsSw zvF=cA!~p@zoj_{!vR*@w2G`A!>ipLj7Xz0W*c-gw>Ur-qp?g46)^VhK(!a0- zc*j9$az(=L-Xn71IujgrxibtpwdmBBV<64UNfsgofFZ3&Tu9-6if>E`J$GFcxv7UzO1$a*hTZ-naoAr+#1`{{XDiwP zIevD1E<=NUEJ$*|e8^qMf==MAxXlN4n5;4Qq zk-9Dk)F6S7(M2!e&NYt7UwEf7!}RP^NKzkN$_b+uQmD_#?3*Vw~pMZ7bbzC$U#W`9-@pP0j zRh!QDQr!QrEPq=S%gaW<56Rf5ew&X@F|Lo8^XX=WP-+wvVN9As>#A+K3|z*L*IklA z_~74HGE#QIUE(IDCN3Fdv|2;=yN1*@F@Y%XK3sgg&Rd{NC$*=rZ%cIO1rzti`z%hk9i?5$A+x-jvn6-xw!>Tqf8=Rtn@jF${B# zJ{L_T51SfEBG4yFBPP1l^g9WFr(kiIwc+--rPfZdthfS;l-ImzI?{y~>s$TrVDi_+ z5DW$Yh#>6yEuc$ak#7I~0)F4Bq@(>@n0rps6hvxXkxMcuzh%CHfnFc3`PB_NX{-imd`;25Qm@!|?CO;G`n3mXRfZ0YawtphYWF5_=Y)U?rqU`!l)Sxt}inBm2C zmXlXTyp|jfStWYi&?{V3Ibl`BNgttTA4drF=5lpSh9Tk^xt-7f^n_D zy(y0q$@9Gp%XsoiN4X~8*AK|8?H@$G@4ReAG#hNgLL)nIbau0jz<9~KuLXQ;Zk@x9eCHT|)KYZZvvT)fG`kaa(CgDI3umT^ygVX*S!#EY zg{eN;DzkP};8{@!v?}P3R;i8^KCbX%Ii?4&cT7T9{EE+;)AAkyCH*!NTpKZ_<1x0f z^>Mz6Kl@LX^UqHD%9FQ<=E*}mk+nC$MTp;J7NO!3M=+(Ux8^rR2<9Zq3xF%G1myJ| z0@V>4g@|HB%+^&6JFf*t`s|$OzLdS8K2M_A|86y@HoG>C3n-SDAAA#NrdyZBFj1{R zY~I-}H-vaZDBXot%B3O<{<$*%!CP}(u_D9voJpahk>(NF57;@?v^Q<~>zkQzgImNQ z&yX=jVqfH*90+(}912}Zxg7-rHeXjA2>dz~gU#{QwTBF@lHXdHZ}sU@duE;N&<{0|V>uY8ba6qZ)V+*^r(4 z{^1<|T-zfWU<QbKE+2L@bERc!h;UinSI`U$1gx|Nhl0cAudN4_N`iyD1d%ud z*&1y7g|@N0X1MXaM3_ww)v>2=C#=~Dzm}F^si>DdA9e#O=RrC<`VFI4#~nhfctW>$ zaB2g^>rs~;M(z!kfj(~l&xj83JsY1dqoNO?eGgp_p+nLHQ^-+w)zGQK792i}ugUwN z)dbI{maaJn_XH$yWSVvKquAK7Sq3Te)}KGhTMK)oFedTLX~@|S!<@YUhK0GGOxpND z^FjBz*nmU~sbJeHLS|ch80Lw1DZJ;-)WC{7PC-WYV+zlLd6mQtZ#r(rWJn-L#^;Rb z*=vv@V4%*~ho@8^JZXD}w%zDdsKKUgJ1hVH-rD2*WEAZ-pnFW#cf7RoVKr@j$N#P` z9D{_vIpo{kXA7hUwWvDU_f>`Jo^I+LAy`en6id6xz&|bOOH{KsoA&u3KYX7Z&TO3t z009brZr&0Y`(L36&c=Gt86(UI9-`|?Mk|c9n}Z1}(zjMWM(7?i>=l1sStC+mWS)&U zv8VoTZ0AN**4jh3ZCkjNWG}U#PI5I6MV2GCJSMS`%rUv>;VqB{T1lqS!;*TG&04p0 z@Ytf7ANe8yeNR~W#m{Bt#YK3-w?~1tRsCs9NL`eUpfn^3&;PdxVT!CMB-+lB{Lrf1m&!cLi)XF3F_P%$6!~Ld zzrPqT%O4ESU~M+O`Dq5mliCtn^OtHO|Fuz-nYlLe;-Nreq$Yu+x$@t-r$WVHz(LO^ z{&XUL&MmADtO!}__bf3Pj+Nbzjox_TC6CbY^5RS3fP9jBEfSNdbvbZQRIa;bGL~A* zdRBKoL|XvTaDV+x(C-KqN_E`-=v{7k#sli7ffwlwWF6iA#Fx2z{m9f~X5K+*+v)*+s0tK;5TEN720&{JNN&teJ6@m#y6V zR!$7j6w?zwd1R+ArUbT`hA>rr9o@C5P?Z`s;-}C1iz!-_1ftaNwXJp1zd|Xs?dq;k z_@h@;xb5S{Wxh^buwVA($UiRaAC5*d9SEc0`}p0VYDGk!O9G>w2;P*cp;va*a2Db% zxZ10IkCekdkJ7UivBqiVJ{i$ijUm&hIC9r+e)>t6MCb$5`WbYM!)5VhYi7ByGcig^ zqtM9nUI>wB-#ok^T;>nZl*g4@N6?PS?76z2T9( zcI}Bfd2KGdI#8cRz?jUv0$FM^UZ!Zp_8e5JW`^R+SfA-3A+gGs>NB0GwgEd;}lbq(n&gfBBKw&(E8B{MP&DIjxs~148N0r(!!$j{(`Q-++hcY~#lhd=FL}K*oDOj6l0f{w7 z`P__AG%q@1`WT0t<_SkpJXAzQqg!h2A~u;N#BFaN_eWKbzZ72LOdTmXIbkHvVodiU z5%N&!nu+9XSB$<&atsny8(Ru%_vu6ouJXh~901h{_Whna{ zi}wK?-T6{eJ~Mp%FI@9{$0O&P!y^*)8M41;0oa~Q6^K8mGpV6aNVu7aef`yD_Gz_7 zCO8;Ap?##X2guY#X30M*iv_?hh!S-rqB=`RT!B zs_?6R-zs;!@X=W^494`MbztDDz$F*bUa)T;qkqtq2TVfXko)krNF~Dq0^otSFD9v7 z1ka?ZPoXO|k|X<2k|_SlBmXgE;QkxiJ##f}2p<2b%Y}OkUM(usZykKI7Y)`JQ;hp!`@-hQG_lzP;@(fU&s?se{^uP{`)GNnU7Z0{ii(>hgU&W4 z21dj&SF6jRt0HV7xwtIvyNyfQR}SndWNvQZiRJnR+c|!9C6L++2(1GP=z_xntLy6s z2r5_$2-yCW?};@uHFLv$VE|m&dg;30@B7y!;*(<6Ll)H6Rwg7h7#g_2`P2y&C%bZT z_TrrICtAgt#KO#@!bpV#Rqp3Ptm!eGaoe5CS@DYthHBv_P7L(PTrf!51yx$#>Fxw; z_H|t&XTc?&TNN%@0Nd<*lR~d3jaCafHojlWJ6)HHH6F_YqG&!n5$+9JiRdz}Hb^~D zEE1Y@g#C^+7?=cY7mV$&1l?g&EbLJ2n^Ou?T{0hZ@F<6(g+Zo@7ToF0Z@0Qmx^jRK z9;sIg+rC7o z0m)*PmQ)LPrP*>T>$n|=i+?satDw3*o3lw`V!$j5Sxs$%hD-b5pbP{eHh%er1{Z2Q zjS=QAr&#pJ2@9?UW@dEEc#Rvu5g{@2AGWqgD+uGcW9C?CTl7M+4YS(?SfaerBEXilcW7jf#@;8J27Cc*Lz}>+Egi;95Yy@-1298$}l&=?DWu+56_2 zLeV?w)}l(e3Y>`NKglJ|^N{KJ`mN2j2@D>V0aiRWp91-tiO30U@m;pjzYH~6b5^9D zVabD$#T-g4JgZ~Z{RoOHrxB~bjD01%q5|`Hkb5( zR9iMHb0Ou-@UX`!WOvMn1`xyul@fP$QJi%a7IDw!R{AqYIwpl=-N@y2qe?CYp$CG_ z$;nk~xm|_4UU-w(#29A1um>|!j*V=N1bt8$5uvt(xXj7Dh)La2*?#%fB8|ROscKWalYwpgZJzH?JJ<4^oJsY9KpFc zE4mYd*B5IeTn8yxOx~DZ7STs?3(c-r$RH4{QWa*3_JTM*?}J3XGWnv8Z3ao&QnP>0 z2p8?!?|ztkn)5SW$CYmYZ+!y-&-@NSw-|~1N2DN~+NcE{q6$+@@6|WfxKeHp6l-69 z+tBk${uJzi=ugoIqi%VJL2!AGWWnHYLT-c#q^}W%)K5!1K~#LrGME;v4OBo6q1y9q zgM0h$G#VZ(pc+o%7d3+?K9qDK#MWH6=Sp2kE(eVTfGCwt(aflH`I~}1R^aoMNukxc zV}1_Y49ZQ*p1gh8BzSz<5SF<8F{ZE$@&*A1=fQJEP6z-z53h&+vWIO6A40pH=P)XH zZXIl0Qnc$t%w-7iO<`p~2!-XO8sG-;1I9z4dD#1ew}J*?tys`mF*_Y?vJeB+#VmwhE*Tu#p*>7OqK z@aJDeSj4l0e!F+LL;QM>ltw$n76=TdDZ2qSxgG-(-RB|*ygK)AmQ>n@Q|6mV^u4vK zk6YXzsRgMj6EhQ|(Ka|3Hle8MXt6ew5cw0WY9@9{ibOrn$tm4yiKYS6qMszoHz>^F z^`Tk0{cTi=Uff>7^nAeG+LS%6`^zVi0k*smZX{Vi=d~ZQ7RVhRf1=&EPh_TV%MH-& z5wItsrPHR9tW{E;*)Q0f`&|}Xt`)O1f*cb=lT4B+3+47e#*R?PDf0n(Pi|n!WTAIx ztCzP#_P@3iEv_`TJaS=Ev$C^MPkB*Mto5Wkq)Fk`3ZnR37cV}!D(v_v)$hGnMRmIB zTQ0Y=6^;2~66jwXW*baik^DGJ0#u?x*hqjDG$14*L2wvBnT+hNxHqHAkRLTjh=)>4iQeK-ehs@=?+bi36_QfYE|T3I+pYj4AX!d zU{DH&DzsYTkqvj!!18w0U8TU%ReP2)$zMwu$2Sq+2FpWIUqMg8db!jwr+SeHv`A7F|) z47$@;1q*IsD&F7AgA%k((URjTFdQu*uFV%!McX{K>gvZlKsOYw32LHK8UNhgKOZO)3n?BJS&fHpVku*Mll5l9PeCeMVj4r@iMZK zOumnVj41>`!L)SG6SOwz_NZjgE=ZX3^rOfNl%UMeu0HNwtY9~!3;0?rLD=@kq73mY z>Kho)LA%;jux1EtoEtOk4s5Vry?;rDka|uwVD%TElk*X;(DP^{dRnAFb0scLA8&V| zX^;7%68z@~aep5n^rFK)zL(a+ulvpW|5W282DdKP`(4i0m_DUzW!tHbakZRwn(M1h zYo2aOwmI~?3?@vrEUA8{G|Eg0)aFCg&e174^wK05n)lSMp#VngTign~_$w*}<1@_$VktP;cc{wmfYRlYSrpuxhFb?e%Du*SCpJ2IZG=yoY^@ zLbkfx3{(ngS4|>jBI3Z!Q_k~XPgUrZm};Gp0axI7DVA@w5T0n;OA*u16#S4VG*tT4 z3or+XN9nwfDj##n0zp;fq+0wW9o^z#vj|`vvQO+d%hiLOZ8-LJ@TXV4%auXF%HPP! zqzpFfnMoHlkRn3Y+%_-d?_K>|?9#XBmE>X5PDw!>cpG~W<y1YCw1;j%wE(gf9(cf6%?}PozoeGNqE)*ZL z>*vY21Zh3cxj!@G623bQtD~i+JV6QMrD6Ww_g{7siy2FvNrpHkRl43f`UMQL*FgG_ zRf@_?^*XBI$Z&o2#^v<1iC>(VRS=z-wdvXto$x^C3+M3V-DgZ*iLm(5m%33Wb9bz& z%9&rP3nDO5&Rj@!VTT|d9#9Fe0|;H=N%dCI@lGKqR-tK>wKLLD(w3WjHR9=t<=0R# zjtR+@);eWf@=@;SecNisMOZY zm=1=V7XM)M-QMP|BB_CYENedwLFG}vF{H_jJMK{$(BQ*ddK`XB^ftyLm$l#uLSt82 z=Zb9wuXO2`kD|HLgsl4k&4RJ>V-xz*f?zftyLV~Zb=D%L%Ld)|g7pYzC`2m9ENF(- z$EIWNPbA8%k1S|+hFDO9_M85V7R1*LZXvW4$1P(Em^ZR$Gh|cry2nd~LeDBcC zsR=*TOeqCklTX~eY=iU&pzg=Kw7yx@t|dJ6RMn|(`!TY!=-K}M`K)}4rB@~b-2s3X zeg8E2V<%PVCL2Q2E{UJ!|DZa6-BbpIwz0~(H9tjSHL}loE?l;Vg0tv*BID)6*Spdo*A<;Jzr8qh4sHIlFEbk*zm*jJAVO12V z4O5$o;~g4U;8)hhs}>Uy$|P$&Zt#LznjUT;8-w|ZdZ;2wDpjhYd_rB`qeDoH#7m)< znFTP&k~(OYI%>4@vAe$#9Hf?)C4qr=VXzknrvJPqa7F1E1n?^ovRFo${-w~;Is01gH@Tsa> z>gD#fz9AMOEvzK)*snhm)-4zq-5v5U+c|z-4vUQRi?VgDxA)*mX-jWs1JyRdUs2m< zpznpD+U)tIaLi{LZ_Wh=2e+JLOT{oaTTc|^d3xguY7zVXRTn~((VrUbwmsfJ;^v#v zUq1CMuc&#mVY|Tf|4Ntq0G{UUOSupQa4Ap7VzUlbq{P(osE~a?VS=?)v)+F)ADHUa zZER!`bRq!uQ@5gFd=x1w-~5?dYk#Cj=Yv0Kf=&N5pDGz>e(y|+wKy)54$Ne-9&#N8 zhBK&cCg7S=YNdZh=Y?)Y?r5xZN4q$-S&gGydIh_PisjadhDL-k?ia6IqsdiUTh}NM z-RmvX=tYbYWfS198IH+q)`sSnr=h}-m|*3*t1u4~3n4#q{dP5)vBEyKEoNMfSXoAE zBh5yy5%;xeEI@Il@tbgjf}R-c2@llZx|8lST20EIhFgU6ZwEYSv{OAG^tOpqdi}6i zGQs>xvbzCPs=+3a_jOoO%=^*H>_QeJFbDXTF$&Tfj%#bW_ikRv{F3pPoRJ@uf3XXe z%tw5o@Beh4!1+=PHuWP}060R<#m)`Ea$Qs$pBo*y@parGFX_;$P2LEJOv65D9NB{X z$op$5`}dNRUV<6X5?zhcJpGwwQE0*8p@x~`Ph}L|zvWew+3(P)KS9h_&KZHD^Z|^ z`o^gtQ=vskG%qg=<#_*GT?0x0ZAT0}ia@G9&%;Ey!PTaAYv9|pKJ|Xg_3lnQyNib< z1#!@fRGdg|7`~V7>J1$Z&w>D2rU5`|j>$}L+04bHWg<{8EV?nQCO*zd zPEXkUR0-p*SSID+hlv9y*tVBH@G~u36HSxcmzW)W1cw8$M6!QQr|Rvp8UD#c2 zD{J<*b0mWP62Ae1t9VwTF+j*P@4qFr&B@+DJ5MFC4pSGQlRhX7+Yy``7m&~=y!J5} z=j|V8ko{&;E^VPu3fh>tZF<|J;S_#&>3hvC^Ua6^v1x7D z_Z+NzuoHtz_0Vg?ELh=uDd=3KYb@TK+*asfi=$lF@f|eFkfc zC8usWoV?s+9M4kVn%T1Cb@>rk#MZvNH$U?NAN;HiqQGXO#EPNhpPd_fwO;=eyhgnF zW|C64?q*xBSm2sMxA|mzf@XmQZ6|1woW?@#{kOjQ`ED@-8f-ONSD|OGW@uz&cUFc0 zTVGC8gMB1!vHS5}Vz*uk@&jp5SW!LwxX=@C15t&uV!JFBoXU-5foO4gUPOhx-}?g) ztxh#^TYEcjU3T7uLjEQ&a6h+fA6>1qO?K06ANn1H-JTlCw(D={{x5U{q!@u9l5h|Y zjrD>vnvl(nSdIZp=V2~nbhmFtKGbaGf#Ji&mi{6dG?H_O!kLeWiJEN#bU60vW+8?q zePIa%-x(d!5LJ< zn9tOm%nF4-r4^Um*o{lk2=OaX|Nau_aX`j6Nw4R>Y+{fA4SNz76n%N9s{AAjkWH&+ z)R4kM724;|-=F6a^NNX)5i@Q2@$*wPMtLMs;5T~7CKhVv9PW5Hh(NKo{b^zb2APC3 zjVseHYhAT!(eD<-w2c#kr!lE(K;_Ju1XM*F0TCKiXxP|YqBIZI4A#F}dsyfpG7Xf+ z1fCQ=QVKrhpm!mCG*mrtDE}X4=DRNpoQ~!S{j%#d`~!`RNW-nBL zJ}W^A*#JLM$X@f+sXoJDy!A=wd)ot;s{+VluAy9ZEYr)`qAXs%DV=*YXznN-H$`qh zurx&-od&FK)e94k_!Y?!_!t9O278J#7zstBA)RKeljB_oCf4__pMopx1LCC>G_owA zxw(1zabC^En1CXh%K%hg=a;W9l*m_`C4GZ$Yd2T`w>4DqzSbR#T&P|a2|-MpnhOOegC33;tbzGkzzh^>2%INo+`PniT3jybuY?T$ z1Lj(ko{<6V54;)X-1j>ymK=X1U-Xbt)86@5?xt&P5zEyeQm*ed0m&-JQe3VB{a370 zBxzwm!!~BslWLL`1>Qt!Q98|KU3yPM;}f}6Cw7C8ZM|lsjDg&Na=WlOK_fDaux-FlCfGo0 z@*oFfm$(90nu-xJZN^C0^D~KXRJ2(h-gdemghSoN;E{|MamA__rFEyYvisuaU)&{u ztYP=?we@dyjRhMbj4KqgB}g1zpFnP3hZA)uvQPGwTp6 zQQCNw>vC6s?wQhe+xRPv^>vx&<Ji`+vF%o6^bV$<_7Po=*Pcf8CF`x~Aj z#V?E*uMIZrQlZyxkCXJfvmR-qr#0=Q0o85fGSj|1W`e&o3TNFBdM(&vF9Q!7&yT{} zs=aJ2O3U`kHWU$`P7;zx*j%a;;cDR(nsm_!-B)Z8DCo9#wGFSX@7jB3(K`Fz1?6f$ z#b}3whj*lg<)M|ZVVPw!w9HPvo&|?&-1daK0XpkfPQ-NXA=!uYOj_kxq81$x<7!24&RsL#8=HV6|wi;Hgata4RBf9 ze*g~^z5eSO3JF9!E=g|y(CG;S)SnB)Q zU+*l}EpV1DF`e%c7l*U9PNLhrriY_Bc2I{WbT6*W2ez&bUgzs3oEf*a0ozG#Q-;Z# zP<7tYhBg4ts<^8_xgp_~M!8jPBK6jnIXOA?EH&fI-hCn$q{ZjHB2^G+upkb6x&5T7+ z8meld-FpA*`WEzVW@RHzsIr?MV!{jd#&te1ArUpqt=2QA+$UH2U*Gdb4-3Y`983~W z(TQw#l@tnc$dO}Xripq8y6lckStbt-OtwC0Lk4ncjj`!O%+AYR`i@g9HUGxMsIMfaESgZn8>N6_`rl`3QtGXhJFv5|1FBH(I*iHgip zm$#3Ly1J7n>}(ihl8N6=EIz^6d|k>}f>E2o>xB~M<@T^x?uo25T7W==7vId3%u14K5B ztr1op4picl!xz(wVG)=AX7y^rJ2U@ zQqt{4E##*8f2m0vcU;pZ$-7Y?%AJYKHikE{kc7XXtWfZ9K$CNYA4^YjJLfQ~&H&Pf zz{}p29*GrP5Q+%Ts_Sytn%H$S<2D_iHBC=ilo#Z5)=+k~9}^xF87{})dLIpP$Q>ly)pIoYh(q0wG%F769>G{s+ZYSSDRBEqiHH}|jk86c-O9l4r z(*Dnf-J5N}-Q1pR$Z)=dY(5`{0s1uQjx)lJ(TuP}n$9&Fp}ssmA0|4^{NBSVK~X(; zy@t0hC@7suPu~iHIloh&9Nl?5z`9mf)bWUSvbXw)ho2HAA>Fc)-`hKjr7Wi!k&$2} zC@zjnCdoBE!FgUwf|p`SHIS)9j)a97zBlJIH9fO-_VR5DcPVu5PVm=NA1}YnEdrhP z_s?=Hz(jS{dPnk`)1@$-UP&cbEDc#jB_Ed>CsZ7?(nI*jAiE-j!-v#{{IPya7itMG zV5?fi)?}nWBTefDq}c_%r*vrn5+higEPBIK$$)(i1t|usUSRsEUfhckgbS2;WMg5>MzkQxXi zPG^CO*`lRn&TnmvX_#@3ZO>_6Py0DVdSFmoR#7z!;yN9VP|WVy}B`cs{_{!Pzp3UMQg`%JHn(f!PB{-6=+ zhz`3T0;;e#OkNU?`t!hGbd@V{<*m7p!3tpkD!3q79&y7cfx@&sU+aPnFeVdnVw(22 zpa3HQD$3NfsKS2v^VZRFz%NT{ijnz97yCFJ<8Le^H#lB+0=?PA$V$$Oi=X_Z(q94| zJl(IafocJ%ns3F;pX{!Pw@EOz!;FGs+h8vkB77=5m`{!y#GN|><_rkDM?}{?ZSU`t z4Qz(vNS+U@(*9ZVpiBdOc*JQ3k3frCPhD;M#SAyK7Gg`bkqn!Er*Bnd<3QYMMsxtf zSm8NIlu%tE^{iI(+M@7k_+?I7+jIV%!mQB76{8fNlfLq?&h6ehHO@jUgwVhzvlisI zepi>6Wc)6YT)ov3ji3-Xk&Hh%NEwM+$nRr<(h!V}J5PqqkZ{rbiaOL4(D8*5%IUlE zu7y-zL82#I?|?>MH;zN_Y{2Gyn0m)TxeAP@(JbxZ@G!GJ9wr4>zP$Aaj&*$6Iltkw z^&q&yU}yBE;XvFhT3nriI%#>aOJ%nBZq)OqHTHjRYm}Gtr_8bb{J?y@aH_I`9;-6< z_uaWaSs%B?>yHm&*$vxa8`_*UzJD#yBg5d7&LphnD4O&LEXd4ZH5fU)LM&}!DecO_ zUlkEGAoOYK<9?tm5I`De6eMJ|r+|=34X#dana<`fcv+uP{5>){*j>qj$EPds-ls-E z`gyPTptJ;5_-Rg_UU$F*%2wB?o@zlXE0&4^A^=_`C`u^ghMVcYsR!t&1WN7M@UQqz zsC?+Em(|HWGSi}2&P1n<0SgjNt;ihbY;|tr3`GZ5!GVK^pT50Vnh_bGkK!djt`~4_ zyjOB=7848U4jdg4fD)SYW*tgvmA*W6f=zM2={WN?aXSAKT3tS>oQ1P=K{?~;oKVJG zCSP65ZqJR?G4bP|kP!(p)+@$`)BNrLcj{CthdCY?hw?>)!U{^r*@?(SlJ$dtaF|bV zOL+)nq5Z0xR}mK#k~bC;F~hLrRw*I(2Vbd1QC!co^=XEzvHs#a_ad9kWzCi6ZOt9% z9?o``5$z#F^OncOkSZqp@`%_kJvi-7yDX){yWqnazc2gUFYmp#ST>SJKZUWon8Uv3~5S7PWk8oIYK@^XP7vP0a{k26oW|EQWCEwIbSA5)4@I9$l}B zOl^0I<5p)(-++pG9qNGZT53&Jn!*d@&fCO3Z}acXl;Be zEa?_$!sY5qwuZxUn-Td9jR$;IHzbn&bt~a1-u8i4=iE5C-cZ=8%e4m3z&ZiGZWSZO z(e%*KqalIKX9zwl@;dbods!UQ%A*jg{?cI0z5At+o~E{XxZQiB=1J_+$DW$|=FAqi zFFa9|$YBj$*RE|KBQBoubQX-;3847sOa$!0acjYKrMvMrBz=LEWAn|K!NF-B&yAl2 z**|dpzd-%lckdCtH)D;xd)xZm*R%NvsGvVeBQF!oxB|~O$s)8nh(+@cyQ0XjG|OZv zt!94DgRbrsS1OcfH^7P0w-owp%8(`TqDeDSep2?dq>{nfCh6fe>rJ!1`V*bw_~`j9z^oH<6> z9^i5Ml2d;bS(a5>iszZg@@Y4Xc_!c07%J^c+*qL%;_9--$s`i4xq2`%`*WNpx7^RM zeJ4^o_5wetBneHCD5$h#3pIH&=mKdxzg9+CewQ+Cgqb;)3c8m|>c;Jo7+(ri{m+G# z?Pm>j#H4yi{!%o1^c^Z-JzzpYLjIc76qJ#iHkw-esw4$1v4nrSX7#>!}hddiwq#vpFwI^*B5Tlk=m1 zbVFD1DK-j_>yp6h6mzTPJo@wA%T4^}tRAM(eOy9e8DiJEOKqP$l568I1FD`}_M-(9+J}gHD22d+!KW>L(;*C7von)f}p^ zLwRj}cEitC;9_GJ^8f?5CPT3+$M^zhFu(LPf$I}qIy$!b!Y?A|8ow$uU))MGFA~g1 zfmxFM=+($YapW`5{}Pby-ooBknlG_P(1T!j!ncEnpE($P3?-a1tDKag@KJ==`NN-i>o}vH={xbv1nl2y25_%pM_dG+!vO#BIi9 zlz%@_+4ISWV)!7tWZia6{#4HWLlaSBD})FG3sb&+sSJv#lfaE0{|J9=UzCodFf&DH zI2VjzzhUkt;&N*&0F$Q9FtJicxRD1mBHRGxa`cUk&aN7nSqWb8Z{1<^TUs7!oDu~f zSI7hXrlS)RgGx2_X?m?2srOC6Hs%dS?CCNqi_nx)89}q^tliU!V|dVaDBiEM^Fe{> zkK#1cAIf-K*dT7iZh%bjaKHf*thW>c8}v0Bdmse1;<_nPHmwGspjfse(gpluU?wvO z7G-a`k4NvfX_2MI#%@&9?YRawhLE}^ueAs9sp4B0CRyHuOzkN_QlNBb+k`o%t+@C& ziVG%<-EWYbaS#G({wgXVArL5*ZGquPZyyMkB0>ExascdN-1dSV6q{{$~@Kd)JUp|Cr9d34MvAwY*h;Q!W)OsNmY zr~0a6xoGc9@`%JtrxoI4KcQ$L$z)132tu1iod!}eurfk)HuDHR_cRef!=xQO>E0VN z-)CunOFiD6&W2)C8Ux~ESvSj)t>URP4KR& zaHo3EcrW9vhL>Kz?Lj>#p5Q$XQ8`c3JM`tN&-#R?4}*E@8;rg8`iGQu7cb>s(b=bq z;i3H50)Obwx}fYDQ2mxi?a0vsO}4N+;4^kxXktr*gqG_)!%U&$%RUlJS?iMeyL=FK zvQv&LtU7!M!--6Hf1p~IkDASt-P>ZiqAmO`cs~zJLHn`hb+bXT5}@xvAoBa%eVQ~V z1p~CQK_kIU4OEzL~LbnylDrVGl2Ad01G*?c6BTwKIxs)mF2T zMKp#1afx!vd&D3a+T%w={Lsb))2Z1h0#<7E*S>1AO)NWU36aPb0Z`tB6BPg;Xf8?% zl?7RVNz`6PWAXUY4X!{h^y3O5kp*0Cx9@buVfZ|n_-Z=RY&(SS9g&^(w6r$cLRpMY z6YzQJ-bI5rlRMIyO%@-)1-)F(hs$(Fy>l>liV)MAcR9D~CU3#~rB?7`fg5cSeKVBJ zN&PbcB})@m8WkDO8_*4&HMb+sl$DirPH$)^`nbZleFKVbYbmA*!jUw5d_h=v!Ofj2 z6eFYEUOK=&(TvmJ86EGV`%B%(Tr6|4v9W!xGK5D)MvjV!;UAU6!}&Z9WxY2>=x%;j zZ@sy6vx_M7$(RGz>J9b4ssX0&)s@}1 zRMv`?qi^Kz>yVKmnK$NzIhRuDEdlNCxX$u-WRF<8G8E^_K%LX=B~wr^OfM1XS!no8 z3$ZVtw!PGK$B*^L{*IAE4`jS0FuP^GI%0w1`T|is_d~Xb+2ClQ9Qn_IXoW{-q+-gK z=FNLRNtO0sRM(n~A(gly%h2fzXjoyTsg=kSwy=D-p4DOi=8RtfldP7XhIqkFNX~i# zxjQNW0>j9Jcgn>yYUC+kPyS)yZ}|J(lNfNjZQ+S&$L`ym+fFaX22k+OJplU?xfMKp zx?9w=rL;4RjC!1Cp|L|=?_5b!v;Hyo87_?#Hki_c`6GqB>FixPu3A)s`=^K`c35b! ziR_(2r*n)khkG2C}+9~6|8&7vg%Efv3-wLF;u7IQFxZWi_@y7~`3ZTpCpa}F@q zpCT!taYoqpj(hPRtA317>OGYrn$ttG@~>pWktxH(l%_b9&{C7bKzVxSC4!T`>2TRSsUvYLVKHM z+b;uih`&XiC|)%{H#H57z|Xf>aE4M|y=Oi+#Qt%5oxfRa(%K*a=*Fk+{mqniG2Ou( z%{b<&_n&~?Cjs2gkaf;men&@|&zr~!aVTr``$hz+&jiQ&z)s``H`TUUwLPQRa0+E` zyN>~*V1PGg!`E38WT6_{)>IiJD`l4nu-n1)EiBu<+N{eM>dCNgT{N?FD+#GmJUI_kv~)slor~d@FR1E3T)7 zs}ctc+U@V`6gB^B86N_T22WY2+tp->nZGC&v>i4yG(6lj0;5-TyMjRPSIitn6QE00 z;{5$@Tk=!x>eNO%f;S-_Y;5P5{|z%3-@~MVT@}k{$mgSd(d57UWhyfXnhct%i?`s$3WiRRST!SDc(4X+A&Xv}$xg>WYmj7ElHQN@q{l zBo99e1!0EGhuo82k_xV*8#7SP499UI#E+82?3r(SC!=HmF1hPVD9n>uFIoJ znuGwn^!4)bJJ1*E*_C|#T7*{q^`+tVxiDvn((ZtQVRrX6ZcL1syN|;+nPl%3VYb*T z@8Tx8!JRk&hY@m6H)hTSL;=(mfK|#0ct}rO%#5CEq4}}j}X{%n>5l`h?mVA@I z)+FZJgob7A`v^f1kvGT?5Y^!A3U9B*RxYhO^pd=VvE?*%K z@s1h?jDx}1Mc0yEpS)POHGzq}GSn`H?;E>??8m^W{uaF6K|A>Z-fSiFB#ZsjV}sGH z${brRMt2(XQkjsGx=Vm*Q41b0=v6lMQqs_Ap9cqJG*$g7v)$}u(}IE`ghtkVDQKzN z#~3z22~ozkV1k+}AoC{^h6i{|f6?vfLh<7p1UVjlk=~wSX%@c#|0b;1ZajY}G!zeK zVxTKcGv}|oyaW6hA-7M+$k|*R!E0h@Oj|0zm-u?@b0>9ZT;&)A{(KfqwxG&gY}&#F zHOJQWccl3mnp+h1`O$bb4QXD^`Mz998ffZ z974YhOmp-;WV8vn=@QAsDd%5gg9X*)_mx@)$zA#e6&N>J*vD}LvOejK+P%;o2^hIE z4-Ku-s>rnla+%ac^c^syj_mRSOvVvD4~SW16mej1@Op`N2a@PVj3jvnD-{*2Kp;Zy zSieX0(7rC%#>dwpo@dC#n=S3DF+8UiZ&(fW55xb~o$Et^%{n1E{NTW&)(GwCyd$w3 zGmF=X;`^9;$6K7t?9HE_$H{rW1r(VY2w{FqDU7>_f~u^t?xCQ1jqRWd|#!eud&EA{u9+#1(rMA^iRNgVX`W78b}@UGkaM6Nb-{ zcX4SfmG5Gk;i(TT7eiy*(xd)?;X}SDe*UR?_alg|e^sbqK*YglFx$UELTy8k)zA&j z62M=8k3GU34N)a7=Xb7b;B&5yKI_XE>6zMmsEhx3PJ$E#?r_BALuKhIfN$mg$V9$b zqLSlQlmYykC+ia6lcMFNc4z@4>zCc>Dl4xP7Iddi>JZ`|lcYVdziYDYEFyX|UjeU{ z;1M)c{S{^VNSQyMy^AD|uyvRB7XnWBoF7ROi;~?hO#q$q58hs2@sqR7sQNT_VqJc4 zF8c|OG;{gJrc{%#F)lxZWN0U$e;-8*Yw-RN7a5ciHca|Ogr4)afuW(?-j=H;6T4z) z1P;v~HdrlwNef&L$8{8S=aDX{X-|jvzeAi)Nx)XUaJn2D-yFXxR*zE&2B4OjIz5)Ps zhY%RLxNNGD&7l4hJ(X(uh~MR!TTxD%#IU$ME!IpP)pfO}bhkaAFGOIB+?CooOM{9S z{ks12F|PNb%}m59;4$49JHV0J5xhHs2ixcu=3EdI3Rv$n+R0I)j?6pjVJ(+B1r-z* z&%(OQJ9j5qKM_1#BO1AsF)*6~Tm%9IQKUhBk3>Xe7q73Y)F31CVU5g>Woqx%ISGG? zGE!>K9j7PNp#oXBcVXGg*>k?d1imu9IiUPNcQ92z(pwgL9=^XfHN>M=N)gN)h_nLa zx+$=qoDtwRDl8LDqfCD67bjGwbr&OMIPNs3%@^ioQrfsi7PG4-zjq;rcRyZGg=y6O0_LT9ruJ+!S}7lAQio(&0l2kXJ& z9|hK>vgB)!C9GcUp(9G~=&~CXSoY0*lNl8Jda`@N1{r>I&5@Z{aP@HvmW68ak9jLX zdCPrM@mIAck+rx$f@8bawUa(I__>j+29W;FaW$m{WOkK0v6X~vS2BDMcYjmG(5Ee2 z)s^+%yWlgoUs$4?eTXkV=vfjRN4bg!m|eb8J0ISKh!NR&C%g)TcNtjdODDMZ^BKtL z!KFH~TX?hNUHuYIEWvKMu6f+GM4eeM6o=K`D3rv1VH(+j{~DiGnQ_a-u(*G|%#f;QPs z!Q(4U$euStn}*j37i03Tnrj&#gs+wHNZcq?w46C?QREYt1G!l@`&3Bt2_YZgr}XR@ z6o67WF%?Y~0lpdNw)vUoYOv+poEoigviCcxv>HDhM?oS7-(V1Jh z1aLKFv+o8WN3z*vKnBH0rQ!%m&roeF(d=$kbFEJdIgC!=V#Z zu=$2=*4taUEG6gab4BTd(qo3-(Owo{T6Bv6uIYE*$U|;@qJ1TM8y7~7=f491?!iSR zixU*~pFiVu9}!=L=JUFt>MG*Zr%7>ih^4yyVm-}@O_(>kuy+sY@P)?4xdu2Cs@MWQ z%i_%73P`o!v_K$KsjMW4!lb2U?LJoqj8SUh2Cs~3Hw&nZLWVh-f*>$Y*L7zT0fYO_ zxr}+Gfwbjbz5QXHLJzpTU6{6D6$VS#$~m|f=1Xc?;cmi>S9}xU#Y@;0Ogj+5p2<8C z&$#eJcEBua3;^ydjb{%@gGqfVC@Sr~XL-VA8yMUclMh()PBG-Lh=uq&6T%CeQv?VB zjaJ-P(VK63@#`8(Ml(2w8}690W<8$prw{ccWp4+`S{b-aWc z`2^L!9H7m0XfafznItn)Lk^MQh6!Xr%q6a6a{w0~TQ$+~R+Hnntb;v?)Bmx_UjGwP1|BT>lUXX}$fz4PAtn^)tb zU;JY@5$Bc*o*O61eO7%znSRwIqj1Z*W zde(i*oC)46XBjm;Mkw6yX|CHPtC}-u2Fu&>W{$A`hF*H^+PyQJS0W+rTI)1IJ2Qqc zV{JaQ1227=^enemuD#fq`Y`%xK8HGLT|>$0Ys}5a)_+b2GA^iYCTA+RGqq3DxNGmnaO+44I*d0~ z*si4Vm!cKeM->2Wsr@g3+LhOo(f3{H_NnL8);GS9wy|O#0&M-Y>N5Ei!_vI33vdOFU(7Y>WU= zW941F+8zl{BN{qRcOKn}pa37)LD3*#<*&7JQNMh6;nU!JVz!n91BMPfWZlcNR@}ME zUc8l!ZTV!cXgZR`tYilr&TDLge}&`tdEx=#pj^`R0HDWs z;;ktnep0fgli|c7v-g%|PsIi*Dk_|fBW(fH+N#AKsvRAVtKG?xj|`d8Usjy1Y_1m- z+U+873I9?Oy`|nllT!3Ur*5rS(#@3$_|q4q3%OHNzf(r9lHc5pf0Mt|1S5#-CmOm= zZ{v2s5KoD})Odf(m}?(J7de@u=$3v|QCV5%cxvZdoilt<=;c(0C`IEnPEhNBp&(?l z0Ork6i`+r-~xviM% zE^t+Q=iJxb2L?Fs_z(cN`4#vRb-Qn&233TyE~m8;?vJdsDE=1+uE#5pD0Zc~9KIbK z#etpL8Y&qF5_#yf1Eln*fE8bnLcPk#>D0~vdj*m3y66o4eCD+~KA@$>3O>mj7lTNv z`S_0vvw$w6sp1pG$i6w3IZFq(%E#() zFe2Qffy-o}-Hf!Jd_+N0UK>gxw(hOTzXL~o`#^93WBYL`pOCQD)Aih@rruqf`VeH$ z{>d6xlq(RxCNLk5$vGl%(v2YC&(94-&v6xpA(KusEoKY5)Ol0(J%7AOhZuGTm&sp-lj;0)|FuT-8%9hu#urNT1 zAwtfqxcr-h9v?z!SDCpM9ta&op~N{G1exOP-n4lgNGA`whGz+0O#B|7_TG%gN&SUe_|#se@(VayaF+9_QJG z)H!>B`7m>=P?8f*Y)OXm!~Hauwu`N!^iOe5ETB3xi@0(A3XhB4H0;z#(qXjVW4fXM zP!qe0H)`N5GIY(g(B{blOlJ%D;|tjYXu&bORN$u5WX%$rQy8#C7DpDYQIkd*MXa3%I@t&I2~MMMnlwXiZd6D{j`_ z-j!H<;P3}Yn;bCa`-#&p;BhyaPiK97Z@xGIjk5t4i|Pj@@O0`SlZ1MJKpxQC6$1W+K`IWRH?k>J$YBg*mTB<@$wps-^Qy}O>s(?4<;KKXyF(?*3b=j>b9 zGzSQq+WmUu!gNrYIjZvnMd{)0uH3LD*Ju^b%8 zHyQtu5@LXV^k6fOU~>7BW~H-vBY?q84^PMK1oH9es`GXPvc*@nPYV}-KpKk4%p}&^ z0gj28P(|x$O7enA+cHMN)7Ui}`?~7KYhjUg^F7wN{9SD1abT}y#w1A^BFWf1H0Q$w zvfEj=s$F=5Os^W9%6?V0E9R%Xk3N2&imMuZ7O4XZjrEdjFLR%MZyS8Z@?0R9;YqM z+T5YAJ78t>qz3R3PlT#=4Iq)6Mv}jI3*V7BuyX}sJhLv_;n(35?@MWpJVssZ46#f_ zFrLzdaJ844XerN&6I%uyJKo9Fwyc~<9vs2kV@ zHb{RdONb-l9bn<3z4y9?-@Jz}u)A;Gqn4L*I;b}p2|(@RgJWgNaz^jyoTdrH3y!vOi>G8Y(2*IY-+XbFdvGciHR}T4joC5RYF&U( zhNS$g;pgRK7ZMaa+iO(1n+u_(q7ra-Z~09UxM5>sLk|QQ!tl7T*4J0K*r&4)%A;06 zce9IwX9t0UAWXav#1}UF;AK|2>n1*Z-@i{)`?og`b)#6nSs{IPOve$m!i9Fq$tI1j zw<`$_Ysib&TJc-o7TXfLt@HcO_pUQhiWq-8Ld^XkB+VW!yJfuTWfaBcWXj|0{=Gfy>AwB z^N;PX{a1}j5F*zt>Qpj^rstLooB~H(3nuVKGWXHxWR9_v|ZBOO=JAJhzW;Q(AxdouE;dGR%;+3xFF>7i8j5hii7FaS8q zvTlvLfsH_6p)&J-dtpHM=380Sz(xM_DLOehq_L451r@bnp}BSzh=yG+Mn0VPe&lOr z>wjvWzQh2zEjuh)L^Z&DfUb|_)%3vJp>}&e&19eG_(DQ0gAehy?I(Eu`j%+)8j&VI zvznpd6>8~%%2Z=t|1x1xe^+9SFmJ6$2>3CfiMnH;4g=YHGOPKU`9cfweRo5>;fO}- zvRLg%KJ*-w-9n`SY*%b_u|Oo?p*@}eULE|;fvN0z(?vkDjG&mUmX#zKAe^-;xG^r~Ck14j zS^i(GXEZJ2bx=JFYpHWtaf17AFd5~A{U&w_*S8w0I;?6ud6wynzHo-&PeV$t*y5>I zwT)J+02Ca9#gl3ploeakzCSxG5v`mU?%g81R$BbBZ?$ab>K^0;WK{x#8dHA9_=f=r zJ^T)6B7%a|Np&rP^Jy9?Duq@3{KBs<&)1e`r){sAy}_*g^B=(q1$?OWQ*8z)rC?*K z8LmZ?fS$)dBmf^00cY=PC@|7I8;EtCPPJ}EaD}mE51{7H`MZQfoKyRjKNaH$*emt$*a3Wt)ue?1SU|b}$~A10 zx2k6~o>KXUlE;JMvTAJ}YU&}A(?T1;+Mrj`Zz;_18`dU3*3n$ObxuSqTTs=XMEpM4 zAM$zMtY=bln{m|0Pp>9=vIGLzw<*P&dk1!q*Ex*G2-Kaof5y8w34omtowf6{wePN4 z%%tF>aXB&(b)+s1ZI(Jr(EfY%-GTdD87a@Z?XoeQ>$F=n{gzaaZG=w1+|$tQX0!(E z4II}irCH=ye|&rY9?%I00$LN7R2%r+g^-w*RtleV-Pm1DDk8IQWMqa+it}8s&vIL% z$w6l_Ptrw)_W;Q|hLO)9A=GPn8W!_#=67eS`(yPF)4nJ|DLQ@g#ac=H!SQ@)e0AGl zrV+a&1F?2mu#F^P&Z(gMrZeD`hrABJUCLH3?FiM(N-*lIEweZUBu8o1jQ2 zyZZno9-`6*zIUa;4)Zx<{j}daGJ=7OYy1wcJufaG@=`f6_bVzsKA=`m_J)_u?T#Mk zgq&h72aLG>xaGuop82_h+{h;=_2Tlf1{#C}kIg)UX&YbpW%cf)>GbM}4Dd=oXuReZ zn+sj-GrTsC02OI0f1>VIN?om=U`qCFqWE456|Z5DxJ%H$Dur@{`_ViMo29g-B#iuh0@7Rq z9eTp@MRGF7hdM1+n2}_*2~r0Wzke^Z>zj|lb1j=zS62-N;%0&-DyX!qo8On*w(ZJl z{^K5M{iJT7Qp1GvNUwWn5;%1^*`1FkDx2AgOvESiXvDR|_ z^M_ErgZf$|?3MV1lG#SK14xAmBWnFVeqU-hYL!aCZp9-1FxIOh6V<+QTEixb9h~7W z1F}CJr_Mn|e~=z$U=hgthGFKISBPeS+e37NJf7f|zZHZ4a%YJ0YJv_j!aA3U#lR1b zOm9)8`Nr;v64tc%?eQ&TFJbioqV}}^Phu9=`BDbmj|VP5KT4V>i^LT>FZ^c*FK+7E z8Lt6OhB;D_3&0epw51j?zLi}A4+3oXiOI?6MB|LxrWqH9tv=A!*47?88hM7>7og#S z+TcQ#)45!e+r5p3a5G!5GH`!zVrf>agjgNc{n7b3`@TeZaX>fUuL9BV@KX>La#&q@ zJy?re+GYCANbLmk|F0El-)>y@zOB%8CwsV+dZQi^fHl?v+H#|w{ zD=X^BTc$cc+{x{u`igrzt(hZ*#gpGceUqyeY)t(H$u?C8L$6a#em%1A*H6k~Nc%sXAgX%f}1*Lms*Uz#a2lyH8aglvM{zZ zkkl>2=i%*vCM_1uG4y-Uq;g^^{=Tq3mqA2K44T<$rQ|rzPMS|&A+FVGNXLYKGn}-- z*{YE@V)j$Y#`r<~W<+OSbkE7Wp)(D0@)r{vEqRDZ42;{IV$9a)*?LuK6?k>MWH!a5 za?4=KWo?Stp2jdY;Z)kfSW+SEFItjp*cOFG1|rR{_WQ`cPQ%SaWeV7fNEC9DHn6iU zMr6S@qIbZylgQMZ3zC?epozgN?S{6KlTiDGT}qe)e7ivkPT>zFn1t zn&Bt!6n6Ci*esl!w|njwWqI$cmJ2yQIS8!A0VmB|?jMA=BBrr!Atos4dQJB6J2Pd_ zn$ht>4c0tw%{UPeQ3s`)uGRr)wA#k5wzwmnM(s-0mXD85HIC%;IsyV@e>_+5VS0L6 zR8+J*$tZolI$up>vBM_K*K5HSnCw{M@nJbC*rO%pY=?)oZs?pH^8HQB-&O)F7NCXR zY^BFH2gA}+mU**nBIT-Mo~Qfwd(j(9AjxB;IdKS>G}ZT?w8sO+3NvXjGy*jIrkj-J zX@0=aqZ}bXJ0*HSnT36p+`tCyik3$tKoX|}RUJ%=_)r^SKnpiY(GyMk(iVOicCd#C zX5yNinKoHZn1ext6{(*Fiy*|Pq^0a8U5UxVUbs?OR61A1Az@0oOeqzo|7DT5K(c-(iMHJ(tscR<)3g+ zvR{T3&rNkiPy7Sbi93iusN<-Fe~|NEuqlt}7Ytif4;a$z5k8UoDsL(}$}C2+{*-u@ z)(YdWO%7op|9&SbC2uXEsj)&&U|4ceqayFOlsr&PL1!=&y`#=Rm^o#KEE}`BHj<5Q$;pZ)-s4cOlk6|u( zCYTk=M^Rn|XfZc;4-%&fpV%>1;(zgQxm=B9;}|`VrN?(|#sv9wEye&4pw1Ha%XC0k zR1^{of$0P7M8$$&B5AMcY@y>RfKqg)3S@B^$AnLSq3rv_4aCI6KxUd3vrTOVka`1R z4_+&j&freuc4r4NbM^EP$f1nOZe}UVZnhN@IUc_+*>&5ImsC*v+t`)8g`AXyJ34`5 zd<#|Dd<;KT*nPyKVf)2{W6++*$Nm=;5TWpkf89aNn%vM=aSQ5d2OkxMWC|TB1=@YRFm0gXpFP zL*j(A4617CRj?~p-vTXVf=OO8wp=h*)LK4uY0pkR^*cRzk)1rA^UUj{N-kbb7-iXr z5v9=xjRm6$^R_Y~dk>D)Qp!O*xx8heLfy2J66VbbPssM+jfq))1}hk}^>_?0d9@1s zxxnHuWXME!I?ghY@qpnWlQH>~egbY&cQS5NA!(}cGLed0c9pWys_k1 zA_P>%IPWwALE6gT+gimr7~6nAoDglOp7N_$G?=BP^D6U_5<9+jAFHc`(kS-xkV$}K zPFStZFDA+|i4;o}E4)dSE}6?uzzORpmL>d?89HW2E^&Z)^{e}J=Vs;CEQ@NOl z@JyvOG`L1o?vN`lLm1pQ2sp2L8Mzc}CrtqXa)4MS11?f(qV2_qJJjBlMOS1-7-wX# zQtfPt#XSi2EU|g&JGLoij@)$Q$gw}fT%R6s5f{hFH>;NLyntHsQntMP?xn7trEtfX zU+Kxhtkbz<7R-wl$oa+QAU#E+$sP;=6&1zc#)U?u$o;sqZYCzC1*uKY4<*ClIXH@( z4oow;t+SxK?}Hf!JvL?3ef*#g&{4d*L}TiA2Fs6R*#uQ!Vt$NtvnSNIud~7Y4-((ZC5Fk?@nB^P=OTS{7F? zTN|KuJM1!nOBWK__h~#65j9N#iTF+D_ZmP3+mmG>W8W*GwuyVdHH8>L{mn~?C2sFi8uSrZ#=enHka{i zSMf^KYEeFWMO23Z+?fm9;Lu0`1^qZh+yK-~Q=o{9nHmj-lwL(>)-rPRk@Qmqo%hlurx<+JS*cX#hX>iWA1GbC+Kcz`_}LVV zCno@C*KmoxZ|PEeKd5))zyQRu|J1JF-@F@KN6OYZg1Qp_`Sa&;vsqdT^=6vf{Cr{# z4s4*lW6YY!!$v8QB%3zsSi?7z^mMuz)#YC#*R$@RP=a3}KL_SY{`<$Mg07Vec)Ys$9FU zQAHGx4nevFrCEe@cd2wscSzR)5hexI@b zFm#CPx!1gB&ud5S^5#&#>Y*eK=(#j>D^r>nHruj;zJYxNxJ*^=~wLs1Qd!@RlgB+W;P!25>l-d?c zqcS2GR*qKrWOg?)1iS0mY`nxi#|PMYR8{sUh3&J#XClhVzH8MD@)OD_G^L2r1vQM- z>u$0~1Qlh_T#ndtOLKn*?MD&Yk{=x!9pZUovDl}doYVwStz#{-zEZiF!?DmFm1I_r zpivZ8lqa?bG1bWI&9sMg#@v*#zJ>x#bCr7#TwzAZ-5HU?9jW>vgFk)Jpy%zZ)FfT- z(@g5>pM!h?1w%oqtOvA^9Bd1K>N+)AZ+}xI|P&jaxpu3^z3pLuUZfLsdCJ$ zb2E)yt#Hjhx8;;ARlE_Te`z&SHi^C=%rT{(&{Sm;TWk~*UYU>^{LZOdIrb62CPN?X z*|@}fnlDdy2BT5Ux@njo)K6ImOB^QTi$aLl%`3#jQvxBce)p1@6pWpoo8BG&3F`Cx z>+>GYxvU!;4l-q{cSY>}tB4T*z|BXB(O0m)&8VN&1LNG_@@YZEXKg8YlT7IpTsPPY z_Ejv<%lX`oTUgqN45+xE?8`5~&iZ3E(=f&3pQp7}Qx#?fgC>~a0E2_`3fJ2IYg0T$ zyw{aXr>4=pf<{XMMt*xO9_kp`e>T6FZW{~H;dUZC_J5$DULog|$2iaKso6Feat;qx zNP)e{Wo+8)TYcuQpmqTh$S*CqsqjIht5yuJs_e-{oD?O*#W~d5A9Fj<8VWTJysi%$ z$W>(uBIX%=smRdDpxZzs5klmhi^rPQ2No}^A90tjqEOPqmsJS`l9ZiF7f9bpPZCAc z!xV|VuP6P2T1Zrd!l6h@Og|N1amdGB_F-i%_FxfFhQ@G==rs&EN(YI;;MJBvcWS}0 zHVy5O5?WyYiCuUFjO6ERl<^0%xSbvamHA9|_u&B?y{esaNR%X{+M^-rGCoE1=BNRu z4vMGy^k#M@H5~Pl+T5)X&4-4a7Ur4ZnFD@$ik%wYHP%@tqHyeU3PfgL4>^`0gK{Hl zIX%=47lc##W*Lr1xxAm{W2|DNH7v7VE#GK%O2$=3amn++O4Jj-f1nQSNJ~fmjFSuA zHs@&FRjN#q8jtUr{$PWip)2+kVOM~weSAUop1J@*e2*Cj%S#O3^R45SIjPkq4S$w? z>X5XsE8mXQ%awXSmm#;P$vcr%n(nBaqt0)3QQ758abk+JV32CtIh@QOJ4IM*ITKA= z`086T$2)Rcf(Vue*y6P4RMmvV*Il&^NC3I`;JJVTxUfD1X2G~Qut?<=DeSexr~F>3 z7hV&F1FF!BP+#n+SN;&8kRQ0ok?>q-jFgljb37OR0Y`XDI(eLo4vMqtVxmj37&e6$ z$6G^B!2VB~uR54ri$obsQSR<7pq@@OHmJZzYvw;S>V6P{Ch$;{)St)5|5MetZlM_= zx17@w}7Wlt*Tdv1r5&BHx=*-INbOSE{;)xn!iRJ zdbe%)GN41Et5o{;)!S)lY5PA?ZELiTmztm(?tQVDYryDJ!dhQnKTPqbf7On*SyaSK ztPAp2=dh1oI-Wlh-tQC^|HN(FM~sYb=ix`>zrq+QUt@5Ys)(JXETO1%?mt#^^7W-G z(G;G-w5xpPa#L^Q+f?}L>u;Q=s~bn)T|r+Z`gDRM#~BRCpl=K#+Ziq4YWf7jL9vm!%_0q= zT%Jb+KVD$(jD3+6GC$ks)p*?L_6>=K40NC?=VGh9%OpCxFg>x-wpb->pR~V$f=!`G z_Yo z!^lJj%RSA9YeS9#Mtk$eQ#NB(fJ@Gup|)1FP$xJu(M=jTl*IKXCd1o^)ZK=gYj^El za5?(AxDp_<-+N^E{v#q$J7dckXI;0{b79TGjw^#YtRS zjJ%1n+bn|!abRwa7mL6!Y-FJ4hXZg)NnY3LZ5^e>QcVMHXe7K@p#FK{r(SAo2I=iZ zZ3%EmnJm)tNpx;!8P=bRC0k&buqB6)X2>Tz#N;0Umm;cO!2tK@6ZpeH)28%3^Eq z&tb!@)@>cynp}xxv$)2$D8SyTIaU+NZ48c z$NnH+yH>@f2oW&y=)C?elA9{TVJw;Cn;Xz3IuH6Jsch=KBya#_@NLAoA-$B}_P`QM z1q1lL$N2xfTPFEc;TEY$b8o4@xDze26t5vbxcti=4B*gK$}_8SpTV2}Y%7p@o+1I0 z`#uqhQ#&2J1~#`klVSTeL22m^id&_}6^5YkjkbLooS$;Oug?NxB{lhJGoCABfz ziB#YZZ1zVt*q*MQjEtfR*p=et==k;xm2>(FpR`n6*~*Bj?nrI6P5qHdeS3*<4?(-N zEx5?`m(XTE)FGa+rw&->DBBcdB&IFcavCRU9=mLz8Mb&PlUqda1=SbCc~fq8M9$Oa z=e%k~uLwu?zFT2(FX550kiImT@=>&p1N0O$STTK|akN}@u2~39Uu_@xj%e(OK6xyR zh|5mYMk{PFJprlYhY-A|omNFaM$172yvfP~eT9&GY;@U@r^37nT-dieR|6CJi=Ztl zqM-ru+ul1y6HZ8Vr`rlHXIHwzbEOy_yTcw-G_7Tte!nvL|7Iz?yj}FZhNxIutXYL# z;?3REVDr6N-0NLb%@NyvT{$u`&Fm}R69~Ty`PII>9?k{p$H3XCHF#8Sf5@$8NMMs; zFwFARD`q+p6%|#(B_T30sSjguSbc%=nE6X!_Ii%_dtZO{z)0<(k{1J7ua&j6^Tp&! z#s|NIoNJsXnEe{+xu=-xr}?0dBI7TH5c&_7DS=T>d)NezxFBy|00P0D=pyP5r#6kQ zX{z}Pc%3s_u0$_)eBpOK#j6u%tBK?LrkTfXdpioGVp_8AdMMX5=wy-=Eezr40R8S) z0tK%vISXMLRqIz_&bEL#4u}i~d9f zQosuSo*?CqCV{L}m(Y{VmvvT^A=y;(Uho_{yg~QZtD@bg-`Ag<551Wz(<9y%^?bvK zrO)H^o(jMhTQ{TcgHv|tAyay+*6&cUYgP5VDFCOlCd$gZB2Z63>pz?^~U9%IOzE=U%nKN5K)b7mYiFv z+QlC-F(sVfnm9+~iS-A=Cpsh|2Kwgs4INQsM}`7kX9`bF=MQu}L%9Epr0BP;2ZRSr zQ*!N~U;OcUM|pU3bn(vs<(4M}5*?IsRXUJ*<({fnQ;-SMsKc8!V5`?LbZqOZx@eZ@ zw1=fmo*!-ar=(C^o-T&Ep|~Bbr-y#9Hx@L+D=PzDpgV5ln@nVCg%P<(NI@i-P6`Dj zB{6$?7jQP{CAGX@hD(#^k`mGdI6jcM6f<@uj5Fs;isaxHT*Z0&!Te&u$E#Q56(@1*q#@EQnc{Y;bGq1 zVcRSS{$@5ZAMrLwr0)a?JElpCAk(jg`1i5^{XQ3r#x;jcZ{=l37niObo zC%k)H4LTK%p01DAoFpdt?-pV3NOMJlA?(WOnT#1|lo_SL)h2*qFL6AL42EnN=^c;< zFffpv8P=rmlkM|!w}3Hf*HC9=wF*mo@CKT4geMbUf5?WVUbK8HfXCzt9G3zc5(M{d zZ0~s^BwajA0u(H#RkR+5-Ed2K&xcXV`>FmZtzV;fxpRJ{lT;?ACIsC)v9VP|kllIL za98H`&lbGA#joG=f|ArLRF|x5_A}U*tcoVxrLG#BilEo%j^|Qz*R&fG_H)rDU^NVo z@e_Y9^-T!ZspX#HTmv+r!~%F+DIwL#>&I%g*B8YTDF0Lw50gNtucQ&I5)W^IZM(h?Er4;I%v42jXWxv1n6X#f{f zJLF_UV!Jyp)Z;i=X6DvQ9t_ZFu>&R(qlm*Jj8!cf?{D-6b z8hV=#xB0AeW-w{i6MA{9Qd;eI)b@RUKY(ArHkZQyW@Td7FOWd|9JW0-GOodbS!ikX219AXy$!-86KIBJ_9%1afXe0Hin^gxdwV6A+6!t@m1T*akr0rf`6aXJS zcPq9fgl|x>I8f{1(TJa_=;-Q>Qhc*uiu=&DB3o3M z)qDm7{WwAQ5vYs`{pem3TF9)HxgD9ezZ}@!ub`6X4(ykLS1>a(V+a}9rriWQIZ)n8 zB05gcB!Ut-OIVF}GC3JtQ;}0rrgrRkR_s^eInRH+3kJNa$5K4O2D>1&3?&*OCNDDa zB4NC496OvV)hy&kvX|!|{()A#2gBJbr(V9?H} zr@b*I>R6^i^LLltEZZ-+J`n;Mxmulz`NMm$T!ZZ$CKbk8_+bIvTY@t^Rb$2w{8{9_ zX=iUNoSMp1uoo6xSf~b0tty^kU)hYojla`CaUSjLJW(}|27oBZf^`J>nw9cC(Oe>i zhG~jEWD*L|QBkWW&U;ZmbI97dx2CHVks}Osu(1|S2hv&bury1mDcIQ`fowW2J(jf* zpWmZu$e!h+&}`USirr(A+`XT4Ptd`s3_^0_);1p=3qu zO%GelyIckGyuAS{v1`}}KavF!*C@K!0A7*|KGE~az3!|-T2$=)0wpz7(8l_7iif+> zE#=9J*J+BYIFA=scj`?U9rjzE_{zaJ$RQBLt5K{V0O&Hei3+icIt!|68yg*6r`#0} zAzI7I#$Ec%q#FXtSaWn)yZ1?;{m>NY^kH7=e7aCq;3I&1($l&u3d3<;E#0Hntwe`L zeO&`=gAVIEvB)qn#lM1e|M)ZGEfMxd@AJ~?F##QKlbeHk!r;FE*cxTNd)`zn2t8An z7owo}IBQ{cSQiH@NrB8#PL?vZzMz-vB;=WeSKQwn3n`Dz3MS;Fprh*oSX=j9D2Mx<@74x|p0nkYzgm(fxlpn}Ew*^Td%`Q4x5kpN?b zn>-cAZ8~W^Q={Guyu45oAs!~})7-(?b(v1E_-6(d|6C#3hOx%~&oee1%f8VWt{CSF zuUpJ4Pa)6!#@HV<#NPRiAB8hV+u>DsL2AyPcs&kV7S6nffcE<8{HTVmK&w3#fSUbzs&Nlne|#**`w{5@JH7Vt8FyTQomo-j?LM>(X!w zX5zfP@BQEx^JxkM3x`|3LsUmaG^zU4k)l4S{@Aq86)o0a<^@|xRYpzpl_@~n`7Y4b z=YY{2f|F1~-s3CZ`s6-WhSe?V&ea>!My7}JX+WDICWcfTkwJ7s!~%|&YbEZR&qUY^ zvpT_)r}3;g5Akqu7XZv{&^Rr@F6dV4I&fjh$;szx78|$j(9Vy(Ym#Dj;~ZMG=Z)?# zm|B?>{Isn`^@~+5YS8~E-ci6R2XFHPlX-S_elq}NeG5;ej`JPJ0O7LK z`H5SeL+JU%EQ3A1YjpInd=meR(T?Ckpg90W)+8+KDbH>sAhli!F5ktbk+;;Bj%lBk z3h6T@yQ_-fvR=NkJDQn`L5%Mw*2idj5nfwh3bFco-wJT|-!3P$Ro%DDzDT+zZ8-Hr zj=Cfg9_6`SI^{<0c1x|!dfUGb9b=8Q=<)`#@mr=%BmYaUM z9%$ka2(T6aR}l;q?GA8r1>c1LkaPq{v@5$CTQtkYGQ_sb%TCtG)zvjp?jeuU z=GOx4+A=B3SL!y1tEPN|@_R4nbW&Ol4uCcD!lEqN%Ey`Tcc?ru%At z@@E(Nv_9cGci_3DM1_<^%pjsNSVF z$5XjHOdmh8aB(J7A+|6dO#K44Fp?l#`bTMQ7*m>v*q_!q*-VLuR+$}7m?BDdX#M+u>RRmcr7I`fe~$VBOJv6K2T zRS)Fh;&h3(nJ z{w3#*a*QY>;r2A>n&v3J@}C!!GwdHGM5%uWVTbn`rGyg8t5D_>$tzJR!(#7eGf{*G zRD_3j7+zZ%#&A85=b_|F)P4vrU-gv;j#1- zfzU8P9<0;r#Y`%R%rv!UGQ2gq*19A`mY~I??x&I+^3>s#IT!cnyi10PK>PGlnxf|LolvSA7d@2WL5mU+c z1ZS}T`!57(VS-ATcGdJQe#}VvG?&HJeyd2(=R|0zHjL4whLp`aCm*|(Ub!I^yL$H! zO&5FraNuT#HUB93f#WHy5Z_uJb%&x0CA6=-7QG9LuSj)UTc*On15F$*L41DMCzG{u z_Z4qFpR(SMfJ29y>v{+bn2RyCa;d<>gkh~{`Lc&_G@{JHA50MsEUxZ}TCi*MlQMc> zuN=6h&g4(=KSp`Rgfj@eH%)FQEJ~M#AjBXCpeIDx`OJ0nd22PcS)Rd{H1KB0M}CL*j5j zD9v0>C=dbJHx!DVAT-iX5So-D_Me)--^pRB$u@qXb6zg1h{X1>qdny0GgBnC?hg>o zMEzQ(&h9I3Rq=%*39Ui<2O-YG`#qffir>~!XwDIW8Hz~Ck*@D1l3T+W?7q+H*m9E5 z)lG7#*O5UVVK-)ZlvT>UfUyCL62zH{g124}GO z<8MZh%^8HMCDEqVy$LvatEb5H^&ek}=A<}?!sCQLAtkGq!WPAVPs@7Cb~pPQ9K|5) z(cN&0`iFxb-%8rZXn!t=x0&^>US7`ViH~>M5$G#*@0E+)?=ocNik0#ELfOu3fqw%^#_zu`||4Qdwr^aTy-#T#?=FVK1v~Gyi^+9geAHoTkHo-2&6Gmkxybn`On+EgNV2W zQAqixuLwCT)#frA&_|?<&rbk!<^;O#K4=MO;*9q@XgDFg&r|VvobvENI4dR0G5%TW z2}h|Rtva#K-hH_o(9p|JRU<5; zl<2;yu-rh0ZmZ|zx`*GB)ge(vMFO{#_qBEhYPjuUTPM>pxItpm%2*_Af-Sm3$+>By;zLv>Z#N2v_f=Ru|7q1;$33*; zeX;`$p|J<&H21V=TtlB`#C!weo|>KW*OO+MjvI$Bwx!HNk>XuD@|VkytvM8B`3JQQ zbFHS?g+q^k}EmvE z;)m=e>lErbc4FO*MqDL5*Ia>dE%}-N=Mr@6+q15+kikM;x%1N@dF_W&t!oF6X(uwN z*}8u0iS_X`7~c7CFfuns^-PnO6;{miJ&{sxV)f;6&RKSg!jdd5+p+RV9LN9}bJ_h? zZI6%m&OaEO-iKf=??+Z}LVBLdZIL-R9-;B+T}bo zG%V3tXA1U*RgJZznY{wmbdVh)1C5C1+}!2w%QxWCQx(Uyu&{7wzYyZp#D-sjG3w6I zRX$CvW2(r;n!k$NIdH;v!J!H^hKA6Sjk7HewT{NphKnta81NfUE}gAb z8grW?5?e+U$7}SGz1GL0E;F}cF;>&wiboB!v&MPWv~l(j(0LR@c92`c`vtuXsV7Ez zK9|tvS#!bJO+d$}EZa`an`5vtXKm>e&lHD=%$yutk2ctvH5seJ*%#*{-0` zr*N&&RJn8n*yV_?@|~)mKyjfbU1ip+bwB6S=dPR@j*L`A8MRu*7vR@s7|Dg$kyju1B_3YS`=Fm+4WXj- zjAibiED4kcTT7ctdoDxAP~jO4II#|6Pqg*SW3A;+z1F`10c93C4t={h@UnnDfv-`Y zKm2ITVlwXK#{Qj}#?r)?MDZqzbtc|x&`&TEe)eXb^2BXpYzy_fN^pj_M5g@K%yFxX z1ehLSYns3hFQ1G(=-7bwvYIa>jTn2-FcjHH@}l!-<|g5KyYPNl;!WbEtmhc_`f!?; z`3_AX@};vj-l+xLO+tulgu%M8_lC9C^yO)mz_`KJ7;N;(ar*jkt(kbOLBIc&%!|l7 z#0lo_h%B2tEOm{__O?ypZW@?p=^GC(mt#l^=OX9_2ePU2wDD+x&$NyX`@)FX-}BH* zq}l7qN_yjlvG27JpKC(=PPDTWhtvbOa-`Lm@gHxIyRSm|O>lKytJsfX&gC;$!U^B; zw)4IUIg6}59+l{^EDhqoBdHADqUR~`NPWw`x{C95q&XKn%HNqp$|n2mB-gd>{v;Vx zE=+5A>PmeKNsF}1!5Pyf8u4;rOcgeFv!!cXV9uDg*JV`hbQV%0pE>eDtgA^jI~+InNasYk zH;612&ZtNtz-Zm1=bJ;H#L9W?5}t%DKkTA^9Wv`&pC~O5DE40_jS+zNB~`I*=F8l^ zA5S9l1<{#%bRRbQd5HS*f&Eyb3HW6B$xnBIQ0SbI1BGutSzeH`!&DSM_=W90IU$T1 zFSc}LU+^8~s*$&eu1B+FNt?PjC$961hta`|V{|?{5Ab!LN-gfC0;jM|fOY#752L;g z_0=0sQ8&J!Ox`M=QSI@wV4P4~qbdGtI0Z(t2j}4Mp_1?rye`q=-NfsaH^$#u&^P6R zm?mNPbR+-!bo0zS5#6L)0jxuziGy0!60mr?VMD%X1gs!ICgjy4DcF{4h3kG}VVu#H z2u5QGx(G@xiwHO)X6Z+tlvG8$9ioU|XAJu)raEol>AGBQ{4?Oka(rqf+d&U|hW{&t zSATPhahO1a5I2$6Q#-^i-qk0`COsxAN#EnoKJLU}Hv5moS^SUt5f`LvH>JW4rY+sa z!()@UT%F>Jbi(@Z^)=S>ZO*Y+lcVd+lII4jke=Pk3JpjyxFq);kfiDk3D&I(JbeGP z7Mq`WW6LAXbt56rxSqY?_3^|H3%+n`tiZoHf>2;C>u-q4l6g;0k&Q2|`XHoU)659l zHJ{bpKZ#lN7|^UO`q z#8KEstycEITyYPhTO0YmArs><~-&(p1x`h+t&u`VH2j}2X+LMP$zU%rUo z9q;X1ML*g6SRmKo-FEd|3p&(nycuKGBSvFsk(5WGuBic)$ee_u`ciP~RK|8FumWz+ zo$yvXWB`$sK+3q8J(|tk>SUxqh7w8mH7!#7NoH+#VQ<51&}05?9a2k^<;OhQ3=;}Y z4ms3j3G{2ON30!gpj4KWkJUvbd05fV`;(f$(V8OlX$Hgm5%VQ#7G^BFM@Pl?>4Wu z&~rKKAnZ2aOj43jgC8V>LoO_-#D|QP)Q79R`laFhkKmXZ&nwiuqTej1N=0^u!17*K z%)YKJki-l8d&EcrCNOC-Q_8}3v#+^)<0R0$%I*ZJpOgu^E2fdp(W*E0Ff%sb4G2xq z;n^1BWnh?;5=Vm4QWU>VUx{e?^Kmd4WF#sQ)+IkVD=XmEX)Q&tCiq(Np={h?@r3Qz zn`K|sS=L;}$}ldELYt8V=Dh^f?x{k4l|N2D)U(e8Ia4xlqbyFtx)tpgI{0gI(ste! z{xGiT5PH}1WLTV-07O$Y?W5>l%AeaLr%VC3nXS_8i5LP@gNpGB%Y)OQ!kBd2gsymp z-B#oAqQo9FB#A#P#G3>w=5Vjgc!M;-faIc#*ban~lNak!&)|Gi=T~TS&8|JR?iqI< zD#B#sr+a0&%uAYJ>UO_V2W?8*{yi@i)tS?xSXV!@?sOU(n9zwWv!_=l8i?NbQpqdM&+z}agf9s2Z2H~ePCsKpHxXHRY${SF z^y|;XKW}<(MvB7MsLb1N3g~;(@RD@w#f;ydL@(Og!B|RNYtEBi+UL+<~Q? z>(AP8+4V4Jc$G%9Uw?OG}9 z#`^7mzC7@jxW0;*Fi*1Al0?`n$v9ii`8a~{Oz3adb{Geg&dn;+k+yzE-eoSY@?dQ_ z-}y-#YD2^O3HOT;o+zx8Kl29fWU4Qi25ffS-4J;TkjL;qCM0#_SfY*vT(_*ftvo#n zTvzw-y7iHJtmSzeup#j$n$9O^l#YMM=&!urK-GDNwPKA~K_G;dO-hRZ!p3y!` zY$v>@x*(Y{=4j+|lMx}^5ENDU?-M-;1R{FS5n?I9!FbsQgBCxo0> zLdqyL7XG|rZ}Mk76BIe*)YI|7G69+E-+`=2_Rvv&4;Z zxp}1tF&k_HL0W~AFt55L%zpT6O0^4$MCsw__v2 zjli?5PgZViMEPsu_56;%i5yfS+DhCw>?w6nTm#!ko`~oIct$TQSU$hi^xks z$`;eK$H9+F`#nlyN<*jLsVbs;e7S)zuqFTiy`tAarK3zO`TjTMq8e z#OuondFe5oA>8 z_wW3nc#qR=Q-U;}3gwBK?TWVU(9r8tX>IOFRviM;?S$6(cHM;#$pGBWgk(>|9Hn=_cu%l3K2IVe_7hXV6QRDs*UFdiNxA^ zT}KUuPLr13$s?UvKm2)dTiE@GECf%D56e=!cNAv5Us6p;D99qhBdz)^K5@Tn4Jq1< zQ@0Lu?RkS>`M8Xa?mtg5f{WI1E#xwvW;bpD33a7}RTE-_(QEp@*;)0iot>dsiyc?Y z-SFDqr1rtYaN|dUcq}0hk1AqwZ=~NhoFyel*$hZ4Y`P(Rp`2Yz^>4YB+crjUSern) zKnEYfb)aFt_vj?XEFksczs*1}@vXSu zz^#Cm{ii%fEXzhCTyt4T@50vz&S%u#SRHd;+C@h!ZUxBemQeDrkpI30M?_hdoqMI% zjEkTTD95}}&`@=@cQTqqsrKz}kz-X4oaJb@cSC8AJ#HujA>M|+f|IoJqF1#$;$!zj zb{YoLpGJuy?5(-&Iv9V^R@*IkVgA8BWOlHdb1X(EMyBNb*LloeuI8L4$$GY&lVVK6 z_2og2G9EE+EG5lv{dwt4b8`{LwH(w~%SKc&R@rUd#V|Z@i19l%{b~DxEB8UO@C@2- z|J3EiCv-YEXl&MHEFD@AD{-PlBFX&cd4R?u7uGFdajpQX{^(iGP}s3Gt1Z@vv5Gwo z66kL26M@C+jQ%wFzi$I!2P0{QcpD$o&J^V<)fu2X9Iv z3)q~ePaOR9X*7P@&_g)gt4@<163u*WoDyjg%vD$KgKLLquDNT2|D@_*X7_a(61%ub zXBYcBIQ-VKFR9o@WYY5u{z>d70@uRB6_&SI^RI8NgsJ8>P~H6vxm&|X8CbbD8uI0d z@7}T>PtvOGvjCDmKjJ~|S2+6~!-l$_9zN{uKmX%b|F~E(L6Du?Rm=NrU%xf!_e1{T zUwQM$G+47pj)XmFa_{X z1`Wq{hKN94wd^{!6&!qVaEYhHx}L#n*L1$lR+NEgyzy0WSCxH`OU|4HUv*5PRBJD2tlvvH zTz>CyEK9(B+MN++Ygyu@$ zhFHXKh-sjY8U=^{!D~I^vWuzFG9%H=9I}yD>h!1y;CFv;&nd|bgViW)67UWmV~+9H z_YMHtAPjJ8OwiZuepB`_?QmOc3+1$FILo+&LOH znYFlm$@p}h`X1#a#Ec)*^EZMv0ORILOy`eQ^HePs3d^WI>ss~;B6fo}nn3bsy zE(Z%al%(W(m-7e$%mB`vHQ{x0)wAElq+4S~M=wpe)&Fp&B$PV}nj%J~@wNiPDmllB z{833=O~tB5WTl)#@>VhGM@u()I;s=JyCbl3&cz@|+(4>Cbj_+nSNdm-PFeO5rR9c; zZOB14L}8hurq>l9ou?fW8a8U=e{3j@G0t#2y1lWu9*LQ;=$?#B!U~&oXU?CYY-IgN z-u)Ik@@e7E7vVqjQi^SHtWQ4QQLzLnqk%d10;N>5)17nf;6*iUj)H!t+$J>Cc-yrl zEz*jcu0PmuUcu{J(Y~GBKPKsF)5N&(>Ue|xh0rDd-R7u%{QSt!Uks6~{{iT5m*1_{ zOg#mp1~aHw#?aRHVOdp-)PaeSINd+nnr6oWvBamM&X#Y6*=*_#FQY? zQ_(O>Z#boqi!*eZgF4#}O3_r$POlH-Ke%{Em-pK9`XK2fA8&uDW}^V2s^e;;Cy~ie zF`lB^w+A4!ebe@Re9IXyYgRX_78$1cVa}3ayv9?pgC!=f>N0Gd(QC%e+Wn|Cvbrp7rcAWU-ucZDN5bu20`57%s9B zp0xN162q1G#_idrHcGSz6$=kM49ql|MLRd2i83-ux@x^4-?o+ho*9%xq0dotcXIw5H-( z$&=LLFUX1XUd!ECu|`U!Sa87Aa1MIXa|KmQel>T*I#ASyZAw2!+FjCA4l-I!(uRJ920a+ z=Bi4+c?(Yx)vIi~nHhcWKxR-DG5# zySYA*8PZpXvz|d_toq_G^Neo%fvf>wcS5}owQmlY9=3_Z({V{oKDG1@Xf?}B7%d_W zA_x_@c3Hf>*x9MnX(xqQ*JR5yUJhOK-atF3qYv+!F>w>%>3X~|*!zxWGme=@aI5=- zKvf(J2E_2J#>0~c9 z%4wEvO2IKB9oGEJ{+YDdOklU_p^ zfy3Z6=@*BwwvY5BlFbbj^`y#>AIHd9egYass_bTBN9F@1IX_BC0PsotEoOqocwnBU z&uCb}v4qx%bmi^h1GQr6s%e#HeP%f=65y)?#8=d4luwMsh?K+q;g6 zDeDdKd+QlG6&3~68P)4Kxj|d2h9O|V+Aspa<7BOZI3Y!sG#W)ofr-fT^oO-W?hbUv z-FVh@WlQri)a$Z2uWr9{`xCO8t|M__=aAx%8H>)Ha&gAgcsMbP{CA#gW8-|b9c4is4-l@KvEr*VBtl1)L%o%Yw{^aSZ zRNgcGt}HGd&Q>Kgy7%m{-WjFQejw8xFAa7zAHMObrf%W&Xh4+&DYVR;Ldzw z;DlDhzmF(x9rOu!={`=lj8G;$J;Y%+*l6*nNRbbglHjS#ewCrQG}G&Wg$FnD1jBA? z#>$L_?+i|CCA`Fm(9x>FB0;9J*WKtMySL22Ud>2e%&f4)sxs=tx_*UxF6BI}z0owb zu{X>k+qNq0XXoT=sbadQ(r(QdB)|zPGY1bZb z=)N+AiFFKdHQCWt?L(;iGmgQlN`?-=fVpI**0M~?zZj7#yr7#08&%EeLaFi-gz)zk zN;okHD(=m`JzK+{WiW&89rweZ#Elsjo!zD=8#QVzyCaNqfF79o;J>q6bT^(oC z7?srkj+`Sh(aBOXUAD5bmiUYk4ZcSC3t>Th?MO!S&`7uI`G{d2m1$8lhSvTmV_GSqYE+oXpnP$-*+ur;o}eJ*cE8w^U%ZQ@xUSSGM>E8lovi zH|=eFM}3zUc^}?Q zM@7*2qRKzOqBo6dFZa7(K;?!_hbj1CEp)w1zzebXZ*#n5>zl(N1;Pu13Re)dg84do z9ERjfHgku&;gIQHZ|0)fRd#8-7#l*ZoN8u62VpLx!jXE$zTLh@@={6Y2d9ohYX=yPlP*F?UL|D=%7f_(1By=RLgP! zWpU-PT>ha|9BymJSs8`oz1AqgbfKfb-lmd~v~y9)3p4oOL!5;SE0JTl_L_&apUhsK z7Aw|H3QOx|3F#!7A}@6ekKw*T>8Noks_-Ai9Y8Ekwi>SE18lQ=@*SSY&vQ5E z#yOix5?EBt-C(oXl-DeTCK}{8Ox?0ndGo#s4J((Si8w;z!O?K;Ce7AQKM2; zSOB4kw|!TuQ&w1Hzw(?^Wx!g;?FA+hJKsQFhcEZ3xJY z#E~z_Q!?{vi)zbhtLjq?fK_DI*yI$f422)zE22{eixliYFCBHthopEezW{1{p_{>%~^(j7@6J$zh+uIfG2&*4jxDpzdWf=+YQDkkqM_Taz0t?{F&W zlab`V>dD$z^IQ6?S)k_?G3cd2N=)Bv(-?YS45a&(dSdLf2xPRA{>H z(94Zg_(uVFH1wlzqGi9h1hHe3jE@yPo27qLgROAm&!Zgq6+ZleIq{|D2IFr_+NlX< z4O{R7<3`ci1QnF3cGxE~I|x&O*?K$=YoN>%hLP-+2;qM9fjgrwxoQpvWoCxvDtx4x zbwt@MSF-)KltzmZazIArhw45g@O*Rpij|ClhqAAg9V?AXF2&gRdnhewoI+_t&4Drm>=gG=k%{zHbf}zRHWk=bogFCGsj0p97&w zvdy!H`xC1t+2^v3Nl3~r=$bLRUs*TtwL@B7?kt$W?;@z1KJVmiz z?i$E6u?AJZi;;U^zWuhB>5vv_yVW?t@V4 zQgZS{Gfw-Nal0?9QdiV{Kkevn_N%s2_u{T`iik=y-|vUj^rKzd2YeS7<G%iy^11SzJ!s{FM%uk2iA7R?8NZ0-y% z2;HhbQdbYJMO#3*13q$`4&@I0)rx4_uf5VcsiC|>_W`yulUHx>pRM5CO$t&h*H{Q( zRY*w^ax&~Z?>#S*IY=p=T{gE@^xG*_Op;S-EW^i!B&;x(3=N7x;t0=|&AS7_q*uII zbF^Ox#RTkD^t3d8QBsNUhcd={LKU@e8CzJgnUOd~&NH_Yu=0 zT89?1=+m~&o0vToVE=*R=SYJOQXg6N`>GMzWvFdtwe-hkvTyo-sz~>gn#YyLqIlW}#Z7>hA1XHP`meIaMXCnFG&qy_cvP;p|T% z_x|Yxunob%Bg~J%r0{@RybmpBWb758L-f@`-J1DY?tak*#*ov230)Vbg7P>g!5o$6 z_?ui?L#jA!&vBFQ_9o~HrP_%bpM?XnmS<$!l8Pn+235IH6R>>)0uQ(VFI8On=Dgf$ z=3SlxFJs1B02$(;6qffQSBr&q7Kz`F$#dl1e(i!kh|%&H35InFMGFTK*Ze*xuHp`p z{l(ejF%5@*|J3o#ZNvQwr8wQcim!g|pkd^q-?aRZ$zH|G1=Be}*?{t%&f0IwQzFU@ z`Gb;$Y$vcD^2QPL;cQ7qwJh2_k2$xUP^GPRJifHE*-Itw_H$Qu+@Rgcaep0kQJ7>X z4{pB9S_{JSeSMvThK0=Q)5MO@G}0QjmBt*jy)7(*PjP{JFg^$=e zv*+YEM~UlAG74?HDao^Ou`FGG#*JXpc$ysGDGnQ{wRRpyVEf9dC+r!?%KtpRHGL|sB}QQWsryQ*DvE8<@{c5q$F&fiG1cZf_7 zTJbxpuWgelO+1qS<|gU3#;ZDQ#7G=gw!N2;ihen!)$EL@EuJi*)F1R%Wqk77HrQR^z>{{62nNz zxbI**Sw-X9W6N`=Pb|h=58RN9skkh~r&yccIk2g0*+@2&m|f!lje@;*nA{?Ao^W3U zPk3&fl_)v87AS)AJG*nWsZt-shM#h&>3~$irr5+cafI%G`h-g!Eij$`N^+B6?XW zn;&}Lqn(XuCIhc3)+2b=I4gW-;)tbua(azJen^l@-w^!MDhkfX1jzyR`oeMs-&p^N ziz3?UCY!hN`567rzGTE`xgX()oJea}?cB{y<)g3e{e3Bq?^aI?PfB-io&c_2u0^~4 zT~QL!=NH(vxPo59*FliHO{e)9Gqt#?gYuH@55Wxyp1j)_@8$AgaecKS(Td>`OAFkz zjJ}#k9sWLUBsL2`>yZP~irbh?12s-`Lf6UL?m7WdIXB81IZP^F6wisA7{0zuAcRVn zguU;dQ3~*HxQQ4jW;TZ6w^rOTxuY7kCdJJ*=Z@q2CvQ-Nv>o@saZB$zlZqa0m)N#e zwpOS<*PUjaS3$ckoks~p4UC&w+Q-Vy%cGa{y>%770kWxTDNXmcGL2mtZHE2p>#~{C z44z;8n&sPDNSUu@WDcOBkJ+s*wrTd~F@#-QEj z8XtX8?CrZJLQ_VI$Hrxgf~k^ekN?*WLlu>I=qCFs*B^3w=lQAH^p%`790%< zgHP{c`sl=8!g7x&_Z54ue+jLb_*fl0my}BfUT-b_$CpR=reSs*ngP2ZXT4a($*%=P z%Yt_(mmCdE?(78M37zxa%P~-)zJtD_!7W|nH#NAvfyRepqXw+o2zT+`(_xstIz=A- zAHkf9Mvw0F`2{MZ0i&d>-K}Y*g-69j*p9Q`bx`MU4Smn*Xc@#(rD-hcWC{LNUEv+d z0rjM@&=zp$;8pc2mlPLj-SfxO6B)V3ESd`oze~N$X~hrm)2>gh62t-n?fO;w*MxD0(B$iD!a&395#JRS9LyB8Msv}4b^m1^OJ9TuS|-6ydks`lZ_;uHR+yD5 zi4@zzqS_MEbt?gmO7xT8Rh2x4Gq2XWJ@`$H!TE)Ls9 zox)Reyc*1g#4=Dk#;(5cwUy=9>+;WV^YP)q?))FC#dEVdpo$AfE64D}m6i4hHR30V z0C!fuO^6cq3G;|U*g9le>-*?wo1uT|(tmB3GPg>ljLe9WVtix;2rJJ9K$z{9_FQ2t z%%25^8%(pSYr>DZ%iD077-0cO^upyR^Dmhp zp4BliUw>EVC~Da8@!HB;_*N8T@F)P(nbP4I`;xr8bp7lnJ!x7#>Q7SCwY4c`UN{ji zcd{maB||cgO%v7d@t1ptXMT@2AKNu!Cs+_@^(Z9ntgi&GyT->j@kM>3qDP%8gfa8g z#KnOrN_W6&y3tI|$izg2NSG{fcnd%n4Dv26f8_$wr>9DV6aUI;N_#osMrqp-Y(0QV-7SA++|Ti{K`E#NF$qCF24k3q}dZ z>Ba${rp4woSPc)lNLy`2ju58VUu|qTsX1ilNMc8O_8hA-R>Mt|49z*JI93(j&DG*M zV?@p9EJsj7=LArAtZI9ayV(md>QKkxhlvu{9H~$|#%&rqK7a{*Po7w2%)18qzLl;q zUrqJT2!vWP?nN++6X5pW+yyMFb%lnuA{^HmWsqw^rR9WT|sIjW&xV8rO1?lCg=rI^Wnto%fY zZ_r6fSO;A1BY1_wO7%*{QGD)6^wtGJf;&PDAtE3BsA6J+M38qvXejzUng!+ZZRmI} zryjg9YpmJOph3t?Vp3?LHhxsQ{q*Mb;ImiJu`Nchjgl8Thb6?8?G2$O*`#P0Tc5Uh zf#IV!ntL^GbS-;4C3NX4DG)dPW?;NH5tG@ehN#l4U<16Sd+RH|WPWtf`)8}&M8ytI zo|);;DGSS{(=$7n4UF#J8xi`5%Hrm*PN(deF)3vFI9--tR5V-w$8&FV(*9j*~FX8>%D>_gWBU7>> z+xa!!d@ltqiqUW7Z`?i-APTn=-!-nXvA&~F->}(V1PcY+-TgdVo18a} zVQCZ*$~)#J%IKLTzE?irM3Ny66WYv~Lb#e_T}!w@{*6%p2-cW=9GmRmDWbM;GZ zSxVO8O)62(2BduPxvT0h*8pa%sn z!KJP!E$_*5MOu;4f?#X|Vk-1UOck;9{1H=2t#@UKp)^Ofi_3VsxapT(TY_yR%T{v| zQ)|}URcSw>Dn)bjzv61g=R#3>3~Xpq_nj2YlALh@)z905*66>R4z^w+=L}?m=Lk0U z0rs3NDfqUfs3Dg3p?+>t@{E_cTolUdkpRA8B2uA2qsX1C*rOCl!=`j7?fO+{9E&yh zM-;_+wb}Fw{I5~8?6*w&4{;>A$*X^Cay}`)q^Q!wdwwLhn*ZaE^3|tGb!2^}T?Ess zBZO}BG$7CVuXMX4x}*sFbu`6Sbe8^6yuxU*crpKwSO|FfC`nt-EXPjA8{+$PFPY57 z9+uxw_;~H(dE)hpy>UCE1(XeSJ6-=@d=+Mku7c^2ly9es_jib@xnyDQY5V`@^$+3Ql?`u&N*n#+|)bOqR zK$>!@F0QbC)3E_DOK@FT1Ow$vI_*+f&%+-hH(C1ASEfQD+QumrX1Y(tLg{!7a*V~! zffMMCZYSOqZ6yu?fw9jP){)X&ft$Vf{ad}i%ioUSBKgcGgjn(b8@E!O`kr+6%nzTS zGKR7!(;yjQ%EVX1WgIm|=+C*; zk4MQ@m%L88@7+7CxF%PjWyXWMNfXCsrLnQk!?6mHmtM-wqGRg)6t}*8@XlLJvH+z^ z!gj1_L4?n4^D4vY`b5I-njVSTF!9+2645nFFJZ-eZa{`+vEMrJ_493cC&|lO>X#rgcp59e(;sD-zDzWXs(d*xoDZJ32{A#>i zyD1WDI4tA+{ZhrDLG7W!g>gYAKC-37+~fH%f*UqlEu!~ zIJ$8f%>Yv?lAI>x;BDmpi;1>i?Fu_}ztJ`R$#XL8NX6N$j9oFRVv&|GRqOl5xaQ~9 z*U1I6`kZR>A?aB^$XY!{}V;9a41 zfZ2Wbeb8BqxF)Neb$j}I4oQIbM=Msge5&v%@g212ouCrncO5)k+iA6!vcT)%(>C1wY9 zeyUz=PT|;OX7*Nd$_K0HX4@g`?F}8p($<10xZI^ObFu-4ozY0%Lo5F>lzB41J&{?p zTTY0GAiixlMajp^iws)9gg>Iln;KkJfbC51kr#Ifj<2}fr1}Lc+$#Hn?d#mm5<3;H zTprw(er?NIZy_LQ7DC(QrV`SN$;M8TJH0-$M}=>*D62^2-bn2LXFo;ENKLrFxA7GG z@O|mUr!;t5YfRiE3Qqn^A=2`t{MH|?J^6iCVnFscDfg(wtqipE#{NJo-@4M^IcDJY z%u)<~>*Mz4bH^0SOr$0!C0y&@3ma!mLXLz~&M>9H?p$YiRNjf_T1wlV6y^TButc+hCP^dxmS2> zmll774)u@Fk$yh_f#06+Rq{=ho4JQ6bZoP9$WWbQ4i}ECVbGqf9y(TA-xH-B zdue$^3(goSsv=-kYv)C+GCA*jo(3pN^ZzVLNmZwdd^A30RkaD=;#bCy)G;vs@?X{Z zQIobXzH`NDu&TW~QD}W-!!@YY!^Zu5@aL-l_{GBGrL3RF9~rvoExEC*LcPf|xqbmJrAKh3<0c#e$ zPD#j6E4mZn(R9YHa(T)oMl)_OMWZlp5oYesZE*fYtR_D*=M?3XH}egGDNKp?cJsaJ zdXj1`%4%^QOqj0(0tC@dD129Liu;_ag>0^QFQgt5X9pFbr{MCZ_OhT;M*Z$5lBOi_ zo&i09@hCi80E;3#vLdCA6`BmaGhDMABJYlb`P{Lz!}_3p;x!e4*x`>-?A*s)h3MZ??G_-=g- zS1L4d3(%87pbecjOE3R|aI>%`>0h0$vpq~&ibb~xrnkp|rY>{s*>24lYd+ODHD~W7 zOIv>o9bMmJY=r-4svT!$XLBZVy5AdD^v1f=vbfp$>U`_Kcx9`^3k)ML9;!Z$RDhq7 zIPE~jP8XtAZrj}5a6KGQH$mu_-x*TdXnU1A&~#P==?!M;)Q-hyVHAA;3Z1G+(eUDy zp<`_-KLtGkim68GkyoO#GoGen^?mk$xXXJ{NKjnb$+^KLIS&eu*zhU1she{1YnyA@ z3ahau+UHdr>|0i1y>713C#6G{h$2qkL`qT_>kD2D zNp~C=l=~Gn_Q)Tdo7O!^GsA|8>U4dc{j;@Q9O~f)O!*AajN{9+$F&=`sq5?*r~jkS z;5>2%?0J-e`{)7O{v2k}wc8A8WH5*ECk2fce$rmwr1D{=nNu^;3XjBl`BBVnlh7yFY@%3)CEL&R{;@^PfU-s|ZZ3Q_{bd z83Q#i7Kp=!Lti#yIcBHEnOO_Vah7j9X~L@#_5FTDhRR_5ce;^TXVIdPiS99>Inbk`QcwLiv%KMioeee!5g%5RAPAF_qzJ|yhcGfC1})^I9*^%LjE&MoMiZh z-~nL;$F>-V*=5p7q9K4k4VT3lIZghh#Henp=sPDmXU1roY?gvJG5Qa2!U5uBW1>Be z4zu|3&k|GI?1r^I*I9kJBOe+NY}!+6uYnGPTNBYdFL19l(E1zwqEdgiKBaWi3k=~d zaIX3nS#f*h$4*LjD!A4>C!(F}aGVuymf{)*R=zB={JhT*TG1DcsN*@vgWI8W&t|7b zxeElCZG~cNg@svsBC%ZTKBp33S0^=Tv8k)J$!<*{s$S~a-U0><=H}Ldt~Dk58;*Ga z^og16aeCKx(llMm;gb`3=&p9^ksVKgSer+ZT{Fg#a&vD!ybRy(*9rAuET7|NajP+x zG`Wsv1#&K*3%}gsnmDGV;yk74qnX-MVEMM!lI{Q#?QIx{(h8HcX|TyGRae}?3{f^u zXgHANNAfhhzf8Bo6K<2rYSquc1)^|Xql0MAS195j7tli%$jm2MMK1&K?eOI8Rs4pA z?e@3#Y~oT=$sSr2T6q>`@zbw0a`&b&dac4m$? zJ`n*5+xhZdjuV6lr+lW-=1O>+ z!-RRcIERJbclGni03R8d!q@#1#MWkOgNFdocR5_1IP#^}oIPMJ@>f0#*$yjd7JDf; z(GdTUd=lZ%S6e*<*n}d$CRDYlxdC#L*#&EHg?pa;NB<^Dy5KsNK`E$C87n*mI=GHM zJ2*}g*^kwN9VJWgNHDq&H0uW=N@|G94h~F{7J_6(Z2C02@^cE^&lrff}rL#+NP@P~?{cV%0`$t<1D%D`)O)nghEu_*^_sf6JC6FcSiE&lfHj<%c6g- z-B7GUG#6NKSifc_`TS-<$${;ZX61ef;T2IVO8pb|5Jx-0OvS99Qy z@g4KT_FiBQIr3B5ED#hs&D1S;FM5a$5j&6twN(WElhTi$hq#d6JVD?h`{a~)xNcrb z%=r+^b;<`vhB$Cn4|H3Xpd9T%W_R+EIn&7Vrr~eik*T%wxl-$=7R4b=ka{l&r~h>KN)_0yn*17jXA&F?sJ0yjJc z=`t;vQ}}kig>I*bu=W;JehJ}*5st_Vytc4=ap>eFDY5!>s|t;%c27^%L2*U1md;wg z?5jzm*2&2$Wg)7PdasUM`~2HP+VJ9!5LAXj(D6(lm_c(}Fo!6nKIMVsO55zd$zu;c z94U;!N9pz-Ytd5n8A(#*=@q;qNwZNSjBPJ?i5X^*v*NOK%_qwJ49VROy-Y+rxa%}zOWi^P9Wezz&-cWf8e>euRO z+nG~&%N0WyeNAb@aiHXO%~tVTGR^WRaJf8VI3w9@pVq=qu~0r^ag(7hQYZp=iZ~_= z8lc%W;x2@tywq_OI*nMwI`S41P z*P4ozXf9s5Kbg?wgNTW~nz3Mbt$ls{S}*$Hs2_eIf1E5HE`9&emtZsQ2cR2UyRpj5 z=W>}T{<_RXTd24PAu4P7d~GF*QQ76btg)k?j{#k@F^EsilMM5ps(;_6j_r`wTp?wI zfPAFh%jBY{JuMEvKHlq8b->w$iN4k(U&p5|#6jBy>|UB@#tEgB^S`;Mj{2{S6qdF_ zGUsbx4D{sCQN{HKly}NSM}~k{Wmxmz_M`L|#>O;$5r)j(8?0)47t?x2)xh&+zt2B| zxX`puQ1`&m;7rflm|J9n@^?ijbh@(aQ7OGx7hY5EN$i6330j{g8DUo?kaj9Qq({r^ zsd_UmzPWZ|XjHuS&LPLwc#jUb1+)>|!XIr778NiX{n;|d6jpQ^X0|Z1`{C{v6@s$M z=!UMK!3`Esfi!(2%40UM0ZLMRVw4sj=M!ai6~y_dE;Xp}rzf=@atJdw30kZrT{5)a zjF7ER_xb)Z=sx+8TOIO?o<)~*tNM5P?ijzaiuS-#uyFH1EL;CEpU4)752dl7e>y9V zakdC;%9X8JT1t`~L%!VsHFlDAJ&haHx+WgDL23f`vkzAS!SHa*E83eZxJT~zv({8G z+Z9I+tY$$vJR)b2$4HAk8+h z2Gb8Wro;$B^dBsVk@?HzEe02=lVZ1d;vTafKP_!+r~xsMeTfeJhHx}ey*UErEvyM! zYY7NGWn}MM)~zbW{ZCRdTyjLCCKrq1&lDvqQ#mxtg7?$PB@-Zd6+<#8ODDG15NK`e2aDF8@eFa;7CYRjLD&pE&!wcRde7MT| zSow^pfy``EQK7zPU7FK1GzX-p+AK^&&%dUi?GsZR$;Wbiq|LEHHcI7RtKTnvurbpO z^~cgl`IRxY#q&q`RDse@_Oagp0*^!*Q`s}`l1gbY13ovuAT=dW4}wqXiCOiUPQ_sH z6W1I3?Qs(o_jnJqJX}>EUJp4tUhD#4+|?d$?HvdVe)(U(Aeh3J`)2DmoSfl@E3h-# zT?2Y?6_{bS=8!^BPjd~$%O~~4=z>nj=@;iJ@{tl7T{|tWprDWY{|frWQtm&4{?mmC zAIU|SiseND#%x-oW5DD7)n;qw8p)|^M!cXV8;d(FxrOxAu@cVk@!3@$?MPbe#i55c zk;VI~DLG?R(&}zx1L@%|>&9v6z7f}WBAbIrGnwk`Mk=sJWNVTG9ssrbM_{=nG+Jm^ zaeXSmnu{xUY&Qur~(xeJ$HN(Z-2ds|`+usI|E6+EyZq%ZPfcGrL1hH^DEW-E7`=|)i?K}Y5L$7p4 z`=@N%4+<&#Z3=6{E+4}aZC|x)KM$DAjGx-ilSHVUmvSyYT;@u;bi{gAE$>C-QjB!Y z$>8^W(HD_1s=D1rf*`6SGA&c<1`P=vlj;V;qm=vBks&EkB1wOY2OJWYG9)u8y$%i8 zra2$N0)BcK9u>c0z@B9RKcAiLmb;zrms1+o3N5@)y5xg7jqUlTV*{|3*f;)XH}k74 zUQ*W=z1?zZ^UQVZIQ`*6$TaTPw3&pp)~kGlyKPYD6GcGidpDy-F~Z}tuCyqH7Hb8@ zHMl?}G7ix+50`$}u-+(M?Ju{xwJa;Orw45M&=`4QS)ePQ1#js?n@5+s^j5>pYEjKn z{?c}^xZgYU+a+`=4CUC6+tD2Z4lVV{=+ zRB&^0Y7Qb!SD%tIS+6S0s+rFB@?Zv1$IeAw$lHJkTymXamN_z08-P8o|?^_Z4*E-l3*^O*=3M#>Ve8!0ql z<4;%okiy;VU0DN#KkR(boD-s^uk+uDcg|>BaVuN)z~L$s5tj9 ze`^2=S-Bv~por4HULv*@h8| zxhug<_1dEYF{Q=n*a=lvR|c0n>+|At@iReL6mBNFK?5px%W|yU`G<}=P+2+Tp*lnrsx~U~QA}7A|vwhE1cdAAYBHm^NX)NkD45Vx^~R;iZCY6JeOxvk*~v znyG#{UflQmM=Upwl5h=iZ+!82_!;VnT-VZEOXiW>1L+CK;`Eb&&#_xldP`qI_ZwUt04uU-w=_RzR+>u$ zJiYF_N+HIziu1q9@iD7m*7z<7hH88KH?;(5G$!+$0IKBwB444E7 zj6HHK$9jNb*NWc`>Ky2fNuyr{e38pS!*0NIZnchs0nWwB%1wG=+IIH0 zm--^~(E*PrZ42Eb{<1=bY1J?_cP8!r_qd-GPEIHP6npy3b?wzXNziqIFzHNla{ZQY!iZmKVxD?MvrF}!A ziezdE7t6HgkCC_2$U6hzlJST_YFRDKz#!|Cs_v_@+HA!`1p@5mqwGUqFe5Q}P6`ZW zyf?ltThlY1gG_4OZ*A&*=nsX!w8F9?y)cwrTX&GMgfaCMNwKAcoaSsq80{=M-yGMc zQtJ!*2M%}-yL#r18ys+~FKjmupQ-p+{+F`iV~@Dx-&en+$Oiv&Gy@9u9mP#a{PIPj z;_n4{u@PM>oQMmhr;Jm!i0vtBR)28IV6=&yHj?x1%8@}FtUdvJ->PzT`U;BA|kX1Q5Uf zc5?ywnZz4hKm-qROe6VP_ds`2g#T|;HOd-7`i~sb2IZKnKXS~kD$bjqHy;1Ar~Dvr z^5kerJ^4E(=FOF-W-9oewvXud_5 zPxf-SOBY#U_CPRLe(W9)$c^AZAm6r6=U=m0HUKv*-OFsOZ)zQQVQJd@D^vJ_wZ-^z z?5)UOreVzkFZ6)={y^IN#CNiZYZYH|e`1#X?&kcKojEVy|K!b{IxNW!_J6q%`3suX zUEh%u_Fls^-ER<{RpmEGQC)4!Tp~PFjpXWgjd}j2+_HbWNBpJsu3$Jx@x^$6Mz=)K z_6sd1ql6Z32-H)E0MuV!tQ4hc;aJP@{gx66jl9#joStX%&ZjIMo+btA*1e2S5+Cti z-X?k<4X9G|uaEUY3%3jf(mrR5Yke2u<8L;V6x*#6?0_vBz;XEx`}mDyg|-!ApLL)S z%qvp~-!36R$bE0}#_#OuQ5^XSLv3toEYI7Uc_(3cJrLYL8i#XD@m-MfGB-MK!!_lU znxSk13M$+1FBOuaAj)!+8%V8&*y;;SZx`cBw1;kP2OE$C z&hSVa!qV=TdWz+p%{D7;4bZEnRTm28o_tjdPa$=H8JAsRFj2nsJ)HY)On~;MSDr3p z=|D1Cg}fPHuA3-!Pydd;o{=_K^YafOf_aTb)gND6s5LyUk$hL;dq|ubo*~>^X;G1z zWYGXkJX4Oz(n~u)pAHvd8fzo74Jo6n7W!AwD%s&*MMMH`%kY06)Ri56l2%QoJP|-p zLVOZD0UP3e%q>U7l?--Wl0==rzTmlvG$g398{eoC{vqT8#b`sxo!r?Xt1lmp2x$ZM zeY4D_d@{F*&+0yhZMQ_PbK=Uz5rwfFVX@`b@{!W*w4!BP^ z?Ia%moPIY^sBHora!g=RpISzPjWrLp$+d-^aKFBZS~=5G=u8~mluR;QywDrFWv(`o z_wAj8KuyyG(z|W$(`{>=vSv#4w@z0VBF(eV1Ee~I`{Q( zKIgf!NB7*78J26)m}xQFhO)qhm9KJY={ZA`!`w`mq3q2&W<0S&McNSue|^p|(+Cs7 z9oSKQZMcVP#5i4c=0*ybu#yxeV*GwHJmy6^>;3VBTIC5Z={*YiPW!U+jK|fp4@!x= zU9~BFJW{8CE|OrBJ1!K+_|PMVSc=zHZ$@ELr+BS;N7N4IsJvbS>fAnGW(52BF3isg z&x(h4OTGBM=^0G#HuEV9aa<4_(AjtD1lFqg@$dGRi>)7L-#;nAC9NKzK6m0t_rROy zX*;O<=>zvJXa64d;N9j4tffI`fy|4@#k-!9y&P(vUfR} zVn)%DpO@<0J^r5jZBN9#579r1D!(?`ry6nk%O2cH#KLfuM}bs0=GQJ8@RCfDR9Y&f z+A6=v5-f*vfR*xBmPeNwYu<1MWxoZtYtOd`;LCs|#wV7Y9bf=KF1A)P0eTqc&u@ds z`@`A=U_P!N3o z8@xx!)o&xtTy+L-k_ZU)p;j;`p|@xH|J`3M1dfgRp=RU}@i(5cdh(#ro(GL-ab96{ zs|;9w3vo3_;Q;Tx*|swU70w}AVpL#^h!-=wKO7$6!O z0v0`Z^frj<2ZR)#Tr&uIB^{!x@#@&BN8hdDS?KA} ziX|;E0#ty`j2us1`~Uu6{BIt9xk-h#IZM!B#gA}=ODg-9#!F(>-$ZhoU&?Qjx5dr z+a0m;L&^W+lD_}VIHyLdj ztTwPOPfwaQW!awrMn4rh+f1r(UcQwvu@Fg$>;8~a4=-s0z8>!YUnWp7Hx?_TZz``Z z9Txs)*#7mq3le4n;f8z=RuuKNTa6C|0azX&yupm1DTaiXtg${|<7eRWyC-Hp^)VM4 zEP{S%7e-8h^J1sq%kWGrZi?i#W6qUo-yXWd|2I!=CWx8bGQmkhvj-i?a>UuJq@j3= z00(G(VXAu0!r}Nn2IOFr`30<_vpMxtx=E^ z`#4u|dquzZm%#Q{(pOhY!CmAWlpTF7xkiXUke_(2G6z8MGbdK&-+cD_Ie+cSc^CTI z{tKrnfy81S+{;K@7^$HG8A?j|+2qH5#34r!lWNZ#u+HjF1Uujqp#bouz0O2HoA!J6 zAO3W94EUIUR|A>hK;jrSjaM*qE)-&g$w2f#in_jZhu)W?KgH$W-Q&tyulH*&2%V`Y zA$kwkL!M&5*E6nSB z1|icZNx4(P(5<~?e1Tit<=cQ*ECC~q{#7-m!89~W1$+OtWvpvv^SqKazoY5o_NqR) zAfCT@K8_p&Cvxu-<&)~1I$cqFFzkI`h%!9@lt4-7;yua6MCVzCX3F(JzacJu|GS24fd>Gvw>!61PJd$P0H+FC0lZuJ`5tgjP)G> z16Nnre8GH4+4f*8 zQyOpX(-H-^S2$5z8xKafFG%+@cYxQ|)rDazWG;{KW37ALLV%!H8^gD-OZzgM^_giw zX7%;JU#t>2zrqaV3$O=UA5lsl1^!+upmP-}&}}+h&-M1j4$eRu^`wvtjawj$NY(k^ zJAn)my}eo#h(3e!`%-G#UIOOaZ{sc_zU~R}o4Ujk&E~m883A)@m&9t&!VR$hdTIm= zq`BmG*2hsGGZ2C7v3IE4DPC?aTIl(`dtG2{%{|dNxD}$V%~XX&=e%}Ae}|Gi(#iFG5@?#=C4oasxaso>fH4aa(e*Bjhip))q*nB5SX^oCB-J= z;Lnr=L^%n_F|dDtldp+Tvu=(m2Q9eWQNj^Vy>AAcDGFvy*O5_%MULqFcs{TZsbks}Ku^28ujleFRw)&ns>7&Zhh zb2gtbi2CPcFMs{HkDLRa`^g)92d*HH-bYv#1esmX-OxCYD)coj?!l$q>$)v6(@Tg3 zDZ|MnQBG8Lx}q#dt9s}4lcOayi}h50$RSQl1snI0-klyQ?K(OPQtL!Lvw8&~*yXUZ zwSj4qXs-MM|Ts5?z^Om@iryf2vkn#pAMYRQY1;{Uvl z<$tS=|L9_s3!q#ea{h*V80$K65L#2H0bszYGkQO)ZmCs7?27Q($5o2OSe7>41^5=^ z9XTU;!J|^=?Mtm8P~=fd6D{bHyQiuPb#z{~LRa*nK>DnJCKg$mNmFs8WzzC|<#?2f zP}T6{Mi^EOoONympE3oI>|{KqG?-xaU*I}c&qx~EYU(csJ}!B*HjaHkkUx)DdT(aD zxg*&BP@g&tcDn!F5cc1H7G!(^Bzz;)#~60@bzlNx+STK`SZ4>?wRoWcA(!3~*H+_< zP9W|W0J2&5beS=$r(T21h^IaiR0h+mEKs-lFu<6AGegR@?KHE$D8P`g-M9z0#nRff z0_f)<;F0hEbS?R8Sw7&7bTeClz$isdQDd9dmlsQ+V}|L|Rl9v``EO>S>_Z?6Ia-~z zza{ZA%go`k*Sdfk9SX`brVq60orWr0T|K5d&_C-w*?*)Jra2fHn>Y&+Yr(S!EyrQt zS*V#9TlNtWZ(4$0K??Z{Dou1(qXX|P)Q^CG7Ru}8^_{Z>1Se*$$8!-_KBC^d+rO}K zfX}ZBoI~+IA;2p99|z@LB*A`vdlVoL)L3&LhGJg+? zye0!Uom+k)RRO)PerxF-73hym<1QgFdOA9(^Pt?Ie^%gudA!s);iGoqGq~M!Gp>hoEt1fRZYf`U>$!qN=2dP&z7#t&B{9L zLodJpzrN}R1R?#b`=mHgwiG4nro;OC1(E7{acM3i)roQWz$eLK>no?r-O2;JkKTe7 z%<0L2V;!;mM3jlin+HLphkZpB4Q&CtNJuk*j5>3*32DB>WHTdNHgF8zQ$FgqtQmM0 z*9pNaEueRacThi*Sau!12m6(H6X`ezimq=5+04s~-jf+F;tH|=5yWmy=69%VLJgI* z2s5A>BorPSSD}C9H22@EV&PB~`g{hoW1tn`NqWkouZOsSd*zG%)rm1z|)++Udn*<1j-lU_IuFeq7#6mWRxkpNB7KYUW=#6&8bK7crlN#sA-^ z`nnIm#@Q17*#t?c)tT<|3uU#;G8^3Ec$n9MGpOMn(+)Y;SlA1|sxX2gYF6D>4>B$6 zO!U@W*6JQK0G?@MlZxSH753fdoceRI<>0^c>KSn;;sKN*u^_jB-=llb-R|+Kv2d)M zUB}tpmw~OLWx$QCX48YzwJ2V<@{*0Ahlju|9n|?@hdJm<>VHqL|MoL3)8CW{`}y{8 z>D>9;b5Fed%ZJdUxR<^Fa7n&X4=!|wXcnvdyhk4NADW37qd;_>Mp`z^jV{4{U#MAJ zD6uunHEsPLm+k#;nO+!BL;9Jwg^JSlfP7d24ZnaK+5vdz7~CcmV&6mf7u3W^%_Nz2 z7I~0UyjOdqkij%ic7>eCpu?Y!JD49Imb35QzQlk0o_F}cuVa-G%TWq~!eFXB$qpd3 zWMJ9lxC(buAZJXD?>~ChJ}f}vh;E2{754Xj1N2ybEdZd~VX_js?E2q6!hil5O%t3T zpY$wnKkDE78^`a6fLCKISNJ#2DEN1Sd`^NVJDQ#IKW-lWt3%!cc(PcdDEi<2Sjw+Q zp;{yBDE!ZB>i*Ba`{!$^e#My}gL(x-bza17Rc-{GT;Nn+C-rcUJ>jZdRC|g7l>@XH z1;*tiVrJF%L3f?R-UN6x(_O%$Dwu%8m}D?EqcMKsv+K?AGkIA8A`1C9RVA_R%ky~?|-DcXAZz*%+J4M%H#qf7@||XHo?i8+w=l* zF5?Q95p~hDZk#NdX299iU+3%f_>Q}(D`-_+jyjlgvc1^=u2RmkJ(npU*kyjGywvhT$RL2VNx3&iNYzE?z-cN#h~=>DnBH3N<>5MV z+HE$umJDKC-V5 z0Y;|p=>b^_isHt^NnfCR1Q5r(6atkjj|H<+!LuL&yT1E)MSo|Yk^AK3UANEx3b+~f zy#j+9T#)Cm*=I5|s_G=R)vo8Zcs9v|cVJNZ= zC`g^5DhV|CgKvZOvKe`S-r&`BNQ4aK+jk!KI)7#{t^)-@Y(RamvyX%20tz@& zlT%efXC`y%K^w9u)-?;U?ffbs8NyG(j|;*dAl8#~3Qthr@Zan#(f&ez)vm@J0ZAY( z(ye1fFX#wxIExwqD`aAyz6dC7JQKQhXV4~AA;+#W1p`EygTPtb9)g^v6deo5Q8|A8 zl5xYHJAGt8b1KHvweZ3BzXD9uAaWAHVeZ%o{v=dAjnYcH#}y?qAM28-T5Br);D|gFG9+|800ebaPlK> zPz{K{gQm}CW#&ssxnS4LpH*5HJb~ZY0O*9Bq2@yY@Bu`?ph!P*h2#{>&cL0R{wM*w zUk~I0xmM!2{_P8*xolzpAVWJ=W9W&A2&xVSvS0Rie@MXK!bx2Y`}4b*{Cja>k?rf> z$IY^eA>75n;m{FRBXpkzu}46ZO5T5ji`^udA>vDgXaz9T%L5P0mjSjQ+90^+)fotS zcm~)d90I!Y2jH-I(B|~ITfM9J`ak=s9JC$asRr9ZoaP7o@GymP<)C#(8t1#?^En!+ppsn}{#@(Dya#u+(|g zh&PZ`7t|!wYEEQg%wHj=lvVb_PN_gk#LnM7zZ1KrNoq2*S`N~Jyak-w5+qo{4Z7Ze z*EC;}=HT6+g@Yk+P(~aC6ugs(bMbBRj#R@pccn-!3V1aQhp-%)Mk#T#*Jm?gAZz1N zfCMkpR6a{?FMadu@P(1Ts6RpbPk{n*W5X$LDeOEi>1K@#~SR{QN{yqN)Sa3J8*HO{2QT58N&fTaSn-KDx=inMWux`)Aygjb~i1h}oW z0ZX%3X}kwDer4xbRO(c*WTV3~#r_;-n(x{|p?Uv#i`CnBt{6kOSwHffep98NNe}bF zP>R&roq`IL`FoP84%3fQ+I>1^H`3%XG&{YvMSGWF)wlQJdbTuwkDF`|$g!YYzzI}e zg29F5ybKE;Cu4usnxwgAq-gJ?=omGFO>H&JPtLT(aPj(?4&CD!3zT8aGv~@b=9j4b zKgQlVEXr={A6FCuP!yt?Z;dHMoIb2J^j{HKGG(o!BOEWyl9t`Zbk+b=&!viGV? zx47-)4khhDPh1sKjChBYl0xc8vwk~8lr7pkU!}l znA8X?LE(CWL9_=*8yo1DWoX*N#1^OVeO;!Ik`aJ}mGL~jNeIk6Pnv0|pzouFer=3U z6bVo-u%ksMh zfF6qjhM2{r2K`mJ)pNyGa?%Y3z!>c5#7Wg>lj#B9=G0oRM`WWuF1~E1?!A>c+g*)nOcKySeO~oOPwK^hnXhIPt{BW+DzKWJc}0SQtaW} ze5hrTcgjV6OZZ|#;;NwBuGJ@5>MgAL3>F7JrqWY+{TmQxvx~=f7c|BSqRZ|ch%ElF zITbOO#*_mM13vT@6n0a|Q-h2nYoortv~1TrRxHo)nmpdte+a64Y+WY(W$|vR&pAp5 zE4Msn_T8A1#5z63Ydvup>$-!#3mBJCZ>-m)KjAe>%1k@HQzaM24mU^zF(r;k zZ+yRc%-E!_K>lpt#2{VmS?+l-C!AK)*BfP(RVZ2_{hwB}o7;;TEO24p{^9 zPk-J6Z25^$pB;gAN{Q0^9vMPbjUn-LytV^xO<+pGaZU5s#!JTzJub#{qH1e-&uCE| z5hOL`jl`KlId!VxKuXJ|6``H#6|u2D{})j|_S=aUNlkxhBsqB{K+8@YNGHwLuT7{` zKD39sIi4-z4G`Rv7yxhJxpJQhJ15Lj((Wra#OQ12V%xwJ%tY~?tl|A7?Uc68rLE%u zkNqFeVVNj=oIW*y1})*$CdHu>{waGa*^3V=DKyRNYIl&No9z!{fd#vcb%6@9*lj5Z zKu=64kBE_)9!am%2wwC=)bHY1B7d0+ki({`W1?6lYY0(ZLvBmKY^zzI7!@S1P99BtN9WKhosnth;V| z_Q7B0ff$rN`;#tEn#nJ0&ZocRM*AzP?yODo`x4z>S!>1RmR>71iWpmGGFWSMU6mK3 z1$^(FNx%uAV+{p2q|bJoJ`XXt_AM(r(GF*+5;A|ZM(inl=D4z~5WdwN_A-ySfLR&7 zi!~iHO0CO{Csw(GgYCdNSC_MX`;pxXN2D!07>-=7+7->RxI0W^exjvn)B(33q@A_s zYDjB7jK5A?;UX_#G=Cu{>W%>Y-xtEaT3CJJY%S>!uIch#pIH9Koc4d)>k0@SL`$07 zWID!wypndu<2W6UcEswvlD>!GM?`0sD@k0a1Q6=r$TxKXo(FQ?y@V0o0EMjJZ{hkp zI=AYLAmH1D(mbpAJbB4d<$;s%iE@uqJvcqGL*Dy9M0njv2gUt?+0vi4M4crzFL62D zRiB>iaNcGQcJWlTzIrYfJUS zWE6>@Ks%L71kOsColVwHmrU!0y+p?hGVFtr=jPohR)fu2HW5AtY+g2m5b$c~X~Bng<@IqIk1~!u|+nGPu#QDzp2%t>yt%ly-pHTp922HD{4G zKMKcUu*H$T<+KR9i2egFX99VXn%F2DY>L~!5Uw({Fi}%5N?bq2{|f7aP)^dZN%*oU z@W|(NCzr0vm5OIdq!e?!+);1a)oXUkk+W)IYc=S&uErY=Ut$#we{T1sz%S28NORdN zd0O;n)V4-~yh<_9^>t1}# z$Iv6kz;oiv^C5`j0#!!@u2~M{v%2i^wfd3n(_Qkr=m+sv&2INy-B%Eq*yeMR^sedj z2~-Hjy3ICX{Bhy-WMmpyFCXM)GZ~a)+9t^Q)E@Y8HB5xgo?o}iLv4$m*jNgaKPt*R z^D=&nAjmb(6ZZsb6=ABqGAL?1qz(gS8vsEQr@XQO;+0_*s0{_Xx0?sNarq~o#v-3 ziW-cbc119{xjKJr8nGgA`_s#yVgdH)OI7U7k%QTc==~RcxkP1X4&4=X1Jd+G)}K45 zbcs+Gj`|iwx#fskePWw2I2Lov)hun^3mDnt?9X1-6dxui`RnEsm|G)|b-guY*o-~L ztxY%J6mV`eWIb+t<@~CCycDZ%;i<)8U|x4H1?{EItog4Jjk`h~4gTVo($#?lRQ(BZ zvj0pYfN~2Kknd((?PcH{ofN8OnCqf3%_m6i7(2OQ6zQikCW5{Tam!>%ykrm)a^Cn` zSCAML-QB#awnj^OV5SP1W^U)C+6~&g{f$qO#cQ&@?Abe(BArXMUoCk)#eI+O4`Yg^TO}7`1_1)3= z82To9>_vWih01zqm;;!NePXt`rX<*VE>K@s_rm90UsfnAy{W`u5#r*V>nwc`OMeDk z$myD&@hE(YK*zXRsEX@u8@V?zTX{;ci}>+q+FkyR++l3G(3E}9yo>#?ceGnI{t=B< z`Vy9A2ULXrc}WDQp`(w--^rSPT6`_hXKRM%au8>zp*dcsW=ND}xjms$` z+v*5jp2OgHx*u}OWqa%8$OEaco0#~7g#=mjia z5xO$Rk21KZrWOd|+e#n)`aV7JZWQz*DPl*tFGf3Pbi@>_oH0DDH1(;BT~9f4AssDS zw=IIRh4Smdv6&M=4OTg`}oMW0o%kVGzZ# zDD*oE6Sp%|JvC*A#ZMQCZPh^KD3r;Np5bf??Ww)u_h|Q!!CQ6OUDxGyCqiBh8t05y zP1(P5ay4i_G;(IkbLtQ}pKdzRZ84{Cv;2lz70aT8f|$|7b@KeqbuX5A*+0RN^oX7* z)cY{Ovd)N$)9fZNKLOD2l0vqD8C_=YXeM-bgAn5|tT1d-R3ijSb~%$4M`k$yr6+N3 zx*FP6b4+*4J(Q*^a3OPBmE6WPDKN<`wS{VTYt>noGxE8-A1D4}Moe!|IWQxQB2;9Q zFKSJEp`a^yLFi}kRnJ`T#$r?SX643p*-RAr*)B)5_fXU3L}y8IRCKrkPfC^3Q~k&} z=1ViG=HVA~40g}*xFF+BhR4q2p`_AVhJ$v{#>9=uei;F7hcsB7DY{Jz?1;dZ__VB&^AvY)9ZG^1h%B@b`s_r$8{o%&3sG+sli}^cn55zOK^UX84VY*ioXoo|=5q=Xk4S zYBy(5wfZL=vG0rS3+r@g%iM-B=8HMQH}v00G>srd%|eOlLecF}a_j?&e~(-L4cfJA zL7KkFPR{fNk9P$Vv4iDy=h!}v0hr@#J9dliuI56Q^$}5aotr4P-^JEFZIt`C4;4i? zj@t809+AKE2_D)pdOdH4-o4;IB(=qdE=u~KpM zY5MPk8fy$NPg+6jIzKp2=euu7ob~brxPTwE0{v`&=%?-#*&wPaj81B|KPfp$-H)LY z!oy?HiNu3aaHr&U-(JYMl^TIp8>wAzRQNG-zHZYID%sr~jA)9Z-zl=Rj&PlYp^8&& z*TzGn>TI6Na2sYHcu^_EYOI~P)LvSDX=0f5tM+c#(D%jHmBi8_cmv8a=+Xt8>yJiJ zFoH)hv&%N@v>vjG&0OT5DscE++HZXAcwJ|ecfariIT_K zJl#i9*O?4tURzRkrb3J;n%YKVm@r{Iz-IPp>m870M;#9;)_y}g=abiKKDUo_t;nK( zKk*2RB&eDUaivrvrB>Y}+(p@Ask$ZRGY^uu?Kv}Y)#=9_ z_vA^f;o@g_E(9j8bWT&~PkPXB+pCgO-g-7Oj$F^_F3}85wU+IIwxsbW0Tp#$k@3sjHx`t|+|ws! zb$=X+D;;&fYcZSciig;H9I>j4c4-+|v3&uH@pZa0eZ;lKV-;W1vYTwa_>qR{i|%LT z?8g6j8**-s-sSltmD@pu6l*Wi^-@&;({M$xu3l0{aUwaLM?odX{-AOHB!a)zgUsSb zF=ofXVb?d`JhdAnU$n?LES}UKpt}hxQxoEY1B07DkuXO#Wkn4$6rcbC;(nm8j1@N?x83#Sxq0n?S~!F-_`o%to#Wd_7K(k23=0sh-iyjl=T=c?qZznYd6g+>Oh5Dzq3g!++?t2gpA8wfLxX4S?%K|1A}k&u1$uO^;pIh5_V z4@fHr1zG@S?(rO(29C0A(0H^?lfOgmAQQ+<-s;E8>T;F3v?G%R)g2;ORZhZW!Q3&# zlkfm8iryKwceHs`sfo*Tnj51Gj;q!on5tQh^<_;$&kQ=J=^+p_Ze}^O*dQU&6Z0UL zYBT+CLfA$6s>fg3z&ev6M(0%K1*(Cx18*u7QE-}$PiCHl^*Yhe+SMypyYiaR&=06D4d#Q(D+Y=*_tb{+Mh475|=zil(PeFdQ0xhRs?? zSYf*spfu< z!X55_UQl`uqzKs!9>yzEjtGHZ&Arf{p5AG@J7w?3U?(&~{;jI!EUOx*+VOG(G-HU7 zb4d_&wN5>7ihCrpK$}cJ?LafIVdo2g(^`~Sgz21r)qM0uX{FRL*=_D^4YyZ%L}BH& zg}>ZC)9<#6*>jogVZSz;lfKL&OUvyE5IWDHND?(Fs`O%j+!GwtUe&`2o~yY+6vrsV z1$>K(3{jZ&d@!e{`hE)p?x&_LIqCpb6d69o%SHqHvYAkCFh`w5Y1tfNtVXeCvj6pOtdtSj4xh6Q8-Cy=t zz4vwRkr(Sp%rJ{||Mc^KgK?kCzLiHKx%k{=S8hoQ&s4jwvkpWzh0Vx?$nAdVMG*L5y0tnIgDWt!TZ z3f^nK#X#hT6w+jV0EiYh0D>GjH&WQc3Bng&1k4k@C_llH0H+$ z`tGq;35eYU5l%xCabKuxZ3IwG_sIGAbz>(!G!z(ZDm!mWZqM^0%m#N$?fv+deBp9u z&;HiN1ko9CwH(lT8!{~|uHP{6lgj|p))2x*bENkRkPDW8bXNxH#&fl>nSbMRqd6js z=Fu1t&9c$--CkSZZh8AX88u6Q8Chru7boX$yN~IN5kPt+M5B<rx^tz-#M2GOAZnonf zd?hZ5$r9cN?L-~HmWZAdbFZF_8w)%Zi?a^?$w8(p!t3rl6t`T)v!D<-OCH&gTP=s7hT8koQffH?ME~e;;ao!3hO)M0mt|= zS(C)#$jxf0io0im7E4OUqb7By6Ov1RUxlMSFk$RJ?@hlgN+1IzA`?^@$l{NeToZWs z64V(#{Q2~N&ZyufE8l>Duf%d;1CA!rmw|Nb%h7M%U#NMg`RXN!R0DG*mvK_#Y;gV+ zxlPA@8{;<~tEaEGSpJ{@Ai0gpSR`oeDcxg5R2DjGEXM0dZO9F~E|IA6_0{O*T$5)i zlDj>m?SWtNxLUqztQ42-vb!0w6R#P><4Jqj2Y6bT#{GMj1$BsWXb#ovNT1yEp@sg* zwR>AO7HLeC_r?>7d=~m^(#DRHif*=6a4s5kUu{+EWi6)^>)4=I~BY z)+6@S?O*bIDlSUz+_IB!fPaGYt{nU{xW}1@FAsVn8`CE&qr?6wRio6Z1yX7bn7v#j z?7@IqCWijeyK(GN9HZjN_ZK&coh5_6L^-7}GoVJFWVGM#O_GRMsnOaqq_LA6{o~45 zBjz?GL;P)6Jv>$Am_PYEFdS~W&5`0P#!Sj_fEAM*<1!-wsD_}kKEv||oP_DI)MbS9 zeSPwL>+J)6S*W)r^&}M`!%*29yp4#CxR1z>I1rT>GZp)xKOAJ>AFM?zIS!7q$X}7C zdI+?@@`!l}bgU14s98u`^UOu^?xx_}P43=09&lR0*?Y@N6N>kk&~h-hDa9hX}F)1LdMM`KBW-~mt6(nA0155_ePe}#Me%;UvBXes{h zmxnp%XAw}9BK_7m8!eN;QY(biMdpXy2MQna#Z9Z&)OoVy9W;(ZX)?ixN1!iL0pc>t z?U?g(Jany^xLXD9E*+yro{*AGDt-G1;ykG{1~G6l;zv@_d5QUB>#d{#i<~8bk@2KH z><9R@wvfZu>xeA3%&NNyRNQD#Z}q+~8lq%ni?synXOZ?ZnWioCwNx@~^YtGfNU`NfhaZiD+&K*BsLj z!$u#*Q0$zLZR`T{1&uF4#f|Wl*2D$4OVvT|#Xn%g{WvjOnmRvarv4Ap;AJB4GZ zn9thryqJeNNO)I;p1s;oKI}w5lSh92Gwmt` zPlJW`nH-K6Xyb13QN*hC)pvha(WDiox0s17R;GLY(zek(ne4r$>-qgTE18=skall@ z)UF;AiIS-7lIQ%=5JbN4_wzEwa0d)F_n=KGM=C!kJayhyLzmo-NL)cbRP73)xiM_! z1*Rssss!6Y)sRC^7n z(BdBjRkL|Km)npU2++vqo;%?okJo*EobLGzrMTxyVJd13vNiggDRpv5Gg+Z zM*8s=&4aYU_CZ^i;BibdN9pr@^_J|x5k#SDi;raI35X*Z9H2K!)pJgxH#00O7usPH zSfJ4y$LcprWm~uO%7qiS$$B2K;4gOt<{F5^Rf~FTkRf z|7tp?pRZMjU?l*T7TeG&xs<`ziOv=A$2=4fP0i z1nX&EUn|>%(78i@GJK}`AQvFxRD7{u|v^BR~VqN9R?lI1F+=1O5f&)0E@q zb^&8Zh`xbn$`)YRh0rUOAlogo8|vfk%%;t8)Tcbm^GcSmIU{PPr=>)fPIdpxNE*tN z`Z1!pkH>&NZfRJMjBFP6@JhJN$vBEtJzs`9*n=c}_jyTos-al8{=`7P8qh66y z5p5q?QM}r$;WL7BcPA=L%zW)O=*?c#M$aP7JpC$xidq4msKhh97v|*4@498Sj2etj zFIizw=)LKsfSd8!0(&t8|&HID3nDy8pk7A52 z$ifV9h!pM!uxZ+F!A4gpx+4e@a39KAm$s^`4%D|UTTC_JbJz~6#R1roOKl9o_6AAL z?gOSzQ?-kXq|fe94nrJCJ$#~bfzsO2>vB8}yHN z8efC=%EB;yXnk7Rw-pL21|&eT9$uOYfd@IH&;FnaJczDGqPm${IH&=DS~q+1D0MT{ z@z|l-MBTH)<85+c#^g~9M>w-C#E2$=TEI)9mO*Y(f5Yh-^rNi)pcvyiX8a1bS=J^x zcNm#CO?jjT9S`TWaOEb=k=NK?{nsoPXIQ;{CC6p|-mUa8E4pvif>1L7qdSwr_O*fa zG>f`#EC_E+KlneGX=I`({k($fpj~1(GDr%>++f_P)e!Q4SbyGQs7jzK0(gJE7cb&L zWF-{$k2f4p(N0L-kI$aS_NQsc+#Du+MU-FlaGhP)S&pfrSQ{`qwch0n zt~~NRu&(veZGsPsYXGEG{=7EQZvoCb0-W|M^3&?P{zjEy=SBGD==V=+O%#EinkQAN zUir|`lP;@)x+MPKOE}EOgU;7Xjc|zZ51_i9CiPT(XrgEp6;1t-+C1I~)hH!*O=;8Z&qW)M43)U8x=tmo|X!^#yMc+lhYJld0rciTWwfA%`)D5YbguH;v%`!+A6 z*YS+{Fsy%SP_*DVz|%4|G5F>xQTm$X{$~5!;f~F@-{jX=hEy^_;gZe<(tR-KwVJYd zQ#=vjW>77=m|fJ~e!_QnNri8;K~Y&6Y#as$+ES$h{E;m3dIXSnUt+t{?m#xsWuP(k zq{+z7-e=FsC@8LBIQREv&cL*@WE4_FhN%tW-$YYy7mw zWVdmy(&<`z608*7`Q+7DP5R`Tr7f2QDacw}r{2K-3EQ55T0?%SZ@P#qhTc({9TC_P zyfSDP@oVXi&!P)*%UX|qWPAG!@Sh@LdTQ7-4bO=Q{|Av}mki`to!JbUSP(#ZIG6>^ z_RL;1>+yk_w;D{L#l-2L2bCcSgpX-bxu8|P<{>x9{Dvf*>Oi^_Bl_yGn0t=W6dVe1 z84VNj#Tdf60;tg%COl485b7vbP|;B`(kMT~$zYK_MzxD!M2Eh9O;X>ZO{(?7{u>`J z&(9llIaCEyP|k7W->IB8PD-&n!rhx5avtCIrmBHC&L5#VqjKLV6i=*m8Yei^g>9Q{ zod;LMPq?G14hXdhqAhhGJ;o51hH6jmpQ+>vnMUU+x-4xTs1zWU>3pHDC>>8L9v5;L zn|dPo3E5gu>D65;LNHb|Nq5Z|K?BIxNU*^#XE=_+ygxo7%EKYXCN^V1E`=KJl7+>* zj{yCwXsozrc{X|Ex0`-vXNLgZ|F)RjQh(hz?Q48+yapMGDvgljB&04>5rl+*$1|Dj z$z9ywZ@(VVZ0-#96Lgdj#+Z5XmUpNt^Pt}BPV0@Zo$w!98ARFjT9RnNY^MN)-ot34 zuJc{|B3l*wR^qi{J`!3)x)?F@+QnfN(wAGFQu{_g48BO%-r7!^DlHbp%Sd)kDRyld zFJR+`P`gI6iR@)|93=w-nJ0MUmqWP9YOIFT!kOJ(o# zf0O*rsKK4f02|Slu{;R%cZ$3yqfp643^3Yae=Os7U4o#ORC!Z@eAsQc^#Ap7)TzB~sl};9Id}r&+{K;ezAt&F`6%&{9)B|bnLlI!z zcJ+qwmxojjslielNbDgcm*_nOjEwO)zN3Mz+ z4-s3n4Gjux9a3Bjis@-K8~bC#gm?C?5Es?~?j?4Wuz4`v&LeEi9$JW$;~dorGYv0i z_13x$GCg}aRZ>RponN0^3*C4r$_~=Wh$6^SZD1yo`R|QYH@BPa{{+ z1%wg&FE}-RF62VMMp~iVRMAqxzM_+N{3bD5a<@6OB>IjfrK}547rnxz{qxyqS5LYP zTw0U-Jnk)C3C@z#E!mH6aG%PnI}S3t%-=G*bFRK{Xt)@~ZyFb3>Fp}^lkc}p$Ey!& z+0J(zp=I{I9o_Y{)K2Y7<5ry8A}hn0-E&g?iBM7Ss2bRr{lSFj_XCg}W?`Gt+KIq+ z@_hJ=u=*=Yqoe3go#Co_By#nCbtYUG|I7vK{_&#va>yDOB9b ze%iX*z4VsCaMpygCc-U`pX~pl4Wc%Ig=KHN_d5f&4moK< ziEfbNj>OEwkUU1>3@m=aG@+{{qfp%B*-#&g+D>Q68XjYd%8ki+%HRgfX`2^s97%}L z!*8U_ylE&G+3Uhm#a_Nu%ojXV!Resj$oZXEVWyl zac2afHiP{<)s8c?)i?@kDBfGNUFA>mzKFeCaN0?UZW zep!2A7gSwVtvO6=jmdqKGmiM8M(R%IAV#MgNxJdDVv2*gX*?_n-)nDb9Y;pvQLeW zWNl8&>oZujd+x&$$6tksgaR8^S}!A)@;b9r*iK*5?VV3n(zYDD#-@K-(PaQ~IY!^A zMW>qlh2BzK%UU1x9X-oITK$hK0B+Lk77oNoL-;aHs>^LB!E~F>M}DM782tgN3w{4! zo+`F$kZj@)HANfusfj}esg8ZGKyn0QQXC0Io6b&>yMz)ilx>12QMtYVRq&cZ}^i=ZaeYD9WREc zMC00)VU^uotrVn&_hch_LZI1*2mtz4gW84Icq`i<3gIH@1bHj{ckdL=8APto>B@U{ zm&>4O=PU-XDK3gVG=IvDU^50ZZ}~)m;&Bi5T0owvXdgpJw$@wA&b0WMgj3JhK^5S~ zf{0oSeIe57tSw+te$m$IZWl>EY)Cx@X&D$7zpte@;n|tjaRUVY-ux;IzM7WBBUX{2~z6mEFZ_6LV&@yH%APR~h=R<@k{1--=n{bW6f0@f*7u!rMtl8^usD zk&hbnF}_{9Jq8`salqSJQjP0;{E8*4l;`#|qbX{7!t5K32DJ&i7~~q(qfv3bSuX<^ zs^?e6Bw=W?Ei26GXqUCd@@BhK3{tzH6H2mDDE%qFO>Lb%8|u8ekh}Cwd8qFSv45R= zOLunFYU-tNlPX;Z)yH%@Fm&4Q0G_Kq6_;3^e9tGZqKl}2Zn7hb?HQH?$=~;rIB!Q- zu_0u|ihgCb(HR;S*okHj49EloLkbABp}IvdQE@9o^G)MmLOC}FP6uvT@2I5viY^j= zVl5(3aItPaQOY}9Hy><7#X-DTXe)Q<9+gYlL`P}J0Ay5s^H)Vw?iDUcMtz&edPVq3 zi@HpAgyRh7Cv@M$Y&u3IdM!6&?C>AeEW3+u%?6MJe{af7beq7H|3g!ErQ`3KI%zaq zAkP<+yBtHguMFo_dE>^TKND)&Qt}_29$8-K7YPcR*#Hn?rs+PkL?=tHM~=QQ%qI4P zWf1T7N2$BTJF0zg)UVxIURXy{;Z7j{-g>ycg7t6H>vWfseP`i-!cS|uzr9@2v{lB4 zijekjwp3+hih6(oIV9{s)|>GgA!~zigWZSy>pC&rT4mVC&ADxwb=`U7T_TiiG0T(* zn7Od4kzSv$x0++0%=X|AuX7q&^j-HJ_#f?`$nVwA%kC5rd(ldlWc0W7uxi^#oduud zbhxN~qD7ZQ1{rtX-!H?NDJ7BiY4xW6?#Psj8Uqn%NO(J5gPcK;o)U4fR=)R#2A!XC z`j&Sb)P7=F?*m{xibI|J)r z(bi3<8!r6&(nJhCn9a?uaNy4vAH6-h+S{5GC}zB;_}`ek2KX{``se_hGU8|7^gD%kg6>Q)R!VW`%+wbyHxFp}cXHeG$su3IECBFFp|J+UVmzEO<`x^71B8+rC}R4=#u5U+yq%1tQI*#B~-PFhu>*-B>SInbx8Kz+uUeRs zfi9qQuh}eagiU|Zw0>Z9ajDc#Icklw=k}^dTd#i4x=ywgQqXKqLP2BPw$rH|xPbf(PKfu1BZ?v`kWRE*T38QDOv9GLV-fYmG(S9cixdco9_Ywrbo`Ntc z1rVu_%)w88YI*;bN8CHzE&*kW0dF^qccQrtZUG*Xo6_eHfE@AZVQJIwYYzZ&>zkQ_ z=6A`c4;fwN8g|$7>8WR5K-DpMfosF@ka}E?#V@{thKD|HN0djWa zY-s0TTPA>$Yk?$)ipvE+wkoCzt4JQOi?klxr@=Pqd_!#4xa~Q!4L32|%%Dn?+mrc$ zBvuZxfAUIT>^I(>)!tNg+@^M6`5(RCh*wmzhKe|Kj$wj*Rj%ikDzMGt(DP(8G#6U> zKY4B6O_nSN674ALKD!5i^;T}a*}g+yu$CwUP{_|eIW3_MqnfG_ze#VW@ut|-sLSzr zx#X6(u`wL>{Y@iBJ;DrJd++k_DuKrFLNCE_lY{LJrc-IXD7q}7B>GwxpS)h!9I;)} zJ^Sw8H`p>F@E$p>il21>yTai!tjZKUOOV!zd$a@QOu-IkY-Hgp_d$ptK<(s|jQQ+tVfSW@aNxiQK%)6~ zcM-3=!P=Wjid(<02m1cHE6%t3X>AhP*wOPKDLlVkQ4KMC^$>ROV{as9>^mrOB|0Q_ zEq!DTeF~SjtYGI5Umn>7&LPQ}*kNGY{QO<1JHArxXCKsk`=3eX4l0Dtf`q6H8& zZ2}7WsBLfvSB;%rwFCIv$xq&}%H{ZxQy1p_lU5qm;iI-0q@$2vRDYONb5ZqGm=Bd= zhR+;MqU^qbi-yz_qo3?&IqZi3vl#vbl+j0qu#xQjPy1Itogbgocyg(?1}Hcj6Fs+B z#va7C@ty^^dVY0KR~ff#m~Vnwb&XIGYSu40$wt|1@vwt@t#aP9_Awkj{ZYA0xEl$@ zvgaR;`L#*uIwIQjRDd`AnlzkmtnRF2OP+}QD?gIkn31@k9VeRQgDBeE#^1bCo8%vn zwGL>ybMP8g{|(6e{X6S~J-d!VePQ<$ z_5uC43+TU2Y{>t<*eds!llU74vqa_Ta1bhs8+!-h(aQI1yEwIaB@g=~7!b9Hjh@Rb zj(}ZezpNC-VMmZlL{H4m^d<#`mTMlW@^Xmu16BdC8xPT{x1}BVyFbfY^e96k@0gF0~vs zgIwdb#53lOZ+QbEf(X4ju@m)3f}g2sOzr}_OYegA9X!ZO&gh&Kh{E1BZs6jiIl}S< z#L*^5{Be-N7u~aW%uI#Y$ph@TD6E?keSxdjs|<-YbZTa%IdTKNwvB?QnW~_AVK+Q` z_5cw3JBw*4)=N-8nSCSIN78w@4Epd52TqY6x?&)XIPxKWSHwYNqLg#A>{}O)kaZ6m z!VYt{T#u-*$0C7et%Zn$5SgW3J!O5qp+%9#oVdoyZVOtxO7Z|5E-yc$?5!K< z+bYWZ#tPi}t5@h^EOBY0MC4wr6Izt$D7n&d;2uIF&tNmQxhQ5!c=O|f{6cAttZWka zQ@$0$e?=fkdE70_Xw&xz3BjfvVPe_zPaN&mPslG9bO^>*m`<{+ha5k2V8pAdL~Gsk zY^BAwWnG#h+FRd=J~Ye8ph;wao|mTfj(d9x@4)5!#0QWd4v5nTzmnZ^`=}MTlY4bP z-~BY_T+m%A*t&DKZYO#YshGz-V5qTfr9ehd+;MJ$+E(GqYXBEQ_sRUb`}9iL5)RvI zO|9Ga&lSq_~%5*WI;XGm%O()pyRluOn#fraiyV6x+G%hYDX!7kRC9he!t>ao^_ zsIe+m94_r9+)mzd*RG_}XHXFopW!do@w{seC*@#!2C|0(b<4R7ZglP9r0t!43(fvR zlGHySCO++H8PC{XH+5^cT9!?)sE^~Kd5;5s?)Ngf=di5jI+(7JZ9^s*w}BSdnfKH@ z8sCk4p3sFamy5NqnVB}R_8XIS1k$O`Cg#)TcsxpMzqQI1tLS&%N|;pNdut|}hl$hR z1*)6@i;n5W+~~*Os+Zx`EBy>Wu4ko7Gq16`3)_h*)|jtPPuef&Sn;o6jEJ@b>P%AUlf#^vI#Ga2vM-d z?JRB15-LWGwWlRiO~Mw{Ms7Ow>~NG??X2?Y5pScOJ1>2cKSyJYbFg4l|({-;9ihFv2mRT@adC+1C&4QC|j8xL>Sw)oSu( zvhh~uXqtLTS!SO`3uUE3nxe%IK3*rH2B zBgamzN!jjowm**Aa$)KaC!n3b4xaGEWXNT7plc5 zdd4Ohb-APTFert_qF*OKq!_uID5Si{YaKmj<#0CUw&(+abN{4R`?oq*CItXNxK8>= zAMkTJY~(S+eD2zVRDJbD*yiN`5=U#C@7KEX*fE##O5?xC?Ec;V3gm!(jtuAptsONj&;0kF>^{R=BJ_%CC0k!O@jgGg2z)!r!m7Yp+DKk|wh_A<@UI;M^XB!~bJiD(pc zoxP7FF?|pWjOQnTCNGOyxA-H(Vwoa2q4QI)h?=sGJu#w~$ST`oiYXE!h{z2bOJy)z z_0gAU7itgPuY)Z;yYEYyboK2FlF z2R2P+*7%LX9GtvJX5CmR5b*;*J6Utd>l+2O>z%P|SvM%u)yfwyTM5BvF`l!**6l7h z{2p?FMBG8j$fypkb8JC*F6Tfb45IkLqMT?6>9kn(_cH@#FUyda*XX6%+)N9jK_B6b z$JCZm!*xMUDodd5>8r!qco)QfzcvxpT)g2~`$(IS=7g@h_sUCAN;a0%{aQ&xwh5Yb zNDOV*-CYEs({}RGpnB2sZLyEOtnJx&pMTwA;mCGv2`>37C;qX&Cn81Z4lagNUtKWW z3x8^+@ zE+)0zv-A59*=*a9D$R4p8bap#Afi@uDkll_%0(r0^kR{J-LykemyzJt-kdaRG|UR< z<}u-+pDz9Wg0sq`9Eqf-x>XLQxB-w|YWx!FnL5jC=Y@#uP`^cX&lWlRQl3;!y(nWJ z+-q8c%o5HCx{-m9yxy-q(5qbUT}jb^C1wa>EfAX+O3}XP}yllsoWXQN&SUn z(qn43to5NV9(ieGKRYT9C7H)p-ywmI{3u4_c%Lawg>4(t#gDuCY!$6Y$oT$V-`$}!QG9UJRc054ek5b_94U)jrdT|}5da~oHY|6jJ7WLTlhS?`Rj4-Pj_~SI#0uT6^ z)Yo)uPWs}pNw}qg)~}w%+wH{arNt2j5#qU_BoR+@u-iqQs`e5gJUJeEv(STeePUAN z_^Si|`pZRzMAK#VMJE2bs~*YG^A8@20RPqs1eTSu_-aq|k3&Pl5J`UiUeIx?B}iMV zcX6niVJQJvh)}~_KG0y`F_S6n8@n?%T34szy9G0*&C5r9wo~gmWOPCr%Dwj?3aS<4 zuJ$0#B#Js9g&>|^KMXu}dztYpWtQ&tIQHK0*4rJ;)h&O#gcZH0r$_JL`UKBaS z(-Y^+5f@p>=S#r?wl=j|F=})|47%m25~2y!Z?13jzxa{eYlnHJ=yO4|`w{Hz z5$}*(k^0wPhH(d|ld7C%_#BeL-UA@PY`otN`+E{xdNGafwC>-7vq}ijF4Bp*<#9Z3#+*oX+(Qi?NsX6z0*;{XDCOlpw;ktAl{^GLC?z6ux zD?;e!T=gzYN1<)ZMUn)ELBagLD{(?z5y^kIH9l-Z`hd6jr3@vPbv3-glg9g7F>E5z z^(VKWd4eLl4_{kb^k){^b=ljSjE&hQki3hX7m%}3LUQ>~x%jTlGb zHUu>K%-^CeIe^XLm67K}`W>T$8P+G`Ad9xu(z9abZx!f{1;g+qQDzc{#ziFxM1hd7{@xVW>3B?^tYSH-nngh6&f~-X z`YV<5%!48N&esQyyHJv}g}+V$IIwWec^-+!=gS2x7nfWKwD4=7P_T|z1nTtOQOE}# ziFWK`=Y4K0d~l$Nk!J^HFhp&WbG?1+YAk+a_oLBuI4rL$LK={$&X&O|Rm3V+ zB#YI$JxpFn(K1T~D;w$thd#2=CK>QK(dYa&>A5|vP$QMb=l(tW*Ehm0 z9t0}YcRVuaV?!GR)ty=H`e~!w+{guKD@{t}aey3Y*V?a~*#r!9!p7aU?mNq&R*;Hv z>CXq1WA&X|1zRD$=@fLW+t0A>lF2Z!>_)lLhyK?vU@s0CK>pF=3>On(6*o_N&ccUC zg6nB)fQ`+u#Xb4?RtgFHACIhS0mRWfkuSEs{NZfkX|2{GqX6Hx?l1J4R!7Z2@+ipP zb}c;)2Dy;w^Kjg=G6kf&%*EvUSMET$FP9iH=$t^xMQbBJAVk?>N=K*&BR#3WT^8y1 z0$nzS+}Uvew`$BOwT?TmZms5MxbVTe6zS#i9UsWc4oUD51kRTKr#ql(v2rYNw-%!c z>tqNyw$;-0HNhLFN0e?UT!0Ho-*l2{ttHX9&B{CuP4i1vQ|KX&I(U;;)ZR&ye>|-a z3sYdbA|~syH^|P~`P7rr3UsSTdM9iTtR3L{Fne^TUfR4uipThL%xht-=u4X`q(xoH z_#dAfoHv#?;jE%mwq!L4v|}zc@ZQAt<@51v&Rl6H;`wGIv+8~l$j(DHe5)c0dRZ&_ z-13>(bjO;4#AE50J7QXOZ_-nQnobo8e`ajr){&Cl^+Pa;7EmbLcVwrH%A9MACE*OT z!dUJ)2zh>fdi|pBS=o*$)5IAFwJZ%4P(YyxSW5|fQ$mwA@$On&Giv>k-3JxPC+Unv z?Y|O#K_!#`c!WlQM2p1s{A6A}CVVJI<1Fh~*I?-A)QJn%b#_?xsSACZAG{g8y2+!_ zCNp>PKIa(+Hk1s8`VozMQ}BVvHW9>T+tX46fH*~T_pKxzOP?? zp7eXixP-s>bn1L{{y5#yr3e{=^obsiq!_m-7ed&!oGv)EDV-r&5x+eMFt?_q4-<>x zYe%-Zr0eX$z7hY=#S3rG!^2fMB(x;I3v_m0Q9 zfB%O|BqRwZvNAJHWX~c(w#Ycic9M)DBMmYudz>~|S=qZ}WOIrnnMGzsDr9E-j`uY_ zpZoq^-Pia2Z^Z1SNUuLy^y9ONnKq{) zyHtfZcaxT?C(J`BB=-?Bj8VuF80$UPa@+%kO9ivUUd{DTP#2>H`HPKeveQ}@xJn#` zkk81x-eT`1%_+-6`!OWHQ3=n@WoAA&BehsGcH>>`^7Uu?e(Z~nlecPM1aqh79_7fu zZ8IjNBTf$~kEJK%3O&Anvx z$>90MfSxvuLm-li2B%hLr9>#Z*2T_RCw259!yOLOk(~+04LaJ7anDeDGVQ7}6xNFn zKVb1`DsLHztI((hrQ8Hpf#`^YJ2CgyJsq!kzofNiNHROFLY^$kW*SOD=kswX)~LSq z=I-`eRr_V-Uv-hQo#pF-JSMB28V%iXno+e!Uc$C}qyBC}DJ^h!YxGrNt*aCblZ5q> zieuX6c%RieM;SM@yj8P@7fz$%&b4oU1EN-q7LWeZn<<&v%LaMlG)ycZ)vt`Lp2X;k zE@r>e*xMStU3X%(Bx<$w!X{5zr6<ju9f=lTJ2o=!Ht z>Wtf{@Jo{E`;AXyse$ywj(vOE`Hf-4{yu{x@+As?BXN#$5wuw=bKokyJip=^hUGe&~KTllPvItSK)tIx&9K^_$_SOd7(U;rL zoPb8dX~@W$97Fe7fsI=6hsnW5!-6UsFk=*YLZPN&^mQBnjv0zCj~PNDz7=#O9Tdhy zq8C{f!2PmQnom7bC$P(gy99B|LA`Zh3f4~&47fcgfp}l@?Up(*+Kd4i8hep&KS}~E zentT8mXDy8v&`S*z~Jhx)S8i-?gz{xO0SbOymPGs6Kq&&s+$H}sWjw?UsX&$Ue5|c zFVZXuEe2NPTqQHUB)(pVeQKamcIJ@5)@}2z4yCnOyyQFhql-#<3>C!#kB$j9Su84>tQ}R%@TqSX(`A@$2d+FEa zH?Ih-KM&Qwvw5L)<4$wsH1?I^2}z@MayqSbMQ;jyJQIVT-h8*ep6PmHKe*a@okjY& z2I9#0?4WJf&m&MgXxTvMDKc2~fQc6MtS)!%)P?Gc?gqr@YW@`(Td}b6^~RB}H($=8 z@{e@JRNS$ZfB_*{n=$tq+uPn789z0>C#c}4U!|lfan)qDp6+)QH2GpB$%P%9Kp%7)6^S7@3pMbwAW@d)U`_tHESr>DOr>!{i#ZujeH|A((@8#|K1Il@*m9q zvr$8;t%qa1PfkL;dRqGs)lYDW%5F1^ZxD_gQ|@`*e-bVDVwqu-V;>X@_y7j3Pv=<2 zdB0RA$G8b?hnH}C*n_={A~kVl5ZfNTQY3!y0@`Tngn`>Frnoc(t+PrltpxnkN|7io zRBYhR%Gn*(Bh*3RU|avhoF?m5&zBHE6U${n{7IcNh$n@E)e9%4OY}!qFTNynnR^+D zKIZS{c8WgaO+w*^lJYAXrF4Ez^g3O@lW8crX@;9R2|U~N^{N#b5DV!fCH|V8YYg3N zh+>G_%R-3Rp2kJ#tbWsBvWm~pZslGc5L&OfOyAa;9BbG~_VSa5oO`h4W#W(vi@}{a z-c@hjFJ7maWg9T$J}WFy2HduI|2{O4g{}yWKOIv1GxK z$Y7UyBg5(CvdpC?jHLSQjrXF_9x?q0^@;~1g+7fuGxO76gI+Ir&{TJ|&Zesr)FHccgXhP`@! z;Nt$W%3o*tWq~;B%g)zYGKF{iq0TiKZc*U$o_=d+WPo`;s=0bsV^2MXa>#nkwAwLv zfT>eY+$`XU%MLm~h}M8E#)q{#c^PA1`Z5I{nq0Q7*j@Zp5@%3jHI^~fuUklxDW`l8 z(r}2SeL4){|@4U@TGGc6cT$^!-Vs z>h@>4yN2!e*e}Fg{M6hbqWY!UFGkWri2nN!rWn6w|1_mX3t71h8qr_z+Z~I2@_!5 zo0+8*LW7_#X>(kmlm(CmE6lzPGDP0r#0t=z@j0v>T>-Rm6BS?P} zTkW3C!lfApsGfX((*sBIcsK5Q=vswAB^2gaPHP}PnG1Q`rRWnO-(S#<+8)9_6ucK9 zy_jc_s@fhn0SkDN78MlfQ7-EuxUjl-(~Q2YI^-GSR#?&a8HqwEOBtMO{8@%(hs4|u z?|AV_NO)`~nG)0e>@+u>`KGC-wX$9Xozd~yoqh8gxOn6o&e-1x4t_zLeVJQ6<4s1w z!mHuC`j@!mn9d98#UZOzAE&CjQh9mY>P5b6*9GH}FR`Y&z2(bBFSk@Q8Puw~8kkJp zrT2cO2zf(i{lK&DB5~Jw$FTWVyz7nhUm*(VB7I}u(cL0b{kf17g+hVI?O?4JEPB1i z;^QNR?@)U7x3y-^n&tHBPbT0-4#=}@2ZS7Hsa<&;j9;mmmFONQEEslqYK+W$VCtlU z>iZ^U=qKhF>Ge|TgkAqx&Cu)yhqvX6_}QzrA1`^e0;TnpFr2U(U2D9{CruI*>8$}4 zsVdzK56!*5M_U@X$*VJ>9-kHx*GPu7GT*MabQ~J=Ch`96tP+3Gr2dtCOI?@V^ z_O53X&^KO3l=KBTQQjg?i6N6jM5iCsD}rKqZSY~jG0z|T+K8mba@4Xf^yd=)QRw-?)^m4vWk%FeTl)#{Mw(F1MQK3P(Cbteb z9{KW*``^2Dqw44^lPh*fpE6Nl0msulysz*+46Dm^6Z_ZViia!m&!r`ke~4Iac$9ii zU_V_n$<5HzJ@#YEi={u;RoQ%ytc8$(ly~08Qfd-8Ri(4_<~b>pIY`4-uo@7Kor0TN z@5|{XAr8+hn$+h5jgj}mIz1E1d5k^JI3(Ut^y;q7d8?o^)3M>#CC4qv70Xg(mc8!M zwG@UfD88nJZP#qn=>BUtr5UW&sf{#ClNCx$9w9TzGzCkq*5^xL4H~VX>|OBHdyiM< zLY%#4!^w=w^#@TAq z*Ry3H5tPH!h(l*d^IQl8W+gCBnj2?9Ccc|rJ_zL|*M8|^T=lVnn3{c#hW7jpw44{! zRzi9}Az0K#Ew81E;&%PnP$;Hv4ZoZ}NM|ewhIb53)CmWNN#ML3iJp`{ z%zRI+r@@L1yGvT#6{sHYHiq20AJ4e?`8hYOW%s24V)LjUjb%XxU;bKUnSZXb9@&>} zO9DnJYo?C%ylgRP;ZNiLTw=jkT@I)o`)5>N6>neWLzdX|S_H?4Z{qOFA%_3nm+VEMl??$qg!Q!2 z2K*+DX~9mp>klko65$>vvnQ%p9uEyrrHq{Bd5Ub)8RlIaHYwKN-d{_~JWke_OA@@r zeQ7SSQSzzxvj{@p3vVp@sg@!Gv>CK~4h}-iSp(C76Z9Wog~X<&m~D zIkIJs>sNoLVX453ir|8`2{(+!*2M)9QcbuOjrmXCZ+q+{epzhQ>jLb`22#pz zy#kZHF{!x(MfCNi%pr)iduqz@z^Ckfx~`PV3r#eGcgLJewPX3>iJ2#1u-0E>zE$u}v0vw1zRTVX zcvDPxpwd%ct0fgot9qrw?848#TzBt< zd7Qn0@v@Magd%g%lF-`+nAL|h(%eo{517#8Jg3x=(I?|&R$Hh2n-=<0Dl%5MM{cp7tN^hF%%w)6Z3gt` z-tI#bsITyb46Ft{puiU|IPG>xdr{P#COmR7u!H>kSdbEXum-=%0 z#~GP}g4-WWY@(EG6n+5zbF=j2_x%aK-KCV+2;P-Jl4bAK z_E`OIm)Wm=v6AA#YVA8$X6nW6#N)^tt=E`L#4*rzwzh|qVy{KnW!T*R?~2h&bP=ic zeij=Q=aQ%0cQ3;_w!BS5yqcxu-XXNqWa>#4v1`CP%XsFR<`Wzm;k7J~Q2HvNjem)G zimzLguVI6(B$-t^_DciWYj@RO_{=!_%C(0P1wXVNMqK{^2AFR*>T{mfDD>(~^=Q;M zJ~-a>$N>pkPPI+44>)@vk7ugCnVSAl|9Dr(Gj+M&oH~7}$lWCqe_?H0lmM7x1sb82 zGUetzdsPBVM(a%+9Qz8vT7YF0e`WNIbO;lUrjmVg;nva=5s5Bb@+pFXp&w*o?Gwp0$ra9 ze4bhzugN$iK0t?2@w^liTT69zT{+C}@z=gj4_DY}KS3-RHUzu3K~)#X({Jctw@9KE z@uFeaDkOVQ`;+h~%ivB1KJ}>k;4TJ_)^ST|c72swY03|ZkC*=&sy zd0{)7rB?TN%(Kh|8!IzK@!_RWctc&rnlhiXe!dUuw$vSGdPxB)a)V>@`fvCl~a(b=LfB>+g-?Yon_`~$&eqou?eDD#r}>Bubp*w zUCSd7D;lNM4p>IRsiXKS7_GetnC|6|m+xK!f5_+)4W*zh5FBe=fpV>LD1OW8G}oSx zYNDHtMzh~7%6P7BP*6GKCf(r0l)eD5!B_D3F`{Lv-ZKG~rP3hGvT{9c!aOz3VN3WY zq#;p-DC&sc1tv$^-9gA$bMW4y;_k4#mknRuWHHi5a9LI~pEO&BJJT}c$c!bR?Q9f< z4?!Z5gVK-PN!C|}tApfAss-f&OhGn^jkYuCQQCbC;HueIdk<5d-+|n0k~c=aLDG5E z>T3=$6spuc%lN)pHG|RDW;ViaS>9GlQtmo${AtcLI#K^QuyJ&|#~Ym{`%;07PJa>V z^2*R%>%{WVWkJz;I<8~M@ij)y_N&iKhwca9Pi*3=t;62R*=94nx@DZbFX6uj6)MLb z@l*0XpY*p^9x2@pe!!iyg88d20Yi!}EHA$%R)}(6a4t z{^Y{iBimNf4*IIX1+UgBz>g)C{wSbm%>Qo% z)ZP0qB>M`bEbP@;7tCnj-@S1Q8C0pGvQEBjbPz~%ePS~8E7-*1HpS{{ z@~ajT&OMe8JwfvThz?V~BRWLNn*h_{Mt6YW<#gf0_oKqvE<2_X5z(OE$ZpTXSh3ne zMHEM(s^RvC8k()sCX6-@#EdYVxMSx=9#yiYIafsH+!DLqcV^|${bsr9>z%GjTsJFr z+<7!W6vvbtp#pDn`4CJ^FhL{~Q%#L((D0M|Wt2!8_O zqs%v)@z-=|4gX%OR6lB1K+jNwgcI^tM=H`!Q;;4iz2R5h!cp}d?o=P-PIcK`OFKZ5 z5{bYpv{dmt;RFQH_VkLr*Fen%NH6j+oMGhZHN;?P@)=5{gO=4x9|sAm(5Ihwg3yZ} zyv2{}%F3y`?6Su6qIPGxXpCRy>6Qd{KAW9n>BeCleKHo>&Rra?t7~%wdyM7UIrgg_ zFAS@h9PaN6@huMrzW)QI$e?=(mT9UJe*4o%vL&Qq2822oeZJI)XtBf}Xgf;z-!!=J zymcv_srS#d%z&(AO7l2}KQ3X~Xc6&=GHi^#=NWUyI-NX~mn{mml(}yeka0WZ_o=f* zX4$4UZ7}Y!uB1%@B7g)(R0`{QDLn8SCu6eiM!g-oGuWZRew}R5qKzh)>JIJe>bVB8 zE;En&ZyY?m@9ukXJJqMTQ+VTrdxA^Ow=dat>dn3sRA=U=I-lunCF5Cl`wZjw`IW<3 zSB7|{Vpc^(K4q>`KHx7bYC1iLkYZLcXvb$W1_(n2OY4q49X2+j$*^iZ^IiHxTV2fA zkrj#Go~`E(^GbhM`z(0s`mw{@M=dkSsjlk~G;@r=d3f{J`qEj0XxKo^iMacKG>$&2 zfljKs9#TT9Uk*JdyRkq_m3!pq@#__PPw^RKl+sP7bwBDhN?KX5c(Z^p`GM;vuQ(QT zv_=HWnlEx-CWGVLhXAl<8fOc^ZC2OkO=p&IreSCvwilenUrrB&cz0SMQ|zj1S)gGY8pD(JEd(bsN#4N zXQ>crUnR;P8$~;32)NfCGOT|6QYEn38UfzWYR#`}X}B6!a@Wg1>0Q2u$OUb!(JAMn9 zAp*RlT6yL=Y%L0CJQ>-9>Z9^N1uFazgQjy);ldp^2M#-W<^8pA&(2)8`Sbkki|{LY zn~&6tF>xxZMmcEf9WqV9Ad=>}S^hYaBw70*F_w(-mioVYS8Ewnw85mHO^fZ3hVL}drd+93V#M`NvfyWwOpOzt znX1$wNQsV?ePKvA0Kn`8p=b1r(;~S9i{qi8RSMgry6@(@ymOyeUHcBp#nb@5z?-q6 z9QQ_$B4N9H>kB|7-h=12c>b<2Ikmsu{-d+qryQesE~ZlV%NxuX_m@Ae8FVF5Kf^4b ziXuvbNxEHbzY%)W_DXop0oZ*PytF&MW9(8tSP2}Bi~rCTaB-OT`%IE}#BT~sc>+rU zs?X)gj#=!7PhCHE*zQvYLO*GDvcslexuy%*>FwG&5Oc$J47;%N)SDNAN5hhh3bMae z%Tvi|v9?RjqAGw5HTbP+bq2mPYUJJd7fp-G*CNxFxPl4!4rB_kUe_2Gly~UZ zRIenNYP+WK>D-4lDGJdp!PlBgd1>PR_5yf`Iy1|mjq1DwSIFs_s(@$HT=JacrNeiZ zioz=9uZGaOBas2^JdFvEdp7=}jpB&Gi*Kk4p<@kNQ?7%)p(cbv-$MG0jWz?TrNA~O zR4?FEyH-@Lhv0H6hD;bvckCypnF-C0rzI$zUw3PMK5h3Qzc*t4aem`zailRSSXp|9 z5fO48J3L)|M9A6X@-!+o)MQrzNsJ$XsF=`-;%8wfsT3|0itP6ZU`maN+B+$5bmz~aj_d?s^I-GnYPv;0AS z?h-+JEbaSnF|G5q*O)d>sxL%^qlxwF8`=v^2a**7_TrL%Ir2qTN z|CJ8(usPGAl>zTa%cRk(KIRbuamd7p1@g|_RlL{aY~%)kTGZ}&p5i2;3> z-ksPo)&)V$!OEr67hwFiH6z3|A%yjJt5`hot)q2``fFIlfc@aL+nQcam1QyCz`kF& z*EO6ToJD2tVYffrv&Tkzn6`+O7fRog;)&@in}0#CC%1~{F3>*K^?V4AE7JRX&YACu z7qh$CNITDG2T@ly16lG#PR4Q86dGMAiY{iP$A?2;MG#kgx2u*|4x4D(HpD#bE;9SX z>sxoUN_VT=)cMv2=PXmAlU-1+;q`o-5O5nOl~VK*W&b3+z5?WA@jK?qGA&Z$T`R@Q z7czxmzNG98xnW+O=S)>E&&pNsk}e$5Y;Llw4}kE8D#fe!T!w1J_&tcDOKjTq5U zyBlUG?ie*Lo$D?JzM&PT*h99RO)7)Bbj8`YO)qNMxpRqda}li&`(V_x9utyY>AhC( zf{N6gMo$r6lYju4?`y$HMhqMhw9w9B8 z_vyRe8q82w#sNNViCH0hbkX%hhtWI}opJxEgh8JHVzlH;6%)7J1Bj07XV#fJZ8kk< z9apzV+#y_!>p44RC}lN9HTCfQ9R+uGSeB~u$qJ5?Hxu_a8vA!Qed0748Y|ae%=bfS zhfFS&SL39%**N~M(u1?$>*riYnZY|QZ`2v^#kYD-r!&SJ^qC2X%d?92@N@{To3{ik zO84`0kH?d{T3mabkoACgO3~C2`ztS+SWZqzb`^iVq?fg)oH0iDS7Q^iJNoDGNaVzp zGfQzW<<&cgcG+s|2KXpN|C@64;mV=HFPG)gF0;(~eYxRuc7LzW3pu#8+cUhJFJav0 zT-97v_XUKVV=oxobAfO=BU&t)w-ZsXX;y=(gawz7wXG9Z1IVl>g@0NO;}iI zN6E&gh0Kp&@6@RWwY$hs2wCp_kUe|&)R}+$cl|glEU_SVYgXT&lZcnV4BrHVZPC@Z zD=ALda4`8FYWU${a-xgiC$KTtgN61Ln5b(k7M232v`(svIm={+vr1T^rdFYUDcC*>0+>K6~i zwOEjxHZfv>Xw5n3a-A+!hWh_*k3A%mCWnXNo|KuHZkl?iTATrDnFXNyz662TmXa+k z`Uk?~N`WSG4!TegMD~}^_8nX$9SelPfi*Zbce3SCGWcxN|3LAXo}{X{^bgoBQsFby zhjB0c^K+k$hB8X&UPw?|!FT`m_T=c}M35SF6rackdXhSj^v*%C9IJ{4Sf*IS7W`b! zoCDJ1Z9sqMaT0~Hd!P@q%}KNsu6QFHf&pyPEfR(UrCcrJThz(Ff7Ab5DF69LEful|a{Fz~RR75#{`Zfh z$DB&f)H};B8Lat_H~$Y>`2Tq4pPnFauAaR}$^ZC+|NXaps>nKtK{2**A=@dF5V0Xy zLri8tJDrrDll1(0Y0fWXLev8)e8;25=`6Uwt2rmP0|uiqC+Xz8>*yt%Q!2M1-2KhZ zrJ=VPFhBpqy2K|JtjGJ%MhAmT$`d^x1;t%pBMo2|+o1sb#9@ z^gFI(M=Rtyy4_B1mjGhn+#7EEWE8XLPN$-xY^g4em;_`KX-F?Tt8$QK^);ZP6R{0_ z34WP9aLqacXP)YPmI{T4RqHVkNIfCr!=;OX}zmEJ9iE!rMohhv80eo-{G%Ug} zED}dT8j}-6r$LdJwbA00>DgLVT>%=}!nJRnmF4rrvyp~cza1s~_XqF0k6K;W+ z>hXu*W5&;l>LX&oU3%>NyGvAw5Z2WL$+jO0VUA!6;R#2_9*H&ABc~cntVI4evp{pg zK#6UcLWBpxBO^29v#egr@G?=|*Fk$ybTRF5!yG(lQJsgz&`89Lzgqsj!TU>jv!w{H zQqVEVw!K(rt@1%R-VmiUtRLG%F=5)>WcT{^?@zVAE-l{;R(P))lf74AyYrrUJ$1ei zXK}>`e5h%bIBBvRRd4Z1?@`>gGv)Xo96(bB1ZyMVfTFLZow5M^uEAvf2~iOo$h}`7 zI?EPfw#7h1(R3ip?6u)@u<2t94#*OqBNFoj8#fzt56NH@mTdM#MHd5JaTS?1bHu&N zN%b%usSY3G<|5hK^t>1Q0N^^~*a(A&Dy7vAn2zvUVQ6o$5_E7W-&5m;x8ICUY}?HL zKU`pcZOHTyACsI zGZ~B{ysFN8qc(|N2H5l{=~cuSc=TEN(8a}%hhoBRGt>x4Ik)*kHS&m!S(}e zdy~_=ce>E!ootz+K&iv2iL0y5!6p}O++7X9K;#!zy}@U1$$Sp>MeG`Y7LN&9LPdJ% zG$uwNN+BVcmrF|`r85^5{7X^~k+9SE$i|8eJsRe*N zzpN)tOHl?Iap0%xv3sWnClsVi`C~_Z&DzqYUD^X%h#%H}Lgf|Yjvk;LXhc*KchGQZ z>UyYdBYuE9V??yqN;R1--RTVRI8uD;x{QvfVyXb|O?|208T~c=%Dw!98A_AEFo@fM zdZeEby8ZsS5(sZw5eN7*8H%w=fy*Nv`dotUBzqez6cHYw-pf#OMF}fIDEhcp_IQg7 zIxB;vg|gh*p%dPK`KT?o^EEzv$@LNT-eC{w_Z0D(1qe+f?)8-RH>cYL+P#mKJy2g_kT?sh{NHYa_tcT z6^;VV>VwA$$57^=ms7{I23nnDMA>E39pQ{djDNqrm=1TxD?C0x>TIQF2+F2amK&Gn z8C`ujm}m#&v{$I_mFAIgvT$2oypjzdn(c6@r7cYNMLVtgFHTN_O`iy2CEZCzlmiuj z8qRb;kUID^S;)XGqaR9i+y;&f+jE)o+1`pBg~C7gz5{o84noGG+bp=vdcbSWmEq&} z7t<%Uk9cxFmsdR`8!2ZrI|9^X4N%Xu9ROD7h$KMpq?K8BsClO!Cn9dkLI#A3J|lNH z{X%{6$nqW`Jb5pGl(Lo9{d{-b{x2$ zq}T0ie!p2gY(=)<&{u*?H0JEm{kf_Wt=!MLjKIT2Rt2+myL=v+(vbKH8PvRug z>HLYVqSGN$tRnO4ZGlnPLa zBq6viy8dT>rrRMUl+o<#^XNE5uN!cAC*ek5Y}W&4GcVpi=bBDnG8_deILh?vhnJK< z<#U@~WAw?K#LLS&aPlR=7U@I6ovn(w1HkEYhFGFpm`n&k(KTN`bMrOGhHY4c-yuwrH{3`5 zB_C=!63jQRmp&7)afnEU+hZIO>iSu>?Qye)mBsDVSqa{fTt#f#538z#AB4UYb1TNq zZ(9Gz9YWTpD6CJKEm2c$K?Sa#3rg+ya(6C{v&Buuj619YmfU^{n75uV!u1`%UTYE0 zqVv3;ySX{)Ah^)D{&bG7AmXZ=HwI@p#C7`Tz5gd0WDi zsw02&lJhvikA4mArq%9b-l@v_kms?V1sf+I@(`_qY$n6asPJG5cG%{aH}hFSvXTK`qK`k8YlE zBRfpDG?s4(g?AWAC5R`|!^P?mJU8`JlNiK|l{=dVzx;w`?z(wxttCFJCQcszL@z!1 zXUYeT5Z5b@q|@E=^Bj*A%r4`Sze4wPlf`G(WeHbPSaBMAMeA1Ht4_K7l+2iG>if6H z=DHrl9^Pzn1_{3FCmzb-A+vj$psgC!#k(O%9KDNC{Jb{{8F?Md%&f=K+P|};u2bhz zAg)9+K2N>o@;^}Iik>wqs4S?f+bR#}6_ec&v*XmlGu{6r9$(ea&10v=;1-M< z&3-CoYL{B_!-!tbPPD$y1p*UrwF~GUA3n8jeZirzbrsz_$K}8Z%MxYY0lH{oqSy_fHZ0xeLN%qs1fPA%H=p4 zQ8^eV6t^i z8QCpoYhKk^e8@(&3aw4|z$ZB{-M_m>xcISe{9fufF6%LUGqX{SxEJY@pA5`1AI?sA3=Ung?OZN2smix|F27Ly zbJc%@cXt)fOcLv)Qnq(X!PO#)5uL}Ofh$*^0&0KNHz z^PVO)YM>3GxoIC>Z2}ko7RA*o$vE7-O|-Ji|ik?%h4x&m|rQ29P8?5 zwti7Q`&$V7zi;=~NkCaPc^u|>d&meQYKwZ*AhDL+3i2-UE3Tj4W(?2b}cuUX>Vl ze-w8TE0C;8ZcKf#{Ej1~WZn&K;BK*+wpjNu)2eO%DjIHOi;II!R}5%5QBIao$2QX) z!32a4Q?rNDG&apAqu$EcUZ-`;IQeYls9z^1w{a6iA?G%!WYXE1s%s1==VTcY#QHm1 z$zqJqbc#^a^wv2*a=ki=HYqf2^9*Phmuxn3Xj^7>GF`u)CW*o=_+yL`1* z_-=LG@R^(BEQ>=HReVJa?92l+AHPpD`j}SgZ_UQmr|G==b020~2G3<(XBHjryVBkF zyz6@oqD|xVry%xXvR5d;E7 zXgmhpM3xq9-JV_{!(}Tw_FG>{XNZ=J{s3Y`Eeg1dJ|8x5G@O6MRvw5Si#YP7j8u8} z0_mGR=HM5yUe@4Iytev$mT_!fl<6QLd;V&+>FjJ8O8%^RE`SJ3&u&`M z_jn?&>wG`Y*ahofP+Z~^%`V{QE>4p(y~m$hu5G70UGdXwR7lYL7;TJmtaJ?55^q5v z+xpt;BNWtv!+umI#hDWwbT8XaJ9Iv%)Rc4-$~9>m2X{Evo`d*!!7ZTub`-xmu26n; z!NFK8=+_%Bt$&3i`=k&I@pc?THeiUi2o*vcnV2-zAAR*-8a%ReE@*o83g z=WIiuuZ;S!*O=De+6o$5-B&9Bg^wa7P_y;n)1;RW*;uJ>KWYXEcXQac)-K2QMZ7oc zT&Z)Yq6{&D{a2lQ4nE0R#UGx@M8w{_3jH)qqp)!AMEK;Ax*q)9NHRShKDh*61lcP+=j(UNc%% zudTb^k(#qGCEt${QYww*{;;ae@CCCS&*&OAe_|-TdR@s7=O9Dn#^LKwn^7lf_ z{M@slHxn7V&#_O@WcmR9kSAF4R|A8#IV&Xw(`j3t2zSNtT6N-6i1-OSFP%159|H9Vi>$njf;no*eUO zZbsp7?;5Qgsw2C*X`5}fyR3(S5Qrzfa5dr za=dC}X6*miP6{FI5KUBwPO(~gG2>x%=un6(f_-kS)M(3qL(lAKYCq7#J<=^ilD3Sw zljX+&mKs1}n5tg1hclT-7VCH;oyijiXv4&x*#vf$?J=T4fFNWTALrbC8|;Bb084uN z6oR{~t{gET1RGx!ckhDTPF}^l>?&s#0!6ueAC%%cZQykT3~q!%;DA>x|-y4OCi^8FT#F;&+b6dh!!|Csea|%rX|o z$+T#gcfopzI{;(#gj08^;->HO$8|!kd39c1Mg9Nt>iS;zqYPdi9DWZpWP+C;S@-KU zV7Bv4l|ViE$I%x=9)`IqxSS9xNKS!W}uaw z6xBs)cp^?dN2-8dr9m??+$UJm5L*W%S7jLh_M7Iwj!mlKnAnM7Ibm|l7s*A7dCoEY ziz1d4RoS>b$8ZI$C|j#^vQU+iCeGptp=&c~l)2gG2jFqmb^fDvkUs%!af{y|j9z9# zIKGsvR6m)oA+sRo<7n{DW}m%3a7E>V^K94Ah>X=@rO&DI7?SOP@*v(xU#e=IkI?D+ zuW!S#Uw|?2ofU9o&pzQIyFQ?Y3O)+TT?Y0??P>8t!yCCPkHY`nmI~!tL*>mKywduU z@rD~h%iMo#%8`HBl=_RIISkUwkZ{^>UIcLq<>wbck#uFq^E-?OTzwF)3gi<0>7TJ| zLFsAMu)6zaHBtxF2$Kmud@{pzQtaQ_(lfrrnt(T(zPg_%b85aHx#E&0A^TLD+N=7L zhe`baYd9+>Jq?;q?gy2zXprtZP}Osj<(Kxlf#<|10<5A>BiOjr9+%Bw;$SHx#nX{0 z=oXSS%1sOudO%|bk;Y;JvgMM&V@pq8R;pS0RQ1%w6tK~7TV~#!l*ZlauVyUp+F$0t znRDmK`;v~uuS1>;41i)$Ar*+JO+D;JbakbF^m_#|ejCK;yq5c|%y(HCFjIQmOoqD( zA}3x_6&>!=LRJBRVq~qcS6iXXPrmdGLQtRRUNwE@%VHelP*@hk{r+Ov(Q8Pa7+JT1 z&Q7z{rmi|a=jz!kN$^YJpb&X;&c7asn8&R~HX=%&URYCXK5(mhzoy$PRic#i2V;i& zY^ts8$vADFtmPD#!8IRWCva8B$O9=+e|2ekk$|@N7Fldf&}PK&Dg{>dy@tq>wo*9X zG$g-+-;pao7q1OJ`U}4Ye3fG|{TMN)l%xMo&@7HXO4hId0F)l!i&DU;cWIpNkH)0E zS}sh^?TxP6^Zw%1)l<=Jjd3&=4Bq9FHrAC#*#Fsf;KdP1&xfr&Yh(Hx)1lG~A@%Rs zOckrymR9{mx^|+U`Y6?^=t|CQTy&w^i}>EBk|o#Tkv3lwXYTEF;PJ`Jtl_g%xAvd? zhHiGr6Q-7>Gpoe^2CqfR*PMYNk@J-|Z#YQYGzmcNQiWzzNMv0A^#06dI^jSRBkoS= z@Jk@WR8jK53aSHF6t!VwGA4jV>1GTcQNR572&p)*awAf~edsGlYy{){Be@ERwoeR( zm&W$OD8{i&lXmLiskca-XPKenS7b9_X(HzMUIea%kZkVyaCY=TUNa5#LyK*mx7o9_ zq*qJ79#T3pV~@yex&>Z2YHD#;%1iMqSkxB)AdDnDRv{WPxh*B5AFtU&r_=`Hf%2>P zLQ8To!08+3``6#W=}lpm{ld&)AEO_ERlwLtXc2(S^=5Li&#c7is*xmKEmIO`UwKMm zlNG1BmG;c90f8$N7q-#`BynCr$LZOEbt2uPlu%_ly?zQ5*BtBIpWKntIoipPkYV;& zViE6H(Zgj>zgtAKW}{eXKfBDylMBj{xLQK2(5=9!LyK&SYjC(;8{6l>q=AK;nGEo- zf5B8(YhTCk9mQvfaGgtfhL@mejZ}_8Fkaq!qm-nk*S~O=L?W>6>M1qt5^P|H!Q)8! z_476^%fJu8{X$eD%7!xSt5B-_F@NaDAv%WKFgM9dJ);eRfY*6_G$DHvknj8X2g7dG zOUTL7ym)4ZYruN<;>ZWj(uQ1oV%`^I)U-F|{>5wC?+aQ^X;>`!ViLOSf#z2_GvX+@ z-wwb3^TApoUy?|52_bD2r0fRcH<+C|e`L#)f)*uI&n7SC0H`QpTJKsVXOBNDDyn)22RKT6XklU$8JIM2$&Tg;s#qwfkMV>{9 za4-hKTHin&g-yvmGmT!Mt0_uxAr*I@Oc8_MO3>Stb=(Ivb36a&>D<@WDN`{nW5x6} zMA0s!?5BWE{8XqHALbI0+|ENxUvtNVb_mWJJ0x6w8g;t|CPx-2x*DJMW+DoAr8!xq z3uWkPEN&t*dG6U@4Ja^jM*ynt0YfQ6TBaFgDH$vUG7*OFA+>W7T|`%to#F-Bi!ePm zq2AG040z@s9$yj0ckGB-7@;m{$EXVo`-iYo}M z$ib$tJ>Cils}YEr@J`>8EKr>T{hF}z;ZKr864SV#P(5Z5t&wg?&Z1fiuX1dxr|G3{ z#EVj^QXXH8@?*HE2D}FBa8YB-=Zw;>M5MB%y9_pj>Qz%EQX&FMQ3No>mF2`Dyh;2O z(Gu)v8a~y-Z4mzkW``CjoTB+utUlN=nknw2c%8dmlN}K(g3BX3-&B)b@jmYF(%-s_ z8L1{;l8bdK13=|MHB;NqmNI+azqrCF2E-sf+A0)M-wO!GZ2KMXF0>COMyA|d#_4GZ z^^#_WO3~THV{GZ`IXmEKYv4&AaPMr+tOeig?WWzGgT%@{G-HJry)yq7 zxYi6A7T4pGUvl~iC>`WF{alqt>n$Flt_3uQR|Hs>rc(r-nV6mJaFVh1 z-d$0dF8sLFwAhVrb$omqLRkIRr zpERWW=C!S~3sILom-30dJWXeFdi>3cX5GR*jLmtp#e5i1L6;Dr$1Y6X0r9d4br{3aMOUJ+4->6h|BU zDN;}}X1e^U*fJy(vnb*925-pTAC@noO20Qj=lSki^q$b}8UBwd=d}vH{D_Y)$@DHC zW?up93S1#W!Lgn%)DNV`~qH zf>LT@G4YRDkN?F8lPBhQEmU^|MXK~~OatE&2nmg;(t;cH#}kqFB0%iPUc&$0fL5>v zx^rh_9Gd{tfGz-Q(YBqDtmWCJ+eY+$fBz6bsgMkkNIp0HTNdTtT#{nsJ(*@PCX)UA zIR14~@}H5(pDz_2e|AWG;=ga#|Lec`|G(`2>&N@&qk^6D|H(u4b(a4OwxJTBhN+-G zXYnu*0>@+=!%cX;!0gky;b8yXd&puJ0oy8fBQEMt!rD{5fJi30Ot-6!LEq4&(dfPR z2G}3(zhOe73S#&4QD-)=P77|ha(h0^7+(=C);*iw#|(&@kI%wvktG3f;zqWIEzHv+ zA|A8yQC*)k(!~sG(H(wDg-nD*#vCgW5kDKMxrc0qoPANhUV$B_`FKg(HCI)UCPJ#q z2Jg-y>^oa9lq^hI4+_v;%IYtc;COf9R2NKuZYW1EJK3hVDANnFqMmjktnkE;B&hiL z)K9Mom`GhwmM1Vxkrx@&H%^Q9NPWvjWX|H}Dns$&?a?mw}c`4;>QF|BYhwdc!Yh7gq{pz>_vOQG3HY?#*pO#F_JHjwO@ zorM#dF#N{BWU1 z?BJJdQx)dz-FN*r+FJjtOl#?pbGL%P&OF#eluD$j?XTSTL?|Z*L){iBNg>yc0++*D1y019HF#%4JY+z&l=2E>z z&kNGl8rdP$xkOfe-2r?%$!FvUEPgfX-e-YQN@|uqzifabj^3sT*FPzZ0S|)|{zpn9 z!aKS+3%?)F@^hLmfMw|QM-cI2asw6$7yCv})!{On2=~J;O+CG|yc4~IwL;!iim-xw zQT6&lgQN|nj{KYNc<7TauN9ZW4Kr}hfHtCMNSxWUIE5c@uwF=?&Q!h3x7Us-AQAtG z^Zdq0lx_=D%pgN?6Vm%R;4Z7;xMt=OT5A?O?{|)--1c+&F293Ev|%0QycgxsvtqmS z!hCmcYCVTJ*3DWU|HK5r)N`mh9w})NGM% zi_)*y;O6~orR7m*-aO0H5VoOGnR6Qmf_Fy@-{ zX(ml{$XMKctt*;?s_aK2{PU2)CEId!UYKL2F8!9QQX#$8uNgJPTX|pM-*Lk^N8O$8 z8cF>;N@Xbe6V1ALE&ou!gU=y3yY0scYjJY;J zu#5UWL={l&L1qB(?3k;31b*4VIN*E!sm}$Z3SDpL7H~u=MIo6XSX~u{nLqVu0Ng{W z<0Doqd!1u@gTHc#J@ygQs24{a8|vG(l6 zgyT1$Z5e_8VVeiFR3icW_+gkL{8#7QIe_WNJmPL@te*i7%sKw8bqxGB=gj@g z-cb3|nwB^}D-%7w&0O0ybIh3y-aL1@7K(})F1l;BA4BFHj}e5uM-^2i+imOI%Z9+G zX87MewF|I5qQe4NVrsU>;rdmHz8*+W)+yyKyszd9uwyK$&pMVh z98aV&2%;ndG<#4n5^c@lR0;V^T?Nf6Ew3-kclj{L_ znaC%#y93#Wvkn<*N^}M};uio=JjxH?s`75VYvZO*^~m-kH#@)?9mlv=%NYQo5AG?) zwwqsRYvfQ?m+=F$9MOpSKnBRpEXg1(U>RAMCL7|&;*WJ`8D0W`ZZ1x_cfkQ@BrKsu zB1-TddmE`gc&@_qM8GQo+lY#B+>ZmOU!5<;Zsbh?v7_Y*Wc=F^IL<(D^Zb5evIbzlcv1(-31Sz24w z_AU@kJ~!K{O>qC35@a8#*Y4V80F5@;frI(TOb2G=c2x81IxsIui3b9FqtlrRgLYSK z-57N{6|p7Yk*{RtbpLmEHK3V}FABL$r&YGD^h{TVm#?x~A`yNSvxmsXiD2M7#%;{@ zv)6Rv)7NT?r6`x+($6~6ALSUI{;Xo&3T9U_feza)frFij88$Pk+QJ4GFzjh}wKal{ z4r>6}#rxoY=;Yq|P+qmCx}_BFV2oXbd)iiPsfshUxWDVuH?E>jY7N*t#M$fe)Iqzj z<26k8GcSOWNcVvGs4XgWA4h8Oy0`{f6I2)r&`k|Mh6UdaO6Tf zT1o+v(iOvH<#h%z&^@+yvhOu{B43+~^ff=|=X0=E2I0_-$_3=c+;vw#=HQ*@_9tnU(S?9nmaSjyboe&36UmD_lZjd~x%#}8I9BJ~0bl#tsUc`CjrcM0<3Q4R2B%C_8+gT}OZtv{nB zqp@J(TJg*-oHv7;6R}H^INa#5VltX*JWExz{{yJV~e-~q(-m#Zo_WIhDu%Q!ka}i?B@8S}ivNUc) zjdJ9%!KwgA4tD{jJEJe_{d83d-N?E_>)mQ@920`|E?x>Wq;2QIzrKR*8|o{02Un?< zd;g5rzaUu4B4|HBTZ)C>Ca7TVBo$*exeRqG1mR05SJ&A-B05j~sEW8vP{hBk^ue|R4 z@_}hg)Ckjn83UD7YXq8vU3RA5efd7ng{RYCi0R z4>{VFl9^|B4Jvb2KX4AAv#jbuJ)az$+v%#e=(aNh6Fz(=KxHhILX=R}&T9gipYa;8 zmJdc}4=`^)N6s05cCf^C^VLeZz%4pN3D;x8&E>8G;1J`@KB3q$Rt$lOS_ryYF^}Lo z59HC)8E+P0$d< zppjA}xHydOByLa8MFm5%C6zD5MP^PbOLNM9c33jU-e}`)@!b&BvQmeg*L(`GV*(sb!6Qcz z9c4$vy>d&|Xjnf;G21bD)3v`gl`epqG%eA?%$HxR?qbyVkbU-a2`I7(^vpq=FppV6 z7@J;(wuMOyZ!ZB_QQg5iLh+2ERbl)j=$$Ex_&+h6-c2PxBgyg>yMfi<~SzMEkUi66VL5}6kBf6~z6 z=1Y3T%bgTMFYfP(SD5qUNM2?wY7E*S4-6g@dlAu?wD{Q;t_~s7Wf!4$x|N~f-wfdD zZ?@O{*l0UhP}$}v2zyr>dKFX9LeTwnu~b;yf*k%Vyvw`}S%++Z)_Zg&%#m!sE@DhX zdmn4J1kx%oGG<_&Pj@(!8#_`{6za|D3GL@WH6AX77a9_iX>u)Q^xkrbB0y9!gB4a$h#E|3D{3Atj^ac2)j@0-CTsH z6rx{vmGLug*T7+iSE<^LPyYsU-coxM)viOGR*D=>r&B;@&A3LqkE4qO)?&YFo5lz9 z-pI>~$&-y?QGAb--({0Vd0lBI#nm2HY%K@w<{I3T? zLP$hXcu9LWA$6x9r-({AFB&mQ zDzGXovQ7qs3nFW|@xy`!z zs)5c_d<(Us{+5n8id6hyQ#73ak!5QZv8tPjRqQqjKrnZ@<01qy=XTGPF0R;K0|2xNF&X?m7SIn z+V*I=L)k}Pjl~tT)YmjCbtZ&{cX}hT33V4J!+3$?yY3zk&t<6Okd{35)>oYNLwb+x zhe{*VCuN+0{7+u=y;|Tt=ilna<{7tff$Fhz(PUXUz$Vwiq>MptOFAOb3NzZgw0%$J zgPXf%4D%*2|Kgj(CO^bD>1IEB2R9L7(Z=5lH{E`Z&YDK-6WkkT7u*<>LW;1L0^#2+ zFW6$Xcb_wn1XbjT^bnr0QEtmKsiIs{L1gz%C!Dmroy+vf;DJ#fMAS;?)JQWO&|)b(C<$kGsUCAec;5mrEC)c zTL}vg?0o1$XRD`Y%849J@5l>@KTevxP{wH+Q>BdK@uMGF8qZi$-RSKGCT8Xa&D%PEwN1@o8Xm&Yw{Gg0z!_XP464A2zbi1t>Q~9r zJhMmJLfE1vH6<&tRQ3`A9{Xs4HS1{mJ&4A4lx-%!Fc)|rb3TPJEBsY|Y>eLwH@1#{ zY*R3kwpN0hAN)1ZBz;uHu_LBt&QiP~z&*;%YIP0!&0dn{;q!tRiQ<}dUHDtdy~FXh zjtQ*zI(2d)GIzvzc(()%f%SxLs@9jaCej;O>3)zJm}0-z92ash3Fg6VI+X1;!M8by zGHD_X%Fge2{#kjJ|5kYsV!zoH*)M0})(%Bh=v74Fb=glC9!bmE@B39muIZdpYA_DV zT#J3U7Kh6|Qy174Ia+YY_Isx7^8AijvDA8nWbDnoV}=cY2<$;eBiXHT#y^h61`>t- z?kv9=*sz~*l4ONa=QmJnP7 z=~+wGjE79b?}U~>jj2TIrt1Z`X1&*d)5tgU0{Wk!p*wjC%vv&M_bl!ZaAczLDfnHA z?*+{mtO3dIEwwdsf;pej&9xJ3m??6fLq|=leWP2sqH(y&f?=|dh*f{SJGe}!&Qe|d z*Ba)-4$8l&CxV;+RnR+JPzP8LhQ#BoXHT{?ob^y+j96Ix+Y$(}9X~5PlW0H3z~#@~ zTWmDpSX?QPaQY&);%$Xm(f;!@xrVoP*Y}(V+{Naf_!^TXSjdqJ4J|>};9_c?dUQ7{ z_ffhVOEr;I9D#?s#3Mn}7W>>*&!W^JHHo_2s)8B}?s){~<#17W41=NN<#>?P!4vd+ zRSIb!dnzUu&OLbN%#wrKbBW0Gg%s^k*TG(0mtpw9U=7Of%siPDC>B+k*NPH*eifeH zS1hB2On;;XH+6NPL>%uL_FB?dAB9Bh0KHhm)#K!4rB<=V3u0UYEsuir2#mnkvox$J znnm&Hj23U)N3B?7nos^z!Q6mj3{(Ysm!i|+wX^?cAs~aJDB7y0>a=m-_9gw4UwZ~P zif$u++%Qy-d$3XCzet(jw_1Mt>)_E3ZBL{JRbtsz)R);JVc}gi$MzpCoa#-S2OJ_r zLogK+QW%zM!R=qL9VHkWpm_R;`bY|oyf}pC+G4O54SacfNzk-W(OveYW6LtqPE$HD z??&;W-1wuppB?ZmM`1onj|@a77#{unh(UxZ$+y>uOirN7DXi*9gL3P^V!=>R#)|MS zt~9mhb7EAW4aB=Pn>{SlOb&s^lIu&{-nlEUd?tL2GQ|t)o{MEKxcQi=4lQ;95sK

    ekFsi3CU@wIhSzqofWa@arXJ? zmO5S`k!E^3UgzJ3NaGD_kvfM?UsLQ0M&9|n#s9LNc}op=R5qM-LICS&M)lV4 zmtTfv4x^$j4?|*QOw@6fo=Z=Be!N_qiL%%TCeB8c#QO(|RD4hKt-x3)UI!87fwEro zSFErP%AdbGUL!picjW@O^xNdfK&eep$q=DIj0xQZy6;}o?<+lP>Yq|7#Yr8d>?^cOiUb>ko{yO-4;`^Y)H){ksI*SeAx4)@qklx zJ3Q5JZRL19Jq+Jbdx&I;nd(;Sm5^joBk<27GF@dg{&7Fh&Ok}*8I{26D6XrN9@n-C zpq;V=-0|i2w+SMHT{$8LTZOJF;EqGH``6aZXZqsyWelPC4Q$h0U++@pp$gT^#KCY` zM8||1!n?=C4BuU!R7|#|ISmnoXZ^ii6EqIuVj47J4wj&B{yQ#`7@?Vq)6nv#{j zPK8lK3Et;B!bpV)HF?Wq26>yUOA0!j(*%BgJcN#YC-cEO8YI-zyBp6ywDJa@cf#jX z)SoVgR)buy?r5PxB8oK2zk!$iNfLC7w_XaNnh1%*Aw*9 zHVrdfL;*%yFxiWu>a%pgwBbyFzIO_5;+=YHtXY2y`rh2TX9MnVi8pZp{AHgPDETeu zd`FHH8jx`rS+7Ym+c+xR-x8+x9$4?nZBfoxcYT{QE5XcmX>+@8Q@g0vZ*5B;diy^9 zOv@XoJt2g1@@?hJ;-u(~i^pb{Vfq`Tp6Jbz1u*qA&hvtob#dG0GX&{Imo0CG1>k8f zN0^i9R^>4bM0+u>{Icq1e3>5a@*9asBEwp*IjmAPO9%BK4)NZ^Z>sK>*k-&K2ZA|BZ5 zNcYb)p14>hwGp{SBkhVPABjJtNbYQ_bo5psWj23PS-{_lom6(8W9_T85i&j;=G-<& zqU_QBDU}|Q|MN%=4xlEd7h6Ij#x4MYssq2D!3zdwl3^dVeLvxc>;pRBRN&86D-E5v zoDkMH3`!~VN909a=q>{Rbw>42jKR}7o+e7H&oA0|xttp-W0WVA;}a4krlFN8eQQ{7weI!YG_8Pd}t zlYRakb4A!qlSosourQ^M=K?QEHCj#0Wo=V|{$ggp1$HLi8(SpNO5uG$P0|v~iD3RF z^-GWd+;*QV7k6K8Q}{+^h9X$Tm%fSMzH1W@cRc|>oJPpA&@;|3SFXeMtfEEIiG0{8 zSfN9iCz!Ol&*99(@3+vp=0^Q4530es>|SRHeJx7NH}8Q=G_Y`H8*IC`8M0*B`{gCM z#h_V?dRu)_a-6*i@xul->kc@NMNejC@C}v*e=v>Yj4t=6moY{Ir1^)Y=;b9Z$GTXg z+dupQpgkQF4pyJ+mHx)Gx!s*m&2%OHcr}>5f=%x7W#qjY%*F?5qUEIu2_cy}4H z1~zCl{#leBun&!GgPlElpV#%YB4WZzy)s7+~LdhzUz#`oVpG^FD=(!aQXmCPPsMmq5A!H zVhs(Np-Sa9XC#1Z9#z*esblDq zWBI?3{c2ZfmE|Wpz5hK_X_HdtVqm(&cX^%&S?yr!?RKR!{0N!*Looj3Aj}*Dt&s-5 z$^jv{3Zdchn){o%bH`r~vKKe#F%vnS3zQP_sDn#=2y4h@M|p7YW{TfYU17-{Fd~Tn z4B~>>-)Xw}{V&!?LLRQxe7yRo^RX5LtgR6{Ooht5AtUGa7k%l~BPhL3Q!g_sbKY=O zUGrY45lJIAA}DBAtv=1g_M|rTyij`VjV9@K+qgh#(ie(cV`~B%xtvI;+!i}VIcB)^ z2JYxei~o>!;n_N5iZ)8Ty^aZiq~owbM5gcEJ^c(T5C~4PUO*U8|VC z(f1wK1!D|5-{Ex_<{1JJ<;PXfyxWdo*J=%!Y0`Rmse37pdR2W)v7)GFy5t82iM1hZ zx`1~qCgnT1Qv#{u`2??B@cGcJ0V~^rOXcBKzpoIBQ-iwA&)8wk0l>07MntEaH{J6_ z^hL7qHC=>CYI<2m_EZmT$c>b2^9idQC=%YUkw=9U{k=vJDCeP#^Xklk0W8`wR-}g5 z`Y?&F!m@B-qPosEHOZV{RY@E1 zWIF#fv`AP!GvfAWi5p^-nbykN6?Q^BrTVT%w+) zKT}#BS|_;HJ~wjetR4CnL$R(X6@$dXb1Ns9@|p(0SvCslx|@{o9$cyx#|sTEh1}zZ zY&=&0NP8Y?+{UUBH#f{$z~VsXJALjEbfR;v=YF{v;22vzx=YV)pP!H1q4oj3j=2CW z{1HB1IlijZ`AeXuT(?Ihcn(~NjV^TGlVDc@1A54bL=EWMhVs)kX8*Bc;(;AAl}-2J zUpwaOx{N*nf7t))c8L>=4vi@4X^TPLw)h3klnACqP2fI7vKn(V%Y~)s$6z44I@y^~ z?SO^xcNg^cpl}^?SDi6O5m867cY~k}#RC zL3D$y+^^Nw`hx=j4OnV=(FzF%17XuvwC-?@vQD%Hz>%V@ZbCv^4;j(CT7*fqtqo0L z5Hvl0$mZ9V&>`k+fQ*o}!-4qzEPR-1ubj;QA{8tH!kGrm{?xZDlw1NQ>-CgCp*pAm zPtaiStzOUkp;Oe*mKG_+BQ{C7s@uP9qsBd6J=Rly z&Uv+^{P?gAzSnu@2}6FXq1ss_3C6c4vEptOWezb@nDaC&1Ug>Hz~lBc1|eI~|K{e$YNv%XQN;blFX$vuCtfw|~< z9l4=x16+QezFy;ixj$N8oKcC#md%=tt8pbD{J5@qNSaOHcX|Y7&C;hRfXuF_=tNig z`z1@XuEw7&v^gK;w2-wXt}u75;o|M-`%4SB{#K*9W#O8hErIVUfxg9a9*4FnpRjcc zZ*|t?J`}YwSU2F~mzIi) zkFiIR3u;%p8blnr0xQBB6V8!iYa(*+FUr#&Dyq#GwRq zV<;-XCo7kSEP1R4Yq3qjxb}2VcthQ`Dp@H&(|)7ds=XiDo)4!hR)$c9D95=~d$`KO zoV{ZmB{zhuJBrgtdM+$?BQ#k_ulqJoAiu(T3O`m8K&}{brD%AwGn^epHqK&CYerDJ zUg#A3v>{OGPs=NSC#~sNov8JK?a9SfN;aM@#VYY0mM`lmAO#LP`EW)Ob>>WilX9VS z)@iHDkIX5@P|Z(@8F-Y{n2`vmp+}8>g=J>!uk`=5>hu0>)g!GIvWby3+@Kz^9S!c) z_(^uibheSN_^uy~s{gg@<82m5kAPoN4B(e+@6Fz$d{PI~HbuV1xmX@K`lPNm=h^$C z=TgQ!c32=Ic#w**`t)9w_h#TM9!iP+E`(a+(kjgE+ZYQHh&b-c)T@OqEgey+R><>U_NU?DBMX?oE0S^}A)K@2&fv za?tdCmLhP{7e5v|o_~Swyv1gN=eW{nc;`{;ti4fuZhFnF_) z1oQ!&tbROPKT4dt5;+XA{w*3p^sMB?EkGTNzLwB>2`fjbJjZxVM372>(ZMnw$5s`pCgAr&B$Ahqi`Ar)Q!|jJ9xki6{uG?!H1NxoE`?j2%68a!#z8XwE zH+%pGIR8b6oxg5!@w}Jr=3{yjI0KM<)r4qa`@Z()%s?y0rlZf6Y>G4VddN9Y zbJvTMq3&nM8PLqrLmpQF!pp#;%qMq%XKk4JX-}#b-TnPQNneP~NZ#vt;NsFR~_@<6h8iEKenLk@J{tUXjHk@sS+ zdY-SkN(p$B5V;DhQiCD2ays`>xdVpo=A!J+KQ~6=)<bQAU zfT!usmv0A?oN(F_<`8BkC8fvD}d>zCT- zpmgV9stk2fLrwQckX5(V$DzACTFNhg+-+OK@P9q{bz!dbb<0tOG?~}&2d7oq{=MCZGmBpQ zKW(_FZIt8cUu}5hzuNFTfu9YJGzq!qsI3*HT=ocz-{W2MuYxe2MBbI za3PtjFSq;Zf)gdB7u%x*3!hKRKnA_A3MAEwtkDu-;~kEl6EoRk^6m+mq7MM|NIi>x zND%}#P$E{jX@VQk3P>{rb3AiU6f9fs^2~Ho8WgRtDly*0UJp z*NgNf4DXOS7Sm+STnk-T)_;CxL#NpLL55o#|2fu?p)gI2-~_TROE^gnO1SrNqPUeZ9_HZSYLFt=^!-HzmMZgJ(*n8(2B7C(FCv-$x)q&@0 z@JMI9Hxt>mbuWb+f614P>}&NNJ&q+*TDz zz6nJIW-n=rn$_bEY6s~KQyiJ(P5V^@u=#V$S3rQqTG?8M6W2U^m(xoQ z8ND}r&i2i7)3M*cj?js>`+^!ex0XB4FStTp*qcYjpyxazHjM(1Q3k#rA${>y~` zkF~%Woonk|8!S0-;P$71B?yqbz}%u!)^DDqxzNk6ofsohzgk<|qzgL>B2}W{EBu48 zl=9Yu87e6h5l%QzI9c?@yT_fMC;#1hY2EpKU&5jT z`zt<~0Z(DlQEweDu#G%rb0}%Noe;A+E&|u!;^72<1S=!@4z>L8_u=1=pdIiSUX%Jz zjbwe5VeSrbsXneg_+FIZZB@w-HRYVD(dlzwHjvY8pRO_CV-;Iu`CgM8Rqqbvn5Mfc z;^ZjR4;eBKo`FJM!*y;gw0z*YXO`N)G{hzV$fG_V7_c^tiBj>=5_0dEHi?-kJ{ZZw zx8g`L(v{5ard4v?V2H}IO~wDrG1kQY$QP5m(2;YPRtPT@|MckFhjpw?aVA#qYZAp0 zcB~DA5pA^w-o))Ao60)k;BHRbPvv}fXVQa>2*+qaB*(`r6@4^#M0--LW?a70UchSd z&JVhP)01b#!4LUptx&+2n=D%-oFX z#Ch<4mwb>KYBb=AG5EVSm-%iRPXQ0Dig+^ zZ)Rlglwet7>7_h{gu5kx9=~BZzRb8vwp4a;_NpH(DqGame6__z!f^!`ZXDEfg}BSu zq(b;&qdJEzW9*QO9dqWgI7DozTT?`7b6v@wT&)}4*1-kuAEaN-E1eTQPdzsyb@74Q z97{XQHNE6KatBVNVq1tW&ivNzxTSspXcPMA_Rmg{ekgJoqPcosl`ew$GlZ;*(h;{B-F*77G9|L)0)#Iryh7bL{HKpFhd(4 zY&~BMV`WSGq54W%T{o5Io(M6mF0zE<@4xt7>vU9|ifLp{_Sg4oQxXN5^$aluLaC?m1lYpW;g*HuD~I&>3_Me!~)Jr7i>(L%lpg zXLMmg7wH;3uG;IpDOQ>{yl5@$H?iwd@&6|;PpucdOW8?v{@&$ zc5RA##V_pEQ~e&;E(geT#62fGg>iKW=~>ndkg4T;LDN+sM}jPB&aQXenMbhGbbt-c zu|n&!N>ZeTo5F4X0*}W#E%XTCL2=J8)QdeL5~6 zvs07uQN=}oe&_yhV#<4c$KghnB2Mkm-!D~05TLscJocpjg;)Fow)p3l$%lv}3ZK){ zpu{KB-$#HsvK-;1@84qbCV+qQkxx)LMku}1!^X(|72Tw3rBXow#n;!tbi=K{DUkBho;JUMF*6dbiG zQT&N1PT@i6S}axvT!j4r8Q&=k&ZK~?Z>D-j?OJ^bN9{AqPK&rlhe|fjok>S@U<0<* zR;=A}G-%7x`O`9$o9B@ze5nC=h3J4`DlVWjYaI=S$|ZTcjMq+fxA}ok<@DF<^)w4Z zAm*`k#%A~W`_27xX!(y7G%$+o0Fppsv;7&vK+ zfM*BTQ-NKP)Im^FuGR9@i`+AJWw9tk*BZ+Kh#K1bu_?=xvVk20b@u(f;WliGV|(?l zjP&QzeyZ9Fe!(7io_6FdAb3TR)#=D!+tYfnHDD3^e(ZXuV;rC^#0PryA<;<{!X-ab zPlSGf$=B!-cm7&Of?OhC8P$|n#B==Hhwx7n?4Qf&4<_~>{*+St7~o_x!Vx_ve=pAe zQb7OHqxtVYG7y5TC6sN(p!Ny;%m27A{`c3k&IHO?&OZF#_m0qRLhk7M>nkKpGhoXb_ zjeT#0hY&((|9!dq2CE>F6y^rMA0?s%EX#2ssU&d#aNnQ|f;7_M&cNTL6>MsoO`*f zpLBkI0Q%^u*|TDd4_h>)_gE#Ph3)+Ifr9tdi^JXT@5O3ahxJwq5lR&a zk<@KT^3grKKW}cnbYFZOQ(q!0ETb;mH5W{IkYXb@up2@S9xa7MZU z!n9T}CvJlhKB=UW8H(o{;dFU%V%d|Izz!h*QieL)Ac=q=a^q*pBdOxH10MkC9bw{= z&j>|70eXn*k$YqMiBB|F-|O8q~ssPOx5F~EP2K|>NaYsSnl@_kRL9xLC# zll+zm)CbQlsFtXxN{L0t(7L7%aE>b_SZC`%g(Vm2fTo-sA8fTn;D8Z%$t|KMVDHdWSgC2Sn4@9x?aAo#eX?9cavrNkf$TH{JssBx)Gz( zwvwuTS7}}tYAmAq}uOiJkfF!t^la7KNPajbbSKg z1Ikr^YLtKxsy`&o{O}uS1DHVbN8-7(44iH0j1zLnBSfk7)}KOnrTM<5%f1g3=WLsR zr>Oj@2P8sCZdm63YP>mSzBU0OG&~?wPj&!i3T@@%8uY++Pc1cAH(|5Md|uBNnu)Dl zo-Kv>Y;NA+O`P|C?hyUsE=lwW@V}K#X#fG9@0}lC0H6BfYXGT#uXpVMjw2!YASFdQ z>wCD$OvW;o;U$f4)4Z40Io5!>H!ldL27*tG=Aw*g_B=kBb+*hLh%&8(yqWY;mwk# zF0-c?E94uAblnDQGbU2WUwtTZ9SE*KH^ClA9>h!$@Xv&;EDzx<4x2QlFe~|m!*u_g zJ|dwcv?E#hW3|9J@1Cpm97Ppd#yboC8-@*DrO$_yDR_ygpTFF87k%F_;ij zP&}WB!c>3w0TEX2^6eCk&(0-ezc$0HxDvT<}7&b_9qp(YUB?6gKZG z;10KM#!wr0l4XR#qL}S^4DOP9(-(esmC3vK_K2kCypk;+#$10CK_-;aUvZw&C9tD= zlo!`=LcEP&?N*1JLh6ArFYfwm78xI8_F$;=p5P|1Ke5h5|3dw|;zbD48m zLt5!R4!hY&qieBn`&<$i?&|h1x|Q(UL@N9e-O4WA=(CjRiGzWLsf~JWl#VF=u~yz^ z-^7$8=J10W#iXUk*adc`Y)Ke~;(r?V)kSXxkut+ilH|z?OV})qb!l2M^noLPYkJm~ z&zf=~H~+D<98iQhPt^dgfDNhp1B-CT%$oo9$~v{KX+3?tc;LPUW8XS;OKfTMS~pFJ|Wm4f*nj$+|>x&Zi9dy zvAB7~6Fwi{%A;xC1O2s&;GoBE+5?UUgYDQXGR@)6LcZvR1VHokOKBQI(E>7>zjn`nSHoFwZUE*{)XvpNJjL0c|Cc` zTcR&ROi|P6x3=x;7{S8tyE6|=mC7i}f*`sBlviIWouwx zmWS|NBvGQ)BgToU?#ySc)RWAw3o#NUpiTv*pX`_w$D;k2v(?)Y3*8IrG5rz6BbFFm za6(ObJVLu>q4Fugy?D1ywp;@`EJM;P-m3Nk-Bs3I+FMflX^Mflv?3pu;=#(1_u>H{ zLSFK8nB|W=&HI8#)qJm|6LpC+wET`bOI#|#Z>uBHjyfdz0YRq#wt)7DeFR4x+eT7@ zyY=@K4#Ez!KKhlsa*Q0gC(YvD=J0R{>j8_)Yk?P_UmZxtQ2N>Uj`Yu6lXD3Uretn=99BbaUcIaGV2U zo0@M#FYWR<`&zHab`j62JT4sQf+oYv`~1tQ*#tF7iBvbzKa)u4tP#&sJYyV+MJ+iT zQ{0jcuCrdo`U&TBlYbVbHuIF>eY@G73J3r9mGUWPON>xj*Zl9QwCDbXGIKG3?xO$& zUwXWd9EdrT-SaXBOvu`UrW=Oy!PHDoYpWv%Jq^WLad?0hPk`>flGpyrt;sadWAgbj z^bL;3f;>S>E@$%(o=t6<{9O4b;eDD_vsb;u+|VGbS^(WiG*4o4qu8uclSjuHgQ6 z(Q#7CR_ynVY~e;i>U8N}?gbd7%ahkP-aH4dB(@2%hT$Csecvi@O*sB{WU@?soN7v8t-gJj+X!zkm0eXJM#cPd1 za2K%3E2xNokPCSYk>>?);xM?tv&G;Y~4Ns43 zpc8r+lIUoY#MT-KM-blPv{50Pd#l$(6HVba_Hu(qtT}rB2GWwAlP0`!; zrumz8nZb`%fP>Ss!HNNXMHRhhkxhb08q2pu(alOi{ZK|De9Y#UbQj7-=>~(x_jD}} zzH^E$gGiId(fgr0n?IbkEs(L!OP! zlgZ{Ws{3v7_xS#LtznM?nBj-uWcG~Or)4_4Vc-iR#T1mtXDUZ7xI?zDa`zVGbq2H! zmc!4K>5$P2Uc$MRyA=pC4P9G#?_H-&o;|_IS4ztRWg`4{naootxNoi^KaU6FzMs>; z`AV7rW8UWobBT2|XGEZ8)vMM0Pm$o+q-<+7c%LHl&;t$I%b9Pw$$>8j-01tMB-^mA zQyt|B(RJDtpU|L0PMqJ{d@CmYd__ZUt)O7&8-e3mCxtiesBtK{LKmeHS3p)&39*Y} zh_saB_L0U{FWq31*o8G8ed%GE~QHO zYr>)tM-5MO<$dXy{^#=84nU}*w16)BL|7?pcxutGC*SLG3@C|Lxm$IK!vU!wFx;Whv)naYp`P;#LI_a#WXsP!JX&3&@FPjv z@J;pOAroJd)$H5o<(vE{ldN!L&QKRm%sXLlmvq8l2Hb-)4swQb90jd%NToYCJf45>3sB%A-x=<1%6JUPHvDK{+9Tj@2z=nR@cEwTY3o=R+$6 zGP^RV<%6We-G29*Quz+*cPrs^g4=qCSA2VS6Yve!9N!enpx>c;HHE8u>B+aQ1($wN+miY1;4DDLQPKH zQd{dvEC zpWFBU@4ww59gpj}9*_IuJ}k~mTC=G9iVt~SzFV^=me-=aB+n&(XcFUy9tfSvbM=tY^8GYn3YGIY-|qJxJ>k-s#H8{{x@bSF}s4`932!2GrFWz^L}_)fuV zVHQsZT*cbd_I^pQ6E(>bq(n0+BhGfH;rkzM1r*7pG`OwI+vut2z0Wo@`pXO8)9U{1 zP?-`mvtH+qlIr)^jKqcp^_J($s9%CFC>bX zvyk;j=5OZJMNPnHQ$1duz>-yVQ(X+#FO78={m03q9~x4n$)1+izG72|=Ec}?NIe?P z<8Z3<4i@aKY&oABWfeF?-r&x{gmE(SIosTCy0QY2{zf+^1lR_m>H zrPg8Lcw1VgE#+{C~gd2L#$?oncrhhqWpI10c89sH}>kVfeIxlE9 zDI|e&XxDc(WJLXK0y!Y*j7nBSO}>o2iSlJ*FN?J3_ro+)8y-?V_bO*lg5Cb3uv|ST zcQ0#|wSEn|P1`WN0(yD1)~f~a`Bkgp9Lz&2>(j1KX$}9VbMA5xN||_#P(yj2C%)Op zy)h9gEBmXb`OV1pE-t|#>&}%$H5+X5{{RjlhRz8&fT4dT5$ca>L6_;r;~Ly4gOG=e z(%PRj@tto>o~s3;Q(ss0Y*O|Juta5ZG1b~8qrcb@7)$5<{(SnRW|y#q&>)Y`dX{gt z${=t=H%tj-OL+3yY@3V;jwVPi%hdcE;YDs#| z4_$%-s&LHvQ4%(q%_7N^_!(ab^T6kCrsN$T7=8`=(Y7G>2sWkUW7hc*eRjHeCg!xI zaD`Iok4GF`sz-C%-0uDvn6D@HsbW(TQZ76ooeiGRTB?Eh7ZFbR;ZU@xO!s-BV%ru1;+mhYVw`L!*8Y<Zq!!UQdZ5UNePh*dE{^+9AD>yy`0y~R|9gPdldUyo@0#i}(-Z7{SWd8a zkYYtGq1J)#PJt)iSx&A66 z_Y&?F+d)B<1^tgz(M;k#>CgT#SV~!2Q{&?zhkwx6|FPVuB7<;Nxp$Vg>S!<2bX-a5 z065oLEZjS>l9pk<`>F#$Cgj;!Gxeiq5R#8;mEhn}K0xC7Kyy|BTSH%~Qgqh!=cG3C z-fZ4%l_xDXNQK&H+H46HW`RSh*-v1vV`r-|HR^=gpN%KiQK!z~PuT>8Ag7m^?*vPW zj_Ig#J?Q>y^oF_V*R->I8MDxgTPTPhH7(?(lU{(ac{>9NZs{^0D+@r0#R!n4hGwJA z9@q~ysmM zzqY?*S+YM)Pk%L?)dV&NSe(>KKcDWzBs@&pzA(6g)?xJBZ%3~OOTRyhY-uSBnEz66 zR(rV8CZrs`F3W->^H@c(*f@OdxMtIs_R${j;*e?p!vBw!x^lbF5%<4&F1q@vctcC z!a;us5jg4z;l^+fOi~+Zj$7^rk=%!EP{r2-hjWTBMa|w7>+P=vxs(?+s-EEbgDghT z(gkX}OZ?AT$D~lr9Q$DNi#dkZ0g+m6S;MnE;e&;6=ePvRx z>b3R)&TNvyhfcpQk5PX`qvW~>tP;^t$Ut^DQ`GXfB^^=*v|F)a!pCBl?>YG%+^53v z>&pB*xCU^85f>GBp{bwO(4vp__m7=8?cwgl>2@C5Yd$}g&cGfC2}MD?AUhOfKWp8ec4Mi&Vn+sT7G;;44cy@u9in+TUIG7%vahEWg!^<3k*- zIP$o(tU#7`F`@uai_M`?#dWzm5h?4vH%1p8YO-$txibC)4vduVcqR(?-zd9xmK;1MCMsNIP}ZeM_fE!*2S z34C9x)xpb_cHCW_SF76k&uxY*wC(&$|J#sXZFS$M3@H4lj%l84`c0t!m#g&KG!gHrRT4lQTVU+ZcrV zo}Z0Ty+=z&a=XovPXY=ZftuX$$BO*vAo}O!XkQLwg z`U^2~Ks-}r>UX1IXa%?vsSQhV?2f_}n_hoC9y}|SiEh2P|8LoiiCNx{rRt^awt z`Kb)1n<;$(O+!hAKx!kvfRQ}_=|DXoC%B2E{K+wq49ops9COPkDB)wsWea&RxA0B~ zxXeLPk}}}yWxVVMu^uKn;S~W#yKA|b=|D}y7}wT{$KxILtuS8x^drtau!n343Fl=< zas$d5iTkRN3md4KwOiix5@Ek?c55cV}4qvwq5ZXlAW< zRzNg7*~zGpx74#BYV^@qh5s1Ux7^svVY_r|mcX2>vRfNZqnS`mhY3u|cQg|>K|?w8 zbzwf1Amc&fA%<6G{QL?upuKdI4f|?pVzvK9kEZ1T=wjn;{78UYEyj%)@E^trc9S96 zBB<4Jsk>5P{y>KQ&+1z2`M2sCrFrd&;>JQQS5WGdFIcM3F<~?-EDhq;F{Pk=b)CM; zi0xw(3^Xc-MjQ=)?;pQA5yFLYQ+|ix-uI3M#srbyE-8m^$e!DQi2Hi+@z?dnL4h$W z^?A>H`$uo`6Vj6HocI3RM`-x@Ug>yd0+_1FTf6T#N6DD3e8X^{-VZ$(*BAe3`^7yI z^eCVHe|wbhiJ3npo`3IAw%ANiaf$|{7XOhgQKABBt)wGK!}=l}w7LKt8y94ziU1A? zHyKERaV;6TLOusD5aHw&S_OJtYaamnaxMDf4tK*-4tOE3`_`myX`am82YCN4@sf}f zd{|gBTQArljeGlpZ6NxakS!0B5o8vExxR%*^DCkWptdXE0m<5FW9tW@I(hN+tu#Nb zkK@pnU%#^+$(fM=zSW0g$SLZ!nlSm-@>tf1MU#jEE7^`Y;RATp3Ir2DrX8KOP?FTw zb^$EaPJIap+h}sHnv)!Tz!vFfQ$WD%kF2MV7S9)$_f$Ph0={Sl>$RVc$e#oX6HZ}3 zjPW&U#cay!u9sh>LR1ui5nJ@~NV>?%)c!QUw3eAqx9~p&1^&D!T^SgAO4u2SPU)M^ zjU5gJ(IW%B+g#XF(QGW6=U~wJr ztIwUpFBEda7s3St$|*F;8|&>{WI6AcE}dV5*Ntjd)$O~>{@M-z-t-u@s=~|UNc1R^ zoOB*E)Da(`AxsHTuHfDWfDEn8^XOb*|DmVNzqj@mSKK~7OuTM{$8Bvdz`?x}i>w04 zr)3!xEIQf>(>SXz8b8*MfU!f$5A zki@?oUP_`C75~4#tLr2641eF6cRY!6T>o?1t;-MQG6Vh$Q#!Umg>#)aTjzG5+$<ZDlQw`-6Xrjmph#?>6$4z8wRm)6_*lTSh^#^e2W z*=hmv(5{Vup;Di?nVudd6!JC{?7Q{5EZm1ww1$Mm#BI31#Lt{Vq5|RnWL*^JdAplK zjetNyM#ig-?feKrKN62Zu3|9tH;nc}4NFL^Z zwY~aNTVAGs)!f*q*u)J`_XS76#P;y<1j6HRkO|R1?VGdjR4&mO*w| z!P(%qW=baiB_9x)p{wTzyMnpr`Y3XNGGrO^B$#jsuhX;{7wd`(OI3;!dKcV(mEWDh zRb`G4E({Z=hV#jH`0h@pFssc3I-wgr;|!8W&SWit^v(M`+7@nmvntW+Qid2hGIHkO z6VDuIP^^s)@V*(eE#7&09tRICk0y7&biyLTuheDLNem5#7pB~%?wVoyXa$?$;8yOy zw|f_=%RFj)qCL2{t6!E9SNC?;#;F(JKz1P+u@wiQyH;rn6OlUCOI|JYP3{n!k=_ z69TQF)*G}O2YCE>c&`L3_x8Z|YRe(w4ff@Y&2s|WODgJ(y9X z1IER=uOze%K*S0EZ?=U*eFvES`9?dRviSqXlM<~rVKo=us8Lf6zzCEOc9$P7)hDVV zyf35Nl5{g3wlDW2iN`%oGA#Q1G+QDp#I;2LIRs~ubsANDuG_T`31y`gg%%ZDKfVsS z!45!0;3WkKt=SSTuNJNC9;0;kDNxKgu~EGX^a%Z(;tHn@;Dc7;L4h^348n~FL6>5^ z^K9~ne};&xx~)d9FyiK66_Zz2bB|XI9T(Rt=G>;^Hy72KO# zOLEx~XY^l?-#8h=Wl$sGsbHfKdidDz!z_5Y^to^@WH(e1+9(Wm!zH0^Rv=l&XoXgb ztKue5`*06-aj~gn6NiO}JRnDP-{xZDzcuyo&<>t+_e;)f_UCwDN$|l2O+UpV6na|B zMs>sTmU0%EZfB$tg8eeePH!ak7)4;<0G>FLaBjy4^Ss_{_7^%xi(~%_m3Q zOZvflyM~_&M><<(8@+8coOe^meBxyJFWJnXTr_KFrj_~e&y`HDg)TRuy^ZnABYyib zs(CUyPIbhO^w3*$8U4FEE*X$yV$o7hnF_r_lUe>v;8fVC#T4$y6$w_LMny}cd;77c z4k#q|BIpSzQEznvH?<_bC_{$52FN{3X;VyG*1Ojdcfw^9HTu1Se5DS9eLc2=-bJ$( zkYQp@@!i89Ob&bW`}k@6+dHGVo$0jCe&&lAAbCe9o8Je<{v(wN?D%0FCj&lMaOz#^2!hB!!y9Avj zQ#`eKa*TEK-Jwn0MshO)(sJ&7{l?ca$O%u!Z!SW~pvS>~63Hm2gx#yQpg;AdejZez zZDGpepOya|79w)}U(D5ifOQ5CP1`V@JEfbz?&4d;h;wZmA;2wM$$7W!dbil|P-3S% zEW0^E$&XNAbX)HN{rCpV(p?B35%z63^nUX+uYRDCG{Vu~B~}%*$!NKhfvF)l1s=?a*{(in`gUCW`$%^;v&g19Hm1(0_aYiE8bQK zp+X_l2M_#u9*4RAcgYSpn%{qLPI^dSh)mJH5vW!SYnrJ#Bfu`9NFqn!6Bo0e$`DI| zI^~nD0$Oxf3wsQgV~bivnq24$1G(EeSL$FRjUYjZWa&1Veyg}nOHUR^S_5V7qm`{Q z(`bIp6S)Z0@P--u0n_Y-cH-m_t?Atu#4L{N&2%(jJqt)eSSevwsAE$Sbr}7)Nu`#_ zvm3?;^L~Zup4hmDV6+^q4SSQ&&Eg&n3Si@;$P^-duwN6ldDit)vFv+5gfufL>q+#Z zv*JS)_cS9BcRjViNbm~i(jS&sJE#CY@EgY%Y#6%GeN|iq&$e;0@(Myv_-2}OdqHS+ z&BmwJBsAefJSHN(?~um}jYIes|4ALAe$2 z><$h_7NDeRuvXiWJFCmWJFk8T`{4~66@Rlm$v&boPn7s&cEvrUGdjW7da5GUB9qwH z<8ZMkgX%b;@HbnYS!h8aX{3*eKdDwUSvJj4q8ke?j)jr${d=* z&NTqpp-NSXq{niCgsNC9A&AEHJHP1;f(L3Q0p8IL=qV3)!!m_0!~_N@3=Q@~EZ+MU zhV<`L=f4=hLC+zy@#Z(B{S$r-N*zy~-+JDXV3c_oOyP;e#TW5b9#N(kT9yAH)t0 zeHJ94t=F_Z5zJqQIS2_`&5V;B|^2{HI)jUvVDT+facZR_dkBSV~Gx zZ7QIsh((W;rl0{T0u!@KkQ**%4|&2Kqadj;54h7*Nt`;=YRrCBfkTw||N5`}^V9w( zJ3R;k?Upcam!JRjYx|%4?f>?*fB(%a4(K!Hu&llNzyE`Of6+gG3g{1uNP)PZgj7Qy z@;~js{{3(Nm!Ije5-5wu7hXjF#|GxVe9^!C&;Q>?@;^SOe|;ncN*6N#w7Wfbc51{2 z!pE$DpMDw-`99zzbpV^W4iHbr15)8^D*>rL8eqil?`@w>*;hnz>lfI4wFRiKQwP0h z8>EC$@KUpU3@fb5Ax1cym_IhNIqeoXGPgCH@dGg5A^=y%cUA!>H#_QTEiot$v?_st ztb~D?IRWO*1WisMZ~M-5`;^NAD;po(#|rs$nwCxdKv&)*76-JqYJnb=bei>x^(?)O zbYI-K|2%wqSAisGZ*kh@7PM9D$h(=;4j8a(K0m*60|rw~O%7wHG9`y5x!x^KGYG6n z1>Dis+aAhL>tqGwOIAQFYgN1Qg4Yc9!8v0EJXV}nV35|KH~tL51+4%XiqA~)$`pt>G8YnvQHtB);94R1!@kkGubQ3YuOe9F!Cj)h-@jk?cj*%lugrJrJZ1!;f{W!zn` zFl`6huu_4HUh9hmw=UL=%zVUDW@E5Qm&P%NO1ti%`{gqI<6}R)1(+*pJ$hI&o;|ts z;vtO9Xl~7iNgz?*-Q(g^nd5tbJGr+#nutJD&iaL?D$pf4$y;*u?e|iJ#NI#asE8gq z+4Ktnx~BjJ77zfLD2|gsTB(2}7YVGqO3Jv-zR90)gXMulMfS*L4ZiGHCzG4kV-g{s z9Y{PTY}p(*Y-l|Uk)@*`>beyenSZ_T0;V$dOBH~dT6CF>*8tBPBCFb2mqNkWyPI=O zift1X0sN%0ySgr5CM2+om51@{txxD2A5Jtm}e}6g8J^Bq0VK;q5=R{wO+23waFd|N${o#Dj;-Q8G;u@eD;r8V%+T39l#00C=p zh}{c6b|F{RKM#-x82Xl;aw~ zeTRiH?@hEnfO%<6FB*|N`UwO_qOqePVt1L|1~x#7B;1is4#?o(HE3d&f!h>+^0$Hl z2Pp1UIUV%`e!8X;@Ep~xUa};)42Gl;q`(S(&z0)}@4By!LzBNYO)>S&yu(AYau92% zhj{zEm+yUyDIiYfOCQ1k@)ju~06Z^{nDj?rzik{_CgSc|un_ShC{X8H?~{E%c)ErD z5lkj`w;;8k!DDF3fbi_eRo!sS^kfxPWPY*3sk+F$7jNQRN^IIcyv^x<{xsRz{^7P| z)T3}lPqdKp?F@mvwrKJX;8%2OFjjZ(cqp$Fb;M(HhuzGqnr}}yXq>F?SG|>8 zxTjHj8L^fPmU?Y|?*2tgbAA2i*it6?5&^+$E6{Dm$qsCFVv3tkE;Y$AqtKZ<`!pT` z{l*##vyK^iMM>emkDU=uA*O_&s?IU|AHT68Ipq|$KH+c!PGm&geBKN}&xt`)8h8+=Tf!VB=A z#hK7maUnXvP7+3G1D*k79l^^sLmMvVVZ#-_WRcS6<>hzB!K0Kft*Rg>HwUz4W{J38 zNH@v}Ole=DmidQUn%W3PgS@<6(Hs4KTGUZx<$}sOw|p57Y4rn=mmGKzMWq|trl(KK z2z%ZO(E92q9^82GkRhzRab!=X+(3KGkiyPmnsiURInkrCL85xmjQr>tsx+wqSZkmt zkIe}Aj!;?4&;D#M9zz?H#}_}$M&BvFxdOz(Ws*x?GcIaSxH0C)Q^I`;*sV@q8ZY7W zO?N~uq9A> z3AJ4Mh?{o!I5uFKiPzU_duKZop)DZPt$br}rZIsLytzk!j#q{PN5> z*fT~!>L|fSOEC^BLpQnL)df|+B`jjq2(f-S3-=wKys*3MAv#wH;cyuex|HMWL*g1p zV81BYbP#(NZBe z5-@a%g7(-7a^+bwtdi*4OdKITh#|)dyXT^YAtKG*gFqRJ9gh(AN#5b z+Vd@f!!qXctkNAURcU}}nprN|ppn}ez!S`Lh2wXvNp6)?*c>*I<7r*-`_LlO&cqJIk0}ymwpjO;nCYybZf3) z*xc)gdxW{YkEmH}b>3U2xq*07G}ZoP_V??+ooF%Ie@bFPlNM)NAV`jS#WTf4D4$t) zPjK|?)nr%S-c6_41Fj~779w7y(cnali83iwBH@`=|D^tk4AU{I!5GJOMsb`EO{&oa z-Ugzj_zSDL-jba&^uPoYj~J%VLpNc7NL%41Lmi&nVU!ol;=~mBe5%QgGjv2ce_ifw zeKVliWZYP+CL_u7;fJmlz37VVSEDaKF&VwVPWsbLTZWFm_zu8_$Cw`}p885E< zMvLZKmVs&%pWaHKwB9$)?j#ZUTeKhaho6y#P?;pi6_PIJ&f8BHym6i8!rWZ5*EH+< za-`l7|AfmNJLj3S<`Zs6PCBr1hn+pZx}TGudBLSx_B$U^ZoVG1+;B^h%(QRj>ZY z0oI@oNSZ;jR@9`lNy?Ny`oO{*q|RTq(Bf^oP;8FQGwDl0sjDrfirFY%8!Le0w*&y~ zCl<(ok&qQVw8U~}veCSnJQQz0Z=}a>AhE*Q`xC&@z2Mo~bOp^^N;KcQ2)P#*5J;Ce zgQO+sN-#vYn*h_$ys3ti$InXRy{Cb;G-s{#b{Mq&Fc2hoY4A}Fm-_MZH+yicJ|Ni8 z4|COX^ida&d;y#eo!sOOOK5;so4LIFc+2gIY%NrW}57w(pkS3P##vPhD(yC|Vdm{YFF8Rt$@q$r9{faks3DuOslmwWIKXwi# z4)>3CCXEMF1^Bk$G4wrwmbl&z6f@(N^sFC59XGRul9q^GiN7nGi;bQ@9xJ~R`S~2m z(EcH6n|dG54_PE~bzJ2eSg=-f>)5@{*|j;6XT~!K-?#;{(kw4&g3A`~cc2(oR{OF| zglon>2W$Cr59rBC=l3(~+c4#Px+20x=lZCk^aQyx7&SB3P_b)D!;T87f zX(D(fv5K_QKNK{sxW=VDEErUV4T-Z0u4Pu74Xj8>GH>5*C}`+f#mHl>X^m%Owa`AWO=zE_KL`jhCpTLOmeYa;bBaErXuW*mgGihlFiJ*;);%#^{$RQ%F+( z!J*qN{gv0U)v_HZN|}QP3DeNQj_JzEg;}lK^f-9%>|N=6KFYtKTku3XS$xH9FCCce zS9pg?y1bEshUvP2F1j)i?+HJ-aj&0q0Ri7k9>?&xt|W-apVw&?w$Wm@nbE%a3QmO0 z8xNA{04Mue=_KNArNd$R@I-=u=W0Fgla853X*MqsQCZ=Wc6C+*=Z`2c)~B6228cWi zv!SwU<=#Vxy0K$MexUy-Pda)6D9|-`D&%S$qfn0ZTp04I#x#TNjs!qU}eQKWYxaF;i@!ona8<>CP-O1yz zCfAZ_D2rv}drnV|dI4J43&eHW^lB~5%TQuA3QpMw=&UM&rdE}gDeYnTSxhS)>hws< zVLHkZ5%;TgZ$DmY3#F>#66DuCU1B~H&f5 z##_R#tT^q~!ehkTVcOn9p_OzfML8J3{ag;t84#OJM+0*#uDHDn%w8&DeJuk+QYiWG zvY6ajgpCTBe9)|)65cdid-Q4lU?4zX8{ZhmzM)7Kc;qrpvPeNg=y<0_7Ev9+r;D=~ zdrQYa$nWbW-(9fe+`c;#c%Ch8pDmZ}{gc5F;A|rWZVCG~UNkT9(cAO*Y}_*8=U-q+ zZEWouW|q+g`igTWMZry`v=sb#I~<>=XnG|jgoh+7ui-F0%f9A6!tBUVg;O@KkFXyR zY(=dHAA`|~2T*Rj5x_I(vCo*1E(g{4(L3Mv#m8kc-l-$~aZ;JISaAPk!4wArrWDB=WJAEwG)#ARs-?kF&Zfy@TTN5xlUOE?O3wL z)AWr(&42_E|CYN18z^d2T$@1wiN&0$s3)=VLFESNBS-hVzNgkMTG@o|X{`S`qnSO? zh)gU~be-ES^Bx6UGh< zYEhQ2bCBD}JmfP_l4Z>~Z+|V7QCvqff(ggFLg1&4NB4!vmqB(*wYLvE!YAeUjmisV z`kp?&f5oxlN39H+!^iJLD!*7yP~Rm!xv(cm3(w{#c2e%^C?QZuaV~5m5u*5dHIlK{ zm>eezjP+?l;J4#`pG_5?H;QK%qo!f|5ZJ@Fox#<_EBPkB4)zl?5I=csV;?iux~ivT z1`3TPVdGwjI2Go7&c;RPtt(_$ZS`ari@IugOeHKAr#P>iS0KJ(LAeR#DB#T zZ*an2)^QR(Kxuh+!7t~+0X$+bq2Aw@R&cgTyzDTyNR!1qm|~d*S(utjQCRb zTkPXqGp8fk!uYAr$ioR=OV??3m9lMCwGhO(YXdQzs?W?|2-&M9K-7BgJ0FC9O+pm; zO3#rmFhvt49m6c&$ep>aHWsEWWG=M!jD{B`ypO(`fo7)Av)(o!L;&QkpuiHa}n`37qo znS|!|mr;S|Ur6KWD0Nmeo!0&Q{;WctJiI9%gKwM+^&(;vJK@A5@7G~mimd76L6l&dIuG0>{%`s)L7c`8(H@7_}`@lU+ z?wsI_L8FURwBBJ#h^$}0LPYevguAR-;mb>hKhzkHSE`;Sz^fAaU+wCd^35esOyN0t zCJ7T{r1{;xF?dN1^86^%z<5U%#8MB2hsNu2^{Vle%SpRVr@1bbf6C1+%W}+>*Y7&$ zhLcJuTR+29l!;D$B)~XdCLrs!K;4^klin`mHWzjB)*j2D+iq{I@3+P^X zH$n`((#et;!~OOTS6^0Iv!LP~y$pM#R%h8?a12@yb0cn^J#m~6CO^KPL3_k>&sIj+ zEac{Q1_A2L*Ogs)g;59@RB4XZA`T-=tk`;!p1I1hKQB&ftaCT@DJ1`#vd6{bDNMd> zKqk^8hkG#h$!~p)4%}n$77NW58&VRRHwowK!@B5H8P0y#nNb&$Q!T=ryyv=cs3jt{ zR}G_?))|#e!8cPDEU!78(bUg3FQ_S;lpD$Wz!l7M7<>0iP@GjnA}6-JJCDb_#0Q@j z`}h=k7ugx2T-{|svJh{++FV>b?ZRy;YqG-&Z@yVM()r2B+@Zl)=@=WA`{QBMD&nkW z)`6Pd)xir)dKd+tdn^blj|-2u%Qfj$q@?_`9yZpm(n;rcvl*=xn3_5gJThesRW+jZ zb`o_5Z=M0!+C7ZvbL`IFR9ib-NS$#BIbB5Buzb87A21q54DGW<7EO$V+@Do~$cigx zTOJRD1C8zl;$Ll;X;~-X@TGz_Lzzn}WaGOXmPd52IDCaBg=6m~(9pMd6DpW}nDx9@ zd3}Aegd|@`lo-Q1OdGmi1}(l_Q&SbE(g7zeqcw%K zL*ARZwU-zd8OwIj$k^t}h9mCoe7BTu4;F$Hgs5|TSFfnL2rUOWpuOcpPk}AKm}(%` zw3xfnOM~=htmk&$7*&{Gex=oM&g>%p7K6Ubt;FR(Pm`G7X~)5Ed6>;S*DX;70B%T= z{wV&u1&|BGbOzGlLK@(u#orru9Q57>QN-AnG0`RgMLvEi;-FSlPkE1hM_u;a!w z(59dyCCCEVWE^UhZVW~NzT0*_vuV_~5E0+J#mBi7nm5`gxbL#O1acPkMww7j1?G7Q z@~}W#f}Sd3#=PKSZ=iu&`|w)>_vGU94X$yBc>ZNS>tUlyNz~~;==rhDy31jPKfP?W zq7$D!u6J6Qy!3i~zJriJ6poJpSibtrW4L;!yKBshPC@>f6giAPButWikSzT( zb=N>Vz9L5}1AzWM{v^)z9f4#9LchW!3$0ijv_J-YGzxA8mUm7iq+tY;nt1$MFjHc~ z-kiH~L!EB9%yzU$Gp}=v!hR-krqS(l@G;5O=G`g4hJ>a7Gl{Nd>fJ-J1c>KZPFRkx8T!xJHx zH#@W96NB6;Otm7CD~pfe>Pwfn#p+PaZm+WSou>1B)aP7RpBv!)P4Wz?gDB>86i)B4 z_nU)rzdU(kI5e2e3zJ^pTLnfs^=)&2ZG%y!W0JPy!h)u%cVaYAd7hrgXJ1s(abb#b zV1A85VlZT&I!*CwkfjVIWr>74Dm&Vnr&0HSn%lA+-i3m~dPweUhjxQ-c=2*DM%-GG znw@7SOdS9{`XPZN!;YwDcNvYmi8Biox|+=r9WPrRyGfL^b7wx5@%`OE zEQNH^IAFQmVGbqOv_uzX z)I1HF*CKa~*#}n#Tt|;B1(}rE0&s^_9f*j0*dD!=mYn@5?x(&=K*H4<^ryRTqM!lE z85)qDebues845ZGji-L@##7GvlHlRAIEUcS3g*lW(f|4363&3$SLm)=w&xmmD!DP6;6_2WH&e zN!~oDtbYGC_8Un$b-#KjA4T|QYL?9B3m`6M*4RhAAq>x9FdJ~(TcFMNs=jeZa3#Yy z%1>^^qvxs-DfbTLR)Xjs*v7wK^Zo4KkP=Z?y~CGCX}4`0t(pXTH1cY^h*%Q1?7KTD zWi_%Bv6JNz@WPoIjTK4@fiGQs?8SWlQs#NEf@Pxz>vY(PKd^_yf(=(K1nuq7;Hg}$ ze+G2!udQ(6mIJrD0sbaXIT=yG37RX5jT%O%+GQmd_Jw5HD6A`kIDBhT$dHO^c?+fe zsS07*fTa9HNXlGWaTDxde~~`w9b7@to8(P`ybE#Jeb;1a_xtJ+U=@Ovhkc{Mp_O(7 z(=HZpLD#mtz^HZQy_o+J`!|4ujABC;nBJcIsxn{_OFTCVxw}a7FK=9`ar=}gVOVGx zgIh;!{D2~|!(n<3xas7@@&Q#&9EYo^t3nRse3TOX&avw%2;R}_YO|@Pov+46%yai% zdE(15JiiGRTh7;ZhJuxg>wfSGrpON_@~|y$4=qN+&1vR@qj5JBc;j|L%$o-Z9j{17 z;R-PX-4zqL*i@q{&RoV9S6x;h4)^-HKIbo#*R_tox!F-|1g@rvNgBN=+p z>*XYubh`M-`E|+rGv49b#~W=zADVcBEn;T$r#*PP?#$oACsV0iIo1|;zm+EcBC$3$ z!I0u>37_8X-a8q_+?^~wo4h_}MJCj?pIB?RXSg=gB?@$U6YXz%?oJn4G+tl! z+N1RNymI?tM1k&0(@)B^QlPj;I`q>|!ZbYR-@|;d$|f`?wo)G3da4;5ejP^)#`RC0 z$O?|%Y=W=(`T27>mowT=&J!G&V3B`T)^B>; z$X~g1W(qH)LLjrHuBF5=_?=w~?c8+pXaI3%D>Y_3vh67q7TN#sOdgE|bhWYAC-li@ zpch!9xydKA6uJYhXy-#V$r|7=pY0#lbIJ7O2RKV`>RmXe_qAHU0N8s#-xb#4!Daf6 z3?`&t@knt%%zX;L`@2ezteg6?t6eL;j$F|?SpLQGP?jqs`hz%#t%5F!CkbrBU|&_n zIQ$~JH(9bu{XCo+ipc@fz`bX}TY-{n7Fjn> zf&OGo(-yR0L&aquyeS+E|MCL(YSL>5Ho-`+>rdMuThlbVcwaz@h4gDrGw&&Zy-eY` zbOnaE`(ik2*NK_TT1>ZETmq?lcUiPb%8yXJD}rjOS+i=nIHG2k^VSV_AGOo5Z(Z>l#YAWt-+55AdA*$9t%2^$t*mcR)yh3Hc;Ry`Z zi=`s&@AiUh9ysNGN*3XT*H@6C9OXTJ;!Aw_vL%_BOdEb%o48S3|F!*9=kKtHs%%dh zIR0vz#DA&kyb`MF^;WmqKN-IABQ z#0P$KeB}BiIoiWT-Hg)Atum?PM?^)U%KBVd<>sqsQai$)5jAJpH%ixQAK?2$wQ%a> zk&K0kd2B!D)frsARQt|zVczFQi9GTYV7_mE9&iaFC5=dtg0}t_q+O4;kuEnR90mC2 z*=k7hs&O-Ad|kMB^JB02r8VLmP&jzJXxW_Bdv*4WPC6TT_m7IVb&@`f%CEHU-^`pO zPA?)E7vJ>y%-&QOwfQ&fd{b;TdRt@gAF zeeT$(2O4L9CcN&xPe+C4(4iY9VDxqHDy}&uTXgb4SR$<{tMz@FCP5JPay9bMtHiV{ zv?v?`iN6o!hL;oJ(KnupH_IXn^DDQ6N}b!A9Bw9L*+xTGF8}HqLkfd}+mD<+NXfFc zjd)KdY8#`ClPm7lGOb2tcWcZ8q!JykbQ4q^*==_m?340SS5cPD1ORid`Rmt4MBGR7A2OzL z$q;OZEVE9z9`t`Uvdms9{CzXqUv0&_BS%y}^uDz(coZCq@47SpDpHpI%Ze^hYmf2-iX@8!R%_fD(S2Q9#1 zJn`YraRyOd!MqJ@OV-aIhd(RI$3VeopfWP1>P(CHf)eG;W9zLX4Ixp!$H}eIr(R1T%Oj5QW>B`5$$huYR-zB@q^t>?stMm~`%6A2w-z~PVNO5$E zDS{dy7Vr%<6&k!=Yg~w6z7+s$+R)W<-dKqvUnB+U6>m85fDp}f18O?pk9vnM_$zs^ z#PaViFGRzQxgDw2dUwUU<1I_yUjmo($@JNzbt)*oDpKnV8={(OUr{r4CL24_o{qdW zFXo(~lnGlja@w;vgWRIL_S(uf-oLnmgO?yi-W9G7HyI(xSjhdz^$Oq}?bAxIDXJb! zb9q|H{o&iB$K=7ebC#ADCbm>64E9?uyX=dSq@!mRfzjvZ9?7^fQMg^}!&l#59Yp^E zCaD%;WVWcv5=qyLT{^B*yTG<>&DYpzV5yy+ay|)XXJ4KY?OQ})&H@ZBo>#=` zT^OEqd6cJdk?;|@;)32YQ#uOW;!XB^ihYXb!^5nsvj|*vWhH5q=E*Y|{DC9+&6Drn zfAl^5MfuBZb$j2}bFQB9TSI@^WSZzhE00fnO$MiW4=!%lR3Sg{F3~T^?NYpnZnex2 z*J0jE4uzq=#T>pH#A8O%NKWL6up%52A(Es zKF!YMm)%(ttgBdz6f>;K$ELq5k03+b8P}=)a(eS1+nzS8c*9I^->^qE-~89Zxl8?y zTjb9z`gSBp(tQ#U6l5pd@6GL;b5Eyo%iuywYAH zm|Lr(IEKTN(yI*J^RcgA3bbqVc{vCk>KquR=An4b(?#HCYJ z2DPgzY)!on?*5G1&GyVGtCX?1Gh}du(-2^w70(~ie$uLDuFO7SqQZs)sJY`oRs zhVD#(-T8H0T7*>hrU(N0iOo-wk=jwb(s6G;dH1$osld6ZW)U~fq* z#fVBX{`!W!LDBFkLB=P$_LhM8cC`~+>;=1ki2JRwdcC-q&scYO>mQk_>(pwPW| zh;v@t9Yco1zfF6$IsDWU@G&G=+NX(Ju*|9uLhcFdb#0-dZE~ZsCdYAsY}A%)z^VQ& z0ytGkIFEPNbI>yqvuka;6eh~o86JmS4=vZ7D!F-yzz+cOBAy{w{1eNV?@HO_rb8-m zL(jZCpB%mtaF|1qp{ zgo??EtF)Vo#MuC5v;tQrZP!Rg0_x(ucRdz-({kQFE)H*uB~Lo{#ERo9rmg z;X6YwP+pJgMEp^Q#S+s^A}>pXG#xj6S}Jb+;NGi~Ke{927>tkI@OU#_kgYE$68^y# zt&_H=^wgLhX%aByKxTC)_+Z#xUX#gipE_3D`snv*Fmz8)qu`ejOcLFMf4noKJf&Nd zR1;NhkfA>k>K%Cyx1n;z-E=9*z&L+b)^Im(a7sDd6fAXT?mpi{fy3M7*9J9{PE0Gn z?4%pArc}C3Jv1HNaIFdKVo}wUrytP3xcXavXV|fq1^v5jQNF4xpL6qo0ARGVtfBm#j{;W#kUr4urAj&{dvVD!1K^5g1)Bmir$^6n408gy$IL73s_;^u%?ACZHmq2n!MSbV=WIaK^3;+mUMP!=SDMo?XR1Un#3lyUXG5 zQCDy%P$I<~IK!7UZZYDzHF~b`%-JNBI*jWDgu=XcUFY_wTBR5do?h>}^(@YVcw2*C zBJIDcz*4t%_Zj@MkM|J0%~9xi8N%SY@p*a}SGZe$slufuE<~l$H1A_?x0uJDI9nIq zeyag>?kdd4qmj+m&R*0e?_+UEGT{u~pfL0|Xs>glil(&L?8u_mR$NI~8@NfK*~G#~ zU-L+~ZfI>UfK{CJn6hF5a}$jp93&B(XB=s~uPnrO@;(OQr@ayDI;F~<3lB;Xdc@lr zBY2r1PZ3e(xQwGD7BnZNA?zZgkdSxk9tqvQ0V|qT3+dHo=%c3b#)iVzz&$Q848r`g zca0Twvu~xnBNH#NU|F$e08b8TLf4(s(NA5uBgyqsbxW;m%o>rx--rpP+j~bZ@vn z4@0u$Rd~%r8&f7#$4-P{oK_XWzMv0m#-E=k_!|@AriEMsmxFYS$6G3TU&CzB;j-wz zzq$fYzKz&HFTme3G+ypoyAA90x4yyXP-Kd4tX4?s1E2k_qM-Ms9?F8s@qH%}fm^y5f|Y`Qyn6_tIO(%295y?y)A z_l_v%a;LAAx^4hUT_zIuzee;R;TM2G;J4FfU=PU&xK{x*obgu7ckK9SEwK9`s_Jm* zB#-O)%$TY}N-tyOhho{}?4Z_b$_orQhSIhzDaT58l6}FTuBPXUz7!tg=2adLoy~L0 z$$qr2G$L0ya_xtRTr-6H9!$+yR3$$dc9}_YH`Cxf;vU6uJV}R2Z6fa32)Rq;_k5ea ztQ0W(BI&Aeob&zni&u5LzYprhJk$=Te{!;oO0VCg&dHyBk`svGggzidAoN#>har>2 z_|P7tP7zohZ%#JtqMtGITxs699M_*JuX2|4uF7bAR>-4$|qiji)$m=v-5^qdlY-v1`e5rFOPil~|h<7(X>Cn41i z%7JE9{3y)lpjR$(p6J|V!T$mZF@>2WXAo-#MgJw%Xk1)PITI64Kyo8WXfl3YpVfu! zB1Eb@N=hiyC%h6Oc;`99bkp<@Pfho>@vWbHwz;4XGS@rcHvHB_YUFmurgSOu?_`!0 zMDN3P_E(5uf6zHJ?AXUi4Jtk{gcSB|obUMQuVSn!w3#`XYZUitUFm9v}okR1~ zxSuujv1us#d#2IM=+bEKl+)jL2h}iz;4aI|zcWh@H7&PFu|t>BxP>uv?)mHJb`36>$Q(@~|O8qHhC%s0AokHz66uapD}?xOtgi-n^oK97yr zR(n)w-P2IMb}ooxPdqc+MVc7^WmC@vB0Qdjwub-(ua>W`bF2Eox*W(6Eirl+Vnnnl^r=MOMAQu%ln%OTJP^ zzl$Y{gIHZ@dpv%f>zLYIdT`^fnj#+4oQzCd3!7709GUth-7|>C$)-Z>ueK(6cm!9EW_Qk6D+-j;P5JY!9}i$F^=$>f zO51}VPL=+-aE4PmLo>&T=aZCo>_sNO&?Gz+@W)uho*XV`ltjBJUd#*T=;ZRV8qCg) z{&;1U;*lTryDujl0Yh)mo;rMYnZKN>SiI06a52i^gs?9(i!io^io-utoTSTVV`ous zDk0%!-#|ES`m8_mz2qEqEba!8bvaTBBLC&5@Ix$fG9pms%N6sh?Ap7Kl-X>1*) zWnep3?H0Y?3%6ax-BLKVG&gO)f|%Vh*(%-qyeNdb+d|Dqs=X z&GkEAhnGImHv(6z&C!F$A1&VFMipyzJ8f55$bDB;-WGgFu{V`=Wv;;+IDKiBfZja| z-rIkTbw-zjd>-Y4!bR$DGr3)?uXy&l18AI8VprD2*ccl+hH_{<(_a2bt7j}LoVJ-q ztiN+<91T;&Cf(}Rf2lzqa^}dS0WrL8$;i8gZJ~03BtzltM^nr;={{J(-~#P>kwVh7 ztnfRW7x0&3Cd@RHRE+36gp?3nuY=hij1r&1-&B8B_)+?tea5loSaOa%F70K+Gmuc9 zE(WYov*E6^yAW-P_t?MRRn<~Z+_c<4Ce>Jm#p8f~7qVWT2)LDRu_bFm5B&whjt$!>EV#9~ zkIwo-WFx->p>(x2c|R~hUk?^aAPnq7CA#hW?EVIgGgKbuA>Ai0a>Hg%{BY%?)(sbB zkA^)w+Dw-BU9(7`vQJz`>Zs@bEQ$Lq+a=R1H~6eG#mza5+h6EAg|lSCt-5|>h#alpecSFnO{>sZe098C`8Faq{2i;7Zh5vOG#oQAi-z?WY6-bAj5Yv<3eZUMD% zarH$CE2bJWd@uzMlssd9KnEV2UEOf47#D@08zg`D}BYJ1Qz$70# z2d_bTB!$I!bm0L^za+n_T-qTf7Sr&ckcKu5uQ6%)8=f=n07aj^A<<9LBh9W(owoi? z7x|C=B!jDm^|_1(L+xL1$WR-TN!S+!F zYNb=!jhjR&rGsKJsr0-pNq%!)W3s%SmCQJ|ZoJ-=oDMSoRhrN)r$|R?Hp%xzqprEE zMMkXG+OFFxjT_VLqchs@sD@t!c7dbM|BOCBCj5E_+1K3?*P*=={D$yOHn(#k-=-~3&X{^O6k5x0nkww|n8Ms5R%#gC7yuSJ>B-|e3|g55ZG#K2bl z*x#+=SG1$xH?#eDW0!0VAPUhv?lHj^F&W<>QPa)y*MLyR2FMoc^OV}u@BTHEArN}= zIECPjVz~l%WvQN%u5wuM7x|6H#kMiP=`s0x%z+7w-kZ2m0PinN|6oKjob*f>WS?Yi5*aN>L7Oo97fC+5dF; z|Nh4Q{Zq?*FQ9YTISp2Et5%*DNzL6uNcS;T{Z=&A?TVp9qyTUan0~h0f4HLb*O&5@ z1nc$ur@P7N*tVsh5wI1A$rS;=ujvh$bvuz`ycENaR%_`4-oqX!$DEt3qd>o%>Dj+d z*c;E6;FG-ys`~QnudnC7zoUQs(|;Vsf4(SzmYc5h)I3rAKcABScy#{X7tgVgV?nu! zO1giA3UXNZAI{wW??rLsF^|3EqI!9Yzum+|NSTN?@uWTOI|qV)i~RL{2N_BxLpPGK~5kz9d|Z4 z2IF`pPRvArl&n8QDm;S4MXNMZ^F1Tb%`$^6>-wjGXhrMbdZodv{%q5~A0Sas_r>J4 z7gagh_huonCl=|(!jY?3j0bL-5Y`8b>6;;pFa+Fp?tHYm2LRg$gR}c^{=P80_|ZZ` z+&xJsp1Uy-UQ91!khLgOJyv<#EBW6a&r1^7i$Bej zi^H}65)TCK`;9N_LondKl7P(N$w1)H2v9HzlSF9u`A_HX-epvhW)@lT9solwJuuwy zOaNBlqoJS@59>b!qh#*PVsO>e$(dtuiGHIJym4sgUC*+iNtqXGTc6ZypaGI;EX;dm%5FAYiX8x1s* zD+%E$)oGNB8R9^ekbWXMyJrKqF1S*My2Oml&}eA37XhQ6!HNKK3fv1!Om#6{&)+Hs zOZWiE^Zv2NcLvzj4(`#BxDzj?$|?X~A$pf}(%67^;4L;jtnX|3 zn0)J;YU8;wYv-$IwkXfyBlcKWop{^|!c*?@6M#E;rRpMZ3yp2SVU#>63==Tq8(6~=z)F7_9sm{;MJfk8J2frS zAl0HksnPWSVu{jcAlTpTUhw1tm?QH&eX>4avT2U=i>MZSrPBp=covobV#FX!$uUr} z?{7u7!Ic^zj3U<&xyuUc-IM$oYdj>)8m12PvwD!X{%V@%bt-yP@M%j3VJ+uMj1o6L z(3x>iQbj$4p!?5Vse7W{ZS2RII>E%PMGX$Gx|nXx^=T0ioBXokL#TJluJT{YfMT$rLq_>#AAv5{`F5-GnC)UO zF!f8|^X?t6q?FL&HAAq?*CP#Z!oPq=YKX!;72;RbvR5{o=6-@Cs~e7rQ%hL06R2(HeRlU<1XvI632ykTSdzjCa_=5Xg1jEWpTfwtZ-XhfxmtciwUo8T|S_ zpjkKpYMo^DU8RJSz`nwOye*_D+^cl)S;P8hJ%z+BZ+Ik{^7s*I5-1HmI)I^#(db^8 zK=2UDGcPz61aZ##Zc`S=(*E>AEi!_Ya7s;QmDU3#@IV9;F*e%{{fXd_$TW6uYbci) z;vHoz+!Ws{zpA`*5!@PPl&)sd(du{Ow{kPX^J5#1@5Itq^b6sypN22|F)NlVi2R8r z&Z8H0hoz3X?aKWy33QB1yc!^e$^qM+Vab(-WDEY3@{s`joW28V&;@J~I1;g#nSkC& z<}54W_}3;J0ff_&FJzZpdx$M|u1||9P?Qp8Akh=O^zMUj-o3MOQ4M_09yzsQg#`B1 zu0m$zDnNNLL^{0~al%^-vr>fG@TLEphP;8%Af zyB!O1R)yDlMnZVEN#X7&q{9uc_j(({uftI9)UzVR8;pR)!1}?{wqIuTS~eB;+DF-K z?ll8#aWjvbi@7A^?nq6zQpm;GlCo@u=K@S;erAJ>`n5jnFl9F$xkyoi@;DCv1elu- z;Bq1^AC#n3c8RBna>U<#rX7?0dERDE4~OHb`2e6O4Ly!BN$@vNXYaC6-4VX;@u8i6k6sX4Ep1DLS7!I0y0Wb%X?H^IRD0`G&MZ z>y5qGwVkbM7W^5ke%w-y`Lfj99!_KiL^2po!YnxBAax6^YGm0MRfs8|B)bD^cwH85 zRSTdph0!;_7-Z27l-Q|ENOtAL&gQG|`h8O*kxJ`B_!}|DtyOXWIO==7`w=PSFaaRB zGu1X7ca^0F?9Z>`FS{0h8(-09W(v%;{l2=An9`+_nw+Ytr@{~=B4pqmtTIQ<5L~r0pvWri4RBBI#`xWO`7h{`&Mi2N0~>+>`I0>>bROGN9W9Zic*JW z7M)yzO%!JtSp+Z36d0CX)Q31@Ek^KeMJ`W(wql|FIzw(0QN&z41U_g8Kbs}zoly@uu}ZkolYZE194?uhf84_JWM7F>{J+|~FMbswM(ObJqrfmy z$kb@dps3Ve);}dHeSY^T6pxBW2PwsT2vXhiO^-Qejf&nIDV)eVtWG{~wgrnb zoa{pqq;uLT&@OT`w;WYomr>6sLyL!skX;MB^n7@1G-SWTcLa$ea>HKCWCgm zn~A#@m`b8S!h=8@G^a9eX)$*M6Llg6Km}+rE@O)KN|V>2Ude+$W~DcL8`(T<=SM_J zz|haM{KMo>?X!!DnwLP;yX%C4?VvRUWvSahO8XjJ13q9PvN=MsCW;s?YK3NS&0X4< z{2i=IiLx@&T>N^_D)6@Ka<12e&3P11)7vmkk)hu7W%$`-S|k~mC@^S+IuZZed!xco zCLI6Zh>cG+fHG1HQ5U>>+brH->xqN^F1NtDZ(0C|6vkKk%h|jZ)c0D7^$&M})$%IX zYdb)7Rv($*3Ejq22)(T+j1#u}=&{pXBf|6Ow98gPpkSa?3(34;cK4KqDPP8o%w@wK zZ;aATDs9l|0Rw?eRtBRjN?W=ft`0|wlymaer|zCTYV`MQ8^Z!ElGbnbXdd@vtvtW8 zyZY4R@Qa%MgA+d5zUB9LRza9nrS3&pv1T7`1kXQXWa}~IUhL9cTl+SacqLlmHjU0mo8Ow(NF^T*uo5 zy-#tpqq7qUIHsCzqx)DA9VQ#`*(+t)(W&$z{uA+pNO~yI`1}XF2P|f4Yj&LsbOsI! zROO;~__=mqbXKl(;!)D)&kh<<4ad=lm}`wFNy~sNHrEu8OJp&VUpz7ca$r4$UT@(T zo(~-MTlMtAI+C{{??U#C!^pW^CTabdD}<+j+I(LRaae){P$fvv%CbF(syzlGCiP}` z&XPO|cqGl#c46s53~F2vCckSi*DP*p8OHJ3yN0okb^y8WqH+C{h57)S>G8R5b7Os`1_n1TTCbS*$Adma=d(12g%UTl2@3ca4 z<~pROow23m?6|)mj2dK5oTu$Y)*#hZWt|i$Ya$kuj7lqmov$6qmYmEG%P{AyjK=3n zNZqKew#!AzS+6bTR#_d0nVOHAW8ZkHO&N)zouD=2b6X|RnX1A(X5lA%gOqR&o-@eG zt_ZPngzQtfNokngK{^fnTxEwT=-szo8A^9>KJGhHvE+F-K(#W&AqeD!I|>(Iyy2ra zweFrn$29LelTJATzpJ`=E_#L8u+C);>j^OJwA*q(jS~)YH5y-$tY1LkAM)a*oXcp^r)06W`8$udtg&9*tDIgsg!b!J5z2-D4G$*kL6v8?b{-n?*+#K_iXI656u z(;`MAqW(~AxPzjNqFCkq4FQV%v}dzl^XOe%h(i1f61XYwE)U1l8f2TfzR`b>VxT^{ za8ax#Dw@J;UrdWjyZ+}i5<-kta;VV4B>VfFS14t0p-5g~V zb#gK}bmqdf^s@DNv$pM4tSbUaVsc$XRQQH`af-ew2P@=dzHF51{$36edBWju9sVh?V7J<_OnWRQK z0ob+cTcdXXy}dLf0|y`O3*qt(IHnL|!&96Lsy!9QPd7tPfmrDKdc!1Ky2O}q$?1+27 zbrdKt`8`A4!TeMIvjWB_1|3iRW$c0_`%Mf;7Q%#>=jaYppakG`Z4Zl6B7Jq`G*U4k z3VXzv3J*kT+V;rTp%)I+Iz?9sM%XM^EGW# zM%`2^zMIuana=)`r(#7Q)5k@CFZqcdyCq-N^+o zQn6XOg)ToXuK>W!8IOKCg4`^$QDrC-h98VDbxFq1c^Ofkjy+clNmGc3>|^xUf0W_q zC;$FsB(X5};K5@uuU)y+{eV*2sq|3MMsqtU#KGp9!t1f^3+UmoDSa&--)~b&ZR|M| z>f;Xp#=I3e=1wNM`GC9y5df5^*~O6|3)_3nsv^~<*%fS>G|?)Ol0js27j;g^+-`xV z-Noi(Exi7gd**RInfVX1hqKh~ma6GD7m<^mpEa!y?3r*5Ury@Vt0UIs{*KyIWYp%( z4<3zE`f-|k2uam$*C$oa4hLEIl3k@3Y9nhAG8rXBcjX<|uk?4=G9Dq6|1pTvbwX|( zTBK&fHE{P5s9L|?g5IOxa-6c@tPbpZ^~L^#Cs!ee-R`G}m!Es6{dkzk%rbWiIa5(SKwiRG37b|lh+-7>kW$aUJ4i054;iS0 zhI7I9E5TsR(b+Nm*BPYD2U9D$);!`K0+AM>TnvHthifv83ZgPQxB{;fO`;Mpe(t+> zLWhxFMyLuOkeO8nd%qt=4#G3BB%%4dV-a`O*#q?C++?hiJPv3V&2a+ucZrN` zKKoG&2s-dDU6}*?Wda{ZTLaQ7gmwSk0U(d|^bO8bM_MU`d|^Jn!}>D>8G6FRrQF)p zJ6Ri4Y%6zBJQ4lCGi~xXn5ZApoi0${Zat1kN7&!4e=0gfm5x?j>lPW16|p|`&HQH4 zAwj0=4FY%8J!Of=@Z0b)#64_egeGyt{hE6mF4eEfZba~5iF_lwV#)0h??DGy(UnmT zd>pmUZP@hs62>Ef>XL_=v0KqAb1a-^f1ht}J?}zGu-5HO+Lpo~hK^_Er`8s!+lI_A zxV5Nv3{bbB{us#{Qp6J8-XCNQ+H&sqU=a6v3jj8Equ;_g5Kl=B5Ha~600(N+I0{-p zb^NagOnr$7o0u?ZN#lD-QMcg#e7gFDJKb~oVBYtVlP}4r>k*3+ar~D+Kr)>Q&*X0I z1*Db<#gfO@q$jSMlNZIi-vN|n4%+P-o-;53jyViLujd0xZdbAAO2^yJI!2FgbRj&B z&DHox@=ejZiuK)k{VmB@edFI2GV|zg$>8O|8kUfabXu=@d=p{3gl5OLilYZqr2UgKLfM=qM-rS>@DV1bT(} zeW~Ve>!os%Jhx>m2BHsU7KgB-3^(F6_xg08N8BO-il9JV9`sEe#wQ!w4uS6Vb<2AG zb18~W0@eejryaE|)I6i{Al#ycCp#*Qb`N7h@=feik{o{xK>El7e; zpd|1`^tWq+KEMrHao8{Yq+=(IiQO%Iy|1UvHnAJGFvrT;pu1B_hPIUH z<|gocI=~5?k20nHV)g_!79c?Jbl>IfO_@_PV~C$0-u4sBbgYb|2g(PA-myWh0%$Vn z_4iL838!ZlCyFZCjDcjk`gUO=%%}Pv$@=2JCmE7X#A76!j)x94SwkZ+X`e4LDMjzTu+~Ep%Zskc6m@hr++4N#o1AsJZ%QsRY?Tv%7 zFCtGvKOClqxh1`?2pzyXeX4zVj!j0=Od--Ne~ff20UpGw!< zYVA_}>KsOr>q)2|>U{-iLTa2=p`k0NxR-z&RT1hZwoz<_cw4VVg!o1rxWhJq`E3-( z3C|_b#Pv1912=VfW4#_sh|OMjDc4ofJ?gn#+hwgn%~x6j%14Sot7!ZcF|4R&F}Uc} z(Sy)2=BfpVi4eeaD&zU?_SeL-Kqt$x;pjjjaq^B6+Wkn>)Fd^?DrPY&gn$*c)KRNp z`QsbYkF3~UN~8z~SOi4;IMr|MIU#^wJ>4@1z1FLkB@r~~c@D*i?XAYFU{+lScy?7h zbds~YIkYktLcF$VRXm51cl>a2eEC!9El~@kdmj(%IXC`pX|te^^i>qwPb+iR1aD_a zb+CL*9wJ9aJG?FDg1|qfAW_`g`PSDP*S8yU?PKSyq>ZY?MA_&TIAWNdBVhk(*s`Tt z_cOg68iP!@yP}g1uF&%%HfCJfXv63$>rm~S`Q%C5O47Jz=Z*bQH#&0qo3_e2gg%Y^ z8UIi3xDWuAF^>fj>Y(WW3(LJW<0DZBMArFv0CqaXm;jeg?NXO>P4_Vv6glP5JY3wk z0aNm=dib^gwubZzTQ`+!Oe>1TU&%p>wdw1Ab&%Ar3jDr@*cyf*4p}WPqg%H_*_ZU`0 z`kKM3T_LBg*1Otf2Gwb4J7DyPKDgAVE5-)bg!Jh_+J4a=g*MisvZqtWI0j3*y}xv& zrY**6jdUb658lAUUhQ}++zimh5kg0!0<;gD)`oQxf zMD7U@vt1e1JvnDG3IQFHmlz)pAAjjuj*xO1+>0C2IS_Sp=4>alJr((J!Es>mVSq(c zM_1Hoe^V~h?O?tY;iNnxF*3j_bD2G(Pl}$fh$DT+LZGjvY!lKV!#gE0kC2?TedR|I zY#n`~?Dj-%u)RyFt3zjsrd$Fat@meLv2DH7_szU>g3JQaM8BL%d$1n*T5jGGGsuZK z>?8pn0JB)}v2l3Z{Z#GTMrUElAh3!-CK4VId%j9!8@8tEn|OyG3hXQi}G^3v_B#pw8(WP@S$``rYTEDyN=A%;oaDOJS`3vNyKYEIy zz2GTESV%f^Hoe)09;^(?Z}eEnAw@4S;iH7CK?=;;V@*UYB(Qa1!NIOoY*;B%Xod&0 zCs1qzNdM>1{^&?t>OLn!3$sU0VM@-{>Kc2T&MJDdd2~g6CYgAteDqq!)Afo&{JUzu zm=^qkDgNMlNd{k4Oqi8(L?QFB)G4k<$fVWcyUIA`GHGflYYiY1adG6MFW$O;eD-2p ze>FG}Ve8JVc#b*vNY-u&N(u{cZfI;Yce4<*=cp zEb1H+nC%GrhrgmIs^~umGYCmZU;Os_60`ZhM##eB{~r;kT-RXsp1ejtQ5P?LjMv~3 zgfj0ZjXQFTc$U}`2_!8v`Pr{7gN)|D`Lh5UQtS9k zH6_R&lG^bW1jTw6?sk^71Gf++k7wI>z|z|IRgfMbT-W3CgH5pS;Hv`--1#Nwd1O3} z_)nzE^xUe!T)qYxL6t<-c1hNI{;CX9h}S$qoaRoJDc%iSxE=qb|uE!bj;zIWhBtqXkuDrM5wPW4f2 zNQv9DLL^PPX5vBAmnnRc3S0YP!$R zp5F-ss2eEN;G zn`h6bqN8NT{?;~<&WQt=E>zf!g13?B;;n#~yP+4)vsNXI*?wvjWLOy1Mo%JBgf&fO zz)7hErUlhUT}>)4Xc@i+)c0p(i3b&}fC_^L?96pQYOa4B>`5KA`6_^KV3WA_Qqn>* zC+`D%?r$4H_;W(8(i)?`<8LSGPu*F_M9>`EGQ<6GcD+1>)AFpO{S8U6hKS%`ncm&v zA4lB~DLv`-C)?Kpfl=D^l)6n@&nr7GvWtbFvFX@i1Fz2AcVimVYi>sb%H*Z z4}i>Wa6**1Yovk1=cymr^D|OuQ`D~=v8Oyg^ZF`1T;=*W-KVC7F+k>gk`pe!svBv6 z&ya^4M!dza=pR~DlSWwPlY|d!K1)`b%HRNcB#ran_OKB;{*liZ-mGIX@NY`dA7Dz; zo-hEF_^0(qajlNurucD6%MHt8vk_=8@9H-Ni*;MKR4~0I>awYPv`g1tZvFh6j2QdO z{%%oDXLZl~r+Ov^><9&skR4@`&mz2$R#aLe8APVcDvRO|dD%Ljd;)8`!li)Byw=m8 z8sA#%f%|QJ|Iz{&)oI1JF&0V^v8NztX7JP$i;bXuXy^ubiN&9b8o(anS6v5yM9(`|ruXf2 zThigiUCOhU4&eHWR1}wW(Z~`DT&>HwKYg_8YD%fUM@<9z-XIMl-F3NmejtCF^0Irm z@8u_iRb+bf5gEvZd1vP#Uoe4meE{kWMTLo6GK;peP8MUjowqX)-^+-1M8)|$4G1c` z6(6VSmb>Vp(n#*c;laHkI^R^V5|X8ie@F2`YJ>kubXL3)eU}E-QAfwOD}1o1rC?If zlWllwn{-xWSAVlMt01#Dua+R=hd3_p^TNNzbZx(T!CoqgtN^TMUB_wAMJ$YhF^&{5 zyX+_{8?jO%zSDks2T}rbikJ%pcbA)v4O^CX|Lxcp{m>{9oq#U6$Pik(J-MSnPP?DC zbODLZzJkcQ={5s-`&U716lVR)){xI&SisC3)=jF#< zdtG$I;d?*jpS_}=D<_nJqhN%7gM|2p7d&k+NEWNF^68IXW7H|Kir*z^PtLzlcK7@M zbFN)+Odvivu=xDG*p~rjWq6w)p}2DNy;hQIAf8g=t2q{rAf;jLZ2eS40H$wyk9 zBAKPBZ=H5*Xr}@{9fc?l6C~l1X#e@KEg)s`|5qm|*AI5<3c^40r=e zYg1|5LO*;&stjwgkfjl}dWB#_1+_(Jo9_Q{|sC?W0*P^$xcrL7p%)T}GIaqIy0}t6cih@0#gn&;u~= zs_TxMaWwSuyVKQP5~wwE_WArazNBiUiP#yTNb@TKJLcORFRLLNNP!vWr_WjKqRRY! zUp{d@^Cs7xeM5JYd}O%m#;AA6oBh$2o0p2<7qPIvryr=~tyrCYP zu2d-fHHRu@AAqpyg`#m3F&^nkYnGTL6O{voj{O%arF;sG$)DglPw+(wbolwrR|Yv` zVDO9FYwf4zDe10vmlWXJ4v{Db^ zrQ&@HiNpj7qM*Pr6%tFA`#D|;o!R$CJUCd(C7fjHBJHnI=-Wr>^yWipt#5zq$^UGD z!Y2RprM9}uY`}6`T}Ho3f=u_VN#XjOQP3;PjDU(?q-=fyz##-RM&hZftcvQIFJW7Upilj*8Lj~gzH=*MqK>d1KPGD#TnU`FP5OO$c!f2s2CC$YE$D!oes z9_4u0VnR6APv!p2RQn5x*@6iWlLQ_tj4Mc5v_o_tVZw{L5Gic*rf$G2M2=qC1+cYc zh-cbgy~G}%yMu?%^dOWn#ZHlu7<>QU2byvPDLNm{p!uVGzE@TZ*El#_GM814{_JC~ z>)HNXsSjaKlO>?DTo>@4kP-{`b-tPP682c*-9-v~s&Z&cU%pt6Cq(xH60mLG49kW& zqh4Nvc4KF@Sv5Ev+qt#F_Y^F4{*d<)3p?U`z+Uu4(`KX?@x<|aUOejL9jF zKfJdabutf$>f28c*vSf}n8jO{qS6X=>mXA`cD2#m4^t{4K0YH7m3HcVV}*nnXxDMb zZg6eCM3Jq}+3gNsAj&WQxw!hTl-B&IcBl@QLmMWkaLzj|wEgWk{Gl(A zMZL^E{WwFo;)xjr9hn{dK~^jr&_L;f_3pvdr7kOuvmRi7!_XH=CgxCo*5!KBRh{p2 zah%pZtvb-bISfp9SL;>vK9_@JY-gDE_jRq+>MXE-%&AKgJEd>S-sMQrZwFo z<}1{*Ki4$A)^#0=7DGn6+<&;H<&vPx=}BfvS0!nbC#1-ANhPvsr#^bbVR(doMmO=#i{~PB0Q7hO+KB%V1R{?dU=YO7ZHE$QG?ErXM}F4#OS*_o%gd z`lpmBmk&rSH3^tq_#u5qG-96qKo&Xni9EOhy9^|4S2X1thE1|2jL(NS^wfzYb15&n za8DsywP88Li=g)H6t!snFwZ&fq$X~=(_B@$Qm9+gUy+d?E~G?72y_H6TI@&F+qXZp zzW`oIfW-%fG%+$1U8^J$od-12RB&avOt+!d}?oslK{a~8Dy)c0f1BX(s~VYYx} zW@P=YVchd<2-)4P12~N%@wmwa{^mI}254||Et!;MaIpTFlp-s6ovcxuWu3ojnZbTe zGSu)FOQ+1|)JGXqzMzhgQn)^lK4P&#w7PPQ-Yb^v=LWmyBjzlOzi<8YI5TMt(MyhY z^kum~(?>9Ig-e}f1;9Cch!3?ju$#3mrLnLHQg;ATXopEO+$?;?uJK^D(8$O`(2s40 z-o13wUwag6vV4JMMF`OXNZ($fhL)*tft|0@@2-kBE|dfh{1RVn-~g{`RC{ z`eCo8(w^+L?bXMS9VlpY|0IPce5JLTZzIyIJU7r*FmNZnEJShtmROK67(n5(2ygSX zERs2DVg4U#8bTj|v$k+S@l3>||~@U{xWk0}_)BMFW z2bSd1t++e8RzQ)0|I7o>z->j}0CPR9G{>v6ugC&Og>%ZBb*(IICKNbElqOv+c6sC=6NL4-NXbr)E((GR@G1)CIVUZSBwD@T`2?-w}(VzFP>{4ID%{ zvi2|uR}Cvl83OAnSOKOq%-RAE9npyyC5x_e>G&}s^<^L6Nz>E;>c)&1=W$ru zUEDyXeA1gsU-y)HYBEs@A*Jj0Dk>8^2h@p$UoB1n^n07)VIHeA8xt1>41(odK+@#1 zRMd34dW4V%zE2PQkg`KYk*!1dr*a_a&z+!m*|N-RLAJeobK!Zz*aEQ(p|7&G`-W_z zPt`iz(Nl&QaT#BaS^(W{Yf(RGT`!*3wpm2n!Cr>5u5S}Al_=F0(!aZTT1K#W#XTVt zt+*PV6VH98YNw1}PXM;H1p1sJLGo*NUGFafbb>TE0G-2JB^V-OK} zIs8riLTlj0{$n2@=W&3&)|=Onsr2uz0y$>tXbdHuZLi|XT0napn*90xlFXz$>_PCK zJoRjavF|No2zm-~)*AkVkd4Hq^+{Br?c=`!aaY#sH$^r*nN|`MC7m{#{xh41YNQx4 zQ@OxmnTCxO#xRvluWGLOl318NK)1qWS+tig?zf1;M*0=-bq2nik~+ZNm;h7=`%Fz( z>2q!Sy^o=kr07;mh1wrG$!A`VGKO6I9gw|4l#Txi$U9J2v&kmX^MZ7!*JdxdG?fsQ zNTacXuC@5tyT5x4AZVH>!@Zuh;Wx5({s=2IruBoorj zq;CX?MP7f?*M6&1_&>E7fAW`x!FxxHA-T1=KZB<{3x($&CDL_Dz&zr$$Z^7&vbzoX z^)en03TYV1KVdxS5y(-jJHDt*lcgTTFf&85#$z}U-AC4hwmE?4*+$0+X2IFzID62W zQDeamRZQtT20|M(6%d=1h;$(ZHT@uRO@83?^?uCE;ZRSlCDN48qN$xZx`g(7g(3i% zm;Vs4)kJUi2TGDWOVhU2tsVD$NF9XAOPUpM=rD?FH}ZcW=wcjc$?T(QEzYeQ zl4^TotxyD)`W{B=_3q7eU+0}4@+6=6R>(-utq;=ak_gsI|R19|D5hwua(EpHo-5FG2~4_jfMdU(vS? zV0iibNO6)gv`($2CxGP)8Yn>|<{9F^s{^D!cdmfra(v(DI;v^NeFSK1V zQ{+DV)g6|ffw}{cHF7)4(w8NS?VL~d9L ztVMLlVx@Bh`*3Ggz3LHG2k{%mIrrJ zo!hW*msoiPp6GAj?F(NXgvlO5>=49T!`@t8&ca^*{FZ zwm{m2I=_#%cq0V5A3@~v#^=AqUQ?WIvuvm-gaU26(-?|JAIeA3JAVHI)G;2PClb1K zFFOGgK4Hgd`dfEwNC|++TzqMCDBQywV%Dk%$q^u)1kxxj$}FR3ZR38!x{=7$QE7Ce z5t7>Pv3C??jAt6g_pJWdN`Z)G0stuDp( z)iT|35L6zu{b=PQe#wZ z{^+d72a5;o+0;1nNe>`m>Td01{yhC{?g(vQ!LLh#j}yK2v$+gFdi(+@VyVz$3f(56 z(56-ox+KsXaHmuddtJWr`>Stj-H~@FtDJR3_w4u5ER2jU<@LSkKp{d^U42<#3~mC=Qkv#)Y#qh(irB$br`(IXj!t917g zV%r|T%EE(E(ubzJ;T#q37!QI%@mNCHPrDR!H8l*vIzv7N zGh35GBywTw%jzLx^bN2#SQ-SbBhZLcHiz_j+B0EHK7%amS$$3@W7UMY!UKM_7x{K6p0PtA>?b+lw?AMm=ATQac> zDGmhLwCNkaZyxZC{mV}9H4=I-ATpB^M!VtKZB15A1Ir%_SRN5E*%3=JJUnN`~0 zmLDD&d#3e0gNriwS6?%Sws%P_G`rUQ5b%QT8Pt8+oV12h9W{oTMCVEf>xd$=TuC?m7Fee9h629 zM*M0EfXH*tQ0q&9yBsr*zMfIDdf`6H`m4mBx+v{6PzaeIV_k98LvynPPTO`b_d|lmG7uG=vM9H zyUXC+a2H9|-=~)CwC9##T)m+1wAfn39pjjKtXkB7UAscyQ=KKTZcA=~c&Kd;D(V1h z$>#dN)PE1i#S=79--9{f5wfKIF5@x@2{|iUhW)ucsLL6CHB^XLbbVvE+M`JiC0_L1 zVBPIHUsSt!IW3{XHqC5^`CZ9FY@0|8S2U8_xYG2=HJ<`^F>xItrv2<2t`d-pW?H)&L zPMMv3i7#Z6W=iJ6p`fLsJ@-=V%a_feyv!cD%=MOUD4AW#cqE7@AZ;<7oM~2nX63Bu z5=|vRpPt}}KJ9os$WLjB6R0mjQ=d?BEEjJ>y=#eB?E22 zR!Tpnw#adS#A~PQ9WOF}GsD{xN`3v$LzNEVzbeV-tL5wnKZ>^6`;@Mg@PF8Q>!_-? z?R{8OKm|lVx&=YHI|V5bP>^ov?oES;fJ&E0cXz|4yQI4|E!~~IYkTgwQ%bLo9#k{hxk2%moG5XCP$xZdBcQoy( zwdH91ZJmO*M4@kxD_MV?ou?`v3oN20KceP1;lk$AEj6RqH8$7G@(RY%Co?HLg;d(5 zKVZ14sa~p9S|Ld@@9yr)lW0%9+Fj`nbfNG=7tFNoJWGKb|yo3l}s=` zrFMbC^lc%T8?EIxqq1aJ?Rg&4Z@Db$zo*Q(DKvmiL&2`mlDLQ4Y9{`t^V?2==Sq)U zdM-Q`(|*vM_msa3j=kV~5976P^F7-O z!gv82K#A5Js0I@|@i!TI9|=9)SlrGG1_@%B7*KaWABtRf?C*?0e2`bb7&R!Wb)8Ki zQsYVdY*J}qsC9zcxAG<_v#a`Mr7=XLk;K*bA@3R)7fWYQ-@ucTkawn^uyR@P z7jM5k_y=dVo#$?u70K$098{XyQumD2Lw+%u)x>TEE1lpyU3HisNnGI+KSRheNv{S-P^Uz7m`_^(Y=PRbSM|>c>+jgQ?))bqA8%RCfXEN9llvyt z0ftGcTLO#2?|pX~4?PF%SCHpr5T$Ys*U?e)G%>Z=05srp+Q*zIDZjtJ4eIksslH}hgIm~@z zN$!W-6fX^4V65Fout0J7AcsYFJO7FGq(M|PP&Ak%*7Re!D z+Jr>-EgmvO;}mC{d)M>_*roxt+oH*&V-;COEM`c5+&1_oz7eHXybabO>%}}E^>PBa zQ?J+~LIIaD*)zo_V;*-Sr7rQz?*IN$Wpw|%ab2tEKBZhCta%TQ*^=r6nfoM8-9Al@AOxyfF(T@h; zb~HltH75K|&6WS65b*nh{oyTTA3<_demBqX-w;gy@U}l&@W1}67Y@0jO{EDb08OE;QU{L2gfFCX~#lKf@y{<$Q>WTJnS9_Y*>)K~`G5WXMBRrx z)K$OJmwtao6>mS(B&HucQVYoNK@C>!o9c!DExE(OB~t*mLt)o5842CA$-0R58{j5inh9@TJV0H?u!1LcFMUkU*cgVI#+ zA+~I?*|Eq+!9Y?_Y|gy2_dAG2jbynO?y`X(YpgdV(n$yPm&`zVXzw3o1}a$a`2knC zZ_Ny}ICm6ofclC)y!~JYa=sWqNgsa2IjL_5FkXF$aJp}Ml~m})=@f}kxinca zAImX_==%~RiUz_|CXZ7}Sq!d$TmUEDYpi-8e{hQL7_o8;hFFZJUxQYd9#x#R$YG$B zTPi)YKG#cluqwc26pl{VyB>H`lzNG;-MP~#KlfJu!*O}!56*ep6V8Wio}l;HizUaeOX9qYak!CN$Qvw(#|3AF8v<9!F?ZhQ zh!i&s$9M4q?0orpBex-d$F9>i>eL9r8p%n4^@vU;zeITRw1Ib^Z zWPlyqEk44k)6B9gQ@7db-uZbo>Z~WKz0V9^dA($pH8_m`ZI>zJRZsmS`xrG73`cV@ zB4TSnW#evHhwA_w)^Gg?;@e>c#hjWZl?S5m+o8+tI9HoNI%Svk(KCW57A-@)JA?>u z)kn5Dw?q7dl@@@)`CSa49-Dc_D4>1fL4P*6H}!JoSx_-?0!Q@$obtkgLSOVg8t`eq zgF(!NNOI2e$Cdxvbqmt~H~;oQf7NA}UN>Na(F0({Rr_{PI|^pa4^_8DLiCQW@ac+o z9yS8{#a4i}n}|wY3m+zm_F|nBthDC}%nq=Ig&+-oSOJ^86W|G%b9c1cD(-kt?V}8- zEcLztu>vbxgdAn5mER2z@V}?{Gj8zn#sgU(BnpcAPQmX&Hvz$yvlfmXixeCo>#$z|aa7 zfa57(i%Uiq0zIQ%pzEZlikjffT;>7b)Tm33P_6JXUx8LhqzQ)z9s+*U8&Vb$icTle zJLy3AzG(~8Zw&!+Y;Ebe9k^Lp@is3_@KYCyQM~PkbiOqJ@!ewCYS zabo-X4H$<5Fzc&AggAGdg$5SDhCq(a?BO~?eG0eymr^ByPqeF-2E+piRv^CU8Hi(E z?Dp@9AML)Pr!5XyasB)b;enMhUbXJqkyeZ+c+%G(Zs3w;D36!~(?y`m6hclx$Oz3q zeZgD_48lDG?zGJrLzEPVx41aI?FMdk1E)ht%_LnPT|pj9K)R0!1i|ZtlkzOq&;yt& zu7O6PzA@?k4Qqr5KE&aS;MKK4Y!y0{pkIwfw|#{Pafr?PqZ*(cQsWG~HeGOXizGoR z5&vBQKsXJeZ%>`O`oR@TWuvk}ebA^*cV=70+`jDes?8Ng4CY-oyCD(yb@{^o#9* z@vSbYe+q$qp-KMg@!pc_P}gvvvfbUncpN?J6E7ohJTeK|(_bfNPedOen7xlwt?Szg z#Z9-DaN1Etn*0LVoJiQ}R&TJKNI~E4!&RvpZcz%5jxp>HT))oM5T=dO?CVVB5}y+BW1Z z$QTC3ux&=Oh1-z0Lk6qJqTYHck&zvA2KVGCQ*%+p{F4GN$m26~ux$;BnfhaQ@}$jz zHFsWK0x*K%7*O=OU(}th3@O$3!lus*|o3?+hnB-NQ>tKWgbL51KoXSo6+>vbOtZo7XN-C1vgy#1-O!v=Tgt{!A6qNgUjv zFnMi-I}*T=M)zt}Xa>j?^@7^XPWRez)r+DgHiVA3`KAX`Mqs+~s=!NrX)v#+6e#gJ zN|K?GP=s(uJX!Q#G(7ntZ1up?cN;~VIFlmI| zmpJ_nieG(Zs-F_z=yOr@bJ0u6@K^8VR^=qzdy>mSp=W!HInZ@m;vA6X)HlS0)qp|Pe_3%dv01y~*L+!u^ zIjUUly2rfJkG3Mb;@4ZoQV0%N(*Aj&YzL4{-vC#zuYAQr{rPl4p<}T_@9LfJjX_JY zQ5v>gf)Z>tcf&P5I&R}ltlrtazfHtG$oaBU54Q!^`HfW`d4be=;z!>r4xCK9>T`q9 z(ZGtck6)qN&po@J;&?W|>VfN%?H@o1LT|r(*s+g;c##L1l7r=UMxs*4uGy|T7X6ua zQkEUY2}LRtBZ;4C zh6Y)3O=_*K1Q~(dcj*e$e%G!g_!W*HGS(318Ds9s1K2`KjkV0emo=ah7z37i_;>z! zP{lM*S@;gr`nE6Z_=Q+-P!RcFump|#8}t3EYZwu4lm%0i#P6x3ehG-5-uhPQzabiC zH@U7oo*FJ7OHp$)BwY$;OhoESE;4#p}X-%i-1);*XS)d#h zt@$XRxqv2E7IaB8D!}z<(1~I73LaDXW~R_4D5>cBm;=gt&lU!GkmV4FKvSYQtZn)B z_xIYYZs1x=xU1lv>H!V&gzt=Q24luqA#(tTGqGZZ=Eu7$TJw_+! zW`YatX~m6P4R_om#UU#{vWj=ocY;}cNHTkA$1zXvYsmgUkD zPyO~0nV(rfb;j${t(24{8+fmTgk|9V!3z(wykc<(xb=-7Z)(v=Mdb1XRVgyJ0P?pF z$O+|w=iwHBPpV`yd$GuGb3~Ih?DKZ*f&o1eJtL{9Ur zlMoP9i;R7Yn@T8icpt48n0@C{L{Fzs7yt1m?UZG+WAiQR6z*!z22RylCIW882tGd+ zPfg66n*+c!-rYl9Iwk<^A6hMBeOTlxYx$iW=Gv_K$Uu*!2(O>-H@80`qTZ)yGwTv<>z_pbVn1Q->hs_bd_q7F_wqGd zw#`pFK$AIT;Tm+l!}~6C#ENBGf6o5GNs=s=0{)yK{3>PJ{QQJ8|A0$exYHS6i5!tk z@qBdhK9XZ7qk2AT0a)5+`kl4n)|v$!C4TBdwl%+?6?<35j5TR`0duj@I{Pzl(;=Y6KRTLBEuc5xfd#1hR# zOg>Axpsq{$!(Ps>835jKa`yk#R6%jiQ7?CIOvtYi)?|SMU-;1fGJIj zldrwdSra-d4(6tB23q`9sCl{bhxV&QDwJS)a}q!>4xmsKfstrPsKWG+oK}g zDrT|N9V0;=H6SAMe@E{D=Zd@E>?tiHcQkvp~4dI6+4Lc$1d&|kq@`_x*Ilz&Xzc|2+$(HA57Y662^ZppP?0PeX!Dx0DzM)UW zQSoAl9Y)?T-gn|^(pGR1jR;m(IOi<4hM7&@@?<$6_b?Y|ra-p{EL)MLFMg4Ey}zl; zSH?y=p zxwy7N;gR#OkRCV1^KPv8$Wj1B~k6QLGmfuHFp$c5RUwyW=gx;_15#0(fXR|YWwWQ|=cLeQm zk7gwE;#@EQ@%tRhG>t$S5@CccM&EBhD3Y`u@pNeYb-t4MM(i% zD^;h2%d2Di61J-0U$o?mt?ee57`WLXR)$Kze^fXFq z57&*E)p?NQ6dMaJ_L~`tSD<%m*$;s7N9$6pGA%;MR?d$#VQm{9(&(Vw+(6m`a8tFg z3x9D_M11vx=H&^WlpuL>ML4(^FQ=1_(25#7FCmXjpPDPi!R;f_^&RfDM#S;RE zxl0}f7Xtdxm;DQnh-}t2VrrvG;_vDf(Id!ZmBqRl8UOLx{g2YPQ^LT0ri2|Ht$yyH z!Xcdy3sO!21vMtuUsJ9O<^V2d5}c>apoZpqS!?$an;O;flCctUzkDDXwE?&oh}P1= z12>ghi)U$9qzR79wB!#vu1C>i2zlG~?gl2e_0oz`qc*n=4*}7LNzf&H59fd`0PLtx zXA0hL9MGio-#DPjHgFDTN8)F40umCq#NKBmC+v6?{ZO$FTFiUA?`IBtWy8p|keRER z8v_xI_)#zth2~0?;lArx7->f&b|m#tJV+mRu8HivlLyZ@fRpLie<=RSgOZ;oltifV!``X}t_X9PT;}yPrNy z?U^q+@IzKYBfw;`S^3w~3(nbX?&K8R%$G@qj6hRzvG+MME92z=iJWbaA(Nz}t>npla1p~nD5 zH+rn$`LDf^f172?z^g z!siYBnkt**7k_CR@vx)chVnj11T)g1H~km5wBsaPy?Pbh8(Px%WE0$=iEt^`oz7?7 z?MvT0(-Z{-I%*yT>#d1A=dV~si|3AcP{`=y#sjozN`WpfA0d#5WV=f#ccWq-G2kEc zaakDV#2Iz~=uX5m1$IEOPI}fA=jrv{^{(ffz-m_A>qcR@gOlKg^r?QeUpDUb?;HAR z%_n>F51{eOFTB|Rp5WwpTaACcKuhLHj$Gytk+h5VJ557q$JyONwHjJvrr+6GB zbS$dsTnwFGjctgsd2h954@%NCP$UaH6AWgjDIIS#BJ}zAKht+?gPVgHVWwCEo`AlK3H$W+I zi^O%SIEXap4b~EMH#zVi5+lc$KxQi()%$KhcO#?cOkiRqQDO-w2V4f%>;N?Cp2!ir z2F-8+AMvz+EQj|(M?ox$Ms&Tv1W)bz;_KY}2up(KiR#1I9RY#8=0UA+_~R)7L`SnUAa6HpIi-n#+Gh zF;C%9Oy&d9#Rx>6gU7qPc(fRB*ivQ(Uh4AY>T72j=xgz)fIDuY)B!r6*`Xydjetxv zA7+P2z^?m{g9M*f--!PBd!q`;X@EHQ)aUEF&PzwZOg?I z{Miqlcnhr`L9b$V#sAc&q)~))MNj3Uvaj1c2Kkq@1B*`2G>Jv2gTiz%u%QE_sCQfM zt(6VShNti;gCX@ySq@KL%)G7`NDz9`-!Hyg0b2RLtRdh9GgZqc1UW>|R?=>_3esI# z;|-9Rj~1F6km`Oph_R5~`H5jvSiJUigmdX4-pjK_JAl5$>;?ou$wW#4WtOdf|Bv{U==l@kqtwUyc9r6G) z@WqwZZ{*$uh0$(m8CI$}jvLe*zSM+U3R^ zJi)}eYHp$~6?=OE62uIEg2wF-f?5JWLYw|E(25aExUc1T?Qj}Y;x3x0Wlu?o{7s4@ zgl8?un<$=$28*?k?_w=gGp?4plGP=N2AujHKHcwU zSX%&$z~z*s_!stgE$FCmPI8x;kG~XAX#F4e8+&<;i(h=`p!Cy(emopYUC!e&>j>%z`5DP^grk`!DWe zuecYGeVtq3v0d&A;(G18tBt?;rSkR9YS^EpbpAusU;y8IeBPz`8ZeV}f`PucvvrVZ z3ookiT??qMEa33No5zmkzO_qml1z;r-JT0e2%Jd^NiLoDCP#+}!pEEn=GnGNZ>6Ai z_{adVlI5TgP;_#~A!FHa0z*q!gO*zE1&Vc~JW@-Gs7h|ze+`BD%EDnghaXviL_N

    g2~}kjU=v7lR)8dB<_>)j*W|!*|SNn)y$pa zf9lk~2Ok%SvX15aV@2%Js|nAE7;v&YSloA1N5nJetts?S>7QD=QC#>Y-~Z(!ZhlVG zpf^_Jl>S@_J|)@jhIyl6N|HUgO&(ArffAu;?1lU6F*M1Y&lulFBzcA5C#%}C2%Kl> z!ZfaB0YjMg?o$X$-lim{BzNU*ab5q(FHs+4zWK7m9EI;(%bHU96lS-xs?k+)JZzUI z7dpdRs*U0iohWeA+l0f%v@PT53cKg{uqN^-QPRD{;i{c=211NaI_=0GOfkI!@4I%W ztEk{jR9M8$j+m~0*UKLb5~)9*?DzOFRhF#^)KquhU+1B?ySa{@G6LwZu#HWz9mh{= zX>LAsrYW0AUXST1_lmPX~^myM&r`Z;RmJa(qJg#ecy0j0=?QDEgXa6r(87~$?C;amUaN0QO~NA?@4OQ9h! zm3vJhNGDV~fZd4yO~4g051-Z5lyL{0f4~QQD=@Ia3K9pdG{(F3panvmmgo^^Z#2;L zfeWrj9c$Z?!wZ*7^y0Dmao3!dG1vFFo38Ng&7PmoUCZhfoQ9oqn5ET`3Rzn{Txz`P zeTOFyByR+r#w`&Jz^JNnEghl_Qc?7^_MeN5fI&U1f-Ws^%?7$OYPpVW?d;S=IJ9b# zqlldqzQfj7gDHZzpIC!xJ`!k3`Mi(*R(jk^dt0Tdg*Nx2?}5m4=_bzOQm(Zv83jTv z*QM`$CZ*-ViHgnlmC-u-p1OKhrKM5s!Mi}rb2HU;vxN31RCxlj=84JAKlQ~?cr593 zJhBpQN41zDjXi;@tub;NS|Lr_OuQB6Ophl{^;rW$sy>a7ZidgD>-Qr7^mQys;-p>+ zETW9-Gx9^CwFCV*@b1J^m1r$qq zmqDvG`qPsF^-1+-pD zOKUc48Kul&_VtYgLNvfig$=}V z?4;_{QzZ_YO1QQq9fKj{qBG}a&u6XuwO#k)P!>*hXMfZ^yoYP=0xHGoBC%tupb$Jb zowqW?v^EkyMl4WSb9g@GU9=Q^QzdQ1fAv9DNo+vNO!aJ{Sut6<%tCY=tH)}tWG8U0!b-ACy*SpYh^uX@ zmnF9y!(=~Wv1so_mBGeHwxtjNRVe4y;L@wiqndZK$Huh6^h>y09ts={G{SK%gYdSa zX{<{9WSNP=JYd1z`<9AmOwX(ziMkaiMTqRG=RTIFx=m~1v%?3|JDBbhd1}a`+x&6B zuvx2;EKK=LJFUb-Z0-UM%It(-1K|1i*3^tzDCNUP5ZWN&@D}aOp6xH);cSaRB+H8j zV|{6Lu!~a|a9$S`m&`{H?}CvSxOi-a0feQb;CLbAPU&AiaK8iOF^GV;g(cx(B`p(9 zN%2WP01&E}bu^kPQa+(Q{yK`3T}kUUvv0CjCdf{;rc+ zJ0L4x_9`96<)N#1=!O2~Sz!7caMg)Uq!KvvMIosIq*HeO0-5S7d?->|W(rIqq)tqXM*=Ua*$^>K3>t46dVb zTSoN)MirWQn5hCcn8%0XW(^d-`Qz*29u;j@AIsqrm$Gq#lUk!f*Dapn?XuCq3X5r_ zwNuUtCE|XOiSzUG9cP$G2ImeAclkKliCX&hQPMAWtheyrELpN-XN|N%6~cy)|C++{ zVmR@IP&LCvw%MSZjeq?<89{9*eA0Z`g$!RPV5U7UBk(oOdd}WDv~%-fHfhx5MJ{c1 z{YLCSeQyG2{-#*LI)NLfzsQiPIX*(O9K02%0k~hbAc!o7YSVqG;w)1OG)|H@>a`gp z3o_o5V*fdHu&JxyB$M)steyd<6ClNCBeq&Ndq!&g=SeY_O$$iTvB3mo}ZAc^xFgQZ2^?x zlM?2md|pc4!1y&nGlt6|$hP2g`xGNBEoOekVxJ8kPJGUrGHB4_$B)a*^^_NRB(B<< zPAZ)b4S4{J?Zq9^9X$3TeXdG+0>KE--4kcOTIuWk$`?~;j3;gW>Ilt@hKZ^2Q7&Jh zcmUqQiCOS=g>>4@1k@uK!II&M$UENFu z2IAFu^E0M{iRZw46$*m)SZFon=HsIsm1nu1ol$#2=y&T2bzP85lLYTG_hv zc(SJ+~Kn_cJB?_<5~ zRCs4=wF%(;V3ybb=d3m=!9Md$O(tjd`h@M(os}IH7kJg@vVU&iVXKSfH0SKG8}U@) zKTZaeizHMlYOtkgG=#T*?qKsk8%ok0&#g0w~alSPzfuQJUu(#u9XyXFDy;PfmWM*R2~`b*tQ>myfOI zdtm2>$oCy~n_cZQbxz&(6PVtYGEMrgV->PpE3k-?fhZ*b>tvjoy6ty{6>=yWDj{TJ zV!3>ueG)3H=5}7u(_D(XTBtXf-qkR?cQ(Giyw%>Gt?cne*{UeD5P6^^lh6 z3zb~bxvYE?jTkR-M=*IiCU@4z3iE>DhjB8-W({KN6MXSaxw4K=J}%#TN%YjBn*^w? zFR<4ke2gfJ6CQc}p?$Fh{Pj+Csnu;YbFV4*{I=0_Q3PE`AX^o?73AF0jB1G>+?|3{ z?5cFYEr>WupkIv-XY0VV)?vBW@H(h$;?P@wv$@=0d*b>;<-zpR%SJY8_ZRUk}S(0c2vlE|YNG%_a zwj(cNm=2C7T<`!2Jd!od^&CaaaWxYoSCaqRYaUpD+y3pl@%3jc zNyzt(9(aMoUA6$aL90C5N=x#><*-9RDHqz2(L9>V48M zuDD#wy(1DYVm?Tf$ubU;)>gADToB(?ZgrjJcS9WLx$Cq&+Jh9Z^W;B08zuNKh2|`i zdtNX)X2%_kXb!WlvGSnApAaq;PIV0PPG0>zei25;oiA5LCnbaW{kwPu{(pNuATYmZ zUNgb-f>G=N=tU14VT8|V{_ScWStukeEe$s8?DFOJUq50+&~iO4K}fCn{crsK%0EM< z`&eXa5zM9pBEK)tzt0H%@T!*+;yvCrx(>*H{LH_+5W$EEn|g8%(h{@#_p4eYPy;xES$yyfq+^7mQ!`>gzZR{p;2{}K`X(Ov!>xcnW1 z{T&(qp6>j2clq~O`TMN=eOCTHD}N{z{57rr)o%PdxBt81^2aFa|DSsW!r=9uL^jL= z@ivTO_w069g#M}hG3Fz)uIr_3EvtJKqZR0yvz(Wk4-s}}_GNJ$Y1eN#_MZq0 z{p80G(?$Gv#->kYYTDN%g)qt*!hB72N25y^RO93^0%0Q6wW+!JQ1Or*b(h#gnJZ@k zf|QKxSN-dE&wm{svbUi6m4!6^>%sm@jGh$#9+v=oQbc=(&egL=#$nJ4muu6I3tuF( z{JUJGRSI8_-VP1PQ&Mm!G7W3c)2QxBO36;*zdU$(R%Rt*Rh%RrzEj%uj%jE_e|tX-rUJF zYX5jXb-En=!-@ylc?FR!0+r?sCMMZoiSaZ~5v%1xqA0RyXoxrH=qTX3yfp@Nxc!JN~N*u!>egJT7$zA(@8Bw25dzR#w>YNO$rVINXe}T1|=~Xfv2jh zY{O55=oB8XcR9x@K|O`xAFVPn^uMi!OfEl0|t<+!Dg z9_3P3EblGL;UecKC^Ro{R(jT+8?ah5OX5#8l+6lOA;3MO&M7w8TG?fgfJD%LH@WwP?|tG3IF{ve z$uk^cE4w!Wr}fXLWJ*kjIaMnS{LCi{BArY()1Qw^y?hDLZi+|D{Dzg^9+5p7cTmEX z#xA6c6!D{HphN_Js8R(f@+h?Vib!hMQ~)QgxQ&*3Qdwpjl5O&A+}&!(Jxbc5g%j7- zG|VM`ER8(&EeeMTWk&Urx59Un#^g{2*f}ZFzJI-Lt3bM>bsq zABVWZC^w-k7dTy76iA)0RHUwNP3}+m*jjuvHV91(B#wErvs7smkIm8ri!^gZOw^$& zV%bP&tR{O{yRQlax z&n{QogMWAZt_>LiC$k9pcqPk`M)_zSgS=%!Bh{(A(JDW4*iMd5WRBUUM(27{)Q_$o z-D)OTg~Qw4Kc8+^O?q-RS6I5ZK)3ix8m_i#naZbexqj90x9Z`iYOZ*-^MCk@dvpV7 z2pQO$oZbFbOkb7fe!it;XHRp8Nh^My8Hh3~a)-uakV4r;Y8Tydpf(ao&zzPg_oS?& z@g0xh26r^4M^&0xg2`c!K}o)0F?)u&Lu7*K=7#2cTTJQf-Kej0FtlyPe1Z_%nwNt- z2l5<8Z`FxIX3BKsBLrj*Y%eO`dYe|VaFyK~;bQOp@HDZj@a81t?FeGW98$_FI$rnP z2vPk)0deC_h9vR|C8HNTC&|j@BYBHsMrNFy`JSGtUkm1FWzde_VJDaeupZy-?Cl$| zksGB!*-Xz7zw?^6d|`RvdO347VXfUiM>aV{L*oC%c*-!XS)QO3niuAUU21)!`028% z&`a~wj4Zt=-$*ySj~-@k2Z@eJAUANvw}s`o8wBgplw=zwq#2I9bWqHX?WUiQU!6iZ z;gA|)--^f>UPH|3(HGvg@9OKc5)4=)kMAS0vVv8-%w@Vh>_xOaJ+yl|kd@4a_VnrI zTkrF8XWI*}rR6ly%xaoD3T0oE%Gz$f;ea;at}=$yB*dpGpmIXz;}qvsuj_Vp(i*<@U^>H)9odCf|KCYB}p1QG53h)6^4(uceo zOgtxLiXtRA4~{MPPP=O_Hj`yh)72#vmJrRL_tI`TZb_apw;wWDoIhnKYL>k`0*UQ& zOU%Rmz{<+{v{W|?m3CTs+IXlU#m}C1*bsctk_@GUgzI|#y`gC}_v}7J?UEz~Ez6)d zev7;~n~~=%$no@Kl?JXpS3AW~`hC=9lscnvk2^9L_^oFioo`?ar)f)t;w%n5&{dN$ z-wb-GdU5VT)7cwV+=5hn*o4w`Y^ZSY%V`cc1LqLWrH}qUe8U#z?Lf8ij@wT?f1XYs z%$xbAXV*{3@Qhg?XIiWnQgu1XVbSb_Q#$a9B>sd>xtBBof5H*jk+@~ym%=63mp!fmq8OdLA@&1tmd9`{qHd6S3WK3H1-r`YA} z)}X*&3bUW+iwq>y=zmyf1=o!e5Y3qA+Gvgsz2Yb_!b`$JBTzrodj9(F{MLRx9g z^UNsqv-MTmafP3eXw$TsSu_G1GQQ~ap@%^ea_y*uS(S(>HzbB$=~QR~3|ID%8V>ym z@~P9GJkm4LyU(7t+4i1OOF@(jb8}?0I=e0;255$!*^mP3E4lNCB|b5sd6oRgBCA4~qf=W;i-u$4L`dRcWb+LBoD6SC!LBTHFNn!D zuc5P|CS(s~R=!uv`9_Njkm^{CD0<(S0cPCdk`+shamFM@)QL_ z2=uK5!&j*m+1H96WCvyQb2ikXuWuv{x^j-?a}~eYva%Um6?w3_50QlB3{R+%(WpJ@ z$`Caluv_azB<(pyuO&;Fo|JeC(0lq9B;L!O{erTmdR`gEbql{vD8k*NRiB%^;>3CS zzu4};ox55I4R5{{=0?;>V0YH1ULL(CIo@KdJ=#G!@7om`5UR-R3?C;VyYq$3oo^=8 z!QFAlO+r*pILDOG;Y(mdR-)F)^f)fhn70132T9%biS?=i<+$Xj!;c9*=$!TS&Z|q! zVvDqn@aq-Z^_T~$Www5Kv+@);TO}Rm3$9TwLzQeczYoPOay~#aO zv0iaJ6`(v7Jpb|-9iJjQnkPo61pS@81!j77_8S6IdxFv!3s$>D!$~vw&qsu>pG4yi z=9WW7x?q^)A|fWLCPPK3uTBU&Qg)hQRNGT}8^#ST7Z*Pa3u8A3H_{6Vv{;9RHJ?%` z_~Td~1oh_djG7Isk2>OZS&_K6&Bo;okF6~0_vC$USy@ytYC-Z#~bP zb-m(o`SaQFBB8P#3$*IGGMHXozJuJ!)JHtm;xb+e)$=s}ncy3z71!|m;Z&*0&E9NR zJ@Hm2w4U#N^^$>wU7N3IG&jF?M$l)bkQ1?>W86u|J-x!Z4M#yS+dzA5BQo9+?FP8f z6-;mrEt$0W(w>h}NH33~bDAcxD3%O`8%Q{K&)Du$y0ev~DfBw7*7J?BaQ7bi-pA`&`5PD+0Z0dhxmM9S4+c1K&xlx7?XCFiDVy9 zl2)?~C#qe=p`kYpD=B$`&s-#F{-xSJeu|!(ue<_RB7E$f@UV=G7ge#z1VPzsTrGWQ zHv+2(H+KBY`_{eSK-IO7=~Em`|6~55_3e#~&6P+;oSyvPpURo}W&?-MQ~{uNd7_JA zr4|c)lx2u!PlWJuvUbf9Edd|f1ti!n&54AphJvgk;MVP$k6^hq-{JAx`gqDY=&i!yNi5Pqhub{RHG0oB`jr6QOR1NU-RPM zi=-z)d=SGms9J7{bA&I_vEyROSYL~}=Oe|LQYMXuhbcT2P+Zo@UyF#!J>8^tc0N~? z*N`W3e_%&R$0vG6&O}Y@S$1_yPL$IDqX2`n=R`_V29+6W(lCq@Ij$_`b*e{VTnwd; zk27Z08Y#z`{`9WK(YPMXE;ZR^9_D*ixaT9r!?frth0bFVkO}OEu9sJSblY48L==*GehN}m$i2QG zE*uxnvM6(?v$$gP{?2#3IGO0Pu6A3J&v9j3e-A>;dAepI$n~U<$i55`bDJ|VZTwS^ zpK-eGv>#u!8%c1eV_zLOcfy5m+*Pixf8W2rU$!=G6W?d}SX#ujvmIPA$8GrS@sA82 zb+L-Atniw|pY$6px?DTYxde}P`&Xb19+Yjo?xVxwB=fGdV=F;JLgeYczid7J2YuBv^HvE1A%n(l&YB%r=dl*#)9yyh8#XY=U zFk}4YeDVFc?Wn5WEMWkQ@XdZL!pdtNIOB57~DMNYiA zIT$pD90+R4wlv~zGpfs%H7mR{jiIa=2uGqXdl_6=l{iVzU9=(N)s9qrjYMF^(K(Xj z93nm|PAQhzA}u?bq@X4&^x`#R^(0g_9G1Z+HR?Y37*b8((%scJ)TjW}*H3$Cnj=G+ zziHetvx$J>WURL%R5NlyKQc0r6*X3|)1xFMB{g`HDfwL2awI9-$JjilT*vx_ZpHWa>BxglBUPN!%rge*TAh{=bUo2$vz-gEuI2o7G7 z;{!l3h+h&f{Fj^s^2RIZBTh3$K*Qcs$HOJ%Z6;ULK9tHYZfBDap6v_gH{&QZHt5yG z$!}W3f1U=_mg?s!LGa1AUSxTUhC1j8)FSp&_8c5Xvf8M4q3%Y_nwF}#^YM-46Kv0O zf$NZX%QRSBVCbJ-L*RrZ7 zO1|HU+w&-x#vsDV3Xu``n)aNq(%849`SFSS7vG35jw^j#>PtZ$sqw*Eub2zpbrGT% zugTEJ5PW$c>7pdeYTN2QdHl9crW6z>F!A4Nq(;WGPYkCeC%oTLZ{h6q=hOB(=n%^sh4=Emz{ySq&N?!DpKV+t#Yjq z3hW&i>1adf(LACv6_rb@Wi8yj&FSZM#&UD=?Uo|K!YHHFY;u{o)VnX1r6Gncc9OL9 z^ZtR?V_zQ7%xSHkSn1|B*Aks#(+C?(TNLJvJg6;~&DHX@v=h{zI&Czd$?$W_UVzUJY^p^V}?39L@sg$!K{EolS9cZRg8V z#@@jb#e&N+!Rea^wk7FVWLz2VRZ^8Qr`{H82~P2A=Md5!^N-w!qw%iuPL0MLY2Yhz zOI@s>+b(TY9P^I3nUH5r`I=6?t=1Y-$_sJ8rZZ4)G0Zl}v0r^>Xj}0(4M$3KOqiyJ zI8FNwC3P=-3~8aoNk=8{X^jPB!UHDCcw#}nYBc|hRfTn;F!oRwCP+1=Z@jo>p1VHJ z^fVqYF-Lu#DY@x0-7q_5>^|6Vf;?Bdr=EMfed0VOjz*onK{{LWP8FIVFI#^ib8TPL zNm|dRni0%&Fl!+(#$j}+=PnVHZ_$zrKBPXLc(Y;~vh3PWv4lUZGF5&(m6^v^G7-yH z(R%UuwBi!dkz#RCPT=PKtfE74t)3!Y&ZD5hJ?&EDbXT)-DR^Z?M6 zKfbdeGiuu^r{xG{&ra&|Si2R^eE>fbNB9QK_$WO^s%+ z*9wQ{9_TI5T@$y?8g9P0&+DuhrerliU&()!k~dw_OKg^nU!a zEvnsbAYM^5h8bBydg(Z#} zrhDCU1VKqUuj75_8K$b8V`{q6=1khzk;?^rpomYvKCSmUnn=JR45?4PWoH;Q$8=*E zgs9zLj#rw-mtCe09MqSuSIjhT^{JK7-1m!G=}DN9$Q@XAF@S0bf?Gb{jNfCYOF_J}n>9*NM?#k?FVOPT#e_CW~nOd~N5W~$FgdWm|qkkHD z=Rr{TdW#`PB38b)8jf*pUSWhmEN-`-`j74d3;#5w9CjjO@(i7FrrenkmBJ-h1o@pq z-<~r;1p|?F!LX=7>a7WO#~>8A%T`7KE_<{^V)b8LHiuV5vJXeBnf~@yg2~!CS)tiD z`t!+b)>m}*u-EM^l6!pf%P%CBUgp~#71W$h6$XblrK|3^#M~e_bgzkUU5DqFy17Sc zTt)@EjUS%W7_U5}Da;DLFM62HOf92il@qE+n5%DQP`BwX6F6Qemw`&(cf@JpVB!hW z%)Qo5*Y)Y%Kq`JfTOFM(ACX8!aj~*S9jzX3|i%)LSh=! zcce@7RpxZWsI5@n9S1t*)d{L%$89Sf|9^zNc{J4h-#>n}Xp?kRD1{=)mdci~g%C0d zgPE};yRj5wH-r#n-?EG)%V03J89SB45XLrR8!u#F38Ff^>(jN8g{Jyb_qmk->7G*m%U zODxtUm`kJdp(^USG;ZoZ$ii)~WZz_2MckRU zC+c*wvp4gSM++#%en=+`2U?b{&M)C@-$okXY3rD zT2b;c?j(CR*y_a6t8;@@@9m=CD&dWm(?#7MrV*~pHVA8#*VaAvPTM%v{7y5;v~Ayi zJ0LBA;Yxuo1g@;!t3@ce4t5gn^W}TZV>gllcp5c!W}Ncv%sqW;f}a(&il&xPj2q=w zi{6V7&fD!Q{uOYrXPmQLy|tJ5F>c0;?iyDTs}k(jXZ}x3SSfz$5eESLSVKdT+oif> zC2@5yh#N}4lijvY9s>$%WE8+3gPoFsM*;xNRq63`qA2cx|{FPLv&s4&A zykI(qQ{qJY4TTtR;teOoFd(-_wG=y`L62Pp{DF0Spa?q`_}#iBSlD!mSr%aek5*Qm zTFttq5KNr%*xJH>7A3AeKyb4+nx*9Wa7!kk*8`@;Mpal1&pcY1o-EZnDgRSNLZqg= zTDPKg@!l@n6MGXjDr|8~UfY0IY!UggxYeX$z}W~ruey#9+~f|v1!5>Bp9?+l5nifH zX>J5}J#kOmn!A+_aI3$TA992JB3QUA_6^d-CsU@FHq>-@16^mTm&l(c*&gjN7x505 zy4T!X`B_jgf(FHPW4Bdqh`Uw~(N8Zme$Zb^j`Yq!8VR#0c#Mfk+ii>{ z+*zIPlWcLPE|hzabFvdtplIrljGHOKqh@YW2-Q7fwC#eeXc}4+P{iFLPl@@EcYbUd zTISnKn>#tfrvn={CNCBgS6*NJ&F16O`7xxTk#A$%p}%Lcd$CY0IH#(kNT*iH^A}Wt z)xv#k zwuaP|6^C=&$=4m+idznoX@;*3w|WeQ9FwuFm9F2Iy9%45qg1%qbw&+Mii7Fy71_y| zDYxRRTH1Um!%lXEF}&|Lp#9{$Lv!!NqkaK84SO5Gw;ltvN|p8(AiLWquiCXlaIOx0 z;{G+}^CU0Juj~00*@ufcjx`eI4HnrST1+f0Ds@!7pqSwLk+voZsq4s?2i;m@ZfBo~ zb$M1elhoF4N&TXAQvbGh$eUEf3G%>iF2;`*5{sI4*Q(VGfn;x z+j~?OON+U4%4X&R;g_*LBw8u3V>tCCar?0H;lfIN7QiMxaJwL(;&UwHn<8;)z~Wb? zPsdTgMLvaL1>d?zOMYgJym{ABwDh;GdqzI~kvIh7#bY4-Z-H}91X zDga!V?W04w&DC+O35wh|A||+{#?o^Cd6JnKrn=}e+p}+HyHmhPdh<7tZiK!8LWlaZ z!vW8EVZOM2Szo)wVx)9o_%a|SDtp_%_4|jXA7`qGp8<*n(+t~p<_O}0E4kasdn?X9 zQx(IXl3P`Os#{v6-+S6B)6&Ln(4rKauT&A2)*!sSV`S-4iCrFR5 zT->7FHP-n(zJWch+X&Mpl~Xo0d5wQ?#Y|2={b&941h(v2Wp;D$>n?Reo0Hsl1#SN8 zvofo;5qziHTnHt(~QXZE^<5o9xB|ul_ADSh95Ifq&`uNe2V+od@S|>yiNEG}2%X zY9CxXNF}L1OHaRY-)8UWF5T3=;q^0#P6r#~{E^ZFS)Y(e+UixJA_;XCNq0ZIcWk~| zIyT`fQjBo!?%qMD8mb$@4p0{H^ih)1=LP8L08>=Jg^*_@4>cb5f=VH=-iurl4iy#B zH(-scW9~UWe8;5j_Uf_d=7@$=ysD|vP0Fzx9UF_Jf?FuPoUIF@x+Kx_Mp)wvdPB8! zq_Ip&4>3dE@FJnq(iyS!sm6Bt;3VZ_@_htj4g2L!FHcZpO{CEJUgeYig`_qWzvh8d zb_>bPzYh01k!G(K1T014Y96;=iD5(%-LEM=X+`{e%w%Gr5@nmDV9ekz{RU>x&&oi& zTSZV(x!UUqDYBkrhJbme*!bv{Gwuzve*#5H^OSihYp~tq6j09IGUsS_jr|m&x=~$K zfAYHalRHW1HaQ$9MHDTwrOc80-J)6ZR?KaJvtk3U;p2pui51(z2wX(69sUqu^N|v^ zznO5t2H5Id!93_fbk7}HD3#2&;!LNAVw#JSd$DLS+K)D-Zpwe{0*FI(`*JAlNG1@m zPAdl+-C_;9>w4xLV(8If({_22XO0~ax>Qt&yPs+$4vKypF%W z<>t5Y@xj*3B)@K@N7S?y9YN+~seEf{7@7b2&dS^9#KG2g<+-x<6Lph|wCUKfQb(|? z82j$1!t6J?Cm)nUgs;Wjs$rw7s$%6?&$nk4X3v*bm^?k1yvJ*zO*OpAs99B&N#WV6 z2WWT>ieIVH&VYEp$2(l}mtppaO)VJPEwrEs={(^V6qzl+wbq-9xmN7{AnSgq-8Gk! zQ&m}cc@pSh+gZ6s$3n)&i}Vd}Ptaa=4h3&P3N>PO!2vW+(<42l2_ue`tkp_8))5=oK7QXu~0XaU*^vH9m^W&z95oY|G19S@6J?&4hfp_c< zgF1hWM`q4V1y>M$J0)(blI_VuPNo~?UgAwb^(u>I*EfbK3%Z3SwoN$lgA>x z+v?1?8l)^XixfFqOBXx)x0>JCO^X02q`XBbjPjj->fuq)all1*HAEUxpmpn-rPO|> zbG;_IUxw$1^Deni8c3%Y+>N~upA{i-#mw|3II#kh)T|4fvRf1b^pc{1-0;CfRfeKk zw5@|(&#pEw!G?2O#Q8Cv9njL^SYp20iNd;ncCKn@@Oz<#lIqXm^%8(eF2PCp)q$RO z&6Cqpgd>5M1&wL}LphedMpmZlgbFtFKx1?*uYI+%K8*&^F>P0ri}4JLNL=-4x=9T# znriFsI`F{jL^l4F5Tmy93&UhmB2T;fH}{Ww?md zWi*~ZI-A5+aNVxGauK^hH0Cke%qglw5OmmIJEar_rYHNvpY0bA;efc}8D#I@ z{;3(sclJFam)7-GiVw#M5V#N4npAH;#0dMww*?2i{n3x z$bbKPUU%`AI;Z!KV{rc>*4*%1yiQ@om%a-e5BC@e2DOl06ArFIJd3RPQM6L+RlX4w zu4%vTi$jR}AbdRZyfVt(+j5i^)Rk>kok+gZLRFroP4Y3~NNGaO~u3g&|4P=C#ttN2(1sx!4DkN~w^?lpE#MbBB=*uR^Z2dAXiJ zCj}i8WeR=|y}-jSZGL|!$t&aS^AuDLWh;mw@3Tvl7s0d0s5cy>HxVYy_m+w=d?wo8 z#y6LupGQ-Bik2DOYRAmOB{Qe`ihkGpH9CvIyqcxG^X%-wL)1i4JI+cx@IKTNx9GU% ziG3Ok>Kgx;GnZZC_zunQ^{e1Q`3l!r%jM}`^A(G!J>{&0dqdpWDxUGMelf`rN^L+7 z3DmH!tmuxUYtNbTY60#P@P6>ghF}FKZJ;~=9AoH{-ACN~#lwkMEefjLAG;KPk_r|? zEzfozwU<+#q#8&ip%%-A41z@PLZ+*WM`eRLd8w0|j0^sjzRZoL{w8Iku-EkZSlT@S z*S4vB?~`&&4o(N)NZY-XP{sFQBMP^yhniM1K41D&FDZKhE!9sVC->s5I5ikdzL z{{D+Yt#n^_i%IV3h{R@2L%484hgvNzehV$0e1xWX;W39J_A0Wd;$2Y%<(?W!KgDfa z9^GB5-FCEuMCud3jFpB6WdfBz(d-^HA5GcYeWViqAzJ$xEbpEaX`0mSCAO%`!8IoH z>T@@6w_(iV^D`zsVTe9>Jx0r4#{5!G^^_REiQt5IuO>2qq6V`N38mjTb+%)T9?DX1 ze_2;I4Zm(>*9WmlfFsZl3BmD=GK*s235FVdgla?VPP zYaFvBdb`Z7VtfBnH(Wi=sv9DneT@5e-4OGd$G`IVps>iK5&N5F>AL|l_PgH> zGuM@Z4v{T(3ttmWhK|GJuT*rajHWn5`Byu_hOxP` z2q|B-Ew8qj^-GAzYQ0+2-Pb6|uz%B9^Zc_AR3g`>F*{DqbxM@CzHtFIJ1Nqr!L+e) z0yV1)=o*=PrnvJyuKpD*Ic2sr@T;fJ;Mxt7F;V#LSPltae{?0aT{cYm`X z%Q|_zOUqKr6~$`#%2x|#ipJPeyR9sPw9z7L%;d}@p~&|FH?0(0R7+ePU1$LhQ|Wb& z(c<9;DlW|0^CPLQigVd0%alV2NZwG#2-I0m^+Td?m2N2A}x_HnKu_3$udXl`Cg(G3Pvf9Y5KXD(6;hu`# zyMF2*w#D8SWFD^*j%LxQIBd&A8VfBW?duFQ1r;$Xbh#&hNZo_U8<^CBa z&7~OPdGp4D)?|C6?H>WFx6U8Q-dv@}(|9WWAr>Dvv=u&w5ab55saTIe(O4O{eW^W1r!#rH}3 z^xKI+npgftON||@ayGgi#=F)9^r%i-N~MvppFpPm=I!QkvKiACT*)GfWeGWxO?7 z(6PX<|0qsbTIRQkO9j?R4~?o<5rKtgkspVOM;s3XJ-+1|>6*}PHQh7Paw6cxNH<~U zZ`<*dE!nDxWHx>;PG33G{0;$wodON{7AgVs9xf4)>LK4xjcsObKqS zd4vYAp?qpYhZ)W=0wFE^91~hfaa$MTdx+MO-{llm|V-!YQ zlITPh+;ekc-X<*{Ib>iIHYO(oPNL?F+Q00XE6_eE&(F2feHIQzvdY|N!~tATO;q)e z?F$_jF620WA+4#ZM2*`$ODdQ-C#R#!`B+Q0=zU+7AWFa9Oxj{bz!-;=%v)5SKx``i z(KDe>vGhzJt;E@XEl(R2KK?PBd-b~Hl5vUqPb=?@+mgMiWh1YASoSRS(3%5Ucn$r* zriWer+HDQpu+Kth!Oa#$o-RebZeghStubSpld)1nr@?C5mrl4*85=%}RZjZ!F8Sm} zcfeH(VR5wMeBIaTHE2$9lT-O>+~l^ooaVX9rgLB9aQ(f2!ZAOHn&WbY?(TtT{`_8k zB+glE@PU`i(j~n7pqY>6U17tC!w=;)0ah-}7yYPGuyKn@{tsdYhfnX%ty zU1Y5<4qQG`*A~DqYS=F^Ap5+h^P`De}FZ1|%Hq8nPgvCjCj$l6%ITwsu$$3zz zTE)A#oI5Ail>69F<7@o^OR~=M`6)1YJVwA@E!BN;t*E@F^r?m}`+PO#nwU)_XY6Mb zGZe@Hke&I$;oc*pw-$d1S*@TiE)&z8C4q`*GZUAkmOWbKGgvAFpY(KBL_Z3j>i1%S zs?SZ&v$W&l?*%*mH9S$-bL!CK0GQm%kb6^OTL?x!#_?dCb@RkT&z{s z1G(l;27b==_#U($SQlkSX0ab`XWCM(md?kw=*|ycIaY`PVofndAI2_)_&Dwq)n2!7 zBZ;Y9oG?BTEd#W;0*{s2wl9?Qnbh4os)QLV8O4__>X*h6 zJ)mu;r*?E_o`==f6Q`>k=e;>@vZ2iPsKQBeZ}Fd(IfP=usT4219`;OBz!p0I8Pu)> zt_*C-{Ibs296yPCp^#Hn{uuGQsej#jBB$;VfK$*_0M=HHTvN&UsSxql`ewiVROkGi zzeu^8^>2WSejPd7_>1mk1Iw!r12n60be*=?-$@ zL<7sifQl@c#*5WB>X6d7fmK?;P=Z5GS&MdexRuIxr1y-A#3|d6!YN3-vD?9fo?-qG zaq-hB+1a;@w6@X&f@o&=zCO6?n>^Y%y#L?cD~%N;OT)JX-2JT%=)cPW4tXYRiI=W? zP%s4Y`sF1r(1udWc}+W2l`akXRs7*^S|F@B6b!!#AnyM=p-_oF{m05Wc}4HWFXawx z0zhNiUZwY;*=<4oO(oqDPqwN}p8MO(aD#C;!&Im60%IibXJ~#z_bwzxOj(w$+NZL! zA=YrhS4pfjwmJ^?4)P43i(#!sAtNO@e6ZrYOW}kFo62q*%WYEU3I7^dIv||Hr%> zt#49%klE9yy*>TyJ&(x)#5{@TNirXZ!($KRX*-L9$Rm;ZFji!=o3_~98^Iy|11_*= z5)t;VeDJ{^njVJHCy*vOZqLH!wtLe;4-^u!kz#n00g#apRY-v^zEW3(k;AmoxaTdb zUG4KOH>t3rT6$>yeR@C-3w?G}W9ONxQoMRK)BM9?18F*?DTl<#eq81|Oa9T_oqEOQ zWcR!S-(t|=Z*D7J9I0QXs3a@q1m*iN8gH3W9VdSn{^5%Xugfd0Zh3M4 zsxH~*N7!X6(HW5ti&8baL~^Z{h=IU*pyXpNg*jVw@+Fqe400J}w`=G(m!w#5zrmcB z*8ZSjIlMYJN65BKEmFrGk{U5=SUZjMe-wUB`p;-nh^6pDLWwB{4^s1gbQXV}Mn~8L z!?~Mn;x1igDJ6s348?z)J#7^tBJ=j5lQKq1#Cde_+op(bZ(U>aCKy?|8fC{gIe#-9 zZIog+CW331pz&&b*dE~C?GZ=MsS3DUVCPmY1%+xYT@|R3;9x{DJl%p!7o^BysB1D0 zD-=K9$W2jasrk(U;C7!bV4Ri($S9?lLx=ycN4RBQsa zJ#+AqaZ)h=oTU|YW7|NARTx4RzAsGPAKDH)MP9h$x>z?3%xLUC=zStjwOHDpqOpqB z6jsqn*cUz2EZ>{cSL1P#aGB^kfqbF-?+~|#TYt<7aTO18ek!N!$KFW2n@A&P-mkhV zudfk5G~Z$eorf|;aiW)Un(Z^ThJIK~2on(#-bloO^Mu^=e`d3;6NcD+o&z{(im<=m zy1L&mcWI4KsTFNIU#M3R^!mDyYFT`0$}Ld_IQ%BZBk;WplN%gTn7ktw>qfUfKS}EA z<35{we9Nocu@#@0p(hP$3L-e!6w1gh4SRMEh=^#oHSXA60pwZ@J3rxGs9GLdK-52~ zXgO5qF!9u1v$eJDZWZdD>)tdl=`{PIq%6H;i>R|PZ*)fPZAkUw{pArBVTxYe(Hrwk zB7B0(Sb?#?3SW`5rU|xYqRXTpOEw|}ZuWn7oOgxq>yJT|4YJX>gwsNiU+=mispx=E z;AiENwt6Ec{0@fN@PiqY%afeR=M(Jip(iWgK0)X)Befjl#m%d6Sc3mr(niI+8ak3{ zI^6W~4Tz`uSL*c7u{Z4A@L~B$m1b|D80=UZt7-bfc<$|ttU|OD)yLU7b+2d#*#d{#Jk5=Xd!!KC1(2`@zJ+A1DsMT8P1TS*5wQtsHU%B~<>3fd9z}k3 z1!ZX;B%0f9=%?T8MV1k(A~^sZ{wiQ-0^_Go&NLOL>K+B>-*KHz2!bkFyi(~1xWnYr zD6=jwz(OWVNc~C9NaA2NdiB-s)n+e`VxW#+XB~axGSuQ*<2oTmP&qifqVrE##QN3z zn;trZ0_D6c(t#Ff&{_?vCJM#B{?KHQShuWZ{oCMC@@pZ;(X_dY08{^H%4kV`s{7-C`e3+ zPPxRX+LvRxyT1kaNr4~22`>J)vtuyFD082I5`NF&o@}!LJirx%>}K8SDv%QdvfTeA zjm$PUrswZAx46)CkNG~;GcI#ER_C46pm@8CIJw_)bPqa*Kd2Yyx|Aw_ z`2Zi~G!B^Xn#&!^7G^GsPRxA?t`M?5I8B8vk%|4|5xn3^>+I%c`m++}(L4H#nH<~l zx8@(r#V)=p>RD2`SQvcpTt^S+6BqUT<;%tOH+jt2Kg&j0S)A3#jA$j#{xfy@-$8h3 zyV39Gijy^3!td(W z#x*H{gGc->WuC-AL(Jl-{1LrAM?;+TwC)hA780>RwEF)qf(nliU z8^bhCB0;sSUfl!h9HT!kXYJTZS@cHYdP;T4#hrC<{qTttD+2x-eJkuFeyrw1Pwopo zsItLoVAbm(JD;E*6;}Q~Pc>}4`P(vWvM}6n!7095%!TCQh3C1jr zTJzrD%{$S>VjCNRmZw+jSdu8ylpb|mZR!Mx3LByW6evY?wj&YF3J+2FP3mZau?GQ* zG2y0DmertE%#qU4@msrnY1-ZLv3$hAoX0lS;$Ct~mngfou9_?FrV&AP69>)0gML5i z5ZVk^DvQv%wEhPo<2x;c+zwuMUh;UyPLd=p=9H(k<3zZV#Y_VY>Cx^8(9mVl0g0& z+&q&668iJYBzrSBAG`l(qgz{(h9=5H4Rv;xCU1~(1d)Fdxr+5kif}3WIl#H*`)EZ* z)UGY15QXabdPCS_zXRp>{h3sN2TRx8u7Bv~Q#JW}7-ncS?KmJ*ekt=-z&X*RoNUkL zx%s6nFaI}F?giWjrKJ@qD2F!j?UpSZyTVul_(Phw$eJp!)qg8wEle=zNCDAoCJdvJ5gBpU(|5(ntXz7-ef~+(lPg&Y zx^${f@!77cVt!ZH7;5n{@?Hh~74=vC6d;8xWSDB$j{p4MixU6I0bAZj+T`~%;=~P? zW_}6udRr&H3fpAKZz3PR3TiA9$8cp%eP_;A=(oeSxG>rl{gij0sgi-V_x%~>V$yVp zguyRxFghf)DaY!DxV#DeW%&Ypd%LLiiLgClKRd$K1J2tNCX%8XMgJUn7X)!TZ1AJZ z50FfH;;SWgd*X^3nJx%Zgcj;WO}Dbh>a-PWZer2cj~lK)?@U3;aAl93B&xlKXw{Z=Sn+fIPqjA)6uI~uO z&=h$O8g_`gFH%w@?s|W_5oUg569e1Xz3NV=z|as)2h06>sOUu4?M|*PZKD8B^1s5&Qrl zljd1*qBuLNMhi&X{N3rWI^+5=qz2^C1oU%IlV=C=wHR0}=jZIW6Fg~~`Xd|KyT< z>E=&$tW+C+Bt|5!-oN~2P;dXrRCR0%z4crIRBQ(U!Q8KgP>Rw%>p5vx|z5J8_`@naz) zg)O_xa4UVR=m6)( zIsBW+W44diK0GJrFo%MwR&4Qx@s<){?Ao<$Q1@3OD4Ee(e~QD#c!q;5h}vm5^i(+J zu&qTg1cSb_sd~`TAkS>ATW%dlEtE`FSId>C#^t^(KK12olN4>exVYN)nnH0$1zLvQjNe`hc;EqjER!Hqajo~6A)KN?2thJ%zh#55;XKGK0E z;uAS3&g(ANGs>zNTzg;STU~>3?;<(vfntUg;mGDvU`)7uw8ZP5Am96vXJxz{Q!5mM zlNYi8H-nab{D%5Sjx?Lrnbnsl7cb3Hm*gfs)Qtt@cUPAIhTs0+omG^CfNeks%X6W? zMR4v)XU^1!TQ{`QVmk+O{6_Zl^aG6~iLWCQiCsAUUk%H#!v4$U&iZ_rQk=79DuSnU ze8Wqq_Di6M<`{CFYG4^CItrAt;khkqu>YG%xg3;@=*8CM@6-jPf+=;9p92fVr5csZ z7r4^Do$Ea(_LA(LcOTQdMMH}$i?OSBr0q1Q!cy9}p)?emxVITwmK>K)ud5a$`rA4D zxdr-QdQ*S7z%087o)T;|<+5!S9pvCS*m5+@w3dNUy+dLr4ieTc9-idFq|R^KMzGuT-haKG#V-{TXgpxqT6uWA^mSx_uBQV)Xm4vkBqd^meHi|Rr_T}llQacC$?Kw`QiRmZWZJC zmY?xr4x&aSd2mHkUBv{`OeHlrx$@g0AI3+2uM%Xl(okx2-h0K7RFr|uu-bhiS9^zq zE0V<3VSm)nEdSQ0-bg8oAC*A-OZuU6}6h8?+t{@1#|50&#{K^}TDRWa)&VgT{ z9OQcU#4_1Q?cL5GNr~pb&j3W^z^pu}80NKU+E+kJYoivH+jt||*s{Ko^X#ri0sD3+P55-U5Z-__4L;*LV0Ill+R$?v;5F!(^b zkS$)2L4w_4%d17(4g&Us-Ld88AfvYda0}?VuH>v%hV=30!PBG2P!DVRc9y{IMP< zW##werNGxX($H*LFfD@Luj>8>iWLO*J)L-_s?N@;6S%dGnd~gskogLmMOX zVh}l#{O^T2g508OQ7{+!X+T3@sunk}Qa;^`osOvOeJ7Bk&B_4Rudvj6!n0-f?EjSx z@=ZRWOJ&BSv}*>-g*ED~Kiy_`tMj(B#ESHYou}b^3187Fesb`~^Fuf9 zRJTJg7Ie(vDY0YJu^`D-{k?k$uKT=whvS~yMNTA6RRv+kG;USkcBeaezv$rC79%%? zEIq9XTh!Z67n%08YTZ}R3SJoRpQUcdxx&tf3HZ_ScYZFuG~z7svQjd>gblb|!!m|R ze>QQ|8$h&?O6> z?q4@*vIuMIX~ax#Hjtegf5&!kf10o@zf`le=-ziW)FZ%+w7=OJi|m@VBFFW3{&V(; z7O!)YYM<`{$Xn=>Eytfe^%D`7))Q#>anr)%8ysCQWPKxL3L54$N;%jc=+T=Zk`52s zx#!NDR9nr;GPgj`FSP^^->bw`ouIDBp#3gse@2X~;sgct!>i?cclw*gN*cP~e=XIZJ=l1PYWXcvHgn~oOu+ZZ>i0fv%o==gmQvRJ)b4525>r6cdJ(3iz+k)3 zkaHqYt2&nXkfTL^!NDz9UzR(zd1=lA;Sz_-KIy+VMVIQ+V;_go=>W<=JfGb}(~>F-Pw; z>a*L56#jSRZ)0w0YE(zU)W?1YxDYl`pGQ{c^;d8&>o@th!@$t ztOSI_ei6r>QJsyn7f7foFLNO*tZ<9Z3lNjU)Uh#pmEcNk+4I!J##rmDSU#D(*sPbk1O1Rh$m^R$-EPa=q)?>3`@+A(4lKWda*g94jt^D~;=* zdX2?cNS&cM>-wdktyf%)Ghsa9(n1!J118Gb43wFeM<_-K5LBc6QfkF@Re6RJKesQ zdj#s_jhAVXo7e)nCJ5_NNklv=EC^5L>L!J^V+Ma0icDli(A^ICRJs3%pdMDUrgC+O z={A%9I$!OSPoZ@_OSnjrv&Aq-brI0gskHXGzD?w{^!T{8-^2-%@PPHRRYMX9uCt0D zH}@)^Tk5LmjX_Bko^<0Dn8|c!u9wAOo37HY+47^N71L$V zas*imtu$gSIi=2hm-iW@SPIN+jLMIM1?ab|rEmKvGd7@A-NxK^#iY7PD5-!*`)l&n zs$#k{)@#+Rb_cOEB7lqRfDXI8Mg+;RRucm{(2#P=bg)LncvCG)#0r+f7ni^PT?6_6 zzTo1JP`Xp0^oLB-kzy^FTSmpR|4TOTfcJPtK6TvEL70A$u<@>~Y#4g=>Q?M*G< z?$l^DWJTK#TZ_Q)VU;*Ff@qc0#Y6u3Mx0vPZjZG8Pi)GHV@*zZZ}V*_1Ji83&?(oM z>XFRSErGU7GzjEH#8r%ZdHApSAhSj%)0 zy%sLZ|MOA*W6qIgxur##P}oOalf=B{vSE!toKV1yKiV9dpC{`$l4kQc{O6>M9k@w} z`}q0GJ4X*Vq2dzB>CPcv6YSFI?vQ3E;zvKP(0?uzx!iR$OkwE~=3K4sJb?VGMs-}q z?=gPrR6=^5UQx_Y=hSAOiiWP`KuK@u${wwQRIa@1sb<#QrijPmi4J!ZHEMtvlZ*FQ zCV&QN)8u^ z9nzlmS_@*;YJtk8a6p!th(LH<=O*3foqTH<3U{Cb<(0oJEL-q`gH&jP87lsAIz;m+ z5%2T-57rxdl$AKNKi~8?{XYluzdl_%8}j)YeB&={p=GVJM&ZFOPW5)_r7_hav8;@u zi`pqGR|Q(kXI?%ymAYk5c2h$+)`D2>Q^udQ?mGC<;r5`f@2pCXK2n@Qk%_rA!$aKCG3-v+{0T(LhQ9Tpo~Zo(sg#Z9iXk zhi;cBpq7V4lj!-b$T*wl$B{oJSBWkb6`gXE`5YYLT)1dpI8wuFtUh>f>0=Z(d}ZH~ z0#3%VKIzjZRyQS6xfnU{|9gGRKM>!`(c?eWP*&5-*R6->00}|skaO$_ZzNKlbF@~@p9({^bQ=v1k5eL-n* zouz~#r~{>P-*7cXZ>ZOPlgj>r;i_d+0LC*vc0|u;t5+@J3?4jS{Q*bm>?(rR^E`96 zhwnb=jLUW{;&xf#J{nbbbOM)eVs)^L7&o4{aNI^+)H-F}VPzq%Sx+c3`9uKCSaWcD zyKLBG;*i+HKaiQ}J$mp=pgv;GVYw-%S>3K9+(NR~pEHLQ)Azi$18V(zdp_07|$3J?Bag4U%F|ovegg z&Pk5;Sz1>4MJ<4)(A`C{YMQ7UqWNC<4=|bP=grg!=0OYphd+&a$Po$F9Q|Jg=QRH8 z@cS=_=YM(n@$!tXrXZ(@(tg}7W6&q4pvWE_YjY#aZCehT)7A89mBWBRH#w1ypc7l) z(RyA?4WfE4WOP?_aBxU=^TQbgO(X3@o|J(GRoqO2fbX~n{mIJA&`M*~pkH8^6^Z-W zbxBG6c?G9ZSZwJxNb1B{aYbppMT4to12ge2`|Jw4d7Ku6lJb207)cuUFTH2>&N!?+ z9n30o=cGc~jd%F=L4E!bX9(KR5q8 zyw4Jo=|+HGy#HVQ?;qP_Mdef1PK1~Vr^h?PV6+&4y(b3!R=GH$H`|fh?B>43g6U#_ z4l#wo+ljomO2fwchPtx%4_&zCQnxRA{QmSLK)WeB(5WhDy?EovxlkDH2Qjv&oIzPp zb@tmwJ5wF2C>)g9k7EGW(cSLeE}^C5(ekfER)|0TC&WKbye#DKJ9=+=J6T4iCRY z6o_5GIRnQuKp*_G#B^_sOTu7xjs?dnqT32h#x$PH>Nu>U9=wS)yRCFjXQM{FZ!q7M*J&S4;-bpeaWNQnI=1s|IVY}0A|*UJl)5k5 z<8B*SEpr1Pa|*NbN~nZ2OMW-e)IIvqiz9KhwP)w;xx_)5CJD=CDcrHEhGfOLON;as zRacXdW%tc#0@9%QV+KPD>xEU{>;^8&#r|&Bm8l&K6A>kTtk8hMR+9ms<-}M`F8fd1+~MpXPFcuifP6IwnX7>i%!Mpw1;iK&DqC!`jZG&ro(Mzb4#uo&cewZhDvU^2}9$a~M4vvT|~AGga_ z{vZCK{e8(w5KU*tBD5vr^In-&FvpMlbf%;%Sr-(TSd=wfHWlT$O? zxLiGw(-Tmu?*->=Q^izKqn2?XpAFbD+SPmyfPgZspgzMZ2W>6IY-l<3;P{vq9FmJ~ ze?p(Zbn2_oQ^}W-^>X3 z3Rh#NidGvNo!XB>Mwr%;iNn2SqHJNMwa=a4)pp zkgt5;$EeD>iu^^fXcml71F|&teLW+_4gj;2L^vX|p^Bzb^52cRLV;|o8xc`kx_qXA z9wb>|ZukcC&N#8m4mwTH0|k!|iTZNECgpfHYfkz9)7Z7gGrj)tj%Y$|NphD&jEY>E z86raFK5T9wVhx3zgHk3+7fBRjF1gPpBU^_^btof7ZlUN9WwMP#uD|cj`MpjZzu(E( zU*GTRwe7ilp3mj|dEU?G;q$gjoao%^=WJcVC-&*!{sNO{UCqp~s&HLm*f|EVlV!D5 z`)2^I#|^*-ub(wq=d9Mh%KHB#k}|a`lg&PMNZKXb##!@=R88fpoAL|Ah(0YuU=~KM z>bx*OOB8pE4j|49wnp|`>OdCVi$Httq@*|nu!l5uMoH|UUIP1#DJid=?$G!!PBUJwJVQ*4gLzO;0;kpSTRRM4i_60W@z^ zfr+%mcEbfP7BLGcW)Umd?y>GGNTO!M+$<=!i>_%%f;gh=bA!fcQK8%3SEVlwsx~HH_u6C|4dZ#P#mB9Yq44jpx zUJ@U{cjChX^&s+qb2Y+-T2G>n*91FbM2W!^(KOoVti7j2aAH;0*>v)`Z5`Qpqpbm@ zhXOye3j`*;Wz~5rh52&bkmD|8}6ZQT=28RWpeQ0ZYfU90KuPve@EO*|0Fq1eQ3Rzhe#B5aF%h-8cMv z;mmSa7Lgj%GPwjAo>ONJmrM%^f}Huer`F9$X!%Xk%A6L`OahdmW6SvD307+?ci7 zn{U{kG4*m)E4>fo%+g@4GN&sV!Y7oGo-R8d+@CE(1)a#Lh?!`Cbl_YFAQiWsp!su` zsrt@AbIB(q;aAo7*PHlV+pD2b&oV)JyN-Ihe3I-MK+@c^quC8L${SEk4+q!gc|x-9A=XYTRw8<3w?G6{VNah&Hx} z%op)mWs<9$Nei#e7N@di*hnr9z{DK)89wrP_9&C1-=*?oC#B}#}iWgjAEzV$|{S(tq6%o14FS5qdw zFFxo~bZl6Az47Qt1ps`FgaJe^h(}j8=>Gt)@8DTnfpcTL6b7-P$`8R9LrqJbw!Z^X zI_7tf@&rt){dRf_KhUexlxWQ|$so4LDS#}2xP=@p+qVps9*^Y?G%Dme(6^0(BCDYsAE6ItpPI-XGx{UZAgd5oj zmtO=CdP84VD7@9T3k@L&5A;Uyml_z`^)<}y@jMV7%!oeNWM*aca7xJNb^lgA_aaBRy)z2pDWG4itIFxY~jrw`+s5ESBghrNmxcj;gAVvcSkP4S0%+g){ zRbm6hTExFC%??cms_Nv!6C9_MRGJ1764FwAU2aZ2%LUzB{n|eFhRTWLV0Kes{cx(5 zNXx)!k08K; zE;WSpG+{;LhNzKCy7t&VAcKJ-;OjuYJ#XJYPKt~Z>EVy=wZBmb(Sa^Fx>oDrpnF(2F92SkDVP7Fup0gBQ0qi*-Q1aFH zafq#`fCb{U4v!WVhM<0C7aE9j^?is78DmzZxW~BG+ve{Vj-^MV8GLnk5Ix*|p<^XS zZ`!RSIT@?*%wMU{o^<3@oO6VA(!xPU32KT{v<2R}S|j2KLu4{zW_#&)#BYIXFGLzc zjE@$OqhmX@SJ*f0)F(~rDyWeKWXu~^SXhI_HVs-#m2&AO?3iU6{~s~j7p?Rlzj;oq(ptBzguP%k7VTrgE?4ooSa-=2k>lAANqe-hE8A{?gwv*stmJ zt9?hqK|7gX*K%U9<4eqq-m_XNr?gcCwW8_{ix_;J1yVKgAp#)H$)|`X`(!=5Es(|5 zXIouT^*o$f;eBVG702YOq4>$XCh-d*p&lR7@d&_=p;)obGHB@{s-EBr_rY46ho>$Y zKhB_rhz=`x0Mu4l3?S%a>aO?EKj3^h70(92dFnv{cUscvAc|6x>ICoU4k42mYMsDk{j#5Iecp_C65(nV~Ic z7MZ}@(pLDFnfTn-nDa3M@n}~U>t4G*Ans}wGc)^S`J=}mr&JJ4bB$l zDD%3$>&e1Rd;wCO%23MY&!1zxG55(%o@UigCGo8-ZT3JcSYsg+ts=~D&;fJTV4OLZ zK%n0IWdqewWQ4}$)jGNQx0qRT7>@SrpunW!{&&7gd%Zv2M{c2;*?V?uHQ?K`Oc9A2 z!B)|2pZs3*^jvtr}wAf7VCH9_aM>@wcqPreN%-UMh@F~eU=?1w7%okGU>?*z=?l+KHV4e;VOjg{hN z)3!X>6Q1;~;SHYl)B!v`|6J4Nb%(?L*?u)D0yy>%|K3cSzwTq-KtD|sUvq`KB&7Y{ zqWK$rADD$LaX`OJ543n}loKgA|R-Ax@NRz{koUStC1hI*1*UnhHH7NW~fIV&bd{ zg+P?;?VTyvIXJHFku!o?OWB#(1M^&7LQE-v`yc`WAUTM$ktr}KB^x{V_jKx(5H|=^ z5ehMbKp}P}m&>q&*)L}i5WvNP*qL4dy1e%v$gU89MC@JcoGDpZKr)u5PCArV!zfv~ zE=N;x{qc(J3KAtNnEUE&R_>Qq&una*zn?k&c+LI$b9VMCTzZs$U;c8ce=QGq1wicl zhg*~(PWCQP06$=}n7gxtinEb31h|$^0aJo6R|2lMD6gOaeh{~}a|S*XXQlk@FmX1@ ztK~uB?3AomfPkAc6^yMQCYSd~*#ZGz<)8%quN(tXgxZ^^K%8|zKnOvq5O?R_w+R1! z68Zfkb!i0Pa#KqqVS9HSU?>YEHwOzP_$7-TCEzI`cyz9FmRjflP zLG?T$&c}a6^~dMz@V7V9zg+P)8@gA{qyExpXlcU8tVpF zn8f3HLX!#f@gq#S-dhCqw*1HrGP=!rUd6gz_Z>`}e0Skq%hrr3>qv;#kYi^YUm=;s zcaCjOYhj)I$bDUFb;m~*u|;5RS6zCZlUlXi3RIVh982rRSJAj{kNYI&OX_^5w?;Br zDX%{fWkN`KR3Va*-kOosYQ#`XhoaLCeagIZ{l2EAIQvZ7Za{wRsCFhJwJOvaHncrz zyq2j-m#gci($3vt1ia&xMae%F-m&kEdI$DB>S!{)e-z2vFE1w2uoURp>rF61)UOi- zi&_l%W z`8fu^+ezw`ApFUcKegd9yZ)rv6?^}aejs5vki0$A*2o5AVnlgGvp)x2Cel9!U1lUm z#l_h9QukB=wYr*0-pCdLWGP74$O&?Fhp>pKn3x!&h>?w@G1L+yW(QE&(#{;DVQD91 z=VbYx*OxboLYz#XmJZJLP|7Q<0f}2eot#B1jG&b4Y=Dmbam~uc1`IcKwz!nOmwzPx zU($Cu^j}E-n)*sgu3-KbG$7T?oh}!W0yNyxM99wE213aK5^^%R6fI6L3y{sfubJ3@ z7a~Ruk`POCi{Bp!xtd==q6D+C0ut{2`voiKOZKbvG=AUtl7$7N`NycsIaHh>w(7u; zt9yj6G=~Yy!cKX)03|EiOP0%J0qMDP4j>2+wo8Mr;+sL9Bd%Am+bmS8Ms>JPm8az>Yq5S z=G6c{#&H~UjP1M_y43Z z?Ej#r#Q#js{|BXE{oQQ*B{dvef0G(k&c8{`AI-?$5dTX`|8FAoZ;1c@jnu!Y!5^*H ze}Z+@fc9E?mLf3-&-VM}KxMF>>H-qyk1?lL2|KvF=v z1$D5$Y=9`ujBK1BAQd?yCu>U9KWoGP)F833ad2|~vqf5)4AfE4Xl!%DZwxbv6}xp! zjNtx*(Zg3a2{dBFBE;@cp43;)Fkfe`e3hM=UoNjxQ(>2fZN)oj>ynk~F)qJ8Snv1U z>ql6D$)o6q`P)anY14CanfH6Dq&ChLpiXQMER6sC`(F?IuLu6-0ZRVFiAFPINq^IL z%X7oAlv{W$-GZ8@@uB@+Nba%2o&bDd@V{hFR&F zXS<{#Y}i>Jnn!9pHOEGRNh3YUF$|$>iDmwob-~%eZm*O5wn8E-9!!593bto%|&LYrykeq9?`|G{j4Q< z&W0V|w$X+Sfs4^JiN(4c-sa(tID9%Z_3%6|TQQNv>bE!5JT)yU)4xnWTbd^U*+#!l)dIgWct(NATGTNn9j|5{+Dp7(WAsH-CeEoGIio#mY2>U`GTmScL2 zgX={9kK9?nSgBZE6{`cJ)nPXH0Fiu;*^=yjyM!Dfm>_KO7B^VvWpuoXro+1ICG)2;?be11I{;~?RYK?Fb|+G&*!bZhJ((?hiUEW6T4Gm=xt+T! zL|bHYq-KvIy4JEn;^(u`n`dgYukZGbeN&3joTl5KP0?1f?Tw0~dTiS!s|cg;GOIaD zAofS*n{QVqX+n5C=IEnQ#7p&9g zJV!0zH-^uL-1e&EJZRd5u)R}K$fzgOF=!cz81Jb8VjkkV1?JZc4*EJMuMHKFQa6(Nxwc}w zkMy+F#;H7vhQP*M#`F30y96Aqbz?$C@H;qZRS53Y$r-XJ4anIJ;MBBpnWkcJWZFP# zXmuWjtOHwObi9GLW&BH}^v ztRvM|$;yie&)nrK8e@-2GQfN_Y|b9-6R=0@!k@F{lUX+iP0N*dK>8ZC#XLSBwx#~dfbn$Nl&k8_J)02IQVM(!q1k(ykd&bxhq@8MMy%%f!y8Rv8uka$Q@)|` z_UAscsf^;zIqx#>J6!E(*AC^<|(AlFhRO8Fp6@tto8k8sj*&vvpiqlolW78g@9!W(!s2Z|?9 zp*e)t&e~1croXH5%g?tGlrh+>!N||?xnz^cbtZ%mw^~G-{p+r!sILv?RND~aR>vg2X)!s>CLZBPr%K) zyD;8QB2janA8wXj<{v$neC_KzwNo-#E>e)vBOF}uCHY?UQJ2paPlQ-x2}mw0K%lf& zB^#ZZf_q`a^Hr2a+NC3F(qtuN;8X;b)|~n4h5X~rTA;{0?5OCG(_caJ_D-P&0PQof z^%5)(%U8+nc1SOwq;wS1mCAN^+-_5u4NTf_toX$FI`G?&rMB{dd^>5b2W^ITa$WFA z9WLCt)aR^`uzfYhV{vu7ZFNk8;RScZHSGs^_94%q*_aYi{DBe?jJcn&b@tL!jm0Vq zRFfK|YdTvXZ18La!ySgce2Uu0h_35Yy$>#Sl+BSQFUn+lZuSlsnBwiuqCd;=j6^OZf2MwZ3 zLA-7IH7phlnE@o9&chnKTk8b(ic7B<#IrJdQdnA)><-s9vL!8-&TP|FEjyXKK`5Wt zC_V>6O)@FoI(ED;LXC&_$tIHMQD$zsobr z(6eukR~N8k5FhX`j!Rq}=3ka^NZBs)?k>EZk9E<7)iR54yf^rHJ^CFTDzgClu-Ux} zH`?YTuX{l+AHnF6N+`esF@vo-T z@ttjz2_>2FqfGZUDWdb+L>tv(ogTe>z1=f4s<%_m!yHxPWpU)S0*sd&PSc&rxx<$z zTTfS`yeXclSdQu$U7KnsvJa4^nVUan?*~=fskb_> z+HVj!Q?Q=Lk_K;?Iz5bA)k9K2t@75gr1||XjEx}pI)nMEqO;vQsB0L@9pWp1xIVvh z5Uk?fnpd^bPRHdG<3GL$YA7PF*~$V&qSgj7m2yZ=%-cY*o=12kZu_!y)5uj_!u%H{ zKXx+pF}{Y3%DM!B#-Clz^CkIM=g{6%s_yyXqP@RQZ;T2s4bT`$%nzcKkSfSSu1XIn z{?iXt$2|&Du<^F~k9yXx`mqq0_b_f7uKy%17mx1q)diwA^;i|PmN!*>-qEe!u9K6v zv}zE3sEoXpS9RR2fkL|$^OT)`U8|HuV4mO67m7se^Ye7mTQF;Qwq+nx|AtACy-lh=69> znvK?IpS<8zETZ^G+Fexnl~|1kxspRLMb3UM!ak;MVm8FqO>3@pNy+dlrV%#&_OO zQD5~Ylj#Uatm`8Lwwy-?>|gqx&nW>? z@pv(r*Z;E;;M%kArci7d#y*xpL7k*$BCmHeaKI(4QzAu~cR(-?<+`}HL6;In`an}+mkt+0W@0D7$+<9yThElqC_P;~^|eI`=J|)bqozDxBA^n&qj8ur z`*r$f=nu*Er%>G;25Z^}q1w(X3Vs5zh}*PG*o>|;{fmn=qv9ApN@@=B2GXG-sXTVl{ZaoFwVO(`JP64~4xmk{_h{8Spuk_a?h zbDVZVWvU-c2_^X*Fjzt^7*K(oVsHC*qmZuBQ}&1TdGCHL_WU%sa9NH{zo&$JBPHLUOjb&*fPQo1wb5yjDkgPaLVP(#ekpt1`j`2I zp}qDsuE?<;$%^G_Psq|VktgYSN4q-R6n=gV5A}`ti-+1kZKA2?N7?={Rv1{MI{X6i zj`chmSJUwos|I(;=|3t*dKc>%r%v%i=$0|c53ckx+e!m|^4T-puNkg>j-nZwo_c(G zxMT-#YA&`Jvj)AhW=$sJZJR>c?|5Zf7sG5R}6zo@nv(X^eMT2*zm- zM{AbT)Wg{2Us~y1`7Eqf^Vh_9cN3>q!>_};?W6@tPP-KuAU}cQZ9L3^Kyf+5?JbJq zSgZ7w_NyS6>?xBa)H1Mm>^J7l9`$zvT{?9^eXFF|Qeb54m(GgTSm)+&(R5{>Wsk^? zOpq;)Oa5;3CZ62)9HC}deXY;h9K}7oGl}UkzsoLw@1ruVVbs89)o{}^2Cr}ZxSvgt zzz%9XiPAieeJm^s{N#SB*5Jy>>_JgHx~W%#+#>_sj7W;_Ej2jPnI5iq{{7a|OKr=40J$Ts&W?m1vYM zjLeqTSaKR71a^#3UuBZR8kD_u{15ON#su>V%FY*uzbW|fLH8-n;xBxEcDU7P(4jEM z{jVVBspg0lir0K`svmN6OexUxuDOUL|4%X_ac@Cu&d_ zl#8j=DD3I<6|s+nho^LKGzMBYmONf>YYtcBJb(|bk!iJMR_eW6`tr7QdEbrd+*ls& zDLG$uoUNbZw;0+=G;M#<8PHSlzQS*B|2(`waBqKoj?snr3{xZbW+7o8Z)n|?1E zKv?m@_npPgaKmPU_v%*(&V$qo&)!jxTX4#W6&}zEPXR=r<@S$9|J(w zQmbu9(v6d7j6K#HA&Au=O4K2xKkv#3?jS;hYfl?4!jBaShA&o}n*c@s=*qOS#F?Dt zm;Gy@WZhnW^x(XWx5uyka5gk8$8QxMax!qiy|d5|HkC7xnLGTPr{w4@X~PpDgVPp; z8|ml0GYaR9%VTb58g~2or&&v<@m_*QHp5=&Ywnu`yi*s>OPJ~Fe7nE$*$vh{tKJ~y zy`)~xO*jvA$tzLS)rcCc1hrBJG0PETu~<$A5k%ufN4{>Yt4edV)q}nd0PR`{CKZzD zS|dI74@sb=Q+{T^Ht zpXKFHJ@E5b@;v0FaBDb`;snCN3Xo?YqTMvE91H#-HMzk=G7H!#e-E-{tvYL80GSEI zoHo%jmbXz`65h=2g{IHJoGVI7mA-QckcO_vxGWdrW@w2un0q)b?>u~4j?PBoD3S8w~3_j~dW0 zcv*^AMp#9@C)#nX*D2ji{XA(Ox8@-NbKYF(KP!mauhkwpgnC>^b`|{JFtj@{tX?%Ooq~hYW`;-30~{(BuBN0jd_S)#Rpe zQGkfDua$9|iVI_EFu8t-yC*2DrR2gVX`ulV)@d@%-)d1ca|{ZQGJ}rE(``?t0bgzn zC+G^2%+XOyh+SB>Wii&38Nc6@5mh86C-q3q(k1kDK35p!vOx^9LWKRM36jc_ZJl$LR#!^x=v%JT9u$3>b5@rzs) zJ6^nik*CHq#{r!_(Eri%`=prlob$7v#*$CXQrMQeEr z*4G|$=BQa{)ZUf)6`aT}GAY|2)ch2M(s-Oms<|WUIZ(A0W<*xKM=`*fGVE<&Lw-1?LgA?idG5{#$?xJk=MPF_rj;#X26<8&OC_cJq0Qbmqe+m0+-0ufMLuh z1Gd!F>Tj<>?w9vonO~d@JD{!BVLKxDJ3@|bI!?3i%+Oedr)k(ODY;lXhLsBUmHk1! zKG4N6YpXoa89e_{*dqFD7wykJq<|L~$4H%_0rPpQ9OikO z8S0SC938#_t3Gn|(ooo@dbYc&vWoYSS9JoS?aJb!(s1Ott*5j~+QLM~Z=Tp*$EVJg zm^$#%eDn3nx4!yFOJhzizd20!(ur7h@Xm^W`I~-|tj3Pkm^o$5@iFciYy9k$bI%ww zcVK5(?OB^xjE9KaQ2)@BWmbcPVPz%usR=u|`)N2er6v66Fwy9)ov+YAKKb0Th74Qc zg!szrGjhGBO2$~T3Fkj+ud9tx?P7(x>?ecDvvrvrt9Edl?M@9t=NEnzG3}jd^O~#h zVM5pvzYJLJ2R9Eb*T&foO1a=HS;{lxw(pg>l(Kyrq%FJ))cn-2QN`p@CAHJ+P#1fm zKxU?2_*;>q(7I1v?oDaI@DDwz_v-WxU&6I{ouV}ai{hsgtR}B%OnbBvzt7LA_7ZeI z*uM)eDl*{np?;igLGUec6(2kOj2ED!y>TnVsH()H&C0EFn?I~**G9?sF~2s0{?7d z064tUo;sg?>aRu@tVhyNpGQ7y49U2 zSiRBBSYGMlD*Ilxz`P=(`4Ogzes}aK%UNZqeph0ZLsAjvTUq90GG3piPh&AE)-?&y ze$*|DKOk7o^Y+0;(k{A=`^EVkpEE+4h_Gn<`d0{)3J-M|_`S2x2sbo6Dw9;}KsSEQ z#n7sp^?gPT_qG-fE^k@AMn22_%5^1ak&>V?3Yc4hUBivo&IRU@>qL3^?z140y^oc0 zMVSssGtcA08%l=Qs)W~s2TFa*=pI%SbpmHCBR5n=Mgj^DFOO0oA)-X(nxo+>-es}= zCdw%JNphFav)LbJ=Sw+tM6IjGHam9}qS|3AYFMPdcJf%GDB>lo)8e70Cbc2e4{Wlu zX<`CO4af%q20XJD8mnVE6azx51Pi+d+(D>F50oPfYV@Rovrd1|7hzaZO5`5$^t1LR zF*Ca6(__zu^HH~CI#f9ISeCrRhhJ!26C4o@W8!M^K^ro;GkszmiQ$CL9P5~m%e6BX zm^wG~M=14w0^P#kIU62-)Rz@PUS@2@WcoR)7P^2k+f(t$o^2$s@qX7z;$ZJA@z>~V z2pDTjXat{yKb(uR8YR$ys|_tKmj`n{>?xX)wrgxP#O~ZKr2^y6RjWR?Lhj(ja-@=( zXJ<+D4z1qeC+bv4LBLfn`sV3Tp97NIBktMh13a}B{JyouU~8xTWI*gphHly{2}&dF zB`Tj+a`?o=p7{pb_~GHN7t$}g>&zlPHvQ0~U0tMrNgxOl$r`#8NhgSPy%%Y1$MRNT z%`=)tw*4}Po5}O=G0SR^Z$*PW*8K)&{ZQXLgVH`fGLG$#Hf6G+e1?qNXNQKCbC>~o z!!W;~$5i9_k2V@q4+ayDm!|DARHr2n9OVA-R*AA^bIa9+qqp(X*Ht+WfDNG6Eiq{? zy)Oq{<*Jy-p@f|SzkELE&_el z5M8A@&ZAwx8)0!T^jN=N=UE7AccAS=Ec*Lt(4+AV^mMACc{WA7?J)KUNS)6sD^b}8 z4?gCOqmR_{xpBEsMrtCHe1Ox-xmlk3B}^#e^cSefYzgC_vAnfdPlchzeg^pQ-OV{K z)2JL*ZZAw?hZ53=iGBuey|R4vhcw!^w(D#(*W{;-fwG7hxdZI25d-mKP!o(iuA#{iX?^>LwU=cAyn;zy0{?W%F^pR}2gSS6Fj0ac59xD+-;Lgi89 z?czAIuH`~@=x0W+nW4UjSxsqLj#$dyp8KK-S|1zPnu2i&oweE2>AN2pqp;0&=utdm zWOS%8c2z`NJCj)(2(<}I|CWgy7qoT^vgCyI0Fel;(cJM;JfV)USMlt6(&-JHX=Z`Z zB`@Vvg`(eZRK4lO2^}z(iL;G#cq0v;; zjs>^wi&#y=R99_uW6)n}cU&-~tMjEVY@#^4K5z=Sm=~*AA$R2SD5WRUW@unWeJYu?ls_rou=1lugnZ%S~ew zBbtiB(+fJA`FnO80>N2g^Idv%jQUPcggm%r<_s*Y;N;Wrx{B>WnXFzs+O4p1u7kRI z-Xvy?bn58zmzi`~T!yN({o3VWCsB%)p*q0Chv4z`6uCP7GV^k{XNumrQom*9lL62I zgb*t=&dQI)>HykYi;l7(aBHEV*{Yh3p=F4r~pmT3N6c{AE}+46Rd(dVv&yyeph z6=?N3ij5CCE*D8&rfgv*EPIwNhn8gg4W10(xkP6{1s(M6>JWEt5sc&=Q=F$MGpejf zfMb`@3>q0EPNr8{mc<=2g7!LBb$+(#dArEmPtTwjdti&^39N<(F72ppeC!4^2YSx7A z-Sw2;)sZF0)aHw!N(rr{2iL63h+mR#%TU%eGQYZh{Y@H3dx?4X%iE(?sYqJjOuM== z{6uDYPiLGhK<;alnT0`~R@M5rZ*?MG%{zV#YDyhMJ_SF=?u^mw2E#bl#nzUAuP)#P ze!vS&3dl&MC9qrN0%y4AQm?;x#9$m@4KU(frfo;vn{I*}7E(1J4oCb?)lazASs8G@ zhkC^POsM)w_i_Jh6u~OOzSkgwPxmU5lUYgH#q4WzZV({`2mu#^D4qW|q;hrF2|(nd1j zG&DMJb~>=Oc!sHFi+Z7Sta9c6G1E4;x^;5H%JS`o+l>kVZodetZ%KI@9g3DKhqBOS z4de8H71Bx+*2%(FcKNUzd-N8eTY8^e-JjCxRZ3yws7$qIV}3MW5l7d1e(S$;Ep#d* zP0DfC4Wui_kV~2BbyKRW^%F_$#!s#Zg|kd5jwU)+X~{7r%lG*uxNTEY(|%Q>DNmRV z9nzhCM1VgQ{n4!6+nYtEgL9as@c( zhlhIX!(1RNUHUr4=y9r8UVn#O zmaZnH0%F70&iKaIvg1eV!|Yc`x4S?Z{FDYaSP<*T6%7d_jf9-t4Gfc-sPKE)EHIMx zmA1NyH8bdXW(1>Ykwr9-LtL^oB0-9JPXew4wZ6*w{?gB@!DIz(0kAFFdG^EululFFb) zC|U=n_~ZvzXP7DrlKi;y@L8SW!ucvAFN;;rm%qrM~$3h+Ep_V*a}fOGc84t~bkhIK|K_WcnjY)WicdB-y~y zKnKY!55X>$7n|oe;UIf&SE=R1JGa2h(sEBf!Wx5yqP=<2V_fd`je>4EFS?UY-)r03 zT@H-3S=12?Q+3N7c6_p|6NX*O%0o#GjX~F&_z!cP(O$o%T;~PQRe~CEaGRsg$fSSw zv3jMt-V2@OHQbgQe_q}1x2<$zl7#PT`@=1b{Ux}&qnB=`&xMM4df_NCr<%W!?4owq;nWo`MP*> z7ER1!cHBWkVv)hhD&Gk$?#wBLS=5+%gJNUn`BMrng3msQY1C1O>k{i zd9ma0#^XoxL=o4YL}mlV?WZ0;Fq!N5hzlsg4OkNITbZ)wg^yF}Y(gH;8~Nx4?KWed zLakG4o2g3!%Z!M&%R2{}iB`Cwc-c($L zYkNxj<`ZLz8WT}`W~&&2-W2;K7su|{tt`NUgVH0gMrm@g$xg2@w$FBqZ$;iomi6SO zU1;FNvzZHWCnT6ZO$B{@?4}M4Kb(%{gXeuh?7I-dQ#h2;qHBIQWErWzRO--6)J@rY2!vM3>%Ai zayCB|P}4lCQAe*{wKjNnEzkz;`7_FACE0g&mSGhziV;YAs!yuJzlc7=#|&vI90ePw zNn6#qHw36S1}o{n0erMoUpmUZW@Ac5dc8WfiM1jNucBaLCz>y*@g$JkFV{S-2Dwnk zUfdxCPNM31IYaAa^&7A|AEiSEsVGriw0<{E#opBkTvJGNm)?97V?40oc!pIntA6L3P5Pgjt!ShRzn`HQ!K>W%8I{< zK?q=9kv#2Ys4icSRcZ~j3&{Ds&|D>v69f*(YFt;o-#bZg6Pr^yRQt=v&t|U(-56eD z7ssKOl5aDQnhPwyZAUe5qWc8-Fc6CbK#1<@A$69R+m^IxNi)o{1tKJgM>fL|nT;c_|Kw zmqyV}2Om_0*YxteTQ@8Z@id*QzrewYumT#OyGqDO>|rRnE=0JJOrI;Wj}%Z0k9pmZ zvFASJ;~AGEoR@N5tZdx|le?h8nHjLL-kCw1o^}}jzR5EQBwpR(BG(4U zXS@L1VXOu&e7+NpJ`et$L&m=8_Ev)`Se&PB8(R`AS+)gyBmPDF>7i^htk&zTM30)! z0PLaf!{5fegJ}8kzfw$4E~f5`o(`HUEf#NQn+MK{T@IZ%!bC2PLFZ`SgliAU-vy{1 z8rU4vS)}_`%e0cfz*=_+@TAN$}WbxVYkWBPbH^83twgX3^*Hjk^U>dP# z`Nt#44w2|{q9jax3v#dSlGDvMf}$e?tL%6Z9KMhB8+dOj<0>ET&7{)uEZ&YGON)c#kRlP#B}F0WK;K|G0%+L;CYW4$8ZzX%~SUm;uG zrK?t?d9xm2v#nSr@;G_V9r4)hbVr+Kn!xINen-%h*FtnoiR9sALMh*NiAg`>x?LA4 zW|=lxZ;AH2dr}_ouXn!XHe*O?W_a1gI@8j!+5)ZBn2;zLkK=?k4u-g9l990#1Let|Up{Z_UdpJis z4V^WX6&F~USh*~tqhUQy%t?7$GOLe8I*ma;QN&C=_@JM(>YT;YR_e8ZPY~jGItL~v z92+Kgx)L#|KIpq>DdWJLFw>-tr1yUYhkauVo3Un+nA+;`8!9=+#ak2Kao*1zesl_cKo8e)4;(x_M8s%>7-Bsvz3y{o%F)ldcSNK5Z%&|79P+ z+sL5SGb+P}mAjUz{H58m;0tT(OY(11$yu%z`uBY7U8(zF)umJ?dy_`Gey_@Uy$56U z;&TQhq=^TD^;X;dF_t<^bY4) zr@>)?_X*uABsa!y4{USO@Fm?fSFebWPa!9QXGNqMB|JQ&48IPjSc?}^N`wYb7v7$g zm9;Ne*Q82DqnC--mphq#d5WOpOKAmzi6~ewhOfUdf_&75tPbS~{LDCVxK3i)D&3>O zdZn{|K=5AP^sda_!xJNis*S`GD$<6Ko95Ef7j9gusLYME_~{%+-T3)#~?mL znDJ+6J}ph4#)a&koQo7xl~_j;Sh+)(u)oBk`*iX}{JaT3Rc-pN;dQeD$4dt*Zaij! zJe6A>_CJ%)QRDuAEKqgdoBT|%-al)AUH_4sXTQ#VVf@;BuZ+PSv+odoNUvA?IjUx_ zg(~wlV(|~0t1q_s4*aem2=7~Ac@Uy4RL+*!!bx(D!-FI7auhkpxkHm>0r?_sza!Qx z<>FxWfOiM0Hy))^blhY(4t24~lFi-|C3x^01**YoKnpp}g9)Yp7KNQ7&PH_Bqx~iY z_YdCSZ@P8+x;xC2Ottv!Kb_V;cQyP~W0>o^$$K_;KS;&x?Epvl^|PnSvtNhL0I(q6 zDb5S2-qSX^YK8L>-Xn^)7dWkr+5AArZ(W0F^Swe!358y^d&~qpULJ$XK+zvczNyff z5V3goCsWTpv;88cOsNNt;4?*JCVAPl#FJ89f&%@9O$iALl4*1KFf+~3^%Z<>(MnYc zk4fIJS=`O$vo$3 zeM7ZO=Kzy@92_>93~$zvz_f1yIta9m1Xs2N0LxGWf-$#rnxJFZzYbO%4L9RsKxK=t6DL_Pe}kM_tNxqcOr(OUu9 z<6L2rwl{8(oM^(8zskmt=#|+Ma)uuFy(Zr5x<${M96g`rRzxLd zhoge(GM;Y%GI9AZzM;&7oA2?AQ> zG#XxWe-vt#1~*1oaqT=Wpxn%%Km(j<70LRQbNq zyg4+2C3wot|8~mW*M~N1W(=_!^ab3tt#^ z4MqYyv~6A?;n!eEFMw{Emm&3=>k&b?v<&aPy9wfuO=`RAauU6CgV5n-KnqBg;0g7Vc(bgIw=h0boWa>j1j$ZJ)9;1P@~S@!a^YIH!7@WD;oFg}Jd1^(>8 z<(pDJ`>?LBL8>e?!=uh{ber9_dLk2a);Z*orL6tqyJ6!<)2d z&zCY*`!r;H?LQ&u$558L6xV2=bnel#T)^?@~WfcfX#jX>Jef4brV2&98lK z)I-vQEo@>kctL5Zv20>!)cls!4v+cQ$hrr@OL#>K+hrre^QN^DnG5+OR&KEjAmk*5V8SStSFu3!;n!aFlq<^Q2sxYh zBTH26`L-l7)ijwDctLIHS^BhJy+R5BkCxjoZIV~Wc44`v zx*vP#XSO{=+*o9+{+X7pRH%OJa01V^dh$>Fyhx22dG*GF;M!~d0vTX*w2y%x)l%LXyCn`E z(7@}OP0P=<*4^rF5m{8Wl@Fu!3GfwxtHrsq;Wf-?)iBP#ivS7@aEOC7LiwKPUl2E% z%?P(fMt{^fH^nrYTjdJ~w0O+ks!f(MWJeS#FIS6+_9)lwoV~{3lgQ5Y-=fl zH1p4WIaar^vZnYc2NxH!eV^oR^gh>9Kl)cje*0VqPnG~qe0Qa3rb^Y`8v%vOB(xeUz_3_OlRgy&P1|z;1gSve7FJZ zN`QR{ASE)urPdevo54UMHGKU5=YHFshYFN-D9}01eb3$~wWZFTzeW4G^31budPq+4 zBb7YIsDNfK`uEqBR~IwkPU$Wm=uqb5VIj3^{fS^0HrDfhu~YNQeiqNgbvqQ$54)yf zu8PtRX&*L%b4yCQbX8ZopFLuJsHrT+qxGw?Q0bW1cGafkw82w&8y$M$ds5U-m!Q-A z_`hZN4k1p{BG|<<-hPHqe?57j{wa2*;-?J46M`GhMf3DB~00+d&*P9KeL)K z2Bx}=LmFES@}12q4?XeL*8=&X%%xD#%+G+m@PDP@7A0_6n0e5l@0_FR$nAS66F(t( z5-}6^n1S<#77=Tu3MP$$>A3-7m`iz-^YP5J{|@Rq%(%VNID8^UYx^`lYv~CuB4W6> zhM&dbP4b5$^XDYG>OeOiB%Qs*tX42%kMFc*e-e7vyZl<{%YVg?PthvV`a_Y|>&kdL zSj<{ox}>bQZzoP#kyo6ZLwX_8B_eUMJZIVX8R`8FTZ4N3SmzBRjDI1b;KsG(>nK=G zoH+Sj&M^@X0T$VP81n)s4-Y@7mXMLUXzsr1;d^4KlW+8>P}dRf5(s^q)n4C2DqP+F z)Xzq|J=|F(-_>?T>2A9QWC<3eY*Tbz7x?1Cl7$A{I<;k3mzxHZU*TJ1Ifs`eBFPw$ zPZrm*O5%;E&**cT=PWTJWNDkRI{yAA(fpLBB@?k z2qcNPwA5OMErlCms&aE!%h0Wb|1w-z+DM{i$s0s_qpi}UvGhJslqlHM4)7hwMt|TM zxyDyCS&(u1uj?!>1~)dYTpFw%UAJ~@+OO3~cFFJ{c=h>@ zVqvlKb`rN+_>y43-^Hi42)4i#X7IJ{dO20LOK-wPOJPvZl+?tHERB~^x|Od1d{V;s z9tY9>6Y%Qlg4fO+P)xvPo?O^5&xQqjD7oePJKh(aL!mE>;FCOupZ7f8sz$#h0d@}e zr@p+bK90oLc;o<@l>81F-+p~O0&5M#&CT?(ChA4e{O%<#ios z6UfAsC%p-Jj{z-vjNji3Om9_pw~AkV4A&&5f9bBW9M;v*hVavxsm{4)pJt$KnQ%?( zNT>-J((dKgV(GvfLSWiJdaLfA%qXB>!ggbMF9q3%8nypHy|9&NX=33hSL&St%sMm-(2F3Pz28TJy^h%>wR&vxzcrK7DERQa_8J{Bm?y7*I=Rq=wu=V zb}-1lqg?m=kC`9wyANRV%W$e3qJksux&a4U+VZupV-M=jGXIWQ2p*bG#uMoEKzCPT z4jPosLy)6d(WBSOBj4F|OGcYHvN7N~OxxJmth=b<&?yG}ZIvnaVKsUkIHQwvA(kpD zn+)gG&IorT5GgkB@kHPrcB_SjeyyR6uu{G{qB^+@WLJI?RvUDWfi-C_{rDJGBR&4}4# z-(-?KYc(_fvV-+cgp+z3~(4cC5@F>|ynMEst~`CyiudSb~}^o^=am~gji zs_jn(R6$jl_@k8=2<{&7Fsy|3*Gy#kTl{M#=IP8Zt1d3hAza^4S?Hd zmMC;Wi1&old%B3A;7A;z7@j)oa|ZSD?>3=#5&hX%|KSX#56E3N=rk*=O`;HHIybdy z^Fe{JtY+PvoDd*7PMm6!a0~xsp}j|i7}A<`93RM_TjgsM@M>aHl43@D>n?BQ%%)^z z{7;6z3u9%S1pD1ORDMa$op+>wcVCz3@FvG@(mheJNL|5acL1_vhN&OtmpR0sH+DJt ze|21DBY@ZnP|Nk?7II|kaQoW(uf>qm4~`{b26WdE01^GKi1ohWhwSLvTZJ;qFA& z7wWCdSbFXzfwL!ezmUYm<@`#Jx_)`A8v}dHDx?%kx$KU zx(8$tYqKR${qh1Gs?0y&GLb@r6HTl!0S6H(B8NeE#krcwshXPYCO_@Cyn~D$J%jJP z*h%7X;I;OMv$Mh_4BVZ#{-+q;dCN@`+=f|f=Ae$~gKLcS~0JT#DG^>wkoOL$a({6_3E- zr4>QeyhZ=7_t#qIyCzz1T>9><{)p+X?8q_jgHPLKv_PHL#Xxl&2%fB@3ZjR{(&Z>=5 zwnNJbVcc>0jZo#x&M&2x8!hZ-{}+3285C#J1d4_j5hMYE6G#Yd!JSQTcX!xei!AQ2 zA;DQZK=9x!iw5@)U~vfU5G(}OMHZI5o4oJ$-E(f;Q+2BD&vUD$>X~_->FJ)HnVz1W z-rh+vGl^2oh^un`=Ba4WvfKA47mw3LWmI;#lWUBz`qBa(4eto?f+hKX`53~&-MVoh zv+#J;up!jXM(PgVK*Id0qB?d5dj(_uwZ6GXT|kL$ef`P5U%KaWT=KO0pIu5s8g_F8 zDgX2->LShev6~_(p_Suf&Nv9SL{YFsSHP4m_QVvH+&WP2JPtn_A~FWu{m-fimV}M! zo~a%+FzT;}_%KNHC^qSKgx`hfGSAK^1N~zRXjC*cN9AWk^r~F4$Ah=AitzT$hePlz z^HaP5Ml!ukL9Y+kuAX3bTC4q>-%32^!B-R(^9(7HpT|{4Wu@T zHQ@v9e=y&@tJAY)!(UpdmZg*M+kuNHo^G;$W|m1}JMF5b`M}l9wMsgXs2->|#T78# zbHtClCTq1TILg<=mm{GxF-*n(vpd&ogKs?gZ4+z(T&H-wet00E8fqH0NIkZRIm`Ok zW%4;OA6*qEQ6zI5$n~!feMu+cR!Ze2g6l-mh*HpFc{QO_E1~LljUMjNVaND=3oO;P z|AZ1$j5hT+%!s}s>W)t_QRGlAMo0l)+f3E`!t}uZobVczkhoq`$4dG3b$L_iX^g{% zR3GC`r!L9xAKGbrsAJYB=DQcedA{ml%CaauQmqkO7w|tU7NtyDtPnguyq~{s|4Lv& z$-~n`^XL4{@=)nT_p%pEn0uLsVe-NI(6;TH>q*-CjY=M6gd*A}IE&B5S=0U~HdXnk z%nH3n=LHHV1*^~0MpUs&zOkL$NaK>)WQ_;B1^5(pR?B<`?{*VykS;7A-VrH2q??rf zt6@m%^T4UaeBV3c&97(~YO!w-g#`P;%6|`=ve-1X}f4zL>(7D9YhD)p!WQ9+3e8)la+}c)E(LCmXOELP*1thp^IN`t8q%DZ{)u0o@P$JMh4-HygP=vqUXDpF^S4kKyh&KxToMB%qWw=>Pf zUQG)nHA~af7n}ybnqv3+A%+92I?N=j(TebC@<9=$76yK%=3%W#!M)ECvG;4AsO_1y zWNFwj-K2?Ny)dZ%TFf4($`;U{&QF{BE38(;S-m5&4PEhvzPfOfP#OJ@RY&hPg+FyPp+E7jbeSwySEk4n@c3uu%H-n<3) zpukz;I%M|Dvse<>A#J9Z%Cz{mu&%E;o{Z15_t4Lby+c?sG1mh>PIvtfu{n#N|7e08 zqiL3skXy^oZOs`>QV-v2gZ(rX7Lj477KC=$+wL{@-Y_>RO@b_Y{l>jckpo?7^gNCh zk%J27YsF{*iJmmAW!?4Nd$5>yu$z5|Nz-Kt%+RRKkUqu0pM79|9>BUJLZ^}Eb6hIS zkbcT={p>&5&`Y7;0y=By&(8K&44P84^%a_UR&ZcT+`16*jH}WjU|#v|8&5@rQemA6 zUih;TnkDmUMN+gH_f}}ln9!#3GlxL-x;+Pk!L@ayAPIjm}LH zYSP$gM&GPYo?DCKvPE0ZqUXqI63s>Z6%{E+LCVT()gfPHHTC~HlOMb`n7n;(d)$8) zD#4lRZt`8KRT}`m!VNWFK2vG)GhZ*5T6(O4JGDKR40i-FAa5+OtL)UQd1U?v#{K%E z>O$)C`L%DpnRwQWngj!^gDk0A8_}0%97(eP;2|R9anNSd5&aY1T44rF)`F{=EjX$FGKqj0EHPo1ez^v4r_)jIcIzJ69gyavVT* zojAOE{VGY36RhQhGaJ9N{w!kgvzd1aILRX#o*SKCF^i3c!T67#gylEZ$;msC!?zSz zU<@XS%<5c|h40W2u}#5Apbp^K+Q*~WA4ebYC|g-L;mmn0EEQFq+Z1`NgUsc4Cakjd?+jpN1YMVR*?`YWHa}+ z#77C)1IY3@Xs$hP6GsoHz#(X{!y*W0%h}TIq+rZ2*qxBp5mryOvChQsDm|Jp^8+GU zjzTwHTYr?V>IGcFugooFG_OFBZ8`SRvMATkiS^cEG%SmjohqyDlIwXzxtdVoTShl&obS8W zoU?|#at}f?%N82?$vOnkN8^T3rY%k(rV_?Vc3W0S)<-}b3|zU5Kt;t0%FpqABsH4KcX(_G6e^;wF9fc<~M560T!D^W#@5@>8b78Cv{xrL|KWhO7agXEQE&!vbWTSLM zKmhHh1Oq$;SJl`(UmkgMF8a~$GphkfW&Cd~X8yTd!}@^Z46Gj+OQz>=oBA%rMCzJ? z8MO7!40Y9bt_I<8*NEG0wfJ0eqi%!!tv!DNr<64Ib}{eyo_8@n+{7)3m)6~BfvlA2 zd5`|(lAv>UmL|#L#A25F`DxACYVFfFnXkNpEcAhrhFd{W``tTZR36b_0q%BXiTUPT z?1k;L_d1_bG)0oW4Fhwcln@=w-v- zAmeinsbx$P<;D6%VjfXF%Xa)q%a)1GH=@kMUi{GdEox09l#iawqNyLy|Jfu$?%9>G zu;lIj6e9Fkc`Coh>Q_YcU9PC!oFvk6`j&3N8O%+j8j3i_UWwq^dFehP;+U!6FtU*Z`S{e|Cz1se^)a_^OAv3})t5AK9+u;+i z$sw~-biU7vAWl`+c7{2>A%w#Yd&*}5?2y95dl!lp15(nkw|zcA4!Tj?Z_~+Wl61Ga=lZ@Nn#xT>frSVLu~UlS}hoqGeS9 zpDqjiRw(<$7&+9kVjT>D$~o0GDUv_V_W_W5)1XjQME*d{6AcYJ51Fz*j%s2~JO=z_ z8Z{LRysv&Y4GY|D+cA?{Tg;2T4T-wCL7H9LpNHaBi1M%!E7-7-%6lb7S(aQq_MfI< z^YSlVSHF8T*ct!Rqghbwtf|=|pPa@37CQUS4YuzG3EZQF11s(FEW2E?P|)u zRZ`Qf<*llG$^Pgmp&Ka1{(19B5fJ6$=1RJ?!06|LJC#8&@tS zHUQOVh_TG6CQD6c5!SMYMtrMxymjl4u_!0Hhu(khP)rHDOla=)$uNZbZOUhGU$Oj~ z1Y_~I12NO3Hq8_53H)ceXY1BFOob#ph6hCu_ijM^n=a$DRBad)nsF z#FM#{bjcs2N6%nIQY{A_BQW?q6uvE;WUit1oi0sq|O1lp^@_}@Yb@h21f}F z*2)P7>xlLN0Ex?xB3CV&iRwEca@tX#_xhAVb>ypTk)4}hTx@kX(;uAI`2+AXztt#q;lh%Pz zLvsHo8F2~qIO*Er`{Ak4Nwr$jOGb_wBMYcXdv^hB&mU60%ffR>7pcE4Zh?va z_jLxzv!b(%Hp-VJJ)DbGS(Fb{{?I7|1I>yZjTybj_1)<*#*%+e`S0s3N(qwH$@Rva zSdd6lUKS0r`Cm53zBCZ?yj1<%N;_iA#ljKm>;L6_1V{t_36&Lo*^5XKNS#_H-UsQ4 zwSU|B{NTg>*7!=U2-ZZeMo1#?hiR)XOmz9LcL!cXqP`_vMFh`Je8aqwzq{C|tATdR zMKzj@B)^NwYqPv1X0VtXw6zl}C5^1>yTr0<^$So9y|DOH10Agq@arh3xZ>F@GrOO^ zL!QWF8U=o;IQm#Q6Ht5fJrt z-E?ECAvMt3fRPDsITU4xbNFs3KnfR(7LwNXq7@@X?!NJ_Ih~H&u87^fq>W}A?9|h^dc1na$s4vBVmfD-ip7$f@ww3OFORU|eB=kye+WeU+oy&}^*08lO z{o2b*UK?Yh-F{OMLVPYeW+kwNQ>yVc*LIm#P?7{WQG32!cPb|?@kB}n*7ePyC&Di; z7j;rRjA=QAOFfG&&@iL8)uw3>FrZ!BN()TEMsJ$J&Yvf05xM5(b#D7Z78;qc+aZJm z+O^hsZH%}43+NsQ#14nfUyVXe{jo#nqWekF;kl#84dT!@W?S#ZW&5=ANLi=vzMAHp zQVW>!ta-5wz}hKbromAUTm5KyLQA~0#y&t6In+KJn{l9;WcUTQa!NDp+jr_*FHf-c5xf}Tx%pSVTaO5`4zDSwPWJR%30tkjYPV9 zVeWHUeXhtz;&c>g&_5-uH7mIc-(4BT8%EqO9_0ETkJMXD~Q~U z_VqWKNRt=yXs1(e;mvBh_56CkS=h5{^p1_dxRG{0XuEJJ z0Ly^Nm+{IIH7V!M9Yn8>m&OnF&AcI&dveIKcCk$G78%5E{cvb8^CSLjTC@fv@>a0k z_$WcJjcKzqY&>uWJspwYaLH#mn-y$9!LlaHHLP8L?depl&`+ zr%#c{6m|i>n>N^2 zti)7BwT6NB#7~wk6b&(eJk<6)zj^cU`E3#7tWVW7Uc1E(36e7AMuk-=kZ>>#`$0J%wH4%*f#zPB2X zDuPki|2|U4b?uK@PuT9xSkWea^Sw1b zs$x&5ZDvAHKZ6thHb(AN7165?(Md1|m8QD9mh$S{g7(4yFEp9WNdK0;Fg}~j0a=lY zy;XpojFp=}_HV2-D2Cj*HTB_(H_%Q7=;vqyMkfbxWo}QE%{KjdCqj&j`yV1>{{Pwk zvpG=t2y=ie;;1e5>Y3bp*d z{`(1GVL^faIk@uPqL(>|)%Ttwl_KJxFAwCoSeYMavOkFQ&JFX5%>C)pJG-Gx;&8$0 z9BK3RZBTX8z3%F)EX%iVs&eJyNrfJ^K74goQpxjNMe_aPX6m9()~pRr`tJFO&DRq# z-s-FQOrKvY614wkPnrMpzQT%cZ*DU|6Wvr(yp3F<+t-E}7CGs5%38&g8P)4fYFC^; ze8Cg5@4k}`u&Tg_w`=dCp1Bp7PIIyi?VvxT<%c@@ozaM0#At*q@OwKiXYEm?DjGx+ z`{Fk_PCVAa2EtgW&)^xb8M!g-*$`$uJt^=FaTb{9D1i#qBQCq5P^24BjIs8PMvbmbppYN|9 zvXIWCV6Nsi#u29OisRb~jD+~U+h)REa^H?Q9-+Lhi@d&0Uxhs=$@W-`%CD;y2Y!0x>R@(<$%t8 zee?3()6^VoNqFkDZPC!`lS<)t>m8ZlfiyWcZ!;te>%$HUOAmboUG!aBgLvx^Bvt0l z^Mmkh8N0*cIlF$0W$6oY_KVAX*#+pVf@+0fnRRrDj-|tIaJ}}%lnv9g{LyN|d^J#2 zKO%`fCRSYvB}~gEU^hD})a2NhmHvonC-%JUurK3Tlmz5F!d@+6Wz?y9R`H-`Uu$Sj zBtuEUcZTa~P@3JF?kW_>{+zI&)=zk98ATtN9Mik{RQpw}Q(DRnz`f+G!l$fkJSqiR zzk3j+w7X^Rj#*J2`v#6?DD4u2Bp5b>7TcC=pLI|`4p)_R?xt#m6ydR539&Kzgs=L& zg|5Aww>hqTelzoEU3^(=w}+>8oat=j3;)gx_b>7aRefWnMmf|&ll9%VCUTQg@|L@; z>(#$5_m){yg3Nq_wd36{UDxld{HIx8UwxC;q$y~p z&osOH!jcuy>b6nu)@@)3d0(>@#<5g)EXmtUVB_oHDrniUSHx$y?)t2w&%KghviJCv zU5}8DcD#0Pq?4}rCdeJ!A^Njm-No83pf?gh@EW6RSXom$y~$nv&LHd69yu*CLL0G< za|*%QD_!Mq#@hm?k*uC(yGS1>TPSCVkvz@MI~7yTK*&>gXgNmne*LosQ<+24z*mym z@m!%%COJlfGGifgQ%+91oR(94=OSZ8Y=6daL2F69{nAJJBcyacK<WXM_<7juf# z>uN{ppnO!@k2ShJlASxMqE*L7j z9>zZb>ci^|#rn9sTb06t$vX}2Y`uy-{s)?07(w6i4{S8K9$gptRJhAMY(xLs1#DE} z*?hDat#<&bqM+?F>Zh3qwO(hG#_Yz^HOGCux6o)fHFx>gk+HZOJSB5#V;Ev{@E9O> zm7?5yaLT$*qjopxU6ao|c@fQgn^Z&{GIMZWzg}kxu}9T-bf00`F3`0;tGg*{0X4JD zsr^&gD%_=Zfb^hu`eY{+Bx94|-LtSD2_Lc4Si;=Q(pURK5{c;%I=JEhn>_^4teJ#I z^1iUhR?bNBC#ROPtLgHxUl-a&iXflK{H@1RDa!9Cdgm+Mhnr)KU7)j+86|k!&$Eg! z8mKq0^+L6Ga%`^-*4W@pezq$_FASo?Pr{;4VoFGz*Y1xR~qc-c!=Q{}4>wUqdvK(FG`Nt{v};8jEVE-beIg+7L(#4} zIjU1I{rTDQr@F-Y8RKL}Z?RD8gr?Pl7L07OipXBw0>0l3j&64*vfu56N49zS;-eRo6Im^IU5ErX#4Xqz&s={Z_0^Wj;M zD+!L*ZE&9dfz%Na=LfN>TBnI}&g(ikiGO4R2EFF6Mp<0G* zI$vDXgFK`%1iVDTkEEl=Y4%0oR~P%oNt=QMbN-t9=TY6tT8G}>ug14<}H()f+Fx?qPHgE~LhU;F|}vYezmqdc1;TG?Bi zQ$CNQd4ceRA{vHwn)IwXkRW6T+jP|S#!y*5{5a-Xac@6kFJG0}k4tk&Km3SG@p+y9 zqre|BtM_Ehr|FBr_!|mpTUYZka}1V2K2A`2gr_p1p+ioy8Q#i83EZ77HS`>u{@zB?@?&guG%zttD($tPHNv>;5q3z;p|QT*ZzDhY z9Zb$Hex&ZYFeC8X`;uLHz_=5Ux%r`EvgkLBY(EJs90IFfOpiHwle65DOjRbDo6m=7 z=MP)5-H-Or4ky)WiXpXtxNubSPcJxF5k$o8mJ5WIG=Zs$A%vIJ1r4az;dLAI_ zLr|;W!hH3Vz|vT*8Zu^~L@C2u)hhPhzTPxyWLy(xoDY>s6>}7Rc)z?)K<`{`vAXI) zr@q?|PKo7yv;um&GWV<9kh<2CfbAa=0jEd0LCoB`wJK>gQVX$XDz%fAQ& zITFuN^Y-5n3q=`1tkbB&W7QwC!0b+#=vNU>f+Jj5_`}RHX$si|Gl>L2ci#eJf@%*> zriP|9aDj0b9~rCTospeZAW+QjiREAp`2Zf1rVLzm*dElc%kI;E=-6WaQNX>8qd3m?t#Z#6SEuM5mnn+9Q zYO1V&x9Wvfk!LkG0ft(7>+9A%Y-gj%uOLxk)4HfcUA2_#(bznHP+B)>J51HbMLnZyhG`^J=Zn9N$w`3A zeCxEWiv2qQ#DLV}fypd}S(Cp~<_ZELMXq^|9UKcZwz{7QJ6Zb2Ph9aGe%OXGGTksb zH7okKu4u{vGiq3CDu)(&RDH7B9Qr13qN^kkH*mjT<_1dnTk1qd+Pt(K)u$^IGa=`w>!as;c z$q&>sJ`}dM1b=MsRn|TtUVFU0R1gb4Wf#{qQeh_wE z;8=VAOOG0Ke4}pri{jM}Kz>>IblIE(wDZ92QCRXkD1sHcnBBMu2DQN;8b(KPv&|g4 z>#i9u*!2_R94Xf(zUBR7f+qg-=%YI;%_jlltC#6;YDU^REy@_%E>{zx#q#q_$9jmk&H?uqTvd zx!@Y4Jto#%*v2^aJB}6!xf($1VV9x4zHY5zW|e{FY|U@fndU z%M8X5oUdGC@6y70tG%NqDrOD`Kifo{d?g9c1x)d%V7C^r$fMUTgZ?4WRccuoT;6)8 z90>(4rqi2$Puu%8T~>5uJ{sP7!qnb_?Yta!oASbzxX7B{Dl1~LUP&f^q}9)ToMH98 znSC^#aIZTQS84p~UER~sjGZ5UWHc7w4GOEnRCyS83vC&|iakal#u)aYl>)D00-iRK z`IIr2yC(>%ZsT&Czt#6&4aFH)0C#to%2G7kO=_l#V#uiOM^YPWQN_B9Jv;8->ByZ8 z&@SS2dCFb4rU5llLIYTxps{EJ)}3+Nii#$E|Hiy$L&`lfDV*Z>*sJVw6Q30iVYwMz zwf2%gA&V}lcEDF<{iAAp@*T(wHYyGlRXjb7d{yowW?C8S`(h0{lPq@|!aIHCt)Ah> z93RK8!t3qX!+e7;$%~{7#m`cjV|dhC{-`H%+`|cmKCI4A-W0`tpV- z+G|yj1fG!U(0nS`_lK4aVdSEmtKpA5S$6>8Zwuz(Wv~fLRtSKNOwwk@kX)c3xwUH-N!2Xbz;Y^`?JW!v}?xYX%48b-7JyaOi06j_7N_SL`gW(#jN`~+_;fXQY}tX#?3Fq2BZ9M zC!r)GKPBOPePab%TiIF}xn`@_pvRO;uM`{^`v(R?^hLiJTs6`E+r!UUW4E@&;H1Az zEtw{+aa8TyNr2sxK8l%q8~{ZHzZ+PiyD{Fs)97_HglOO-Hlac^Q8qHAM0hA;>NhhY z(;ZYCuwuM5uoW&In_^VN`7asD+d{DEuGr&`sqA*4(e*Ub*t%(El>FtNiG_P*v@(_m zfB(z2@0o0iiu)-Llq!);9nx&SL6V7kcznNU^k;vg<~(ZUPD^vfb1|$8{lkitl76;i z09~tX@kK&Latxb0(V7j-gx1wT)Ju;M_J-M`C3~zK__IRps>w49KW%mj@-@S8 z!#hz)s37x2O>|;p0@cEqQgBZ%mG7qk=6C;4Ea}|luG8Aqe3&q`6RfvOv)UqI5v(P6 zFGkdn&)YoOy-o3M8+%G1-9MZQ{<2bU8F3T8f@V*=AEmH7Hscde4i$`fh5RhnRq^HH zq=fo&wz_|b{_v?vvwb^tMWs5wHBqvl+pAR0> zJ|BG}W_Q!xt!^URGqb7GZkHrh@E+nSEaveq%cNPx(l|SII!%8l4PK@tS{9ct0C$zd z9m}sjw!lKJp8Y#-5;QbZY4r%v?Gpzm$B(mz!FM^_>iLCzZ~Ms7%WJRpP9Ffl z@he1|Az@@sgO*K02=BkFd9@V?`w<&+EqIA@ggD;${Cvur7-&{Y7P7Pb?bUSHbEbFD zyf(4;i&HogrG#U8h zF_N~i#7T$w8axE!*7>CaO>BBVO!};GDUt03@Vac6!q;rZyM4~OYew-{N)yIKZOXGj zd1Pj0s6v=K3@5)DV|@6raKx<{5ez*N%hY+$^*p#sh*Vr4vVhahu!{djA*9Q4Y#M!!U&7`g8Rw1Z$?aS<}lX;2}D&%n;$Y zmN$mDW<230C9ZACLp=mF=+Afu&0#guxd6?oie$7+<&4W}5xE}oDJP=sW<~C}Z63)W z4c&ZLv6RDo;u1z}vBou7M~|2iQEJVY`h=YDmAUYvo2q*I9MRx&(JTj@l#V@u1nZHR z<^_I0g;)2tfZA2CvURVxqRY~Y_FA(nne95jZG;?t{2F0!(n+w~8T}~Mo_IiG=b`}ef66r=uR_w9D ztgEtnFJs=wQOc6~EkCTRso3Tyn<|j$y7qNl8?A&I(0HS}5_tJ-kKzp$YI8lZqFoz# z{cex6?pmcPDV*f*bRpTMcTrs!1%4D1$sR!z6E4L*ZQjWTTm9%iNX%L}}qzxO<1*l3F1E zR_5ymHXAcu0XLE%jN`+&B|y3}9b&7V0Zd9@-4GISM4Q6uPLz|5sJO{=q)`bco-CAJ zq0P*p>5QR?CW>PI8R+wJkN*BN*#eAv2q(E;-!sx#~N4;!NCj zI}pTmtUy)=(^?Ws5bpA>v%t>Wj4br-Ju(o~G@`4S{4pQ~zGxO#wsCuFuf)4>&iEzM zGX0RKVgBpw{2pHyxQe*;2pe}yX`*{PJtwKgR5hExQg?&fh2m>4V&S(F+g8Oh%%)I% z)S^aK+uJ$jgWJs%2{Bnr6|P?v4ke-V@RO=V(I$9y5J(vzw0GgT{c%Ez7XNCJnbMG8 zplZ^aX{;pXnaoOJonR@yb#FRH6~(Zj+t-;~O1!0Vf5QkD68c>USK-wAAa07a4mP)? zDs+`lzQF}vHkE2D3^b`|Gn+B)eO@IA$!^zo5IhLBT~0_H>CLXy>CY63aN*-Pn`tRX zRGzSWjlfSPO{okx)i|qQ*S(rN{Fu7p@2E#(qTA0lqD`ly(Ki;R2E$8B9+Ax7O3vv7iax_i5Z zv4mXZXr7_nWipHLb?x*XCEiD6zy9&Nym;>881s)uIX}C7OTw7B^wn9tR()2vTzf>x z3~%(WD{8Geu=ZWP+Cn)$Jwjv4d?G! z)YjMiGJU(snd1u3JP_#~=e`H8N5iug#`qODkP@Gx8`DrYsKD4K-?UmvDKA<=D`_ea}**Za++#Qn*g>o-aeyt^6~D= zMcIC5xGLS*b#=|v_){PiB&DCLdO%sUc2;6CmNiiGd+Be`Z+U8XfaEeRWFdi8Eie0Q zW1~fMeZrTUpF*2z^IctH?iMQQ0~WTdTUd^eTY##xkG$US%c3ob{`h<%6;NUEq&)ky zV#Apny7>{WPZsMU#r#t1sujN=;%$iY&a zFXhV?i#<|;yrR?3kQD>EWrnSG1G>!}EWhr09?aFT*vtxYi92ib6nfxud`$GbbR9aF zTnNiB#@^6SDRh?7(##L7=ML}I;5ZiAqHpa6H2Vlcc|50s9pY| z*F54?GjZ3-cVV|%S8uX;Xy~O~AZPhHv0RiJ#(?=6wW@q4sj-BotKouKq?Ih7*f=7S z)+}S?;)c=jtNPq_vhO`*B8l}`8a{#Yvl8dE%FIANo+V|V^R*cha>mPrM)3_~dnMhOzbmhBZn}?%niUdexd;pQIW_x7gmgYTZ zd%2wU(%X>HpwvyFV(qCFQFM8d2TowT`dOZET)Okb6wvne&606(j1W&IQo9S)n*W9%YlJWa3x zQEwt87@k<--AOn|5wRpaMfw&IJ>U+-A^n)%wr0Ww=pw)Gp0tdNz}LB*`aSePj-y|I9b z?}*p=xoi>`^QB<${ipDbmlvt-_17N;`YcGtw)27s>tIWV#n_D=r7wKK3Y> z=MIF2PKQ0vo2=9yj%}meDBUNUV-v8Sl_tDY+$XUMf&JW8#=?)jLeB1kpgp7bUlN*S z0>`rj+rHxM;&C{0b;*Er^ZU+M-XPA$n%ho@u$6HpslinwSrkELgB;L^3wKcYX9Bj#$Qh>ABl|7sgZOpQij6ezL)>)N^+wU#Ss0uX4jIyt=o3@ z!^p2Cn&PMBf?d@iji*xwm+)!Pq~3&WW6VeYp!IMF`eI7CIQc=s{dJ{-Kvm+3bJi*b z{wAF)Xm`*DCO>EKO!5*jrgNO@-knTIS&Usea3`a2C#>0aMV>r*dLa9Khv@-Hl)_s`>d;@0-{U_^o0_NQ$TG6Mn3s0HS+7*`1sD^)m*vs>?o%_%g5PLs(3%E~? zxcoq4WT!QRERCH=0v}-f6Uq5DLA%v7F(yUg@Ou$}%c#ntGv>O}(`lg+H2VR(jN0TE zHxeiws;rStTJT(5#c#v~5Vu!cB9|cpmc}4zCA*1Iqw~j{!mCgxCV`SIq-lqr0$={ZHO|>J~OuTahBPH4xc>G2!+uwA#5Tn( zK!;6#n)zGg$AbI=^G{Kbj$iP9_$o}tFZ_QO1-R+3>A$doLUh>l-=ffKI&As}srny} zL02a%Qgt3)I&At=OPxoU4x9dFd3ouu>7T4H9q+$y`Gn}O>F-;9K00jrn-%1y!=`_- zf^>p^>BDwR7VPAXMIehM>hDKcK03a?__<{H=>+~Zj;*Ec3I=Lfxf^k*OUrU;TY0LK5u#kL#nbz-uf?e;gpCu$ZyK z;YP>(58ZOEU=L?3DcU~HR$PAzTv}Y(t`<&i&VR-Y=))zY#U*X!X$Q2@l#|3_(zdX4 z!&cF9cl7|e|1-AO#{M`(S~XkL-rYg;MbNBwh!XaDzq31_m`#mOyjZ3U^mx(Oq~|pPmBo3t z*)dg_2G?g#(CKFVda+xb?gch&ZSk(Zyxt*8Zx_Gpau}Q<2WIm6XeG{enF|P}s7YL& zKpd7NF4t!vfy9D<1&PZpr@-GyDqj&jGJATMqxD40`Fi3|3jR*@qQF zEdi|8LVDWmQvON>3nPK!J}vwDatF?GDu(vYUM2m^kq19&Iasr`*7l+4G8kFJt z2HU=7uV&qpIcvzz`o@@qdCL8^TgD64Y37qo@OiEiT&>g}I?GKY1qG=k+&qqj2JFG= zM-U@ouHe{$$MzrBn%~<+Jsb$X&#>Y6fGS07UAvh1g0Z?u-N7Rul>LF@o79Ir5(f4{ z@|`LP5?7rAUI(1uB_~cNBm7Olda}3-$2lC}PRh{2(v;nb7aShuL|}$BCYCZ6Y%!D~JyokDqhSG|0hlqrz-94iNg25;T?dF;OTrw;xauB1X^aLl>Zg}X=TX-^%k2!h{b}|7<6;q( z@D>GAh(ihNAp=@fDL9F|^Zl6ubdda>1BseZgbKip%{bTk^Zm=zzD^qy+3V+m#{S|< z({~@&p7RIgwN@)!%=--h>~Sy0r51HKhoN(8tAnfePHaG44rKg>$sA^oFUbJ%Y*T>n z!bd_O9a6NYd5u1=E`g|t!(m~>&u2_So6B_+=Q{HZ-yc0n2b*;;zGv%pvYdC()z^>4 zaXQ)K_ZM34wacsbnR}9gH~@3CiB`{_v^%t+E10tVFZT}D*&lq3l~m_lBp^CnoUPxS z%4n4!v`I-nDVoYgZ(Z97`_1=*4}0KdZKz*jnU2GGqRqG8e}OmxmRq_ofwLKYo!{p1 zUq>r`J8Dl^u76~4Ybj?qHn__5Z5pQypdR4F1*RnRJ0o_+%)BRKHd$ISU2+P_eNpXI z&@d_X)`jW!abfM+FegOh9Pek__06@NgrDclL^j$i4ksZZD+M2Hxn4gp3{O9M;p-9I zf4)mu((XSmanpwW@LX7S+?=T}%V>g~Y=@=-F7nm;ck_;c1jdSc%fLIE zY6QkUgF!}lo6Xf+c75Z=K!jxObMjn+w-<$UbUhFa3eLw1*^6z!r!?NdbLI`-fv3m# z+FeEz1sY8Z6s~@kUU-3t%6g?OtRoMJ$hkCL4GIF-tUtY87fh*d7wzstF%yv)Sy%F) z%1mesjd=0*1`;K|i^p8P!;ju(m08Y2Ultw-MNWl+S~XoTcYd~ul2NTho#95IQ>YAC z>Q5Dbwer@iACB~wx_n3RO4wi97Z+|5J?Br@ld6qNYsl}wv`y})_yTrLoAT-&brc+v zwm%s)+q%w}nAeUU#MNa3E2so@4M5sDCfc4cOjNqJ279JTq``lO=>txgPGtPT`j>0Y zj+V)~x{349Z|2=?DotiW1`22EBzlDmgTu52ErzUX}L^sN=5D zMT!qo-#;aOUHkC!NO-qdufuCGoV}HTn61N-*l??ku|jRcr$=}C$PNE5;@�u4Gvo z-gt1^xNTg5yIXK~O@g~y(2$JwXy));VIq#X@o%_Cj zu$T3!s;;W8>h8tU&BZ1uR)clbkE#eefm<~km}$5K6Z!r$R|4pn?dhO1&K>`KLuF|Z z1xfm?b7Gn@rq7$ZdS7eXo4sa-dmqwo*%>?8Kl2)Pv$xNRHv`XIkih6Itap8g&95pW`Wklhwsb7E5tFGcKYX|+@>;l=zrx&l6i(J6SvmRbi~eX&kJI{MK}EeU8TVtJnFLGxsr*%DoVu&Vp><;}Dx zdkFzdT}h=+l);+F4ah~(7c^9=NJ%#iJ4+>%tDQrsM0FT**Nwu>CFgS^Ck<*T#k1?C z>p*8c>#g-kfUNHh3=slR*()8yexW3|gUc#=rt$m^z;O^6rkwab{`MZp_@>NecdBS2 zKV{XrT(=|AdkCvBO}T*S-YYFjnyqJAO+R0H_~?yz`4UbITkJf0JL6ql{4eO@HnzzC#Ok!nK!^))M?kR57Luc2VrXDYv z7%C7t_cg}BIQ;~#)S8xn0mS)QQ3PjpsN?XA5s?ddx0+_CqJCI{tt^~R{KJi{(d^t{ z2*zX^9Blxd!q%AShm5t9=xh;fWW;5RGy%5;(gQjds2=94iI8FpJ?|`(QCeg#n^WafGZ|!;CgYv+^O63UOcs7eQybVSG?|N`frYhFTRwy z%FuVe@sZ)*DcQZTlfre%)*1aWJNPW$>O$nwyU!B!THW~#eaOVSlmWHt3~F_ryK&Pp z3CV*g)KB}ci=<}vG~(^`HMY15n6w6h^c!STu^;5dHZp~F{e;$*h}d#o-!JV;HTbxH ze0j(>%EDQjgVyz?ja+@>>!st460QX4wbR`R(on{Z4&_p-+qqNuoqLS9=Ahvvp?Tj( zP?}T+>$AHHgwI}3!&toG_RF+vsr9z0&#UoiDfIbST5fu$lB@Z3 zj(e!f0lLZ_h!nIU$d&9U$1!V1Dxxi1IA-ANS2MV(OuWlHo^+Kt;wob>?u_VGsClVN z7pIH|yS!hq@h&gE9AQ9_>}!KJTw^L7?vy?eE8?iX$FYEHplvy?hwoI1YFaigQ zk^Q&Xo_8>w+UzXvc9M211#qK|MJq^0kQgXob>f^e>B2Q>G`BCe@XO)i#f4y^IAY#* z&-%4iBdjQ!156?sCua2ZRotD)eSI=@Nou?u%t2~NzJq%2cM8d$nYv$Zn+VlpBjqX+ z9T3aT&h@sG;YDLc!Sy#1Ptu_z zu24$CYh8WeFTS4G2UV*_wc;d6Y`&;9i$M@A>Oz?#*Dq2-*fCY_w5fu{#~&=V$e(cv z3mmfgenp#=nze!5uE+Mey8~OU*}tZdwa7-u4;P-9ZeTMI_S=5?K&UFqhZMA?G=Px} zEGV09g^O(JMIQYtgh*)kw3g^1H&X<~IZ5nS<2$s-oKAR(OOo;K6TLW`W(bO00+vA5 z@r2i$O@ia_^hQgVNu6dj5m$%y`cbHBYy8m`0%gnbXLiX_?ukF66m08C8JQ<|dF-{K zk=FIaEk z#&}a!=qsQL<>RI+QYbJIro$p=Q>UR7cF{`RHC4E8AgQbsTusb%~fsRQLAj;?awy7pl7d*hn<8 zZlUa=DUztxQC1H2WzvT7>C$pK-qg1%GZ@j7!v{&6B&#)j5<1XY)hWJ=tKsg$i%E?0 z71Bja%x*}(`If7MtekYSV6yGM81AQo;d8@%9{2V;x^xeM$-;NqV1eTCq;H3v`GX{(^C30Gscz?)+ne0`Hrg)Bk1Y7u6s1CrKV<8D%S-V+Mq zhIQ@dFAju7ncSk&q36hpvuo>=Fx<7XxSw_)$)0Tbuf6gHK=P4^^m|aUACqhY`6$>Zj-CIOCa~v5k@5W_N}R94#~-p1V^q(+e~MM5>}H zqCS5JnDrJO3fX`@5XwkkJDpz>yrd<$cR2gV`K+1d^dm~=H0%ILOcHoDBhoNbDuW6p z5}Kdz*3^$?FHg4Ba*{c%+P`QoFL#fYgAfXZ6Y_6p43g!Rz0p`(f)ubJ>*ZX^a>NHP zb2%9VJ2J({aN#}9LZ@7h^1qv7TU4{uj&SDk4f>hi;s?9b8O~<=fx2U`+fw0 zkzo=VKD?#LvwCHoC1*vetXL6K!hTEFUXT13QecU`SlSgG6m(u*@YnGhhu5{)v^f39 zbcowzQEazuU;df89zMrMqQX}kkn8zbE;N+xgX@HL6u{K(5-mrTHe_dv!W=pI`B;#D z+$QdZ_oqr^FVyQ-(3};E-`20>1lA|x^s8+Z6=>>OrJfZ!if{{C-%qSGI}J=uPc7)@1GN&|m^$+` z_EG+qHpFo=5x&o@(@kDN3t%7Ks#K`CALjemOrd2s-MudQ8k%7oYs|uC9QQohIrpcd zVp(&fGN-y){PmT4e3N3ed~Qt+s=dTq98{trPj~JcY1OA{XVz7geBk1+qR>FzN`)t~_q%U5 zgomT|H;^F4OSKlu4Qi7KY--Fpq1fiC!Z*wZhg1Y9)KP{VzwZ%|Kd}?FO#Sf!u9A=4 zfybEd2X{B^+8%Rg(YwU^shzozd)ty4b4_UJ@3DpJC`oNt!Nyd&`^Yu}k()4x-=?F8 z%c^G*n-AnH2)WTY@VU}dTCoeO)YEd8+xmNTsDE}I+PErd&JNn@mx|*IlMw8la|dSi zN+ylyzuHh`*W7lZ8B<}QTpIt5eoFOJeQ!kS?%sB2>$|jQ`$DLrhKY_cJe(;(!@{Tf z!dl@;IcxpgkNTe$q3I_E^}<$>0W&hSi!Y%pcra(NHT2>@`AM(iUU(wFwx|uKR8!Ac za%glGiz6f+7pe><7Q*;T$$uVtaoXgHQ;s>Tn6mo19K6dS=8TT|@5! zk5xW33FlIlTzYPlZvAel$jg-^F`aaGMsrsRI&0jJMMl!YcN`=Pk)x8BJp5_k=v&g&j7to@zn~f9prGOxloQ z^&Aam-giV?W+#7MWJeWGKP@I=lG_m%<4x=KBi*QPm$L?4g}v6Mzl-`+0$nF$kvH4p zZ{ir&L|N&$>YrL5 zc`R=F&MnnTGQW^&@ly{{QIl?0(|~GJJ;MsxO$)^rsSyMQoLLD8=fIvzfx-5?qDJ`w zN2f5(cLy)fwC9$w8NXNAArUu*`B`7I?!S=RNq|pLHkPDUW>d!fOj=$)=FPv|a$%m= zf>s8zMpV)zz`W&98C)9G+9LVIkk^;-Wv%%PzVK)IS$X5}7p(wjKE=)W_?FFBJ)M0e z{xbcxjNoWhQ)>j36^!i-p=L05y9$u(&Du>c`63K8@gZyMUGFZBn8-mP-J}YX;>0Jh zPZd!_3-2UryRf%kqf5&3vOJCCSV1ynH$qX#`|vzDMs11D+)JVEMhd@%U-AUzCsHOS zSrsdNEjsdhrU>nGT)GH*=eLmm%t_5Ka^-Dd0PxyM7gC=F%M$1#X+0&uh~ZRMF=FwL zA>yJ6Lf+DJMFMSM=0dLsKcQjTm`W=gPJ=6(IM$-9lH2UTIV(J=J%IUY;xUqoh2|~Y zTsh-bZQJtxg~ZEcB;wMVJf`G^*+%G+RgdMh!$zDEYXJmb9#o~Lt)mwKI9m7}C9Oqh z0hNp0=}A63jasecI+j<7gRz@V%Qnu>yXtF^!MWRx2fMW18Sq*H>rrJmf;YzVAzS%0 zBr)V;;-a>IEgljmu3akQ^14(1BuW@j4gK&nTJhUqv?pSPg&}M7DbtWqPtL2Ipbz4Ni5B(HX@pOFGAf-7 zgX}*}W!jX}%1`ub07Z!hAkL!AQnXGlwb#ohoy=|Oys%Qm%F0`L^r)m zddQ-r{QFPKjdC5s)=OZRPBqD2H+@Wtf$Z_V(g;8H%4~^nv-L8&EbVLV_|s$+C}!fF zu`a^$+f86I!F?L?@v4*~r9{@l2GJ8WVhjX-5PGv$O$DZ=A~a@A@BCTy0iTO<{MSDfMb=XlB5&dHb_0ybUKgD<6NSmkU%5^jof?l& zOp?`b>Ly&fxzEQEKBchNz4p6;lS!ip60UTQ^Pl-i8IF%4RunM@Q|c$}CKfr}veHQ@VLz;D?E=7~p0{7+L0h z6Q+GhkyQnQw#}1uQqRdTxj{)ez5i7Ia$q&qo}{vFsj`pu=~s%3vn)>Z`|>!bR-qOd3A6ecFZtsm>G zMAVHXiN>hrFBi2wMlZjAqb$s3agaHj@j1e&RT+bI${8H}lU%`jTp;#3kZxTIk?(qs z)@v5c0baMDtU<<2Hgz}Ki+K%I%oAZipv0z$|0ka!%S*oewtC4l!T zX4`wFc$ERu;*FPg7R2!uG_K*|Ps&}w@uHPf&tkp|2_&rU%#e3~+_}T?bQyagM+tOpDNr)=^@T#d2>zH)?Ear22`Ff)gKGn>pPd|ze_RdJR0oDeq}eniGjhKYvT=wEBLj1*yeYH7$0pCF8g}M+31X zG<-Js1mBVdNSnNMOV%w=R)WFoHRmC+6WQ1kW}O&QC!*T8StkR%3-n0dY=>d za!S4|)cTIa!mwZY%KWAH%T4tU&F|xQn*vch#mIUmrNQ$XSjd&4zZTP^D>P=@ z#*;Nb@wjiFl(q||Asq+BMG1!FMi+_Cgf>OP4dpnrU#?MYA5^|Tb7r3M+(&RaARk8YB4~->sKMdSaHV7vZ$|NU_`ItwAwA zjmW0QdI0YYsM1x&w+h?&tlwp}=>`k?C@DKo#!W(lb;OO~fc^SOxQOYFm{I>LBO{YuR87X1M)aRyq zbuI+F7$sNlWaRy;lT&bX?wP2Q%KJ{8csH2Dcp!4GOpyNvDsTM6f zjlTlHa8$f->UBqsbIdCD8fjMm{+&mdV`7Lp%JL%e?#I>Rxi;SCwDalDPQT*mIVDs_ z!E!qlD_E6o1gILBel~;sP%*o9Pt8>6C4K94pyHwZG`l|c`$;8}+*4;Nx1Abgt#q{h z!`z<{f>O2jZeU3%dbw@?Iv&>;xcM^ruw5z>Vrr`y_6t=ei~aT8N{ttgEJvxvtJ)!1A+MQCzj{Kx{ zLGpMRwn&@AO_$=P0l*n&F3!ov@n*$n}Z@$LZwo`uxD zlau+e|0PF5I9ilX-KV5?A}XKAB9z}{6CZRTWH5x^K_=n(*Priq7_3~JBNN~I`k7Qj z+yb$q6I>iU$2S6~N$XcV(x#$WbeG==|HwfNC)w{%M_*z(&3VyY4_(n#C~Y5*`TEmx z)D3g-9I>4yVoM3X5V%wMzW`pGaeD3PQOYFxbcrf90$RI_@7rg zs$hWF!R*?Y;IuvyI9=h@@~!cSAV384kBo0?+D^~arBa=|+D)=ZM(HrXXJQ^Oqima5 zPHTwx_K6)bN<^q%o<#AncFZkwPTR9^rHp=5?TM_03rSvOVUKBqqWjyA191+SHAps6V?~Rr~FHe^E6$)Iu zfdr6&hwq=quiFzm-Z4s2P7x4xf$xlPv4nln)LuFb4P(N_`H{ zq>;NXfB{-1HczMT1E?#8T=xpl0a%dkWvT6Mn|&R>dzpf#arS3NW?@!?+-D=6r0~N>Z%3AhQ`kA9a*SQW- z{4Z~Dm@vVZO*v0#WN&@DI#^K|1!@?O&!@K9Kc`-EzcT8S&?yE}aMfr0c#b=-XJ(Aq zIR=mc6{XSALajj?t`a*pZ)=4n1>FHW_%D&c`~ET8#+5HL_~AF06$+*H*0ijfn4>*E z&IWaA-7|!`zLcbP%OxYgXwh4-(5LPx-9>G_p2U%#KUtue?up!LQ&EB|u{hO?sWa7b zOl$Jf0AZ)?i_6AQWlvjTa#TNEpg2jF8;J74NUotY4<$Op-6Bx-b@G3z!JbG|SNJRGv; zqAu%#Bk+JL@eOiVR$W5mF0XE8nLCiqDn3%h-`k7~!DU^VxDH`7lPx)lWXTG&qX&Me z+wKzM zCvTGin)Y!b=HQxIvnf##If%i%JT9K3E1g0l?;MSv!mPlGX4ju)C=~ix;P^>1?6f_@ z#PJhcfRmLMO!5b?RsdMhzHnLb5)v6}0+@qoHFra-psb=M@sOM);Z-8#9DGIFs@o+A zbd`>l5XX+2PGLb7gfPSh`7oW3tpGWP-%oc6!sm=2K_5BWC;3?vSCx}=GHcVy$1a`x zP_+I*2Z=nQjv6m3#PHz(CozCdwcbfd&mm5BKwN)pdV?v0+QmnAQA4mMl+}D+FC#VB z4E!WZqIRh447&42{Hk~h&&^GECR7yEm#}CmjdVL~gidVmY<-XkIk%{PnFge|%shBd z3jOLc+7ZeCL+~*G!~SOi+6udR)Ta)L67SS7|Jb^Xt{MrM2eppq1C+`QtNNC>$uJz9k?4X#$HHm z!1^wJp|GlJTwYxXGXRG{{|Q|{NLl$cyXq0@x{RrbhlXxemr|QZ%(eejdXG;VJDuv2?*#=Xr8A;>p@qh0jPQhg zw(m{7hS2rsS&``01_s|`0Mg1R8+y@gYJL6n)-vg=FmR3w$S>atBXdmT{hFa`NFvamxcnfzc$J_pquRey1zZBJ2vx#XT4t(mdbJ250a+{%f! z46JXiLpCr7Fw6!h2F-fXB8Reu7+V~Pv63TPfZ@piKPcTT$xaY%c{Tup=6iuADl0ce zuOmsP*|Z7NUghg8FmfW4YLSJ+wh{R>+R8w#+BbZOWThu!~XP0jn3j zRz%RCqw5BxPZ*%l@ESi`Dl)JBxD_DOwZ#g)gxug%AyWYH8#4r7-NK z?<_C1&G>`S^)Nh$$43S59TR3xC%@Qy0dKa=p-Ll(DUCLnV1gV_t{=XTDN%d5@RbJ5?!kJ#$ad@wB2505o=LaJV>Q#|tHISin zplVOLYM}`{Ogr5Hw3;T`h{nPpU0#{tutz8=h^maVeT>${Kw_gK>$6o7Uf7SkBU@@wQ&CJ=c%1Fet1_FNU^;+xK`8suS4kq{?nx z+t3^!fiy!WaI9fOjZ9fSxaC1PY*4K}o=)`?+e|?^e?nnzZ-6RvQWaonLmZUO`mI{7 z`jw^y{~Usc@j!T*+Z+mEF}xm(;Xp*Zo!;BA?)VEFA{I57WWX+Mwc}S!w-sn``i;(v zF>A_9d-+#}W77dZP83{ljx_hto=A0~)_^D;us`&4_2b0Z}t~W6c3T*pwR(h0Y!EK1M5-VN&6Ax|9EqEieJ()5wlNV8tOf|)oyS_WkeTLdFcVjnUqRFFt z6QbJrJS=uudkD*p9Bcwe#O||+agkW=&&gzlY9b3I8LnESpPSKyccipK@@yP7lK`yL z!k#cng^+$B0S^N{hN^3?DXo&z135DQBMWTCv}mGffDm``Tp(6eAleZ!p!iMT)+q^V zG832d)@$^TL_wG|z==;&{!cJIzX%}fMxpCFn@N9q0D!L5~0}$ z*%yl)&*4ria|t>+v&?J?pohMGBPtSWi@f0~sc~N6Bqb1v1u+63`6TtV4PlO`+5oT^ z0GD}rtPb{1v4NZxNiwK|q34WKpn>e0(qM{eJ5FfeMbidmPUlcia~>v=ExvOW1+d;I zBpw~mR0M!KLtavbLRw%6q!)+!!li(T#K$8*0i;s+0nF`0qZr0SS_X0gUq!2|D3J4= zaEU{`;g~NV%+2YEGz(K1ujv&GqRZhKBF+PKK2eQ9_6}1~nODXPBf!9kfe-+RrcnSl z)IU**lM6Ru=8E=<<6|N*T+@q-Rmi9Z8pC^RW}(IhiNF-V@54($uAi(*F4?R5#xt=( z163Tjl%SBHhoO*4cpQrn=$@=Vgl#*o{^SLQ26!xbfxETC!lM9iP+I^vGp<*V`UBK` zps!FrDxj-ibV91MmsWGS1EDZSFDgc~F9=8XBnCo(zM2zqh5?<&zL)(YpfvK3d2-7qGI;oRG7X;tSYOa&e8*3s|V4BiO7|>7riJ zERbkQH+Iacm=YmDF)(ist$4508zg0j0IP^HpZO(eRhXT^lmaCJ*z>Sq9o0Q39EWRy zX-oDXRAFCdbz{@fiUiTdNYD40c|h-WpVcMJW}GAlT4H61)*7}Ium{tw@CRSF2Ga^= z>ku3xGP;&JQHbUi7qYR4XHo!Trn#*H{c*)1N{;#RB>(3VI9&Bm^+4!uVtnsFXfJV} zG>WqMY#s`r&Sy4dOe6zOkau8rJP*WhqIW&7=O}>`wh(1rffngSp^g(yfH|F$3Z7vq zorc8-h@9e3rBCkVqgB8iKaJzx(p~6Wg{WZEvP;r|$is**kx&Q*b}^AA@!Y~x0Dxun zXcfT95X9ib5aIw8$QeGk%6J!;n*yae`07=d$}nKR*!{uey`yP@l62;sD*TW)hrtyv z0L*zxAS}ckpiQ=K@Q$UKQkL$GMf~Pn0nX(Uc8IF+rznWPoyYbnhen ztr38Prip!&G6qQT7_l+Y%L7`{p!f(D7eHMZaT@?7(_-!n(Jq9nkH5H=5U<$#l3idX z$aoU+Bbt$iz40+p-L?CNY650vPG7c^YaTL8q*9J_W7H08;tya}Cv=qWgAtWh5%+}U zGa-aRAQG_;2hj^cfd?UOW*fgR7p)ACf^Ydq8{%RwK{t-ZxlNo4n#U-560|K}PbuMN z^^PklnC@qbv_h0*^;eX}en{4Pc*6XUD9UpoDL;e_#326k z{6c*DkKGWLYU5Z;Q&@h>YepPJR6)!Y%qDyUxhMmTvS zZ{PD*3_3<(KY9C5juBePA&-Nqo#Z#~s-W`$#r(S{I!#2zKFU2bJ3B)}J75h!vW z@&?ovD_nj1pn8h`FqmEe$}T#b&Mk$eK78K@X#s7UUSX9!yC+1=<7_jUE(n7y517SL z8A%7{C6{Et1NYQ{z&pXe0>BeQKj0h%Awi{+(GA2m$q4^$gyf6?d0T8LnLUV}RmiCF zCo3FHeP%bX9u~s1GLFQo!zlQN|O)aA+~u;|GNz8AWB3`ibAt=BB8 z8;DUd7kJJZBsWJ|1ORP8BosYUPk{zw74gDB4FQW&pcrOy1-vq#0gUCO#h_rF%r)RH z`FAHIFNKeg8bg-%$814V=VQ)Ygs!=PSebJJSez4qVHKu6a97q(Ut(L#3s>(5k-8vp z3Zl9r?Wed=C5m zZw*J(H6UU^(cOefQBycb(gNyZGV`zi!0B1&L~ zs)hlALh8jOuuvx`A;g`#s)*1{a5*QJ>XC9WA6{CBt~JdQ7j<3}PbeE92|mLprh{Wp zaW7_t(@i!Cq!%>k+i^mw7*09FD#m}8bWk`{j65Gg=#WR=QB&tCHZTE~8Gt9V{tOE- zS<;zaf?5w7(}DBq!-m>@Ft*oq9T}aX^w|iW?5}O!KXy0!1YFPS+z(&AY-_*q{qePK?2XU) zLI3v-!Rxc+Z079S)lGDVrFNh5t^W68?U$GD4gpuY{bb88mfGAWnJn?YytNJ3r~alD z560n1O2-lN%)ST;z&Rodb$`c|PD?#?6{=2`RJWX&TxYRedOLd|)~rGux}?6vI=M{F zvw(Yv>%BH)u^oqcA?H#tNj663Vnhy@`V&uWE?2kAs5c{tPsox`( z9v}TFKlXp&_l)g{z zazd!y7(+6zXZ%lthx;-kEokiW!$kfW-0)hp5=3wjKtnO%JV)oz2uiqa3 zTKJiRgY$!y*;&~jxHwK`2=wjol$(c{jUB=kg6)IA!Sq28l$hD$59Ho&wt$p`2m!(O z{lNqefVtlU`>o`EV)8#VJv#&!$o%lf^RG_u2p9yhKf>-n?2mXn5c?x;?*T{m-ypUJ z3V$xoU$_JpD<6amGw<(@kV^kz9bO*R2QVV@AC7-wod>c0R0)5<^e@Q%%9M|j4dU#~ zyuUyG3DXB9{2ivhC-)oqzg^P@mH!>4|GuWbSNd;8_>-DGKpuZ%`VToj2Rj=xFX!KJ z{zqi!ADy4$eL0(F05`VW|# zo0p53j~xQ~<$efxzg_Y2GIMc5{O#XHy+OG3}cbNYBSnwC&>0h$%qZ0lO)4w45_iFn8BI-Q?)PJk#Ki32} zSs_{fZ?=Fx34M>S_ur&@$oG%fcouaxW7mg^hdZn+;zllJ50H5l@n;e;l9ErK8QEGH zJ6rwI`-AjZSY%Bhn06~yPbL{ikHy6V()BR1cYS!n1;qIn23a&5H0`Y(`ic-l|1Y5D{*^4hN&HM!;(_sl;OxJk>K`SC+}3cAmX%O6a{Mo8 zStPVUY^WXHzq0h*s!+(+WKZvhIR~6$EGRNhb!u=$k=qsJl5j+M3xttjxgr2TlAwBK?;= z{#%Ym^C%hFnf2WTp-zoMb;H!9TPEo3tKY~D~s5JHC#bFT&%xZ#udcG4!L+{ ziUB`f6*)ZCz9`@ zSbvAA$b(Unu`yx*3n_!2HJapKAiYjynEc6L?(k<&RC^ zas8a%#CweP|8B!QNHqR6`f_mm`6R)Aw7s6XqLGUYi0d&zvi$Y>Fnp}M4`KShSBLqk zt*fjq1~lJd{_vrYh^gK+_d8QzLSa4QLXv+@Ep`C|p&T8wByOlbVLrX8sp@(Fp$|J@ zReq<^MBTfvxR9Nc}hQQUMA4^c+3RV@4c&J&Ki z+{9*S5u=|9?Nr~o1+io3BJ=`C2NpDesq5^$}--#)S%;#QAHO0?SG5k%4MRSLoD)^{xA&k8>8FTQGH=e4Jd&Xa+ zVontblO97g0@et+w###BF%7M?87pW>YgUpYNIX=}>^Lk8<;Da@N_ng|e5GObAWAoY z62v@79Cu$eA@1b4-j~}qa=b~_<1TDNvhS0}O5IX@VP(w)58cs%tL-!*{Y-g2)I~mq zD%P57M&v!Bd^%g8abj-kVr*h2m0!v{_V8O#GkH_5WYMk!xqPS$$dD0@Z0 z3*5H5EMuH`3{lv9WGR;A8%OgE#5O&t`#iO(if(CSGr89dX+MTEwCGqIvFbJe+c7zT z2(4lS6Di+O&_q7=9{!j#4p`7g*H=B{!3ZCjVq#${EG%Nzu&PAmLy!yvP#fS{w-&ZU z8Wke&+Ekt*dOe|x8-(Jd8FmXGS>0}5i5oo#-c?j1nCx!f*}r?+-dA-TSvG^HU_Lh< zpr}upq8lABDP~su?E4L`$!zNUy!7#aoUOH6{~GDQTTx9z&O$}}fFFk!ff$^NTz4qr z_oA+y$J4G;S4`kyD zIx*pKHbnlf_w7OKr%P$X@ujv*RZ*V#<pG7xgj)fAoN` zn|k~6770hwXGS)5r{%z%JID^QcqCcIqeQePGP%Qah*t(OG?O%58G%?H}e$rRK zfK?o~J$`?mEE%F^eQ|ex-*F&8KkadLfA0=stPvKJ_`GF`AYV*{Jl?p5H6_;cqrU;Q zG#HOIu+JQ~FIt+M(f~g0!y1lW1aG2VkPL-O^K)_DniN{Pk3%?woe?w@i8K|sb_0PF zxQ=(D8FX$6lz~K9clR-&6epYa_xHzhU+RhIj&bJ+!ru5%G|VN8!pdbxIpgKe<-U9K zlEF}YsP%N7%@lD812I`L$%$fY zV#K4TGZtGW-DNQi2Vc%gWJsy}1~ijtkqy-e5l}(`=)AU}B^j#e_u=FOC2?0oV1V;c zilt+-_xD@m$gJ+S_xF^D>1w#)kR8aGf6%AKS_CT6!>b&_zIDzZ#$@38(FCTFX5bM^ zw2iGb)EkTkcW+P!Enwj0zy2EbQdX964_ZPsBNWw3ZeooR{eec+{J!BRr!BT zSM2{}x_X=~`FH7xjs5?huCP^s2g9x{N<2GgU-1^cb=&KA`I|{Q=88jeajN8Kh^dC4 z`-1{UWE6TauuY)Zq}IdPz_q};^kNirIeHTKodAQaZ%c|Q7U+wM%0bY(w=`i)3T1>GU* ztNw1`O*pjpR@`l@2so##lo8BsWAPgQpTQg+0+b%&vZ{Sd!m<6-KqPr^OuE&wY%jqM zVzjSy^t<`jnQ${m+`(fj0NV_UpPX({b|}O9QnYXI%YZb(04(9y)=rPGZ{ zq(KuzsB%Fi?Rs312&PR-7@?uC?5Rsu98NQ9_uilpwb_N{d!^_ImPA*H2MMO5PsGwO zxl7L=n7`>aZ-hCgF^Z_+XVjkOUlKZQXg(y-E*OM#4X%Owaq8bRJUiCIDsWM?~8koZ9lE={PZB{~uvcE<$aS3NwAa`fh#_Vd+ zP@s0soF@`zQ=*}HyQ(6jf+D>#nMYvd>Q`%bmzCsq-hQ_2RV!sfKej-!vR7ru3Dm7R zd=YE;0uOVxm57vy42^{fTOQ$V5!+SaLe1{*v$-)nB7rOIF(z@p&W9#ur9vYg-wx_V zxqb<=G6lvxKSZC2 z6W1)PmXdttA8WC_FZmH@no~|UZr3_vcEZh?Wd0V~hC2Ao7dCR~oYk!r5=p45hJ}mS zbOJFA_-F;IAwSNIw0ZqEYj2>??5P}Bzqh#pBBx11cmrTcJUh_C-l+=JGSvNd_n3^1=6xNhdTEq7mku|tWKPMp2v%SG1k7~W5q9>_v)wOeID<~Z8;+$y3!TK zhORjJ5;v;Pz*M4H4rDaNhhgolbusBLpCJ5FC~;ywFO~m<59=PC)aWDZpU0Em&&T}_ zqJ0w9l9Q4weE?q!^DxZXIJ4Q)uB7TsPtUjrIh`S+q zL5$EOs)B$-^}2RRTpo%MYnfE}ifX@cT*!)}`P`~2lynMTZQ^i}s<@55d> z&RDD#3Xho-uCZ;iTj@+ZUJqa?0^wd5hccWOI^c)}w;Z1ahFaK#5>Wyg?ylTORTmc6 zF&wt>YAU$njjd5{UYyx{JV%<0&l`bDR+cv|j0^<439-a|cvdh)&rGZ7C81AJPcY!! ztAI+95p9eLddC@#>BhXg&D9L1Hm%j%ALC)jL0@0*Bw%wVmUe=4!WeUoAqgdYZgN~B zXXwWO^xYdsf@TVrG&?%Kf0Gsq2h6&;e?zQaUAv&HC>BKzf47_T*3|Z(7a-h?g0hf1 z9EhXG2ZJbu0OOBTh}F3Xm;Q0QY1yOWv_uc*da>!Io zQ8t8y1)R|~AmH*nCv&1R`rZ8-XhqPg&Lferw^Lp}T;BWSDtrW|7uP#6ze{d`z?^xqA!IR5BhUz2W?)r^4so(CV= zo<7OL!wE!gciq%vVL=`g%6V6}?5-_a{R0%!vd@ zp;GuW-1uk}A8ECWjF2f-Xe@7l7#Us zM>fW&0ue=}3oN<>N|J+BGSk38IvLjxC*b)24C2>y2h6v{;i6d0jXW1?E~p%o=ceH?+8#s5HX!a#4{y3^bLDo>&x)fUy}H zx)2$g#O~mYQ`*{tdd51y#9v7^-m$faRm3f0GBVp8nEZgX=*5*QqpyUaqvZoPs<)hd znO-*tFwTk4Ch1W|9Yt(n0oLQl&ms#N@!5TtShyeHuGUc#%YIum-u%qwIuf90caSw$ zP$F8}w#w8rWR$d%?UkK=M>XvD?d6dpr88sc=4vvXc_EeHW!iJ_<&o)IXu;h%!Z?BC`c zkL$IaE*O)36Ia{!ZcE&;zdu25m&n|MiG#CCKWhGPgk!R~f3Zg~c#1$K8f)ljF2deS z-M^fd&K|5sAgzeJM9RK>qF#|}wVSNHet>>pCqPe`h2 z?kzFZgVvZ=jF2n-EJWr`)yj&c*1vxEX<(qgbLF5v+921s5IV4nE|V!#nk3y5mLsEw z$gU?$-7F6i87q38HJm^lwXTgfK(382twb&%Oeu}XnvwJFPKB_)8;q8M?0a{g9n83P zaSn;+b3WtfD2wi2WS|j6{W|n?G_>xo@9%Nr`teB)@9sNr(v>~UKZxXouhh<8BAUJp zA~KG|wWm@&fPJQ7$%?RoY#lQaN6(IdYtT;jGI3$T$5Fb^_${_Zk;km*JqB7JSbQoC z6>Uz#96G=ds+PyK9g->#?$m_l`kbUumsn8!{mHpZjc>l}4xY_>4GlATXQ{w+7O|kLuK7OP6 z@2S9j{cCkSZPJeO6AWet3|Lq=$t-C9Fmz~P|BDKAiI>K4aRsVG4bu%Xmi6f+baC@k z`>tuJqj%M*^>y?0@ql85LSS zJogb^b^+(S_dxyv7R8A7e`58d4nPC)IH$P>lgsqezHl$SWBYz_Zj5)~?WU5?M78?$ z-HrbXz%ogt@^i$GJZ*zd*W@LFJBZ4$^H`T&e8MxtPisUn{{C$2?alpD%wq%Vc;aaz zY|NdP0~pIF1yNocM$R(!Pfv0PtSB$cvTw`5We?(yAyG|&>?n5xHuzMqMzS+YtAZXI zQSad;{}S2Q=OqD$)*o4BwPKw#&#=(EAmHfFp-3Uprngdt9!1aG7hZN13CrtrrgqJF z0#jsteThDR=NuSxci4^y;4WN&0{Dd))vYbEnhC47T(#ZPV(J0@$`WX1Iq?`C^^#m) zt2QU-bcF`;;zk?jL=MVe>#~d6HxWTxiUM3O!x^{fZPXB1&Irp3GJ4pzze){ib`&Pa zdy`NO&K1Xqiu0OilO8#R>wHEG<~LP0R8Y}{c@{JfX%1hm(qaiHBA?Ovs>&ESQ^r{67%(lNceo$B8>`%7&=kH zbv6MUNcq&q#WXH5ZrMgF%SLT*$e0Z!i%w6m&U+Cp@Gw$4km&z@@358&{ATGD{Lkgf!E%h zh+DV;HZ%yQ*Oyvmjf^k|ezGn;3+wo~ntVn-V_lc@Rh9tQiF`=MNipgRJymVK@rHG7 zi&I9)D_UTzr=TZOg$6MUrrOBTv~=gwfm@yDjf1oxeiK~7FPvj^X5#7kwW5QX%KQL> zMRi%fL>1Wa${EW3xNo}!bUc!83*x}SET`YVrJ=uUMk7kPH%Nh!Wh(Lu-`g9IC9^Dj zOT}c`lpeYP+ook@+C0tGpdD9Uk!U`?Gho;WZR~}1nozH>?9DSMSg}V04y;?ZEljx9 z7;u0HlY+*{`caC@kh*UwKyOgpQsK_xB`#&dAlb4baC?RZ)@?*ZbZc1c?{>vj7)-HW zzE8txayumT+<=RZPK2y4_syzPOHX_#7iOaz-Ftif1S~0W;Z<&R%rvv;+ky1wsx=qC zZ@(B6n(*#P681+QNX}cF%PpVqyh8=*Y^x@bx>d-m6LK)xj7E5}NP3fw4SkcHgIUqcm_<`1=I#XJ69L8UvO^%h5o-t>f7e?jT>_IY?SrBSycPTMv`*wiXE}=l& zq+YkCj%pyk-)C?sj}28u(-0L4xQfoNRa`up5@JAeYdGs&CL49?RPG{ur)%KRdAMlU z_4RV!K11f|%$#LGStluoW(^ZcA+A6J*Q;%u%8%<6VjUXgw1zo!D>xQz%+-{}vsP&` zx5FrO9}qWkum9}q2L(cZhmE-L-72Jp8oYqEoQLz9;&3~g_Lu-7s?-#iiPv*mjz{+YEhkxbW`k!mjw(*rKaYSAcEm1UzU z(SF$5@(>!eS~sE&7U5Yctq$YY?cV*5g`UUf=|zXjEk4UH6~h| zZ`NBs;#O!lau>+Ad(Fnz3xnL<-+xxfvoi0EfIK|ieJ!CB&F|al+M+PQ?e~Y3-=Df9 zkV;jSQ-%19o}QhNOE~>a-ul_#(dyBvbwOjKNK0zD z#2~xZ?T{ml@*(mBjGKlu1#1n_SfeEdHc+;XsU{`EB3 zNFOW2PefQDDLLYhQS-v<_Qwwwsnc`{mj^y5ZiU78Zs`TM|EX!SMg^v>l zP|IC(;Exme8$`ohVNohTY3om_;db8=3=4$;ZiE++L1LcNOYKWcC0(>0i5CCy4i)0= zV~ltd14Dfz@pK}QGXKS~U-2zb2$36~AJSgV1MC%MHeqNxBp7kq?BEQTu6EkOx z*vbu1P0>!P$*2L84JtbhFk+)5pJw43mP-ethFYUFNS?K8O11E-L1Nlv;T7SOc}~WJ z`JoF=*)Sm>Iu-!~yN^knlRP!9b;xoa)nv$v5sOQ5?_lFu4?^}sN+iv~kKbvdp>Fwe zku}M7Xx5Vn1kt`8kx%ju*2|-|*>f_TA`172>Fi1ydqS|n&B6@{8K#xu7E})*tg9sF$GW)()+|Z-J>K?6BtEkQ&BQ082#@QooTKI%sC?WKg&^i z?o>Ki+*g(*EFT1N(Nti_!IRd_1A(luchmelT5mUJ!-Yxn?dj%ik&7m&*5QZKi%B1G0i`gJo@WC)==07q0R}W7P<(x>vqP}^7eWnhGW^JgpCQpF$ybC6 z9;S({cq16LU`wy85@eM98=<7+gIa6_gU`g-;*B^PIzBc$8RrHNzitK?qXHfg0~*P4 z54d`~31_xegfTixJ?c|!PTBg94kfi3>_OsX^x(S30~+!pG%I;4H?~hOZ0vi-?urRQaQT{*T1=;?Cv->Mv@IT3k-!=KIrhb3v_n8(U+i$c; zhwv{_|B@*CFJAD^O8i;J59sfYI{#7C|0O8M_MZsFUxR{dznQb&px}QnM}O$9KV$R! z&hn?G{l5YQe;4!To&Ku_{vIg!XT$$FA;`(V`WJ*C<6jYiOdRz8MF_I6`~@M%#Ky+@ z&j>+jlixlqAMu+06BcA)`Ab;vBlia^$jI=4?=yVBf=nEL0}K8Ky7gD``^y!Bzhboh z2WIT!&i~LeiS76EMacf!T>IBd*zcr2rTkar{R1ZK-vJB$uE@Vo=)VOP{2eCi&qDtV zq|~2m{{Mp1XZyX@``4rt`(F-G{;DJXhmQZ(U_tiZTp%I)Zzc4Bul`}cCuINqp8xfO z@%xK^itDfH^A8@3f55B#OZ@upYW*vP{&R%xA5eXNgV6mJ!oO1JKe+9`*3v&j{;w4J z&s+Krg#WiK{Y$*z-}egqFG&60Py65as{D?Z_oqmHEBt@P3VvwZ|01>juPub^zZK*^ zu@JJc{0%GkF+y+W7uOu3YaU(Z$xK%A!#HHFcYrjV8ns%>T543=A_~S(t*LW-F&Q7W z+)A9QRh#0CKRv9szY?d!ftLD$(_&hl5gayb9w}Z?UsIzn=H-LTNoit)!6s`g;6mr4 z0Z*-pfDZ$2X0U^lxP%_RsIPC_OFTA8RAfj@Q8gYWlY-UVLz`QAHa<8KDTn;&Q*3Tw zeibU}jQ`*WecokUp$R%2%{NlAii!Sl$s{9=r@|x|S}&Ma=3?`eeny7cZROGaU8Kdh zgr;idKY zwXB42t|?H_)Ff<{L6o5zM8p^UNT^AYXkTYbnuVL}p~ZI;274#1@#L}NIQ!U|L*m0r zt;DI25|nqgfk_=?b%`WMKk3^G=-fUD8;h8*>}Ufsir@o7pJGi6_fOFGP?LPhGxwu` z8|PxO#L#}-T?avla+4$?_RG)^_gctEt3x-05j9sdj4-uNu0Z{=c*04NfM?#lj+fTO zQQ++AaYpAoWXz%g+r^}R4W0xuoYp%&8omH#=vP--He0tYyi6$c{uZW|*2#l9& zf-2-w1*yU$z1|Ch+rF!AveGQ#7K9EJf&xU;@A~({Wd44b|4r}F|AWac6DRB6PId|L zIKHJ*2FvLlf2`!70@%Hey$@yci48XJX>QLZ)IIuk($jn+Q!1rHX`K%Wna0d}v(_Wj+L5w#p7 zvHD$aSa7BC^UT`POMNaNFPR!H@j=nV)KIae=ae-!a6pKp8;+ToOj;vGSWU&OR14s& z!)!8xF8%Ah1hY!+ZRfqj!5jwRV{Hd^lJR3nq>DTwVEfjyaT_ zxUwY|qaQLC`;MnBnB>30k3Oo%B6tG@Bz`8H|92-Tze9@tpZ7G&zfogy{D;@mKVJH0 zWBu(6;1j?u; z9z3fg9qAbq&U+eii&{WB7%XPe8V$0Xatt4A(~e6+vSLU^Zr95BDCs)> zU65Qr_M8rEFq?xQG^QgYaMLh5$hsHOR0psB2nf{fubgTLP;*BPz z`K%Np&SJLIcE$rg#8pK@HU?z7gW#39c{UT2wJMXrQcXGO#2Z;tb_S|v#l);cQa;Jo z##VuqeMf>0a^qNc&O=WLj)2OpA;n=}V$v)R>y$5Y+B& zAnyrPl((y^#TtPffpLhqk=Infmb|s5f8brkTF1!`#SrATk;?6vDUQ=Y-z2L8BSzyJ zu2X|pIM+eOTni9smY*engq^!*qL<%2kG8&_%kRg)gWfD~_Ti7TI|AGt@0!;=a>^-B*w zF)kL=xf>coE3k?CCwF^s=8@8;5q~W1d&^mMmF2DX8hqR>n#e6ahr#0fKk#NILWrfn z_jgPqC4=jockLR$c>SC2_kPQ0Vw$I#%rPQpP$0J>S;xQM)qgqT`qOA4ZDI4}!BCQx;lmZ1g^=;X_nP4&;4&)<-AA-$PC8Z=mOtj&N+0`JsQuyENH0prLND-Vgkyo< zTZw$cFIG{M_~W23cXG0G{7g@8X5nP+Y)EHhYfW!uWn)F}Y-3^SZt6(+u_?q~&i^_7 zo89UE?=h}9KEiGPp6}Aje@OF>Jw*Npcm1)K#UI;Q{Bg?oA>BWY|GeP%xcD=L@gwEq z_>UCEkCcz&KQ5R)Qa+B4i$8kc!$9@X0ssDJ*Nne+>G*J?`=7&If9%olN6p0kE#a1lMgv=x>MSKlkz7|7~{mmCUf&`U(xM3gp@N{zK8~jxC_!hqBfgeqJ9qxC} zYkfmqZ~IqEhj_a_jm2rs@9I2(&cyj0%ae_b# zP?ivDB+g@0%W{n!xe(i_8G=~AHDak5VXUCe+>52mt**ic@x|%jQehT5nZ>hD@+_t^ zJMmrV;(gOHxJ8rl{@S)|PaL=J5i^o?)qyY}mP!4RDh6t930JW^<9^eCx(F?{k-p5% z3AfWU$g8rhr-OmbSkdt@-a_<)K0t0c2dtFO3kRg?UoqkJ^86hG0YZTHlrcrI<9*h~!gL0MG-G ziwglocv8F+WsDpI%P{Z#inXB9T3hO4iX^S)IXyd<@UB)xp`mMiwPrXrazpdW$Zsuh zYQA;AlHcn47FL#}=S>mV=RBEiGqY_(*~5e!qL_Kv+K0nrAZ>gK5nd^j?(X|;JKpVG z$6UOYUXIB5Xh0zo(Jy%4mV&7@26K7H8X6R)VwHmmDfWCD)h&PKHaJ*ShR*a%i zN*PR07zL@;(fb;dytc2uzye^PTkkpf%N~a5pIx=@r_;)d~IjBQVS= zboNG~RRH8ma8b8%ds4S_X8tvzg@tYVUDSNMiz3yK{eJB!qyiQ}(&wX!bJq34GD@La zT(F@h$DxY=7yf-!kUYa%A^<)Wnkr|M$~|yGV-z0fT?(ZY21LonUBCzxGm=3zhMENm9lmVes=aGU^ z&cry5rQ&eduRhpw9F@qipV-)&#L~xmJavv=zM_}=ShKsUA5BOr%&MfxObJb1f+Lq73Os<-z=3Sd&r z996&D?Zl?OJ3qeqe&`B%KE1fFGQO{lKTDo(e?9dL(9`M7M|Nq8K3zWFh_pSzytjA$ zyx3i6dQ_X;(msr=77On)LBO=_6eOkea&E4~)QciezhBs?tMA>|VQP(RvlYX3wyo~u z(C%cgEM)N!D^P2Sl@W4_v4F}8%*sg*!Hm5l;*}yWMA~rgLQIQVv!#P2e?PU6HfVXg za2Z$c>gFER)?2gma(~L#UdM}1DtkT89CNYvj4Q9!jor01NqF(D<7*R6#^i-sXsQEK zGatjS(Xg?&*`X8P({fmcHP^~uF?@VVw!$kc%u393^{}byGm1>iHnNA| zFLwM=zj;$|l=sa{#Pslt5&0`fR4VFC31j;uy`CMOtQjQuWaUI!Ni}`q_@s1A@4}Me z%l%7>uAZLF2%p6We)`mFl-Cd7^j&3WmWt!>cQx_(`c z<*w#nuUt;om~qUnu9YKKZZQfz#JS9c&-W%fHN9%SO?Y*lHTUe3eUM+2 zahc=ksygvDyzd6gRS;LE=)^V>f3c3_MR!HN^Y3>NtPvk*#H??J9*kokD{wkliI39j z@+@XO(`-mwtBZIXEklOn(1l=6*X5EpN=G9wPvsz=7jo4SadE0 zd>oMXO_jc*$M4@j8d_Mh!12^5QZ$66EdX7 zZAgqf+wrt>IVNL%kPUTRCVl{qqhLtwU%!5Q!h^Xi>&$VXV8s$PyT4lY5?OkTcy*NC zBT&nzmp)N*bdBisat*$<{VdS>h>r^@JHaGtyfc2I<+&URq9&jEX;T= zc!Vx2)7kv8SHu)Gld88<%^pju>n1zbn+mt@8EbH1naUBkbv)zd=vmbMzPoyz;u$%7CXH{AsacH*aGVGAsr0LYByFkmqfa}EykIS0>5S)eNL!JGz7$zYpH)qM zxbrYC&eYSqL$46c&I~-Sn3s-JO$SsJr_%QZwh^BH^sx#Qb9uQkfLVPvQGietu^E$lLPeLyqVayfW_0RM&fY*oG@hj2gB) zmp~21K$O<$1+8()FQ;mEg|o}5PGQH_>o0TOx89LxtNs^g~$hHyhlT4Wnf#hfn*}V|Vxup*feUsqHsN8%;pCX3v=_LOLpq#IWVu2| z{N7iZ#yc>~5li_e**00CgR?Uvyodmi^7r(?^$o*i*HR{W%S zhHQBEsFG8fO{JYuBl4u$+(J(FVNNMh#Pmsc)28e6q|Th=6CuFDajENge}#A8>w%+w z4rg-pJ2!DW^2~CtBb1sdup^?O=i6Fz!!Rj+^l&npTIj1gkHvJ|wc@YV!%^}X_Jmx% zi5KW75mmldVIhdIPyN)C9Yi`L#`QD%^EiNSL^m|I;0{(GYABrgr~_nc)*>~##$Zcn zg!80P4lq6LQ{ro;<9QR}Ehpi*T6Tz>?p3UozpuODRbUf8aO6ZDaIRoBp-(Qt8e|EMduQ+wd`09m0;ngXL#(kX8NV&Lwme>d}g$V zEdo^k!#WJV-tid6k90UI1PM#f1}xuqz&V;V-DcQSPN&$G(n^bK-G0nkz4ff?qUx+o zwW1jRkxzA)Bzdq$_%7^e2Gljo^lPpDB`VVc%qL$(uvTk`chqauW;bwsSPyPw=fIji zZ-%${6E}<^@fN*Hv7Ffs*5^!!0@@2Z1;lj-x5pn3dR?Y8fn|?JL2xpuc_QmJuRqIM zxsAoGvQ*958dX9FuAX7}I2n`f`&jmm&siwqhQb$_l4(NDb#9M&+M}M*LGZXPioa?M zx~l5m+?uPf@=cfVfFD*QCqGMSS8VARC}~%>ABB}P7qmGrKb4HR!~S3$bD)^N8&1m+ zv8cE{H}H5+al?lzZ*b&Ut);)f2TQh-eviuc)nr%`8@!_7PjsLJ!@Y#ti^Y70l;Tr+ z-fF_--%eYj9<|^p%%_|(PbSyGdD1|lTK3vGH{bNmfZp(T5S~mF}@LhU}In3 z7Kk?)N+eqbN%C{9NA4XH`w`toa1w^&IdQNreWFEghh!I!JWnGc6?p)wO1HeaA^8>X z#HY*4wpEOm)uM2eI24@J|Gq&3;Ec|P%;{qs$a5gq_P!|^{-ITDS5l2|XZ+N1E6e_c zNx;%W1xd{h2Mtic9wn-MqFN`NSNXl1?@MgMX>p%io5q72C1Ws^3^FQRcd$JJ-LOdx>=lVc!6*IH<>N` zj_5ZfoY4!*yU-~^tVnvba~_xei3CPA9Q-&e%emnpE-2(nDPe8x88jPUOkL0UoS~nE z8i~-eA_~QjcPWUfQ)8s!pT^KeEi0bu@5L)@KDX=k{VHL70}M8K&Mfe_!Oax!y_Gr; zv0hQaPBb4bM>Ivre;yk$I+hyk<4q3N7ahjJv?NE^(S@j}WEN*cJkc=wlDWy_f#mBE zxS~_*`W?`%_&oyo=lo0p>cRdzQ|w+1A5Hq$O@KJuG7I1#BycagCwgih5t)l~OohE2 zWtv9FmiEXaVW*d@0Ay#?^s#w_V#p<7@`>bAR^gt7zfy#cmOh0A`AMRU*}932S)q{y zIe>g4CheQEXt&ME2$Gbej>(98{#jJrw|qP;Gx4P*fOq0zPc`$d#q&p`M~S03kTuzb zIh!+LdO$1u;z!=aZ!FAK^%{d(7=rj@32LS8_VhV9*HEEtOtq_-W@juyRBNM}+UHtG=U$-!W9q{{UyKL& zWrquum`;MCR950fN{mk_(>)}~F5@i7ja#{hi~z9dE;(>j8L=W%%dUc?jNdY5UU_iW zgd)w08@S()%ETtr8V#epR(f@nhmF`9ClXE9m zS;J-Gjt(qn#b}6)_|w9vorqA3&)#&U=lGGCfWwt0ZIAMAh)b znoLNpdh~m&Ayj8SK9^vPT%e>tqPOo;v7&fWQ-iO!>TQln8c~y0pP=QJ(Kpgj#z-pN z8dUBl$hW4(_mYWd+N7o3lPnd>#acS^#Ur0rO}y1p+ouJ(ge<#1fL$yljAN+N8ksIv^3bafgQC zZI)^7xm%1&W_0BrajMHm$!t$Bg^g*b7`jNh5qm6FzLo3DW@O^?!GC`}!aBH+(m(_y zGgsV}1AS4<>E2`wRjoJC=W{QIa4TI<)FrT}gA=+}EP`4i)9^|rt2N+CTDkeI78^OA zGfwEDb{xNCersOYfSzIo5Z-qKV#n`=3ZbWbxMo2`=MRN`O19 zql_ps29tGa6k55cXnazdY9mP>edB#iQY#3OGG@n6V(s3L!~4^f|3jdWF=Q0{G+1_3 z!e(~8!Rd?JJ}_f;h>{{bUi*2r(AxvbFr?f|5&g~3&W#9voN{Zk5DZaLHC;y#gek8v z(hz+o;~q}`;CQRvRe8js+1G#ROMgNC2qPyh7a-mn}4BXC>9de+|p=#Q%LfRN){hQ=@d z*x(wvytL*Tmk#68U%L~TyzF$!9vDhxW*eDEu(>h+wQaeB6nUP+^w{}}hvP#3JOfJ! zh!*j}-qZ;jO0e&3l1h~YPe!NKsrM^3fLm!1!Q9_BTwUNyI)d9|DJ?IH3|UGX_RGR{ zkJ`jLda;3R>V2}}HBgO5zQ*uk=p4fdWDb>{^^nmta@FnrfUMYi0;%p8*iHrT=5|mI z#E0urap*YB$gsI!2ET~Jpu9>>os%FpMOid4PEVqu zyV;ZNz-ky>xI7LYI((`kE?gbc6lGwV$z{zu&G1AGFjbnbM;oD;?VMF_E2w=nqoTj(|}JwvAU zX1jKo6ZEl)OJY-qc*gN0PBAgB%Nv7Y^N5;i6brcTIje7Wlc~!%lf^$L?I1e*)QS~A zHoOGQLXomM&=c{nmHC*Tj-<}-U35IUnr?u~ySL;rH7{Swx_j9A%ji^8G;%^r1#0!% zw2^+P4C>Q%Xe5gjO)@vu@MT}+oOf;Kv zdu0{XH}1~0vd z=C$kiqRA;JQ0b(9#^bX&wRHO-jfP@;81^?BPg`RemHB>ddgG9Q&Zi7Jo8HMWx9?LKZFyj2*5&AV zIk#%XXwn4L_tD}uMD$j^7ZlZwviegOz1FPlLw;y%-LE^F1jjBs$qX=|MC&>;OgV)Lt#yIn5=Tg*n3Jga@6Dpl^y~yg<;_FPl2f z66iGq7$;knhpyFstaKnjcR#>KrpT+u@W#_^7m#ENNc~y#P)eF%TZ_|NBzwTFIPSuI zgoq#LMSnf{PL%0AcdPo!(nGsnK3UjEpNp3Nw=zbF0FFHQ$KwNp9lsFwW^2RsgJ6euX)dwzILYty94~)L0zWIOMQ! zC`n8N2cH~eb(PA9_{wna5Ly_1|#kLv)kmiE9l53pDzti_+ERCD~+36fPZ`T;g?&uT=cz?#keG<@9gYIZSYOyQd{u74N=luT_V{#^bt4|X-3Y-#SrBrEdAM-a7)nKVsLPZq z7_+dQy{?-kv*mKL4&$>rwPt%Bb{Nro6t7M_(Q!Xir!GBJi&>2P-B z-Z&Dt2v-~qhw#GiXmoXH<}vLTXf|rh??KWjL})E9MUqrI$W^q%F+F*rnj!tLHtp$j z$|xAfuHNkkWl)jeY|PGEwORYJ3R%XY*Cgx#0Wp-LsSo>o50wLFlIoq7PI#?n&X^r0 zvg}gT7yMgMV}EkYl6PRjX+aGNG1#KS7g|i5^rb@!ne~9(6{sA zIPPt*{Ip~05go{GAws|gL;>!Uj)6?d6@+_?;1|>MvrS zzjjY?XZ3A?$5pGuH^oJ}i2&9KX0Fh_5KH^Jy)SC*HAhTCEXBz8-H*|jd7>eg8jn;O zY#<(xwqR>kD$J7Sz1eHHsJozbc6z!XqJ`!QmpTqmd*d8C4H;DWuz{RY6FhHpH-Pbo zM6XYPy;5j3(!L9*p0b3UlB1GDEiN2n-ePEOe1b3xYmi6kB+nob2Vtn=sf3=!fu$OK zxt+xl1z_7f(pfT4%WX+rbrJwu6$QvO4nSf~gT7xBM-g=~WVY6VxsuKQWle*T?~-h{YL@SpPdL{ z(oDqt6lq8KmYka{*C^K{#z9H~&a|V{6^v7bPOw=%)|@b#W2~AxHS=KHIfPe_997CF z#wg{L<-=F6wcY#sZnKE5Fc`24w+hR6i*G)29J}}kWN=R*7U3TEs;6>bLO=7umc1e@ z%T&ZF7l474IC>BP;77!&`iPadCs5IDlD37aM9aI%n;;B0w};D#dV0&Lqcw)7InG5| zz;ufT+##h7^O2%~KB?9ja^>@9q0;0N;wKC44MoY2cyyx@=27a#VP0^aNm=1yIVh+_ z?1_c9omdFeP+e)N;0q4OiiUHNDiA*_kcl`BZyPsRnL%_bA$K&{?TB#^Mf%_s!(*T> zSk&?cMO_J*Pgw#Xv_^YW3uZk|8lq!#QuHeDr!Su$rp4YZKf(pzXPDaxf>n8<;Rv^M zD)gKI^lzRIk{W z`Lo0tMl5eq2v3GTSOzC&)FYGjhuE6lZ4cUE7%)EgqO7saDxy)Td?$_Kr;{87i>Vi( z-vQxA*QilXmY^X-yWx&rfi773l(@Yete?vv{HasQpVu>hT!Fi2V7?GFke67E1a=rc zSR4IU9)VCe5lREa{&I!Jrmxyhh7@64TcP8Fn_7fr6Nhr+uI!?c6f^S&9)*y-< z@S?kOu=(M!&HY48{@JV!=oM0+NT;Qcl`w;ZH2fA`tS3Des>w{&H+8EX6q+xcrh0?r zQckUelu1J?3W+C7eV`$2onk+8?sMLoR6IsFOeTeLK?2ZayQP@H5Epi%Sm&G%1{))U z!<-kNtQ&E2-OGS?PoZ#3j<}k}61kxqM+HFVLPC~{qJOQV%*|~ggp&tKm&C`86h#Le zDEpcIJ^WU`WX98DB8973MA`%&Nzsg`@wLKkrBqPLOWo)qpBc6@LvCqePklsO3}9)x zYXyS6e!lKUDSRzZURuZ#jpA5F{bbHi4>DCN@fbBilhjQI%3Ew4X=NrJ(Xt#J#S(U$ zm?m7O8Ad5G(RyC5FUhV;Moklc6mMl**bfp6r|~U^0#R24zzm5M?Ubo~6zNm+L`*na zE59GV56X-*q@CEa4lMat_?ci8un}s=fN1>bQ!wcetEF+{CQ0g&m}zbVg~oHJw|bs= z^jGDj(!`lo8%RjW2XjCyBLl4{AXqGk_=C6-`UW@6=AR1}JLNnV>eXyR%q@&se zvTUabJMLLgA4!7H=AA<57TTUzvNZ9B{^!O;la;iH{XdXAnjD#v!h605+LFM z8^gsrS@x$<)WjsHG^xOr%!6b0jmal7s7qMJhy?evt61{?=sv12?M4rxDx)!W$j72n z8BxAwm4ju583tjLmIx~gA`!@y(i(QsH>nwEO;6J?wwnGt;4Z;6CceDpUbM%4bxa|s zKJn0F#JrK$W9b}UG8ZALjuZT1Y}5_cAAm+1X`JEqr2`*W8a>AS``NEFX-3Mvx(x$M zBdu$D$JI^UeFr6lfU^8~>x}VE$>4UTZ6lK)eQQbKToAW)j!b~-9*NKc(s%nLDw8}# zWQ#Ccb7kBQnCoBJY3|CZCuH~FR5BmWI9-(C@EMnGOEj3MF68DA@dxU8J)6_6;QO% z0LHRynP0G7tOat8qpqxcNgL;RJ;k}kZ$w7KvI{ep?8zAIutJ?2vEg@2v=~4)Ttm9S z50uratZ{|>)g9(A8_uiO2TFn!YaFf6Ox)5@pGrg;=d&Cfb%gqbNO<9(o8~HQf$U{e z=>j}h0gZTaD?~5h2=E*mX`p-0FEQ0#aL*4TV_-;tD&;VFRHWRzkc+CwKA0h{BDWSi zFi}StRhpQ4WHHV2UMYH>`Fc%v3~i6Am&^RAtfZQ#FsX8ERYwjaWi~6ow;0!Xq*)mbq44tJMdH`sIeoC zM^SmyR3*afNe*@1k!E2t@KRv!64xGV2~Mu0@sFa-9RAUp`~whrdQ+T56Bi{hnK;y@12dFZ45?(LUmuL4v@W*_(?`^BZEIB~ zkSCgKN@JE)?MO;^VftCE1#*W{hGIh>VeoJZZF)Z)+Q>l&TzPy+83=PNy&wdO zB~V=h%?5}0E7V?q7r-nRBG4|O-v+LJ>=}hrvbG#1)XB)7)X9k9r79kL!B3+{kd^~l z8Ng7{jMOhsSoB@gHE%1)(&mu}p)Sq9a!E|oAt`*2d`LX7F5r9a!YpmEH*iq$0D=-0 zhGK|qyo*$1OOu0QK|=F&@og9rxM<4mBT&aNr&9B$(E=F578x*d7%a-~nv6)ix0+eI za}v04Et(`l<|P!vt@6tPP<3i=C}owAvU!0j{0VbP12 z^jf@`u7Dq+gUiq1u3(+pnSp~Fy*>UyjL1q!NT|~jlN`j2KDFN4HQgf;I?0aBBA*$R zqKmPe7#{CI^t8E^()f@LrkScLkuC^@dU?Pg5mP^l9U&Fyp=yJ z3|0~f<;@@C%H9=h71a)Cu@r?^skleRW6?!Gk4qkog)k~lM!5V44U|YCqJ~F^m=x||3f3ihUjf@aRe=2sPn9iT@x(plBLb(vUL)N_yPE17tuUOa0 zw#7R=swx%RCQlIxIC4z2beJuew}>b>TmHu~IZT~LK&9KBDN!GP_Ox2nl;q>$=TBn1 zvzRUuA^lY6)L)m!YB9AEl>4RNy(Qx{IJo+-uT8FCBx!(SHc@{DQ>&C-(OOyI6z#%K zE|l_$666o=qK3507s@kDF(HUtjfVAGoQw3$<`SjJHSng2m*)wF^(@k2Ooy96!6Xgp zamEkJ*U~J8*}BJ4RZOB;im4bybF(4}_$NyfNq>dsCxG9vda83+N0N))RF-?3FLNqO zoM8m|a;iLRK7X(-D8gXGD3}~UcqBhDF9Gi;+rS`0#;LJM?}uy%Qj<_MRkbbXPhp82 za9}dkmtp*Q*dq$&*5quHR2atuDqDzNf<wY z?W{mi8OGaUKl95hz9I-plpM3|0x75Ue`*pwY zvP_VQ^GdlvHRh@Q^mN5SR=WeQJ5P{Myz4N^*&mDF_==5)EQPK}1X_ z4OkCDo|Pn8L)HVFa@s!X5jh)4yRe(kK2v*yBSgJ=9n&9kN1j40VI)e)9tI8)W>k37 zX(EIHev>w@NFZ*yKMXyO*e)R`&P^31+;16?Nxb0`^0>+!WnQnoXsWYkC`pHSuV9r1 zkdYGvETs20l!tH8;z=?2aG)51^EFmN6j#?N!{$8RQvn8GVfZKB36v-td;P)~OgA0) z&|X#b%o@YeYHM#7+3YMpRYxlF6inKh;K7FE58oqTk~`L$6!MlcIjiyoK^0y}sgz@+ zHAI2~s)X;V^95^<>h(l|L$p`z6lD>NIq1=$xknw zRr&xF3`-L;%>=VybOQuu;{8ITO=yd{CN>dCEHbg@&Ft_ywjk;Q` ze&qkb(iH&>zJc|fG0&l_yz(Qz_aFJO1FGg*k)zk*{dd8!Pei11$$&H^;R~_i$h3>(S(K8MT7Dau%R6E>2Lh z>__rULVIqNKUp@&str4MC3f0ZLy9nSiC(QSK{AE;Q5z&CjVfG=ea&dEsHI&O79a3R zrBEg>$^N~Z$=S7NDWTc70ESUwE7U6cb zus~tC7;-1l#y}4$!yv4Fm8QIf$Uo{}FDr5f4F!~e_q)l%ot3Xi13_0+Y7J487?U&~ zbwl;?j8v*3{N+?5yZx&hr*N26z`|@hJ%}vp+a#z87C1WWOLVUY!FC;Mb~U9$V`$PMuKuQjE%l2G-VZz19r9;pF%lU zBH^zfiwrk~qfE=io=LvRCV(lXx4=&t>xiJUE*X0Ff!_c?l_k2g^CNR1S~NavZ~EdafQ*j~5M5Nd zg?Hx{DulFYt}2yA-&~ngz?{7)vB+SJbwFIy=t7OFBh>4og&XQ?Wj-4A*a-*%SejH2 z0SGv!DG@$*Ml9!=T`LSQ(C}H@BO^jIEJ1o^vEH<0p+qM5S95Y>UuMB}ay!GJR1! zf|Kiw&OYt|DxHA=iUrf0Ks)YXtF*4IQoU_6%}IH*KI-0w^@zK5A?g+stCaU~7Nrq} zUKRz+DacX?vbdGyr&eiHD?_5z<+{z(>~kg{(M*NK1y0(f+LrF}6o z>bxft%=$twA$+zZ(!{Djc*?ZKM7io~w1oYYyKZXmfQ2F8VH!MQgPI@`gGa86$E70@ zP8X?t+z=sZG*XN9M06-9Uu{}IdUs~-xkw2uaWtet4;GX=odjne*$fkS5)*oLWgKu- zJOb zxH=+XadI78r$J^O>RH#$PzfzzvAgh&MxcL{UQm3lnGRV6ccMI60n;=ZRhPF8*$@~A z02Pk&5{e)Pc>+uDN{MS#u@Q5d<|Rfq1m4fgLB%aHJUyrO>~5hO*D)n%Jp8C_>4|82 zF~2>fI%5BfEU0=G#PK@{lfiahB?CL9?X7PTjyYGO2aC`7tD!%@dKZ?8=K4wax{ z@QX4+h1VdW#l4^zRo)a}3G~J#w-O{nK8Who^rlSkeB5AcSQR9}@qs^!13=6AY}g`5 z;O!EXP34HEl^7>>?5VRxZ-NAR4RI3|3W1;+r`sjf6Tb ztBDbFTxJs^KJ!slyS<85p%4)$HZtM_kRY!4LbL13%dUa43(i6nBb0t{FD_!l^N|hT zO^kSPmO!IZsMyadB`QbYQCQtM5iLYa`p5^tMGX;egoIz|EYRog?ZzA$PFo!hmB29J z8D(E27z#c;l_E~OO``M#%YggSuPoZ&XzV#zO`$?v*cB5at{ydTiW?pc2Z=kKH2BJi z%O*w~FZVf9Q9)F0kOUJWj8TP7xX2MY3&hu^T@edl(csXKT;pgaM|fRU!c6a}ie9bj zK;${hM_6Rcj2WGrU|-3bk11AuwJ1cxAi%>rU0{l2@Z+6f1}o@8KViWa<%Ju)K~00b z6e3|3l_?j?!s_a?TrA@Q6^jj5_<2@7{v%`(5wjWTQ!){=G@r96)Zf2z2y4JjD$NFxL|NrD&OP1X@uG~tkAP>Pm*VB_4*!!P8BO(AYtFyBG>h!E< z7A--%N{PfL5YcYFAA~WBB``*}0~>i~@%;zS7(Qg*jRx|10VPhThNQR7?teQH7M~1+Rsq1$R5nX z)SQTvyrpBY1a?_x3VNNf@wt`GaQv+Jqp*z<6$~D;o_^Mop!BkVPnfLtp6qA!Li{B| z{nJm&@z-ziWmds1R;)64#aL$RDj;}Wf8m6S_x2(o3vGM_y-qH+0P$HI zASqGp&APrg2NF{S512$(L-qJyYd>Cp?s(owKE-pz%DQt`kWL`i4Rn(?=98=H@kI(Y zBZxod8V~W+VG9kaa}SJjv%E`46_Di}K@P8RFa-<3bPXJ4UX@Um4X+0s1~7tDZy=I; ziNgP3wYmgg&67HfOqcm(y7yg}gl2^j68B6nuV}qkc(DaIc5rKF!8p6(JUeJ)aIxT~ za0Ify*F73pMj=^1i*}NU**-DP*2hqkO7KDxGQrVw=qb_37^p_~Q5%eTZfi>cVGC}B z7iKY81qPXz%VOm8Jx5UeUXH-s_(~pSW`uG8pB*Y#D9$j_KxY`#9*ewe_=;%2`}I%c z{d-)TM?;j9F%ZeKJnl^Q5BN)F?-0|3@Cg zmlS>VpJ)<5TKD;eKNqt-03HBKI+VAHtw)@Le z9Ch*K$3TlZSPpt%RHy4l&{7=Czv{hyx7qxUKA5lHp?@M_ z;JVR;I0L@ZF<>aLccyZEL-7Cyd^9@v7oXSMF@Q&~h@|A{yQH06;vRn8ucND7z$3^F z=z1v;$oy3<;Qg;~fo2z{bLL?xj{dWBfz?mi1$|elz~F2Da;5(JGrn4>Kb`##8KygN zR-Woxzo`dMBlPNfFzJW5^6MY;w`Y9$gWj{h+|K?=>O}s$z&Z3s^@5oVskB~qoBh5R&4Bksd@y+J*-?_opzvg@C zg3%yJ0`>@sqsIVB^`jxT)?d^*rtT}q3Fy0kZ!gycaD>NYm^cEGu9RM{z_Q8 zvtidy=e6YB56el@AGc1P{+O7~*Q8}Dj(%S#=YQfceEoj>6UhSCxHL%#@M-bbkTD$% z`6EdX3=&aY$pZh4YyNR13tU9Kjy!*E?oqU*qoUwU$DW1ndfFPmJ!J~*&+`x6Nk{+U zt$mU%82)kY=x_UKmHXz5FIKtFXaB=Z#*GjSbk1)s zgyDsc1w}7X^nDR8{O$k$d${oF)UToiKLKP0z{LDkw9sbGb3Lk4UqAQ1J>$#g{+|8S z?&4>Z`Nv$b5hvFO&TpWL#K>lr-6utg_=4>wG8P%+IoL-j|9g(GTdeYs}7Wivi_$vxwi< zX4k`3Hr5A_ifrc}Y?=Cf(9=h5^e4sHQ#w4}RuChCaIcXnQO!Aid znMMwuIh&ZJDcmaem?k}oD z4i6!7eZYUgZYmS|_#*kuK)S8J&qfH6>H(iUH5w&x=WP(Rrj#Kz;j@1L#+1Dhc`Id~tQ@z&IX+s3e%y zwd^N(){cG=yU%A}?R;`@*N&=lKH!!8X}4jdPw@XyTHz;;3*A4Kr+sn`6C%N>1_mK1 zGmSKIrcNoWbSIG`kLip%dl7kDfYO)Mh?c>fHCBkHK#nL2u{o_MuiFXNYr-9&<$=v< zQOZKGiES8!=OSrb9~7Y9XAuRs!fIt}TJ}ha&KhzteA0&X$O6NH{<4nMl4x>dWz|)g z586s3vDq5|&TBTO^~_W|xxbluA+Xf4f*i$Ep#9jJM!rLJLP7|}B3n0^=(ey*Nv&*^ zMG{ldWk)g}Ys2&FoG_ zYuZ+!5M?>4z*rPNCh>hEj2au$`ZdB8?F^w>P|9Oz*Yt}3Uxda>3$sR);JCktO1Z~` zcJosAWMi5>=K=~~XuQm@%N6+=A^inGe=x%JXh%sHyVjz_OBgLg1@8NcxTHA~;-VF< zM>|@l)ei%>UJJA6+jz>20ANxlMn8zl(OO!hJ24~>c$s0gux}%Gl)^=A4H3wCEmGjT z9SPyVipUA0gZy zNN3N3%Sk~Pc6_*>?2TGAI%oZi!=;pn-@z6`KVzg7lKI!D-;depSL)^R6FRXsTsi$V zQ|@aT0dO5I?x2wxm}_Do{EVIW>K}}&O$eToW(??P3yDV0F>Lk%Z<#Twbp>UP-)IWK z6TZk$*R<~mjb@e-Rx9HNc&0mpe2_q61zu@|nha%Y!yJRVSqWzNAz*olMZ@niq%m{N zG-@LZIrhV`+LNtv{L_xccJtr@9b*tA-tG%E1X3tQ8->`Q5`UAymFFC})`LW-ent{g z4)yn!DlpdQi!1Qd2jNp$#4&u^W!dd{lf6ouK7hYrqdq=>zqoxqK7bFL^~neDi;6uy zfPW9;aRK6U2p7PQ5xYGu0ABXH0sgt7XEK}s zEEMx`0(@>#EQ&82%kAp}_$mPHy6^$8EVRc5@X6Kn_y9hz_9rL6KYC#H!wK*K{{iU) z_~a0k$qDddgl&%#fFLpsvRW@X5VCfFCdW zz#x6uCm+DqG+_F}4e$Y*j~n2V zv&ReYV_!^PFTfXAv&ReY1-8F#z#wnMZ@2+9>rLb$sc8RK>S$;KhjYc=)t;RHnUSjZ zA$jJdX#?+(^C{fGi&97BgZU@EwVe$oKos8ieGwYF)DP%si=iD{Ve=+5HX%PtnwRr3 z%6ghY8bxef@{EHN-y(M~2mrPShLqp&9sK4bkfy{AULz;br9qS_h@vfAW;!Fla!H{) zu&>&}GZ%2Qg)6O6h>jf}nPFJk!81agiy_m<_=ext>aFwM!`EQ9B^E zG|@;j65PQ4zg;`8OsFmYkxJ>4?J17pW?s&h63TJvZ!#NKj9kMb+em^UpU6mIaM z_i(am*>lDP_kR5@whY3!TxHQ+6Zq1Zb8zZ*ATi? z$jS1WBv)7&*Y6Y5Ha{|Uh5~I7YYx(q%nnSUV<6Fi9#;VN*r!E6D58_QDx-I?*pj2k1?s5`Nj192BM4_C%> zfSj*LmX*fW_T0}Ce{!a2>1Z`}$>NtO?k!|g>RQS@ZiLHm4Q{lB8Z~c zSV$jGh;Tn66ZZaYF%K~^s;XkYQJ4reE(DW`vnF@_M_W{W143wpiG(~HiKOB3DRPMK zUgS$C_E!~*KJatE1WVfQ9&dt_`*v5|!Ngh(a!Y_g+a3a&yv9EyBffrHCY=(N0 z-?p=dFduFw`vJ4GFtS45N)NLuUpwNv>txh--!~W7PIrTJ4vv2UIZozGI+zZoqGuqd zHhE%m1MGvu5fQ36dSa6Gm4R;;nJ@bZfABVo0v_pv_1>r|bt0dxR@V_dcy6~5Zq`oN zwH`bYVa6SJxFp;!IvR}{-@G3@0c^58e1necJ95kUXYQCgcp~_BZftKKEuSTQ+!2)R zz(5(!NInw3%E;=f3QR-Gws$Y`K~P)@dL5b_+xEsov(_6sDN|dkH_}1%fYH0za-N1p zAK29S$c4kND$?lScht~HgEBVXRiPKU8V>x%(Bm4H5v?}}v53>`UsBK5Xd+)gOwztu2tXoLLhAi|EXJ6VGMuooAEFNFmwL;l zT**q#A$w?j#AxMBn&i%grYZY6r#v?5ms=LjC}HB4WE&fb*n`cV@*d$7d8emJeI%6y zS;vS0sg7L7);hJ!idf#HjoM12#X9-_+jX0Wsc3KJ51toQMp{&)z{VepSZ>x=w=%Qz z!n|2oR|^RaT_->Pb!8_SK=nY)$N`cieIpxNDx#{I)JP4YDsLfub_tibP#7-B22oui zF>L;s=m4W>w1!7qD*8;rmGvL%{h?!q)p8y}|0o0XpwX8@yh-{R&f?-#!F=okgK8!> zMl6UC)zYd$&?Y5G6ko75)br4veeJac2neYlwyCJ-z~S{efxwSIua>6I=^pp7eL|6d^z!?LJTvxL{W|o)~ zT9`RXyU4e-D8|${uB1|!*W+(Yyexg7C}$8n(@$J=NQqALMPw7qY++zXN_j$RfpM{mUb~4yG%;xp<{x^!pfm9H1#AJ zlQS#-ZsbuP`v&BQK$x(J*kNEYqboarA4%Y4{NdV%^v(8p%q+S(w!;j(BZ*PsH!@Qc zf43gqQtAK!Wdcu!UQH9TQV*dB{6;x3Ck7x)cJ3a<##;A$82Cb$m|A5^kC%acANgk+ zM6^BJP~gRbQ`4e=*T>`2;0G>B`lMCqj(!Rg7hIY)!pae~gB}aT#oNh0+Dvd=8ewn3 zExm2hya!7#e&7m09}O&CV*Rl~|E0IEa)@d@`h%=+Jysl7a^2e-ItMBJiUMn=D{@X; zC8@IblX2^7^|mUbMU)Kl!@`VDrf#K0@Ea$5kClZ@e^2YV+zx;?(HR*t@&TPJU=ed8 z2?9l*w7aMS)a)^X*@z(pS%ps3(X4HDl7Piu?Qk;fb|WMDCQemOP2t9tB$@b;ROpGn z+u>x&o3=r;+$fVSGvE%J(PAm)i4avpvcho112d>(s_%jmqW`kNx2RdeKID@>CkCWu zdNDVJsW!9l84JaBt!RDg6!3)S&|`oMWY-5M(*N8EzhE)BEG=(hV}fsyX6c|j$sYz4 zWrSN(K=j#4I^YGoIa--tMc-}sC5ui#Y)lHBv}$k(Wv54 z-!+sSh&7o0V)RrmbaC zEP3}>&`XyrhBg6rk?Y-8NP}}NpeES}jF1J08lI2b5gG$3yBiI``H#EEwU9_(t>qKz z)r=L_cw8+v1V&KvHF6XVZwY*!W;gFxlj<5^)1qy9zYybTEg3*mAsxg^W! zxwbbdU|h^@F|r$;NH5;?>Et5J08Lk z8q`-#P!^@il%OjB7$P{jTb$$OaHo+ApE2@T@dGx+9@E?Xdf{8&LE8<@`E|rnZ>%WP zAX^f;0sM@LJeveDo>B~)Af`EfDsFG2`%xgT!rl4EP}=DOt3Aa@GJ_&Np{z+Ex+vc# zJH%nm$Wa+34pP|kPbNhQOEFD3sTOrcrKhX0AvEFlyhmquxc!?9m{9qc+k?`l6%*Qko4s+=)kTS&CFup>Ea) zW4ZU#*5VmAkxw$RPUcXh4iBm^xTJ|hp+vr;L1ztL=$pk9$D^_@jr6Kq6mgZ%6Dx|g z4b>*BfN6#NR%c~mb`eZ8;u~*Ri2Y@k)gDwnx1<2=f-Xi-PXJdULgigFGP|9S;q#kx zi5cAFIqL%_IE=d_>vnk1eui%tJ9{)dEa4f~*Xh__93L`O4`3c~XQ@T{c{^B5{YI5% z-1>U{0wk;htRFxsPT;31u-a9HFrcpLqPlh;YIy!hOnm>#1@N1 zH-?!d>P1aZeO7+-**JeKD?ch^F&dox0WHiH$qiF=zOznMl=7Y+F7>p9N;Z?Iz4>rx z0qoE6P)W4GCJ&81o5@PWm|3mwg!Zr7jxjemC`Y*1lad%C(c*JaW3R=QJehGA{p}~h zB6%qeN30l{4FT!pzNuw6)0OZ>+S< z9^8<>Xf2D7yQvs+i%^>@X)DUucyl`Vz7t391oEtkXt&%%_|^0DS#7SQb%j5BxYpL# zwYn(gYeEe6rZ(Pq(%y$FDT^vF!?iYn9$nn%Nk7<62?f2|(Q&wvy4OJ~l8)29<^M14 zO639$>#TOJ2uMF%N^J!~ALPdn7nYS72?Uj2oL!xzleEEwG=Wj0aH+Md6cQwHwT1n* zUMsS8(?^5a;f-p7FvF#`gk;7bmt9MSX}^&>|>cXgE|Tq(iEv1g-dOTjmyvY#Cz^HGMBBO)8;`{sBwcZbVMWHO5VyaE?s@mck-$Fely3G-Lq&dIe z6+yf;7})*eY!80kKTb7_o-_hYuX4G2{Gv@1z5oHxt0v=Le2eH^N_@la;c1um8xJoz zzi|7q#*mCONc5NKb9%s+Y2AlcAND~A++D;J8+zW6jd*>!sB%==I^E9+zXu)c1#ivyZj%GA4`0OSj zak2QFK-RsBRw4L^M8rNY;GC9Rq6UG>+UE|V=hJSY?l94KN-CrSPv&WPr=s?Axqk(4@GUsaIsZl@|Qt3O$V zLm0=(Qm8Kogc}JH>U#tk%3B6V*BWgpaa>eX;H zvpw%XhIf{5jLu>MRwQkF!02SQ^;kJX{t#I<2;_GUQ8=u#+31vxJ|TK-?3?oTt2V$v zv;ib0{KFCZ6N^>~0S_ok-Cw-Ge5LRaXZ+-*VXRTvh|S1Y#ZuIAScVX}}}du+9n)hCPNO0x3u?zDx}D??Vi%L3_G?QVqU zRXH|;9H@{?evOor3#K{DYE_Nt;t)N&%5=U~|QjC=VUkPUVLrjC3uCfnl`Z6hn0dkM0+cAuA z(@=B>Am9*Ka5GeQ&)Z|ZgHhQ$Ez59~` z#%{3aXqwbT4XHD~vACJ-PNsjfr{Tf-mEh`}c=>1M5N9@R8N0@BFJ}c}y+pr7^`eMyNa9Ohhk3|U!6hR(%IP< zMG5q5UWiU9-ve_l1HF0e#6~-~si6rsn>iO7#GB@BvlH`WJ{N9o6tY@F8FA3Z3r|K=C!k{y(7%uZlIFlX& z!PjI@`N8{k0?_ZuM`fWzll!sJGgF78G=WwPKcF9K2DaWY>o z&{U4o&7z;JT+WU0u+?#=?5zLHAjcTe1e(G8;x`^dg3mh{q=%q#s>!fGOJMg8-bkqJ zI6O=lX1L`{MnX54=0e>d<}82EDiUnL4K(PnAV-*9m6FqAD!nr#95lMTD7T9oVT%Y^ z@reZw*C~X9A`Xrm&Vuk;@c~lqbCN>^tOzTvK;d5!}zbPNV#2PS0 zcP%2|K?mOn}!KryJ9<$ zGEmBd54tAq9Nz6j0z+s%{K;eG#x|-V1f6I)$rE852Hrw@}kP$Q$^8v|iq8imZN1S| zQFl}tT-LWN4CFi(ux)6j|yV zNYSO~vkh$_BgtB2CRbJ{s*HyvE52FyokI2qFx;Bj)}&tZ<1DlqA>?>g6i=FsFa4C4 z=L2)$DhU@)V558>b3`uXRCtvnoTpyZ8FKCuc(%Gjr_V2OP~m=>dZ66Y`LOs23?$rq zKq9LifWS)V9{Hx^$LU17qEg}fB(y#Bwx;OqfKys`pt+63+xo@@HuNs8kJMlqobQl& z60jbXtbspxMQyNhjiBD|7jcJkIVDboAFLlcZfT!Tji*2O;H@dJonbL_M#VTLcQ}osP0Qz&n?ek0EQ`ildtnytv61K9+m-R$q*U-aX zPlSMef$tLDev)r5{WHIa`EWB7pYb^Po|!a{Yb@#RSm8MT)W%owm|!kud|6KfzCHlZ za#V`_65lA!BYbGaSCP8jFau8>Pg7DP z@gKYZ0P6OLyBEES(Kq7`(B_2sdX!03XLT_61-?z3cAZ~$cp85-U1%$fV8a^v(FJ0E z?)7OwFJMfo=i#xiqBpUIh+fgVTJ_CBaE+)&t>~W6$8qMnyqGZgi%!U?Z2&BrK3eqs zcU;8<9ToKXfF=KgT3TEDoy_xlVsi3v@`FTw3n#Qgc!b96J^4eM_ul{=CW%9&IE~oQN!8KNHB#j)`wj*T?;V-jUMc> zTFz0S;Ge9YsB2`#-bM5#LF}W&dfplGO6#RGqDCqbZ_)VdO>)V9%P3~^VP2lzy`+6;%t5=}46VKk7)k84hBksfYV1SiZDF%zM~huk zFL2;Yd|NmSeIpJ}oxyN;G>75Rug#&CT`hF6`{BH9SvZa8rcVPY@Kp z=jJjb|MhMzjVT0XB8*gVP^GQ0n*gJ;l{@zP_;0fF1PGONa^F<=&KPC8lN!m-Sc351 zU8@E5CzkXbVz$Ir8%tCuLiO0C)CIk?Gq5w@NynLy#+pcE-;#`&el9Tn-;}!5KEYkh z^Beo_bd9|OP4ol4BW;3;OdA`Crsd*T5A;yhCw;qt?yf1HXT}!SY;?kg#NwA6X#;7% z%e3UXSYt+{X`5s3LlKFGpYSF~{4RAxu`G-z9}*ZB-+qwsn7Kg-a1;zYR!G=c9u`R} z=-+IFNo$|$YBtb*)T8lQP6QgI(3$=2axHqZe{xg94+=sQKZAfHMQ??aDy3cz7`+`V zpDk`rp2A^SjvK01-_|SmYihwO!$QG*h2pB;$k^JR<+y69 z+e~dyu8Hn7niZUhegM2CjiF`k^T^lMVB-K}r`#_Rt2^p)bPxPl=VY zXTHcri#B_yzLO{%9rQB8g3@}l4xnC6Mj{$1M5>q2NLMp_5G7RO|E7-yX>+#aOM4qq zVN*)t9-^U@Uef3URL_0sNooL9*-&3%S@P^|Hk@&TrFqC@)zz`)IoPeZGIOlB zQD98{62aK6l^MOzk^UWQuZV@v&nPUMc3EMmy+2?9n}AgTGDjy*e7HT9L*#M^fK#wE zI|--GZ7j4&pj}H}k?;~kqcVd%6Qaj517=M>qukUj`QF=#%*r5}b&^~88J*BOB`L>Y zU^#CF!85U)t=ST-AN0gq7ApoT(Jfu`A$o{@k%FFzh3aOH5Su_ra7xHbV`(=tSA*DE zcR{WhW4n2cs=;7?O^apG2R1{^Ei(*-nmyU^z8O*noCx7Oi^N0m{l<-wH_P(|Y7r@K z-T*vVN9`#d&pYsdf!+@eAkbYIsxS75@9X?|nqZOQLJ2vc9_q@Z1%2l_BeVVMh4=Z^ z$MHm3mj!BzH1Hu?S8`iAMy8uKhEZ7|-bS`+=hfrbnOo8P2AVKPi084P{6 z2=o^rm6=2yjERc|i14=k#E=yv_z0XMiCopAt}vernk4r6!P_*3#5X}%2P4r8iJ3%3 zkBrO^8f8&Y_BekrXr9+hBd5ktIuCI^**qfYP$T43V!RU8o{-Oo^VvD0iliMz>={j~ z9G}fLNLMp*zQs2qKcPid&|`4l{Ol%@)uK}(VQOU9wl_IsBr-LutuWKEQzx3#&uF9x z=`u4(;q}&J4DTDbM0N!Gz5$k<>E}wY=8Qy|5d>U5B`*p?=q0p^N9Mz#S^o#mz>S7- z%VwPJ)23_0F@RuXMMC;yE0AAR2;3{`Z1CG)wr`9L6`9fnyM4lPE&GL?jwN&4q&3S8 zsj&Be{rU>h)kvh9lEEZcW-PjzEP*A)KpPP`!&%QvGXU3ePs}=EX<0!+#KZASM|x0= zG=5ZuJY=q{&QbkO{c3y@myL zm-Uvev&ZUyxwSA*33}`;Y*fH--G!o&U+ZQeH|RAY>|9MF;8<iwyKrQcTn0;GX3fgL{#5` z?4fGqYr#TUsl|Z`0*|>!wEY<2BG7gkX>G(yj&jWjX{9Bf*o zD^(WV>b26yAJo%|ky@;$6=Xd?uPQ5z)TvM7p2yf#eZn=u@=X^s@I4Gpo241X|0JZ< zl6e+shiaOcL^jqmLqu&WGGoDwX0Msf3XY{2s;f3jGi=|kC(V%KxRuw8(bp|$h6FmT zq?rQjVBKkkya)ARdq>bsf+IQb#9jD+yr)e^sK##Uk$k7G`2om?4>YZBJHcaW$(g&2TBI zEzKAUTqCKieBiE`#RG817(diM9^S&2A7Q0)MY5nu^N6^!z@~_+A9jKD*USv+tuRACx%D)|_P*pLMnudO zw2MX}=UlUvX3&dkEv=xfFx$(3i&A0l8sV~LHH;`&wVJhS3hP-(Ge&MP`;3e#EC7ZX zu1EHkz>U#J#G<6KuulLhE^AiO4A-ONNAw6*{c&N2jTq&;X6gs&43w!Ny{4enxQN-K zveF8&b6!ZiVdM#x=O{X7HBwe8Rboq`1zRBkdc9`2sFjswxEhs}W>B$eC1O}a&kVur z4Qh|7m?MH(wgrZj_AsHLKu+$)#{?I&lK4$yH+mUqgN?N%5*2>QVA1oMF)nEA6@?il zkxLIVw#eKToR0!vwbC9`ig7_JEzNK#DlN@)GPPNnq3A(*!7VH!mrP0;SE5qW3cHC* zNh?hy*K34|K&=FIKA=)W#LG(A1i_uOqSAAD&CIB}jx0S7hJ|4UTV_3>*5Lszz+UGy zL&W3B*B1e3Ww8SnkqvXDk4p2?w!IZN{9j^!f>lF_<(5*yN9tdlE=Muvyu1SWzgD?eaq;J;^#a zl2s_{T~L5|65~#rWlVdol!!5V`kDMbl^su+(ddJU4k9KP#!xQh(nqD+EySU7e!;`< zHw_IsF+#Lb*<~r)r7Ykx@HrP4EwZNpEP%sR*9&^FNOt0*QJ zmO522bDq0K2V0b0oxS9#|JoyEK5m+`t6#77(BIKkUW0ze&nQeh$Mm>O)K_GKY}&wa zCH>!GZd$x64ZQ^Ji;B2!!859M@nTNqHF(NF98sw^ORtx5mlYN{0aEXhf2jA}~iPLasl9Tf3dTBut*?4X zO0>;%Lh&Ss5oTDAEj>PNqs}&w$Uk^Sp+^$J?do4^&&O>d#BT9*8yX8P?-Ud|2D{!g zy{=U8{kZ9F!@r;rie0Cp&;y#ib8_{AuEl4)4!H$ltp!y~J)yA!f%Wn1!=M`tTvk{? zhy>jXQL4`m?#w5ZHYO9nq8uv}%*aPV)X4QBUX_iM_^90e6CwHajvHLF-YD&#-ma<# zdeieJ?n#~N9lnHPoM!e``>ML_;p~vc)V_`0WFm@&gJ`~Hvf>Bt)7ZFvbv? zcg+wDZp>}1+c7+Xy-2^Ykuz>RP7>Og35!zGxo6Z435>E686d6bTtMISpm5vUimc{( zQHa#T2T@hx@iQ7xPx_JWw)kO|BD^2Ca!jObl?>QR6-smvFY30b;r5WlsXTq&EY1Qj zrQTm|0=K8XQh;JcXg1J+?8lePq=L&$sHvao2HwO~=_2x}&nRN~CB78Sb&KoXgG(IM z5t#`KnKOVlegepqdbH7#!Uo&{muZfaGRYL`9kz>1PilBKW)5c%cB6B*6GW9y2E=XP z7f>g153xD5QX6aSGcX#1LaV?~y=*au#H8xRH#W1+_}=`C;2_|SG?LTL@{kl6Q>L@1 z@H7bab%fLwwGk>nW9M(Q&k!oNQWmj7C&Zl#aVKLZoe-HdM`}iMt9q$Roh%3p2m=2XM@d>N5q8-_F2|kGlU+2@ z@EB5a^7IhK3-&YO2>Zk51423yas~StX+-)?8p=vxyo@jo>&^s>$jm+=MpG_VC@X}) z_{ZFMPx;_OWPP6xJLg`kLnTlA7SU>aZ$5B@iQ+M}Q2wvHtLw4jIIiyo^dIiyZUBn? z{SgEK{1F2QV8FIy2NxLOA=V!#xqIbCCb* zSJgRX7Y}1GG6=zxij`FnMFS|Rn!Jhf4GlcnZZ)hhCO}<2SXnqwdr>^EL{mru+2fM- z6Dv&J)x;{X(lt_QZ=0N^T4j?i{s=;W#Bx<>zOqP6#Ef>D=&`vAIRL*QqMO5(|OEBX$3-H2BPf#$n~3dFRv-?s~V6N zlHjC)Gxlms2jvIwEq^GJPGdm~Wl2*~@5WrvY;T$n7*{4eDZr)#XigiIYt!hWF-HN* zcLt%@DVvB-T5QXiL8-N8pc){-*#8=HoRht3at4DwiW5}1In9j~xlT*LQliJI@NGDc z_F$l(`z;2h9`#KOuriM(***_wvxzPZRFjU){5;u2sDz}8iIBw5z^$$btBfHga;sTf z&O}^Q@S}J+72vxT4bcd)06Z!L0`f_ysdtzxziLU`as@*YmTS3=x}{Hn3!TYOO17v) zZz--+bpia9AT_CkDz-(Ey=R5V2?(ILylZC3&|;k|9%V6gT1b;4he~^8V=qx+L^2Zs z!nzg8(M8`jU^v<#t26+7Horx(H^sIw$!6A9MGHt5R^p+oNTIOT9qB#hX(T9%M>{v( zX3^dP zA$cHQbM_mIwv+E<(ZJ}0(#art;x@gBY7XjHMnnFp`FSS!#HzN;vR{}Xr*?4BjL2=( z;A*vz`$b_ZRu=%CxtXtmTFkfrcvcOP^eJ(&>%~V(O>J@_ffdWsDE7l-#*}GAM_@>7 zge%OCmUguS48q_ByYqP)+HS4@(^q>J}skK`yxG6ku*wn5o$xrCjA{{F=Q@W zJaDBfk4T_<;{JLlj`rzjnHB1U6q``erO$K6s;WpvDQ<>&i^a5|rJ>T(GBf-LmrdKX zawi?>Rn_2rik8&=R`Eag>D9$A(a64*8Mr_2+0+V)BudmLU8&Kva@s*(#*O@i(~Fg5 zE1h7@=EIp4G8$(mzRlE@%yT!B=A#1|E+H)eP#227Lz&CSganvqK5 zEw=Q0iS{+~O$Y z-3-d2k!z)9=m}-bF9dX_Q)yO+FGw3_R+!)omfu@K?x_h|?8Ah_YtV-xPo=9%v5RkF?Emz1Sh zYtwx{C?qR!cS>U9@{G-hW??B7oy_AIZb1vxyKbFA+yKYE^;W16h=@!!aIK~Gx(ZKb zv?td~nOX(Vb~B`$V<1Q2_0%-2MJBG?$jYHjE%LM#b(fhuvg17Rnx*BueFaz?%eF3r z;7%Y|2<|cr?hxGF-QC@t;7)LYI|L2x?ykWhI0OlhQ`oDgT+^q?U!A6YwMZiqscudfUho-OYjWpAyR*CLupfGwNi zpHGkh2EeX(kp~O+#)4U(pkPF0W@G|Fnl6=Dbulh3Et_}i5@{-b3Qq^iw&BnwCNV{H zFb;GtY54b8|1`Y^-k?!Q6W7da@?v)y#GcA58a^n*dYUJ8$pFR^}lmm>T%)a6A;_VyRlC9L4YDNj7Y4Bq=(B9)b5)#{;>{9F6A@^XfcjMOPSlIlC7n6&WzOCxpityI?WLecCl!=L4~nulmjg?&au zKWmm6PM%5{LxvY)j(%n!o*|5~a~QCw)-3&ccFJ&tq8*uNkx%G!*9<&T_Rm)@{*WVy z_HNBvS7(Q}_=B!{1UzW9rds*RTxM0hrdZ<-MDOV+-O&3AIX`WmB%>Z!ON`Y(SHoAy zII9j5tiM)>PBh8Kc@>l+Ds4)K7}+RY;xDUMU1ep=;6S=3L;Xpq$FQL;9wsL>g{%C@ zFrfbf8vSVZ#x~cwslUSmN;Wun+8gKAL`ldj*%br_G||Us?U_`gpi2$sVgz+M)F#Q2 zNm|P9K<}|9qksIq% zSz^P+IU3n7-c^F7uP^ge)}@4wybQN~ui;3zz@uRAXl#s@(y&vq889NnQdv|~iw3Qp z?i#~0R1^%S0D}feJ`xegA!p6_CZT#+*kGdyjmH`EoZV|#!=o&ws@nb$L3=eP)_53- zG%LwCIZ`>#o{^UY{!X+S5SoTcQK0KE29L1=;}?_z2Xt*;QV`~r5zvlRN5LYlecroPMh?*%7oR<}EO(GeXk*AmHBcd<+R`(eUA(cA8|0Xm1_65U&CRD(V{J~(J1Z@E}8*z$kr2e}AO5FiR7a45rd4nW($Sd>d z8Dl0 zfN$hcmXd`{D>5%$PefaP)Fx~!wXTrGg$Y<3oet0Ix7^uNic_YGrIeovC}D2Q?T1$e z=0{C;MlX&|63^x**%B3WO5d67bI6rzBSqndN;_55sGyZt=e*h6#aMNweGQ$G=j+5P z!%pP{Xe8e%((ELJ4X`a@(uqC#*fhNer6w)aQIlgP#bmK{_G{5$UtR~T$T2|$Ys?DTIB;c7Y=Qi{_9AXDuN`~i zg4lfZ0pEBhbwcIrIc|z~qwfNY*AlOZSCzN4N}*#fagPs_8*zq<>9!l~h*_+&s7=J} zpr;{|Oy5SSgb>8X$g6xm5=^q-rCUpWVULu zq>5%h3r@ztkV;5;_+Zz51gRbqGX!8th^p>mzagExI_G7)WpSMgjkUtI*C9MMRL!=g zI15+>U7^@0#c{bfB9whIrpq|l^eL;N{bZH7UTihKgYvTVEo1H0C5rLYoSZhYkvi8) z3oMEvQwYCZe(R6IGNRGi_~fO^d5?^qh9^VB>V1P>2*Rr7Y5=8LQsv4PN&Y@jBzBb?+ionP9IrLiiyBQC!Flk5>KN}wLb-xd+-i6QO^#1qf zo3AF%&KJ)sz2Tzx9;jzeLXHLBwA(^Tb+p?}RZc4IXOHLSw-1^R-JM+U4rfRA?|RQN zUHYpYwlMpBY(C*P&pjl7LjX?m-yN#ze`GSU?zg@C z@NjmA)v4k#4`4uE(}^|!TbczXuy4%Rd}2)79!ayQ(1|Fu!N;4uvz(SafBrHpT+6-0 z)Z&V&;^XQv!VvK#&7i~!5Pv>DZ>Q~bwRd`v>1~7p>AK!EH~skCPH#=K(CzXdZ41`> zc`qo^#KRi0a#w8Jif758*%ejB$C{gK-l5r3uUfygsnI+3ll2D7+% zYb|y$Nv=EdtLP^l&r<;*_kD8ilW6TF-kaVy4Q$RA4$l@88tx*t$aQi7 z(LOgw0P*HK0JhZAG-m(CnOJy#W}!&%WiBroavMK5N2CnSl*!H8JzMRTkYF7`?8@o_ zGOYr(PfsG!8>Xw7!_h0lHD3IJGxl6|e3{3u>~D#9`XG#^O)?)3s$y~RAD!H6zjD+T zW-{1sP@M=ke;H{5|%++hutnTl3nGuUf*x(&R4wDB1Xc+W^bRKKS8u4t#m_@->u zbMEP;-rn=VGUI^8U~<;+#6gH@JTln7rBDZa4oFS^mkj z-KGI0y*zzX`;oB1n^oXEM0+sozLyz)>KJk7snjFm*|PL1SEA1q?V^zlXXLFX3$i}f zzJ^O@0LX7Va~rthSD&ZO{mG@)P43mTKwa(`y~l05f`e@B%w?>NQ;q*?ev;W$-)qRs zezhIX2rj9S0UTlpD4aG%!LMYvvQyV}J6vAXTx&@H{rhiX$h)^~o|u45pvtD+-ryzo z?r7(Fu9r!)u&@q(Dp#h+we?OsPckyyJ~*0)4EDusUi40Y#9c4^Eh-)G^?KTp6+F3q zZD9lpR{~istxrJ3@(rft=|jk59WsTXl1BeW&y>Y_Y0_i`P2{|H6ZX8qt_3%%O&{qa zmiQ4PA`khm?4E7B_cx6X97$;xNs}J!)^K>e#q86ED03pPQ=JQ1sNQAKEsYX6hav7F zJ2yT|Ux_7ns!m29=cL^2BZvXOb2^KdO?VimYBrl9KmB!#%+w2TGeE>-&s zh~AhKDj0T1E~0wxGs`5tkNF_fhl`TWpfPeZsj~@wRqsam@!CM8r0`Iyh?27lP@dtn zA=JC-J@_iMX^SLLelSjEgJ_{$ddNy5A}l@y4>zm0zNPW#YAq%y2ZuRB?tx(Y|yIM~nr9^>tXi7VpW zcnjCQvCpe;3-<@4Fa6inSE^>bt~C_y-_wNf{MW3efmci0yi0i1hXU6Y-3`$*$nWj& zw`wNWDz1)@^3n4$>DR`+Z?8@+hJEVo?#|c zGPae6x6NgdhIbJsY>>13y+;zF{6Q=DW!Xth{L8V zN!ZpBRSYW7xZbRzd$^9}dDOk?I2z_=pUYk$t}^-=(ISD?N zh%T(DxRs&zxLtYc4Sl@=)3RLl@*Q5!`U%mfFz!*xE%xhGlg@Vh&mNP{=7W}-C}EBG zOqIi{eBtOi-*uKMtgj>N6C8sXw#N+~<&*aCzZuiPEy$;lkpzA2)8=s7aT-Ql|@x5bv@^R=SX z?Ty(%Kh|8QZ5#X>mz?=rp-L+CacM8F zh?8NTcCX8%v@P)GgGX2%aTSqARid-^LXWS;sXjirZ%S1Y6hXYg*CJxLd+RSuiM8;) zKRx*|Gw%e5&fAiKoNiE061k*x%sponnN{CQlA-?@4i(ZiTFc>lmduC^HPmN7ofx6K zI5Thaol_u0!vo2dbpw$}Z&hfRuTZ<}Vq)nVH*L8<-cgu|K}GgVUq>I=W4C~oc(Wu~ z6TP;t^Tix;zC$XZecWVR6-^x_U*%*2XOi;lz}QhK&cf8*O-w54e4woZ{W^$i2p-lP zh>v{(w>@YhrUWlIBZWgp&@PYDR9YSwLML&+Ly)h@u&sDZmlaa=5>w6!Lsp#h!terq z?8K`ENHw7{VOJbSsOiu606ljkDP@L1o%^blR&61y;_!|7E95X8hsfEG2ELk9tV)Du zyA~y`ch&ECJZwMeQktC2$Va5zO!pQ0ZX_|Dm*k@|=uv)8?Mw)I!Ooui@-gw62E#kI z`6JQ-drzo#j)C9h$fk122WfD{4>d$eiHgDt*@`bBo6Ne!nCB`DWeAzy>z4pM3}$9B z#ph5&_v#Z+A@*4)*h-A^j!HF(*GvJJQ*%()y zFOnHxNEKW5#JtUS0yfrYX8g62g;ubylGL`l7|TVsQ*$XqC+}4rS(@^IkbDqKXqq?{ zwDL!mSCOG*D#H@*Y58|liZ&HB=wvNRk)|=>2ewAbJ(B}~kZXvCF%tbu z(2$GbMuIMrXsms8P@dzL7{;cVrj&ivkCOJa0jaggN9a8Sob~=K@9agp3j}GZF=F9o zChW53s4g-WCD59-8y5A__bSo9aS9fjDBzH=zg;Gt7@St~Kk+2S^0J1aX$8<~Bu1EF z-b#H& zH#OID^9y`JQm-KcCy&JX(}hwga*ohQxCRUQx#7V}y;w739#mYC;OVQBBMR(TQgdlW zqVYO6WE4?#ZbISA)J`u%c#Jt!G{@ohf^WsAg|hh+B*;+an+G48KD=w&PL%Sij&G^v zF!6$xaI$urEYjAl(Oheul*ncp)lHr*l)`k9=el!SFQS($K| zSVxf&Q;|G6Y0@y=E_tiF@sm6=V~swye(1-GE(TNcw^I|dln$4B!uUo=<;NeOvcZ#u zy+~OjoGK}EGnfP~hYpN|m(h#XQm~6{lZJ2)13%Cg*9T6j)LS-Sa8NN)Wal!8jyAk3 znn4IF`A+{xhQPa0-JpibYs6&K9tTzY;jQr3G*xWq!ctWMxe6L9W2ky(EKD&(#|vUp zS~((Cnl}~3$`q3AOv|_{2|NkIc$(N&Wj?>}x802W1{&;L^ z6>emCbO^`?$>pIUjhuS-v-MT+pyyz$QQk*stJH0Yw$tQ~!cn?OFcO~|_TnGRnaP3{ z%Cj&%NGh+li<_FOt$bBG{UJmdT7lBtYo)ElP_0K;ds>}U6~=Z3^l`()qDg`HIdhA@ z<=6n?JwwBAc6=80k{+i@8ib#vK?`ZAcEG_Y(RM* zI4_7GidOU8tjZV;LqC4m_EAK1{bd%(5V(7{C|J9ET#}hbe$Z;M+ie8d+MU=WJ@y%_ zqN+tS4FuWCRRmN4fvP2k>?DD&zOabGh&Wp&qeLAo)9c9{&*7nqSFp3nhQu{C`0+<* zeBslRv){!-&-p}Wl_iU#z6i=|Q$hz}J=r6{<RQB0h`BKlle z@&nnPbQDyluo=wQ3QJ??XG6ighavM^jB64kaOXqg0{LJ*HB_kSb@i1HYBEaUw2BT! z*|!~HjH1MGVl-#jO2eN8IWiJv64Z5Y45`^LjFgM;60Av63}_LB^zs~r9Nj4?iw{*i zj6)bG6x{Q%OvUJYC$J9*UuC3Zr=g90PJEA3MH8#7Got&>vaf6*Hds+~scs~slwXdJ znQ>T*o8g!_Wh!CWnmo?2Z$?@rsEohf5~bk{jVHi`_K;9%;WHss%t#t`=Wu7bg&CG= z@=&F;Dr&et7uO4au|mk>4<2`7(*k+#P8s}(Y@EKr#co!<&r+?nQHHB;rU1|)My6>j zca|=*CA@!Ao=uUMo`!v#&9I@w8@88pR&DQ>x-QRm*Ot95cV-Vc)!MwhU9JxfH{kUd%I-hM;anWh@xjwHsdGaKeYWIGA`nHpnnGX924Ax#NHO6y! zBtiEZkU17zqAmn}v|VKCJV_ZlsO4LQS`@>lr(1W;))wpjX9WYTl!Ci@-k10OY@sy2}STZrMF*0_ia8YwCa>Sb>Iz@zpW_F(!Gr4xH_$U_fY}T zW*QmSFs`BUy`vhU`3x?T48-`A4|o)}4-z7rk+q?NqrH)y)z4iUeRD)OHf92Pf}eX_ zTmWGwYXe6!8*4>3TO$HSfS`?)o|&}cK>+-bC!hx%;kR)DDFn&s zm_fOJoD4d^#sVrMPr%0dBhez@;rS=LAELj+WB3!E0Pu%qHU>s!#y_VZGy(b_QdTx5 zMo?Y;H3dnf34s3s`lBMh0s8OM{sGF&z|Qmo^q3pP+2`kMa{0a;wY#?r=K!B)?}2q0+WY-V62FT(#1@gIc+Y;5d*SebzyNB{z+V`BTY%fdkz99JNUU?qS>z`b}{!bPAN6G(=%m32#44@7~_hV!I zQzu}j2f@e_u>a^UpoHO90`ldbyTs3ezu_G=W_oszZR!3c1vUDYci33znL!Pt`~{SN5AQTqodI};GpKsvU6PX7Y*hY@}U^qiq2g6MV=1i|^R~g71Ece#iQM2jBgy$ZvrDEqwQ5 zO#YoNXJKQe1F;4pJIfEs`$vwAjgFZGGz9)5>-}`{pB?|d!+JpaU$h4VvGot$1JeH@ zJ|O*HFyF7jzu~BVp}c=k;O`Q`ud4kH>VIa0e?wA#VuW9Y_#M=LqxZLH{+HQ*2lYR< z^}ok{K>EL#{{HCwO!T0y|KA)6f5Cx3`u_G;K5&2`3=;+ zg$RN4e=`#@0~zT+-y?ra{{E5sZ_WRHx%y932&DgC;zHmbTnPMw3xCX!esCi2uek7M z;otDp-@}DJtM)sn|Ct~D4QKtC3x68ocToS0-rwWGpJx9Z)c@Sp|0Wj#|7I@yqyPUg z8~n}D@K98v*G{$%n|1&Iy%}qZ~y=%W{###`g8_1Rsc&&YfFHWwVAP-u>&P2Wcbe!Dg9_*cUtW4y0_UvPNkp8G;o|PEFbSj0c{~% zg<*xNQbPGhmMoBlpw7$x}AWjil^;vb)MqHW?L6|-uFQ_^4W8I&dw!ovS}9yRF3Vr} z&il>#d)ltg&DZP9Z_6V1$KLmp{UjL|lucZFD|j|Jh03d8aHGHrBLztJ7`hMxeYd#{PTyl%_1Ji~oH(X}?a zo*_Mp-!%_=o*5@BXzkdZo#Q8sLuHf>lPRtub^+GD(GB;m*@$e6MF0SFnL*(Iz&HXI zm-V`lgODfky(5>sTF6{HjUZb+%9+?*E_Kx+epyfQ+`Pe<5hYP!x%Pl%G);<^&zXLi z@J8GbUG6SNi`J7TBTVJcF$J&kwuc(Lp(ic%rKO=aGu&ecvv<9Djn=~=3{1WXulLoh zDmP)PCAea%m!FUKa?LA)eV^#4wl97Q&G#axD+^59FEkFfX`SUOGZgc-R~ASNwyJuj zs959`H`l}pg@H`#blP5=+wbIdXib|t{*c> zWTuO?ugQ@g*MqOGv$;`s?d~ZoIlPlyyYPd&U{ILbzk9y|ByjwPzG3S^Sb?9JxMq9 zK_HdLJ>pzMbg`aD(x&59b`#^hV<|aslcXTL0-1zW`j(Z3=jM`z&yF6q!3);S%Arer zE?h;|p#G!`Q8$n={)%Le^u@?$h4?EIF!+meujpDIl4`aCMc8XcbLkr&Z5{~*pK7GI z&D2&;;_nm|K#7VZceNLa&)Pfpy_`!GP|agq{UTk7R_IDYt9Zy)U}JqXk8XLjD?Gv) zkBtE)^9!%~ciZ+gCjv$uK0lJ}3O{vjjGWmH8Uy{eEGmhz9c#*1z1#2@=*w_9+1&P) zFdA))r7r9@O&=_S&n&2)XKw@=I@>>{U5gNfb-UR_e&@LoSXzeugop@_yV1KCc`9n> zvp{$)o9ESJ4a-v|@AmVG z34ZTc`_Gd={m&%JIIFdm5!-%&S%$&bwHmd7j4EenrXZ+5xdtt!X76 zYB7G;+Fg&vqUk(uay?m7oyvQ>mt4A~_cn=ZG5GRp4j?O^xaP^AGLub>{r29KHimVM zb4BapQ7xowUx|6!*!Ic9iV>0AUOYz<9ydD~jd~OMKcN$lYWgFnLbSgWJ%1EY{;|>WA+R0mLaO=!3lvlP$_QB+xnfXax zqJ?*rFr=vT^EsOBo~&O-hAA7Q#m4Gij6qHc7AkrJ(!U{jw=#giF1~N_5ui)W)7tN4 zcgA}QOFdk78seq%O}cOCq<=s=qD}Y!6`CbT0AnjEW#@|{f?R17^EPrJ$23+yzq71FegCtr5<6F95KXOplY;)U`eNW}vJw1umr4Yi6S`?UMpPP%1^c%e zWB1*pi3PIrmaINCxpp$7VxpzB{jK9;s*ZA9olZ%-_*>mRncDKlk7!HsNrsE>vQ9Jg z-CDVAl)ccvtodwBjjOD{J4LZ>X4$c@KE6!)BDQ)k2BE9#iX&M{N8WFtg^VP z<5;@fs`+g5s<*rt!(MU@Eas#{58<8vK2q1SbZ!olQQ_muYs&h zwmynyFs?Fu1c2L2kvbQ}-KbK}xZ3Z@eya$9fVU-^$txn`pHzZ;?`l8HGr-LJ`h}29 zm8=RnvJGN+`q>>9>=A?W{?@8ZmBe^xE5~7gJDNwT-EjFTm#;=2c~vfU{qnbwfJP%v zG>;^^;bUp;7>!SAzP7&8ZycskTurf>TD@+sJeVP{>p59YqO@bc<2ogM1HT?iY$T;z zZE5(XQ5*1eY}Fs!t1ZSOb5y?_1ucA1L^$?5arNm~AMw5#_mw)-g{56m-=wLP3%7TL z)5$OuaTC%tm<0_aKbvE^Dxv^~ zHL}J2o?2h~r-Y_vyzdGtZO8IxA6*Ov)35C+k=t#I^2Ar8ftL{-&{CR&r`ac7pAqoi zFmaWeQN{`Bjlvix+LD*F;o&?DP+?16r7Pc5EX1k zX6!Ldh{QM|h{V{YSynNs*~XZ1f9l6PS#53cs%duT47;h)S<@y&GqdELIFhwCGH1>m zw7$escQJM1Wm;Qrd1M#HeNSQjy^sJ&quhuRO-OCD2rJ5#Y?K1g4@TS^VH~=yi?>Ss z2yA+eb*>sclupC6@;dTtj&bW)Z>=V>o?Nv(kNm*_yW+`^I6 z!DnJRU;}Xz9oNXxJr*4rui-Y7<1}7!_7GX-`l)b*E_kb%$1d>hF>Zn!IX3O4J&lyO zP);!-$!x&#(3gJ7OkS5E!Z}0#8Aro)b%>1ljM?u>Juz`ca7Le%9@RYZ zl#n?*FSYr;w)Ya$E3-CwKo;Wb^SsL;AFzvPW(jj0^j$YY($*TCz=NEee26HC#Ll>Z`&8NB%l#aR=>Yd(&QjLOB``DLM9PF&6^0FVwqqhbTQPtVd4Tp+X9W1oi_4-O!sS8T z4^B{&JK#+Rmd9Bxp#LTqx1mp?zS5nBR2|A*$w5v4s@v;Nuw7mDFORcvD5y91b!BsE zf-~}@-uW~WR!vAcLkvT_7Ax|KDdvRyIBih&7%YI0o09Eg0@$eQFGzWYEbVu(Eq}lh zPQ+_*(a|wBZhhm^q9h_==(l8peLjGXn(`Jk98UhILX9y3K@5qyBrfcNDvcH1f~!U z1B7KIT!xl}N8}g|e2v=#-LN`_A2^*e6}f5W45I5+@IMy$jUHR_y+(hrG@w|}3|2Go z#>mB1&nL!FUr_(n=p{ZEWOn&SV+f-KdA5RCN@MWT{@EB7wG>R{x-q`f>N(ZROEXcc ztE*1X+V-$QY3y9wdidGtUlEDF-P?A*`PMHiw6S`llhj)s0bI4S99GQ>3kOGym+gBt zubPQ@k&n?)cUvR5Q#OQ7DWNy&YqM-V7i6v!XNk(Ix|Y}VPGnp^C-bs_ z=76Zp&~+;vnW5DWu_KR@O^_NI+rOuFsN`j>ySm?pzL83Wa~TUOhxPry6vt2Uo50@bS_beD5F| zN52YCC1Hx(-^=Cu*gM-$lt;6TUZe3oyfRO-f}zWMBxVw)$k>w4Din4x^azgeQzP@F z0_~?>8e>E{d~$(ko4#d3l({JxmbBQ0fuuGvRD0*BVmRuViCe3@13vW z>IT+j7OW8u>6d7Mb0j3AOBzx|BlBcn$=8wMs)&q9?0)AE0*fF@v-e&TnTyr5|xBq_*4(zDaJmM|X{Kik!as_95UbgT7{mL(LXJmkAqm-wqBK364|k%NoH%ZGmK_VOe0pp;`mOS0l;#`skYkOh+J!tkvR@x zzAzg@N7eOtr9=eMrZI|OrCNhSg#9~?T>Lzz)58eYbRN2xsHxkA4!yzyyLzYCB!m^H zL(s{G%Bk3Oi!@uz@Px>(1{LDVCFzOc=kls%g$4f#C_*|Em)nK487Fe>#)KAvH2zgfR;51)cb+WFYf2)v~PEOcN zN*LZgFH8sadhQX5AZr(@&c%4)VGONcBrz;4n^7ssaizu7?!cUF>?b?v<8)1ZX>u{K z`pVnhY58g4-N;7brT%piTkT3al6Tv^JFAPdrFm{Dc82K0AWxSOgmcd5%q$mAYBn^@ z7{a2XxagDRfrSB;BqlS6UP6ru*go9gKcZ(@C0d}4lAXM$B*8Hx|H;p0)? z=f+2TPdDO|J~OTs@I<=@ly%iRtuu*Y@{h zr2+5tN^N@56naY?=)gX6EkFkBABV)UM%V-pR*0g*aEVX3JOUJVl&)%?3 z0)D!cNiB@Y8!qWhKF&_SccX?Ne~}n{#zc90Ual#}$y$iqW@c*-k0dual)LjCcnx1y zziy1bu3nm6qhgTu7=n1an15ojVsQzqhm7J#40h$3tuQic=cl+a+QS4`_n$wye$qj9 zE=t#|1&5o?eO=D8)6{X5V|BH07JijX4UraJw*s&&!31pNv~DQ1Gp@LCyu}yl_Z(5@ z#*1D%>fV|^07_que*W?eGvLKnGAMk8H2dAuH?^BlLeqO-ls&3oBsQm0A_@Eic2}}I zs#MF=8J>$t{j^sG-=Z=SrPCf8ZfXRj&^5mMGmjgL1~lKK*Fd9QxP!t9>9O`_^`vlH zhp|1hDfUn1u3nX|@wGU+EoVeCf=$&9KEwaae=(;XbF@C_Cq__?g&4 z@vu<~Bn211>03j8gJxu8@br#TriAW8jY`iDHsDg^l54vkCQ9ggfh&k@;+UIG*c34A5^oYLo zN}%EBWv#!rJ1ZQf-?{0XM{6P*7smiUsvwK;j(K8x-;=v$KAr9VwW+U|zXa#DrL86krXy=F%RBn@x5uTN@ zT#YPpM-S1eX-~?=Gc zOIro^u;|no-$r@T)4p+rC4pa(r+p28A8XHQsQ$Ry@jPf}nNMVhTYCuBNtfRNOObqWKSZmD`moRP_hxT*s*onCMrffVWSL0Fo#4X6w9f$o zL$KfPPVB3cp|Ar(Fb|iCHn&d{%wt^&EZ{Yfa)4WnNGV~>-HnA2h_ET8a^)5@-R9E4 zy%VoZ55(L#v5dKvb{*U4$mf|1##EQmNLNU7jgR~%Gl^JXhZj7ru2rw9nwfwsBfg^8 z3<1~~dLQ_^pT?T%neiem)Hya9a8j!3`@99=A0)0=)J^22nVb6%j+sJL`PE%zTCKDV z9)%?PuXxn3A5Prlw?oF+H_DMJ=uWfi#UU|@SYMQq05H!dI}JmrQwXudTDbA(*CF&9 zlfNPQY7Cn4r}7Dj^$|J^uF__l=D`*B&|}R;(Vm`AkI7DvkV$g%lX#Sq(8wX{_tZ6N zs5#UV&Vl!$fi1FiK)$i;MU;LlivVNSkAQD>8VrJtSZ(9AfCwer<}*MTgfAfp;2b5} z(a@oT{Ul|t<>5e52xl2-q~&bD6aZTIQ1|Ha>7$laO({lSC{`6Ki--H>b$QgMFzf4@ zO@1D@hU<)}W>IFi@gg`13$RR?PfOaL*5M>@_&gJNgA_35E_KM-*x^pTeZc1to(!+% zwly*HbK@~)msb0B<3WTWrKJb?6&&=CMxOzvH!u8pIUW~!VIEDs&72y?Yvz`*h`?t` z(yW9GW@1>yS#MDs=FdqqWe~CqiE+xS^Kdut=?TZJTljTA9Rv0PL^OK91Z7^}a}M0qgp?Kd9RiMiX?6mVKM;^lg(hy*l3vKhXm|vUe5`J_=Ngap%2H&9HvA<;(>*QT2Cpm! zu%j=#oZhZit@1&rCATstn|OJI3@eVr6}X~f4RJO5IBf7my`py+i4 zHiW${zuI z#&lUa--}a5bJg-uA$m)5abt5>Oujhm6)iU;b`{fck5>2h3g1c&*Sr&k1HsSvc+E%HBaVOF`0Y?uzj%z4Qmwl3O(5Z_p z*M35}JTA66HfY~RF?2ZjRQQfUQ4i}i!xX%$xfZLJ{u{G1HP&4Cz&ZGbU}51V91kYM zg7wAS1#agLR7SdAvm z6ZKGEVDH5mz|fXltMPPC_MXfok=u&6pGL9g$-(0F;5wuzrXplJ)9+c-DeMVftPXU! zvaj8HjH>VHmYbfvr$-Se!uD!17U<|97Qny8bL=%!cfxEnE+;sfk_smApo?M@aG?0& zx7>h{X{2`tDsJgCuZmwKE z`ckLa>-UqcltykAv-WCN1hZFmL!fGrOfE2xJJ+--Ff!zwu+<5%s(=W z$3eiMsKq_w!sG4RU5y3f?}<>?9sP)9kvt6=(p{dpmYK1rov41^Z1ZE^2iHDv3kZ|? zSl4-RENGyA8{WnDc$EOtm&(pZ9t?$z(8DjhO-`Ra>IChPJK8j$86B}VpgLQ`jU;7w zqkNhw_sVM18!7AvkuBChzHKJKAbJcEpOlzvw8)RpxU&w~UsP|`R;b*EnD{H5W?Znl z;~4yml&Ye!B4)mLssPS1Ei>$q{hO(EiLQ^MB|!IGyZSHh@TMcCmzT!PQgjSi&-&G~`3n z9E1-i>F$F7(%(f!IO&d$r#Y%pQV~cxkpc#E-1=q_@&rJ zb1;qcq5>_!&r_Ba<8q&i2TopH4Hm|Cz!vZETMFsre@(x^egaRX=433~OLUskk)tZ~Nht}r7;sZ;&1y#ZhR3BDQe$mca1BF`c}tD4asPdWNpvfYL+-c?yZV&YMv2K2#+UUqWJqqQ0%XSjj$q*oTbNm%^~K{x#e< zT}%WQb21EeGIbdVVf2C<10O%cAxvQp^KrgVFBON*=D*#p=6{IE!(1P@BWrS0SSXi= zWq`M>#g3)G`(P2-MF0g)j&@x~RRw|N_l^}W2LaNu^;)a|Oe)t*CWCrt9_i)b++qri z;pQV7L4`9zdetCR*)|Z0oMWqzO5-+a4-ZA(@cK8Z&IpXyvU*sTZV(t z9XDxz@q#c31FrHFy&PCb*o}5J*y6fZ&$TmFdBbYn%MR32>JP5n%58;$R{8=S1xTz`J11;h=o2 zX@cp&KBGXe>48P*=wyS{PH%#aT1p1f^Nk<9fcQ!cW(5f`ft1%B2%d%8O^ATPsjk)u zribqaI*Jmgb|}pBHV15G;=bICA-zr*T>n%v7wq=Rfm0B?2xFcC>;dvIlL8EJmKhFg z#Wv(cpj9~^SVL&YCdX~0D-1ad*dz+_J1{CSn1W6)dP)S{E-)qq6$&uL0AA4f=tFRj zzL+`BuOPEBrj#l8A{ZaRAhY7wP!S+~Yu@#yQ-B>ORA+(7US6U=W}z{Sgudv=&C-)b z_jLw+Vu4i?eOF=QyEz3lhc^tOUTO|gusfX|%+O1-6Uanv=2vL@;vz*Ixr_PKVAOjj zkWoTYKyhoE^`&&ia*IKz;R^&q_Pf;A$^6-Bi6R%h{@Einsre++o7T#KqNeK~5L1hx zi9CS~Fv#er+9Q30smTh$k!VCLX^YspaoLd;(o&z$(_Sk8!Z=fk6+6{U;B05NID?Fp z2J&{mC`{;NAi2WWY|l~KEM~WIwelN{89$VyY7D%DQii{fE%>sjc~oV{XY zqbCCgK_##?ni9~9KG@@xVZKqtG(tCRk<4a+l(|>d8uN!<5(vfx+WVJU-_E3 z=gbz*gSKRgLr|w`~$pK{@Kg)ZOxi`GI`L+?#XBQ)owJ zCFEDKQHLR#O$F`?MoTOT90GX`&L1VQu<<%6Q4^Q72t2o8`QOxA9tpl))tw?+&&bU1 zCE7J}JS~v^%%9A62EHCgjrR$%cT4Ge=j~e)=)X2(Or9(9(Pws2u$sXbybZ4~7c^L35Hf2~+@vI1M+M{y;{? zb{A(%`;9}sf#o4ba}TzLJ2>>MrO6K-YHsI6nb&+X`70~5)~ zluHcn!&km2eDoDkkcK=%8RXgVcSZfapgB|}OdLx3B|Wi9+rPB%6w&>*EDhf@!FGAT z-1Td}mLc)@2c#EmRWhF$b1MBIN+;ex>MnLzaTElwQ%-4`7U|LmglwTk&Ab#UJ)sEc z0`Do)4(p(n1zU{|Y8#ZP@r6jzrlPfjL->yN;b`t6<^ZVKc5gUX%QS~DQ7)$UZ`N@Ttu)@|6jS>s<_N#2 zlv*s+My3;)vQWDlwPZqh*#s~p5z4vTR{}dg@E@TB=N^ULoLVm-xA_!byyf2E^4i|c z9}Mb&;v$eJ7P1sNpBm?-cx~gD7+AvkHP;3V{wK1fD}iPTrorM}RMZg~;HXFkjeb2m z)jF8EVDssFRWq3|&()JuTGm8Shm35`YT48%Tp1ZrR$XG}76!tkqQFxb3{g$IHl zSJP5rlNGWI#kN>ILVqr)bZIDvX$Pa3GgSUl+sqIj67OS_?5aEyxdiG+D4LrbQ=H`~ z`lvxjE8!Vx0Wt{%46qS)aO-20hXSvX3iD8pd6dJ^nCV;T1S}7EqOXb?u_843N7T1) z^=$CPzU3d2h^k>#-S<$}a0)9pDtn0ZO0i~y38zA;nR8)jP^=JKQroV(UWB`r=j^YK z#fHsOX@3W2mufj*nZz3DBLPOz0Q(j8`QDT%dYuFe3x-}y%9PF$iQB|P8*102&_EWf zyuvRYrOEC3ygwVP+x2f&iZix+76>gklLH(mM0PsMXr1fOMB00Dq*Pb(&mc;awj@Bl$BLJ!-#x7g?$Ac@-Ymw9s_LXtY-pb z>OBJm{OYDfaCyf-b$rDFY2dCR=fDeQMCRt=(YecZTq+6-iRBWzuekbE};P;A<~H&0GV z2@u`Ts@b>VFsoX&wE$-Jn%T0?CSL~dTFU{i$UtxzW?LHHP+p)m3sF|3cH^VDh z&E$mh!M8NZ9g~?Vs3cuvB;!X9NBiQiqRPTI>$;+5tv@5p9g=-JRtrYd6^mD(fH;8{ z%wQpp)LsR(g^^xB|I|`{)QU>$#2+A5?-t5jmO+DRIw1OELRSt3(WAY^v!JX~sKOKh zMOW9b%#DJre`?;(6ogByIc*kIcF+WP9t^0>y4C9PAc0b_`_kPlM_Ff8m@!;M6IAX&ce9u1wvdTgWw(AC*=J|!UST_ z0^sB~)_M$IwBqt$QK7?VagNrYRY4PUU{IML)jAKLSoKLPSOBQ_HL7N4P{ut`)Wfy~ zH>iFBgAo@e7j-}0UTdB%TA`?SSj54)699E>f^C#{o`GczZiXe49f=|>*gYZ^~Ry<1-!^=WB7;^4=N#fcJ=I)DhB=OHZ(}6Pbvsd z%&4GAk%1wU6NwVgFC15@XC#|1L&qa5AuydjiV`{=Y6T%CLioKh9si>NH{38?qQUGq zE{+tHY5y^v`FKUta^iR=F;m-NJ1PxaTgcbVeBV z^XrNlm+*|A4Sxs)8L6;<*VW2FHVfZz%sJ0-iEWwLU#GRJSNWQ7F^ZfQEO!wbfh^{-KK%-r$JjY8hN8M^t!t@WUE~$= zuuS7Kt&r;4L!$IjSwILZiefA-0%$VJw0BMZ3$$fY5!7(EzAXt z3br=I`Y>8)w^dWuEwPHhye9;9bAI$nU4)Exswb-XkwHlGlWL)kC3h4Yj&^aKifn9D z2~m8eDKDtzr<5LCV1DUXP;P*sHCC9;XXp(lsM{#^g?sT`SM8^x_Fz$RAPg!T6caxU zN~g?^lJ&4jb8%_xc$l7XWB?gmZKCQ?$9M3XJk3xIV6K5uPfElxKQH&KUhP~%6?8b; z0T9Y9qQZfA9+@qlOVHN>S%&!v1h>Zx<_cCy<0(gH9T;#m{lTfi6B3*E>ztn@l(y!u z`OK*kjqLeX;gB@YEv*ssWkw))?bQd4>EI|y0CWQ4|aND`GG#U-Nm(!gUkK;hNkpc4`HHn`TO z{((@+NJ4mHy^(3t!@70VkJ;st;_@lmRw6PstYA(Ffk?#o0Mnjs348W*IT0$^P~=>9 zp_vBVb7AmWwPxC{nE5)~IIM$)eJk$}MAeJ3TfR z(mxKz>Jn$&xzP5t9gzb$N}bx`4zm>&EnUP43cE6P7v8mPS4hEOeJ?ws3#*oC2+)or zW8M2?Sw|br;hxyjDima?J{B_@=b=GTLxLMKqYgWsoq_b^r)(8GSFr^fs^eDA;PLUt zBX1Qd%6RsK{cisbg+{cxC~fY98sLYio#BYf04)q;E{n7L8D8~FW+C`zzV_yCu-RYD zrMuI}>}wmWadx2JVgu+20$zV4@ky`d7@oIQRID(m^ zGB~Jl&`?>E$6b(yTWw;RW<|qCK};(}F3@`IQ)r3rO!QKOf&i34IZrv zz=8Z{Y8mpF42Jp;kXf>KEuS)y&SU-<;)96pipC_F<$Vmi^zM7w9({MnDY_o6(`B=UFBl?1AK-odD)peaQ}qEMPHqE4koJ+a4th&0vait^sPBqKNy~UR(}_W9^Q5chmSj%ed#mwcXAo7qqahU`y&j`2%e)O3_RVj? zYXhKDE%hpraB&qtn9SFrCKk$*0G0V9iq#s&aZxW-#%lwXW0n2PFppg#>D}H(pGRQc zg-PU=J2zXk%+xG1ow`r8yjr$=fvpczDH@jwfN2!vxX;TL_<$W1sUY3Nb8ptT=RB6UxM*=L$=?YeRM7j;`2G3&xnJ94TdWaoNN zjmTH)ilBOPyC>)=#Te=RR2(;M2;c?X3D=NBHG6PJ87_%Cb#7J-j5ujvUO+g;FWgs&;1uNgMgCs)VPk#ZZN)~47Fo(nPPm!_|^ z{p8IL6%Uz)XFkjZiy0pR2t?!bMinW^tsSo^3s-4P^R7J44m2P1S49=K#2Sr@M^6sH zNFO{{PZp`ghLZGNC=v~Vx%o# zV7MYt*{kx}BbUoGkze+BT2zzc@@mXP8|6_cfER^wqPJ;}!iRUyqqvI6c<DdSM~%C>Zv3O{oFzWq;Rdyv!>@_oyJ}8sVlK5x&mM3(-7acU zPj(*6aDImLG8r8$&F4^*2cG3^5E)w$GmN zKgA8Pu>M;&;9r5yyctjalsCk}`pa>fgj&`ODCu-zWK>on>sl*|h#>+FzOZzlI%RVf$sPddr31oOW+!bGBbr zt+$`QELi`P8@~$wr;YlP-1rll-(RyH{%YD^iTbathks&+{AZixuYve0QU9dgzu4D* zYu)>I)BYzT{IC1^C*%Jg`ucC}V}H{+_{;numR~9VAJ)PD76oMchXVd?FZ|~$|5Jza zpUlDEVmJTQl)sQ~e~$wGvPk|B^>3K@|37wfwqI7tfAX@j{}1UP`#*G${U18W{tq2w z|Ic*rSK&WK?Z2{0|70EhHEsCSw7(MdUu(mk=+!@K!>@t(D^dTX-oMz_|06p1%Om_p zU;n<<@ZSItvHZ&We-^_xve$p_rQlz_4zmA43_1R@*TG*m{}_`0N~-@YV)&~me>Z@BaO_>;Fs)e--{?)cz|O z^0$cLucrN#sDFzX{u+qC67_Ep!(YSxSEBywzW&Mh|FiJ_!Xo?;_3v8^{~a;>mG}Q) zG5lLG{J)01@e5}0$Ef`eus1m0U>U!|EPkhJ{7cvd%Nxh_x8pbV#joOjM(%i{PW*Kk z2j?5;*%wzckTX9=RrT8wxQY$8np#^Z42)^nDp^7f}Hud zCfd<&N08(`F5-UOZ^?Dn-ivnj7Cfsy#ijbv<^43#IMzPZwXP-89_!csbiW(xcXe1J zxGI40pG*xT@Qw|JSU?oVTMx4%Hdxb}J}=x*@(8;@flbqplRw$-^-t%RXzc!G$rtD_Q& zD|LIv%l%>Sjlr>+i?nptKIU|B%anSt!^Gh0{b1g8kM`Qy<}x?1dUnn0TvMsGhq;l~TYtgbpSc|{L~s+|PqA|QW3>tp+@GpmcM?}# zHQqmw!{C&UG!s$EgkICVrfFX&$c$y`aLiFrcg=(M7KvRQyjs3Jp`sq`tn*k=0 zmp5IHTWfrS<}zZXF;ebA;ybuVZ!$9MG0iDn{F1YDa);|@`jBNa{6)YjhFKWEV;!Y07c&U%c^&U@g~ka2OPetmU@)PcR-Ks<`=OX@MPs?c4YR* zTrCPGRxEB;th3j7sI3f7uP^b3g=lkJxh9Xs5l>m1rAS8>*n{p!RQKf;S}Q76bRLPh z@2SpOArQ@94kNVQ{@da8)HN};#HpaE))QNC6ddi0xk^{sQsY=(6T5Uy6EkV-w!cz2 zB^R2JA=uNcHC~{*`+i>zv8&Z5k*BaOUf#hOv%ajQ4rz6bUy=1CgpP$B=VD=>P$ks& z$zQ$V>eaqh!t|oCIAf_`KZ*OZn0G<5#pepygw)nnvapwTiS|T^HtcE$ggpgqIs-_Z?C*lqq{lElim;k!Ay=5JxwO~B1!&cyk% zX^_6#CNnl_d5r5qt1>9=#WCm6;pg2x9!f^myAspE>P>&D5}!*OU(>5v*>07JyQ0i{ z>0>hcOktjrtCTnR5h$Dqb=KZN@)5heeF1`>A+n>V_>2}P97G>p*t-H9Lw5C*Hb0=h zUmkAzYkOP8g)n>p6ywh@B1Hu@75o>Da-t_h)AUZ))mj0kW&!O1dK-$gCx&{L*K>*0 zA}Q|YRe@-%ZBQVA_l#I;HEPm^fSZDbBAko?n*>4${*%=slbT8{K#qb;Jos{O$(!!ATIHzHc!2S2OTN65<3QImK_J79 zkB+xQI`7e_qu6`hFF&=_cj!wXny%0M;1KT0mo44r-I0*&zbL)!LQ`mN=9-liQjd-5 zmjh~@ZHGB&B6Z)Ad&N>h%&-|b!JJW9X!rFbtS0&nG^V)6vhTTR>;U}B>0zt+OqiMY z7jS6XBeOs+E92MK$CD))8rf6BctZBOG8L-92YT$tky6FGQ>lj}`O(t)05HnjrhCr_ zA6__8!%M5obgZ)|2h64Qy*W-53AfN{5!jOCrEAws5c`4W`zDmuJTA_DYlRWex9?;O z?JdN!4)%`7nzw2z$8q1c=)pU~q z5|9S(wayBXXSgwI@lao0qVW}coBviQ+S`3_Mle4no=%pcW_ z)3my`tFuAJ&399#4TR?p3;X5e%pBu}ID%)Tt#;I3rrJ4ka9G|MVQnvux3AuM(|D&s z38&GXyV86%J*PPff&x@%LUW;d?7I2)t+jt+X$(0*&rvrY?%0gMfY8^^w*oH!|fSnNA7StaWnKB+PdXS!*1W})#goKci*3W?l;qQE%@$88sTd;GDxoK0j2ob6yv&8 z)lFvesAjn+rF%c77Iik+XV&~L<8;(Vm6d5OY4h{=l4PZa*(QH9ep(bbD8_g|tEYgM zT=7U$PDzkXrI_!3+I>wq6!ovzTrz#JqqNBiL3?mMe8Y1Hn3fm5ENp8=&s!sF?0Pd_ z)5&YB+av>5hV5olR4TDsp5n3Zl|qlc_3f_xY#B3K|FTM^LoHYZqvxX7s`TpKjI3;{ z1;ag9QjdZSbPkvONJ3MqXuZ1sazN`PDJ?B*J(mNpo_M0fsLY~M716FLiqp_p$W10= zXfwV)(h42^B1*+&(ac}Y?f3ybe}?Ru`%%wa>4IG7V%&~_o9ugJ;8EIof>ViS5(s@p zNIllm7BRy+@xb||JwL|IDhzunjPD9nbeZ)Wf}L2tnLgU@5b1V4kTeWHR&c3K8@0c~ z9sng3Q$g02v%GKpxfV)wn~)Ia6|Xj1b<1;7sawCE5pTXA4eR+EGK6QgfG%{1XYCg1 z72~m)=kv{oT{tI|7E#EzT&yb{nC98$A5mM*$QSHRejheWl>1hk@ z7JYe;pGdm9&KS>nJ&xe-tx)XAd#1f2Bh%0_oUWkZv|!XN{b^i>XY;esW`Ytz zq5zS|E#J;kS4GJ7J4?Gj`V)caEH4DI^j7E*p}#dOTNJX;2cJ?os#M7U+) zg+aULT_)12)DK%kDG%-2uvYI4_YBk;{%)ybLhme5QrEEe#yP3EB6*Yv1t~|T%_#j` z%>^ziRmNA}tF%f+V$gB#@z29^>F+XCFmjW=S&kuf`Per368LyBr%a@V0a#oOx=P1W zxrFc5O;&9t?UCEKHer-aRyXZPtW4g%(f;m1;S*8jRT8IT1haw*ngN(Bzm|FZi4S@@ zb);;=wt4nC9ebchsg-pFPx?WIE;^5$pPeG*DOm*jaW{yoNP)aYasFW>t#o0x+W z2ebX4>)Yv``17Hwx$~etKK~tbsN|%G;1|NMVZVNof{O2(iQN5(o=uM7YVjI7$Yp(< z&|$+0-0NPL?+ z@nM)@c%S&3Q;v4L&!OZBMFw(+6@$=ow#i0INep&4n+#M@l6>GmalHa~_^vJ-VvGE| zLI5 z-%Y)az7exZS-T%;5D%^r+c$-0^>bsc3l*39Be3u1Y_+H4G2c84iCqAivoC#SX=_%( zL`}q_&XuB}Qa>607A_*WNNdXzoP8_>3I+Twy43}P$N=8qJQ^mG_&kg5(+H71Un&V5 ziuMhLh`K&n>}whlUX$E=M!$>HF`th=KkRdmg;5I*wsyfJJf}1lc%CqNn0HFh6Aur2 zzI0Qss+c_PXuj_PWeZXhc z+kI?L3TQ=Sz-3jV6tSAvG}3oWb;7~2k(t;ufknQc zGx%*|MC}{ek1T~jC}I7Y^opyIdpSaV0gWr0f_AP0>r5uxmb<9z6yT`d*;*t71J>qn zkdS?r^uir35P?p!Mi5Kg(OPAaX%SYznkx0~K`l|>%Qv@K$CHMUp@B225sS>LC>{m7 z$rqQ4kSIdc=?MzPHH&Z0^?Jji=w9D^qv8E`L@o3yabr$%B2MpYAg|%Ig`k`h-@s2p zryxgl=y>I=`u?fj@36=ETzv}YEL8A~DAS%Yw)r!6zJA-LfGmycy5Ff+*}XN+c5+#R zRY)^n`%u$L*C6mAlVA5aR6`~MLwV&J0zP-Vx9{hd)8<@hQZ}fe`%`b%qXp6Z;u@|I z*?A{rz$BBpyu!#t+!3XutC!Dlh06>}YwN^Y4;X#%cAGVQkXl%P`N2J})vFh0vY$4N zI9AYfb4Rc~Qh6sGLxBclfP8^caSGys4N+jZsLFuSmrh6y3={T?D>Y)>(^;8EFMV|5 z+$8T0e=?|f^D|w^HF>t&BSo!$j9`t~bH8*+q)&F|jBDcFfS@NoAYT=*SEV`?M(ui& zUXAoa>u1B&`^Hcm-_P>=yc{0Ux2`HzxJeG))|{T)#i_C0_N2@r*bR-cKfE)sJ@H~x zLYH|$eCDx^#|!&TX8ruu9ddhWX16!?w&xuLFWblE1y_kP1NUjf7*XeR$jC{124PWf zj^S+yK6*O`stBTb)Rl;Z;zN(djfH4SP-(iX3RX2TVAm~fT-=<{ablQk3}u7Uw(qWw zP}v(0Fx4&8Wu{#9|8SsHanezG;`KE9=%_p+hdVqCgLvWHZO~~$6I*}G&zEB)4SP;+ z-)4!stE3Y;Uw>T8Q8lWZNZ5-D`kAJa9fmNhQhjMJ0Ln%tftZ=T5_(%b&e-A8awC>@ za&h3`=Cz5GKn*Xy)k6MjI^x7FFYb6=-#H6Hiilp$Oo3hJ27}f6u?$D<<1}eL+oB_m z{4h|Sudxfe{oSKlk&RBa!2vesQiFrgd=;g3Yf0X83C3)_L9g^b^3z^Ia25JT7Y+|6 z&4(j5mZOwTIwyDsJKHboZxka5k82Qixs_u>9r1S;zQ?W+ijHZet+nm96P521*=2or z&%Y5wHJcw-9ca502~gs`a3D6|iEomT{}Kj<2C6{f(7Tr^m?3I@s+Dr!;kTyYOCI1+mF z^RV!=w1luemeQa&m=+O?Yx3e`Y_LPx`*CEX*+b^OjB=;pCizR@X)|V2#IK9R^Ds?c zqa4W6h5Cd>o1{{PbSPvGI?u!c;k_0~jr9@4_T1IW&A)z|#4g|?b%z{BKPwcG%JQux*ABdk=dX?pTbtirxRYZg>^aoj24u8sU(UjwrDV<0|*-8?R>H!u)ovSHEHl-A&@)P{kAcMw zZrhDGNq*R-H(HbB*xJP-)jDdr;xlQ5^^Ot8Lv4nSrrcVY`TNSy3wRR_>9<U%Brp&$F9P`M6q4+JV~n-)vYu^-F^cHHRN?}^IX zbYvKp)|!QhtOiLVokKbDbi%me&Fd3bdIiG7()PfbJ4!DBAkx}dJe|W{lE2>D`nm6m z8{^v(^J4`&Cv6p$Ve7s|F+7S`>Yw>@roVBEmX_qx+UXUAF}xjiq^-;8b;ku<7u@4e zU1CGzx9a91?++?j<`XX0Q!_j=(#`O-Hl&WVJN1~);9C$usQnW*JyP4~_cctKLV{YX z-mg`BzBB_}>#a?5k&n&wE;~xve;pAHPxhV`;+^+Ra5j za@Qz*OCpdi0He6S`$$lKf-|$`!DY2DBwQ(@7CE!Yf1P(+fiyW}9hT;hU8@b|K~Lwo z*eW}Yrk$Kq6-^FVLmy&^U43=$_Qjpckw#iTAYOGBnnPM8pv<&LciT9O=!m5llWAP4u_Q}@TxTs7%%q|K; z|0%L$wFB9@r+!d-D~)TphA^w}@g(P-Y9SH`hD+|fFf;2igf9$>-JrArA*!Fdi0;PW zQ~Ju5Mjx1J`PyrvqMlrQ4SQD}h**wg+=Tw^e^+B*a8rvO~(c%!XEhjra%s^5r)@Y)Ss@h6m9Se1@ zQt5}Hlv7K-qDsVCS%Tt#%0TH7E@kBL60;YP$mxwZ7;%R<(48GWtX#=3RWiyX`|{_1 z@X;;D3wpetpDc-cpJO5;S>t36eP}_Gtsa3Eg)Bawuf z%MX#4DP+vx9zH4LQ`vLmT;wz?Tw$*sF;)2n`y2DI+-fK$ECTMjWBD6Bto==cL8d#o z1EWv3DjaAH!$E8U*=!O@(pmHT2UdpdTvv&0LvX6dsKvAELc`N`T@lqQB9Ok4YfcGm zy+9YM%{hm=S!}GsOb#M%xr}gYc^h6+x=)AG4S{NCdq&DuqNvp6y3n9v$x;*51aTSk z@yWXuhdH_1dA2p#kFe{zo!U}jF0sX#?HkW|r&Q(;_xVR`n*->2^0+KOuDH7AeHgmc}FRSFLH zh&KqBL?F>wCw8*bEw)p0j%HBKr^x{(&sj@(7btelF-OhbsJZhTMwmLazyg(FbmlvJuclJhnS`*NCHbwpM8Xk@5v(+fz3Pl zAsJEL_PuU8l$rAi@)?m+?=jB^60TVxpMf^ZQ*#)4ROwa@Ji25#BaHOKrh9gBf(nCOI4|houzc7j&f-S-lbNi{(#w`QRvbT)I=SORB~-GAfBifzkobOA?^b-3-;P z?#TCA>Y|I`3V-iW>LA;*<+XDE1=`^r-w#7=a)F%!z0TqMg(&=b^p(I?O2cuQYZ?m(WaQdRC!?r}}k#J(Z8WozoX zP~~OBY%kP@oMT&_pbXAGSfW$JH3?)cXwI#WMdE5T7ssg;$4C&ZpRThZo0>3KIOM(=NNXjouK4wERxS!vG>yBjKa zM^Ii3^I5rs(S>JRQCK2c^mX}2V0*wGNw*|)U9Pf;tx9bej6G}&c(6lggNw%4@~`YT z9#>$Tpj5M3nhMzNuE@#88!WW@U|HI8WH#KvZb_kSG449IF6Go7NIzPwYjmY61`0if z5Fb|Ru)XsXk;Q%Y(TG563D?IC{erWbcx#pudKm)b{u&n|njJz?roDg^ybjSpv(d#^ zIWw(s5N8}FzZ?HT^rJmZBL4fIV1?zlpr}QeY8an?#yEXi39~@?Fgwc@R^nByp=*PD z;UJ=&UW6%8fm&Bck!ZWn8ton}JLET0o^EM}4`wb^{cUz>0E^kYvu2TXJfSCjdNXNs zT`emk50`?O@$)VnH0ejjp7j9$m`Mn_pyZa~4VftAM{wiOQR0tk)QqUucRnK6UST^z<#NKpLiORLmw&@(F}Fzh@7 z>^=JcMx1xd((n-C7ul4!|a?%NR4jxMn@1VnK5m_F?7)E0~xAdgb}D}_x3 zJ-;K_1H3Xq$V`~UeIX;kN%TzmLe}%hgS6u=S^A#YDz1wx|Epq~(ql|;PGJe)F##Fs z5O}yfXG-fE)=y)Me_@;Q`<^ff?42S#^gvKngkt z1TlH08VEw!g8-KHB*+NXG6s&;{T;o(6vnVb0tOzZk5&%ML}Q6p(r~zlS=tb_m-r*e zAaP0?1SCl`DK;5b4F$6#x6(0c+NC*4kL5ce3>-1$B5*|gHfTgA6TA4HulOmuRyb=W zk06(M-b#Xf+sI3YlF*fC2E%GJRS>#%$vSS0uq0b8Ib^*6Q3s=BrgLdx(gIE~Bv_s5 zh;v#rioo6;|K##y1x%hPyTAqFS$L9ZHDjGD)Q-15r?9%t(FgJ!UJ5DtVe}P17ow@b z+<~c?^uW=s@yqU{=cTVQq}4GPt>vGCTDlZ>YCYF325Al%hk>AtC^0$iatg%eoLH8~ zZU0b{mO=FKE{ROTq^qh*pzSK=moyqQ#uGhLotv$_8W`W!Re|5;*$CRU;j#R8PN2!SNzy0;#FX9{k%aRQjW;l28(5^y z#mC52AStcRCI+Z&&MC30aupR`>&hz&L8A_41jgw1FWW=&auR>Xd9>Y#?~{PuEwVuh zrks$JnHfZ78FVyMlzgPFREvL<^jsZUA@99(l!^&eF?BasP@uh_sJH z!W3e8GN9;!+3%f}7>?viBJt%i!3BpjP2l2g-@7g|a%@uU{Kd#;ghsKH7{1~}UMY=K z_FT%U23Hkkl3i!vDP_im=;690ROKYGlvq|fWNCeugy|jnEQKab`&PpD@SZX>B>Beu zN??%d=BO_SJZl#tB}9~gd0M!B`taq;<46g^=m7g1N!meLIbH~ZZdmt@RiJ7r&zF&- zN8?25>2f9t1=oN=<#ldoCaKHtv=cl-y+k`t{o+7)>L&Y@X~pD0dC4u%QB|O<(@gXQ zZyc@~dZ-V#WCPRl!18!_6J}PIg(|rS8(jeiwlR{1k1gjVUw0xpV2BvmJAToEYTB?C zAFv^qjyOAMwIZ@us&z*I12!@KO@Z$``>P0qRw6GWzX%){wqBuL09YU9nXA{+sMS;-J8F8s;^ zhixTjDMYgc{HslwV;wrzXL3tRqJExNvcywzqny=@NffvQ?F;lU zpN$E8OW8w9Fr-1oo(+!fi9YT0z-3W84EhlFV!i z+bogRNS{9jDEJ%cuptnk7>J(4nG?gp1}--evW&TMX2KXKh{8DE6JxKoT`$z)><7By z?oMEv77p+#Ksd9)gEvPoze`-oBzQzsttWMd=d{;ld@HSz86V(HiCri99$aqF&9Riy$4<9(d=^+yn?+ za?pWRO32e3$fAShpqIdWt>ic*>IM{ht3mO!&-I>I7!nL>&|YTAIfaIfDe|)3jY{B~ z&Y5-t3B*BE^N(5WRU_BrpWa*IFQyl6*mZh zUl)&Jn_W})1{6=Qq2-~iv9PpxRpvtg!YF~bW%^xHimD)STfxLW96_n(?uIvwc=?LkD1|M=v;;C-6$K;u!ISr z2@QHB*%@Sh1lN+DJI65J50DcZ$?|bZKoPqN64pfdBxJ5cbqmQp3RO!Q6B zuhR?APloUKoTRbZ2fb2u}aG+0Pdyr{a!8o`i=AKD2 zoFpgeqhlCufDuVQD0!h^5@uY~Aqu(3(zO7}=G+I65pHa?7aeqtMj<2XLBPi0KiGqeikj-32PF%tkJdWI#({C{9u8-Yemfx(ha}jS zh>8kEA=~X443AfOABBxWXqWZYkKcafcoZHV&unZOS7X#aEQ!rhS7xX-G%O_f_Sl2k z?~ehM-hbF+VMP^>f){oo{QZPw<@e&OZ}dK6Q}VYK2D{@jvz96Tv#~otCp@ByLj-sA>B#BE0bbt4Z6CsA9>H27Gs-t%K-tqfCiKG~A^ z4LDiMflz%zR(K;EMOH*3-M43gM!*aF;KWPX3QebbRd5yS%8bkjsQV7!TeBa%tdDWc zb=)znyyxFh_X&@hfs9UrhJL|Tgbc+YHW9+HKQ02L?&}IHG#0{PZy~*WpN6>@w6n6W zL>R!j;h6@N7X=mb0c5o73Mwng&6m*Ow`og4fH;U!d=5y84!&4(28@ae+hfu$#MQB^ z%gFi(yXiLPZ)be8X+6I@y zOX9#>1tf`Sqrxe379}3CcF~#(;Xs>pk2sK4w+QpddXu%VZ<&$-IIQD=OreSK$dFl4 zF&4iyWi%iW5C&>`$c+Ot;J{kx8_K01KudMW@G&M4a7N};uxm;_4@w=BtO=PHWyJ+z z2H%9^z*_S1_7p}vLf;5B&h}eXJ?K{5Ba&%v{R71Tm&LpmR|*?3q_he!)`oWdID$RK z(rX%pRt_gCu0oH%jKjKhF^ZU&Kl&XiO}16jkoCOEk+1kJJ6=FwzBICP)~F7It}DnC zJ`xsOQoKR{o)|}PXA3e89%7{mGGXR&3S2U@7;395$W(K}icpcY@+UJAvleZWHUNES#ba+R2uzgftDdFasr+qD_Si9CD1Ep*@zXn z|X^N#jyp(^5xhqEFn z;=ou1j2fdtl#7^=?}5?=eNLDINq(EaGWex>W5RA0)Sbw@vlJX{abf*w)-=VAC@V`G z7;sHYiwAins4+=sa7{?I=&)^HW%Dkr_B^xNABIHk4N(wWd1g}d8}IPRNGhT0(ODd# zf?1+vP+YqQC8@~(@I$}dhzY*cfClFl57a@4^f6m`3w0^k{qBV|z=XUgK91bBJjjGR z>QJ`Xx##U6IK5ar&@d0#jH_*_j-0*vGEvTWnv9 zTC{#(dorDGWm@|C?zneu?Ra*E%&RT#`^jX692hX{6rSZIZy~}PQP{HI^%G#XYbS{f z0)S%Zc8T{bTtoGmVM6$i?C6@0>|C$j$N5?0K;y$+A+o|#QwacJQ?`$o4274|X`bB+KS-zx^?Vf<1G zPYvDG9Brw}Qu!-E=HS%j&uKqh*NX_$p;-(oF^2Wmso$7}x3Yu3AT+_#T2jh-RCDA1TLF}b(vS}-_JICh3OJOZ3a7{-f z8iA=CDuVzBASYoS?3Gx);F5_mNxYC_mL||8@|!I)>HPx8g66w)2LqS~2_b{UF591a znAOs*>A12(ocE+t2cZLxK*eP(Fll5MvV2n2Dz~-Old*eE-RTo@2E5=D&xLTZJ2M>m z>QxMrV=R{m=3oxiJwjAMjlQMhZ5%{skHBwEJW#}8f?_Gj-|5Z5nM?u`6BxPzx2Y4$ zDSZ4|9QswL#SJuD=;+hhzQ*eq;YN%)fr6RC@+lmM1Tuy zdEkuFi^2o)w+tH;P&~`Iwp@$dSGLopa9a*u^6>VH7~mJ>#W|`H%hk-iaxRw%ALp7? zuF*ugHop<0iqRD3#C7Z=_g#9xN~^`Y`gsOYC{rivBb>#nVKDW(i(CDoMks@!{g}+bB zCnOd!-v8j9^u$2BE&~cNy#aB@)^9S!og%>M92h%ssg&5&xVUSi$YEK03En|d&y|;E zi!RI;W5S^2w1}Rfy8)A2jy}s?8egFE8A7{YQM9Gy1rb3s(YV<}eJ)-%%lvLEBATx8 z1OCy`gaiI5{dz{zI3C@li_EV+vd}V^`g^Q_bwuFCG|RNMG-*#vZm&Lh?y*C#0=QRU zDu7(sT3&tR2lf5ic%gxmFK zA(ai^{Z;~^#pF3%{EirwVRUcY$Wb=9y>kI$?xU+)HM3|Szij~V{MJm`zFoL63)5$0 zT-f?cZ!*wn5TJ;XR^NA&&tS>vmZ{qu4I)`JVp(;Vq31`@o{OXw4Lu+zdxp7z+HBA@ z3z=ghB~ceExL8`D61^k@72+Wphl#!eSO|b;h|LK+(cxN|y@*E1EzO^X7*5@+7mY*P zCJP`Fv&Is*d;M-_4w)}+>IyHKi?o&hmO znez+GyTo9P^j-rIbFw4!HsUSBmJB330I;kMg2h4trtREng{IUR@Zn~bQ*v`pb#Y_nOjUX2-j*n<$S)Y11-}koHeeHKb zfS*}WauHjAW|=XW{$J$11yEkgmiLW2fd_Y&;O_43?(XgoEVu-B3mV)dxLY70xCWO1 z!QJI0oRgWkXHM?i%$u3_tNNgt)uvQB_q^F%4c* z34R&F&8~PQMryQ~UFokX)dI)imGw1D%{f-$EIu9?l9C0UYGXZzJ8X2G!GeyV#2a0v z)!mQ!=H2ziJ0aJW83}bzlv=oRly_pfsci-WJox^vi8ds2IFvSgvei$k^jN~x=*DE~ zD6`QXeRa0RiDw*yR@;E@Lo|k|nUxTxskpF1(KWzBU4zkUsI@>|F$Xj0UM+8(;KZp& z7=t2B&MfLR6u6WQ^a!a2UScI5NlWJ%450Vfx zbf=@o-F*}1hNr`4P$ct-CIeiDNZ0wcmyD+tz&66-xTV{)Cjx&SR}b1!d6>8qQsMJH z$cP73z&w`d2hKl>tCO(U3{7Q}rO- zS{=i$S4LSf_KXlkYu|GPmT#jw3o&x{x`6d+-iqVCl~%U*H+R5i&{XSa2ihqXxB<~ zdm~{d_u%%%=SkHX7`yI1`Tv3Su!U$DbM#{!Hacim`-}? zth>}%7-(%84j=_L=4Kz1UqovARZp;vhYd!^D(R;{PNQJd1z62)rgTGDif=%APRt&(X;5Pv)pzpBr zD~5Z#2{S0l24FK7UH6*xz^SIFEHX=X<5lLE>O-8^M#wWsyPbD2;KFJYw+A3_J@U&& za0{{~cG}}oBBgu;Vp%uk_r{<`S!M}xyu{S;M2m*)xJ`SV@y!c%weNnnV&LMea ze7B0s6h^v=eshJK?F?)Tm#BWs@-#PsG?P&`99;4{ZcroL;DZ!aju^WP3{9dFD7cw- z+#o_8bPb&=S?XWIDx)YUg>;fLx=AT6v*nQo0lcj0298_=-9eFfN>Y>P#6EddX39n` zc5N%i3L3J~Zs3CqkxDS6^i_^ic>w_hecI*r@Nnq$+Msz#cvR8rEyu_2Zfp^|#*11e zgY7jw^%1&^xhNB{=*p8j!blegCm~3Gtx0myZ^Zk`)HBT@Gs0Wf`<7f$ga!|T*6NjI ziEy~g+z>-IiZKL)KkrErQy-dkjdh(PhvcUr>$y893$T3dn%*kH=1?ZL;|$g~+`Mhw z&U(Q@5=z7MUVDW+DPW8WQ46pF0}(0PWg#U*@K(?s5t{SP1Tj~7&U`$7yc4@x6B{se z=FcfqSq;d z_T;~u>Kw>pspTqQfaiYCWLiuS7&4S7{$3()lrBY4zQn{CI}QzbcrT$h=;iet2^>$( z)F+ChxdP+~o?@%mIo1nLP6(|6QxzUYt%xKIatN!SDgW0L(bQ+Dj)v z{lD%FETc6S#-d|^`#^ra!3nrS#j-SxpK+Wn*ex3o7jXFkRUkWFw)WTc3tLzjle`ft9$_}>V`cCAE2+3XgPv{N)6r< zS!@Tth8b$GjU1HaDpT5Aguhok%kphpoAO~vO>j38FqqZBFF(&_>`vCn?HYBvyrm4} zv6g?W5n>)zoDuzWU-(U$Mi6bR9@62>)8qTOPAxY~bRdOp+|*_98LAiqKbWJA2s?Oq z6*M2*8$P+ps;fc~&O6U}cV*`$d2B`gAr}ju`Ei5o^El2#f!}a&B zBi+`ODI3{%iJ64!TAEe+8`m34Q|}1PH)cKuCpU9B!=#1kGt@+zeR1MC*kQyd3|pMa zPq&)?oL86BIbrn~Zc^st>8=kyi{8CyfE{QqP?;j({!ZwWzsfa%RNlW{Xp+a{pkEQmNp#xR7Q@&26;2Fb$Mhvb>nO z4?|Jbo1t5L+vLlw4x-3wYa2WtN(Ut|a7ZlAj~`Loe6h+L=(mTh$BOD22(IcSl7sJh zUWL5ny4Q2VJ7uC*NFXx4HCrA@C&Z?6n7}Wot9sT$#<{XC7I%}!A)px@N6}#M?STWP z;qhgsm-mPBFv0Tb0-7G5AV1{K zi)kv(-)l9_JDRmsBMuWTUwOJ?rc@XKj)$$Jjwt7svtLL{RmALCDbU6O=R-0YdhcT% z#M9l3yy|1V(HQyEOq89sRvQkN@#{e%4N3vo?> zJ}kx~5{U8!C@(mrZGJ~SJ1T4VX>|*uJ^hda44p48PtxYF&f_S72k!_6bOp3Ti!lX{ zhyv54P{z(N!;X<jz4Qc0CN`HLYr)f6{?ESJf=C7^OO`wGmwkOhK1*RtlpPQ#{V@a1qZB(xHFr|;vn^OeShk_A z0*?(0uNGS(Q~?(eEeo8Mfh7fb@OVJKXfQ>Dkd%Ijmfd{}UN|z=lrTAJb316zNVt!2F*Iok>KHo&|EG+59AR?>xLRxz7)*3F0$?B!ga zejYQjWFOH|rZ%iA40fyIdwJ|Rn?~p+{<^MM)mQ+6ntyp>Np&ShGV-`KG;L<7+80Ug zb#x_v-0@;Cv)we%ca2EeL|VxnWEeCpNY>E4CO)yUl8%J4FB#jQr*8U!;)%x{B^@W$ zWn)4=*aP@cz4xyiOmKH^pe^m8G{!dP~i>HxUo8^~O^agBYL#T(LeM)0`NY$_G<0 zH;u=>T9<;rHe@S+vz%g~-|P&do(^UZu5Yy#cBD=fDN+h~OsX9;7-W#lpMIMQs9`9T zAtzHR9QKot=8G@PA{`@a22nCp_N7ksV2lkEwzepTKR%^K3n(2LjD~&lx!=i+9;(&A z{-a~H60pJee$TmZGG2JC?6*CCJaX{hr%zo54>B)^+EoVIz+> z@Gu=qF~fVT)8JrZy#pTRMN9*HJ$>6t!%F07NubZpWMFhF!w$}D`?#HXMgOO!k-uu)0 z%B+r7-}}3huix~1@6Jk++kHJe?yqinG50mMD;{gMwYQubUNyHMHRj>&x47B0d)(;C zl{I_1wYhB?D5k*XW4(uYIrHcNA-+XtsZ@_AS-5p;E;Oe`YgN05aG8bo<$hOadU@&N zz1>WnFJJ51OJC2FOF4x58yd< zrt>0ncpr-#i49BuW?``qVjV?VfG9^l)xU*B@!_+Oy=g4Wqpa>28wgj@1E z-^W+MV(qim4c`Pu_!th=k3WB?>*f;6sebCjNUm=rj1^-#`=p5!J~=Cv0b9 zU}3BE1i4^%;$eu`8rd0}X#Fi4lI5xW2e!xQ=@8cMkfA5|59^;WjelfY{FFfSKgq^> zcI9t7`uA3U4-V>?W%9%7Zvfyh(lZeJIvw;6y31dvc&y*)bU$4CF?ruQi! zFg~4)_IqLbPqFX}e0qw7?|{hfG4YJ(d-{DwhW&+`$oi|ywx58AevK6O?AqTB#oxyU z`#*;x{RG(bS2)u1K>Ui;pPcs>`}(V7%75P1pN#)s`}&OW`O(+kv<`lq4*DGOe`P?i zePT~_f2FTKIqxs_ z_5XTj6;{~K2F^Hl$*`Tt*1L-s$_ko`|JWdBnQ+5b)rpAY^qYX3@v{1!EQ zcI{WJ{uVWS9*AGD`difSdDwr&>Ob%6PsabR%Kr>%{9*O?t%m;&9+ds>#PAt&`NO|w z5U0AFp{0qDGu;nX=raM6PR+vD*_?oxiB8HN#4X!$j-*z&ep`%nSh;6!q(Zu(caG5z}bYr)WF)wgicAuz{%x#{9|t zcJ*?aa2Y+zFEeRZqqFZM@V^w5Z~N{leCe+f?tgJI8lOIByvcQWyE)9W+~nc$D3{$R zZ(~i}&=5}~axe$TD8r7RYv&9Ar1^xv7U96gyi6HhNfiaT!14wVpcy%GZ zSwkE>*-Qrt<$06w2MNF7rS9feuh1_b3?Vmf90FS6U(i5uibgXHF;y%Ki(SxAg53)8xW!=m{PtmufJ*LL%#quvOrNh0SSt#1~|*| zt4J%E_BE0VI$V0?%muw?zppbZHahf4T(RWn#^B@HCURkLngK7!p3SGR{fUIvNntuK z7a}Oux(y!(rYHMX7gQaOj)5uSP60r>-*z={HSQ<5n@M95wBbb&>-Z&w0A1y+LprAW z2}u|(7eKyt0##nLMT|x?3*^6B(w)o zjgs^h1x?#ef#F4J+#njUe=KVKi^n5V7yB?%p0d2i0d$?Wj$>AuxPIRX#u|%VF^B`b zz|2cPV8c!+15}qB`Cd$zO-JZwj>0ID8q0LBoAP*jfvIEnoQ4^fjNc|-Hoemie2L-# z>aSE|cUgW6$kyjxiCl5C&nW$-yP6c3f=!0*Hr$D0s_Ubao^}oOdM+F@y4IzBoQoeS zAIHi3BB|JQUUfaqCDdN%u0wJ#ipe>U z#V*peEn1wAal?eKs(E$;$l=8@GWLOklw6h- z*dZk$mXX|#C5GM>1`&DKY1H}!tkII6X*5t?Bj;8O99VOksFE)^=fl9$qIht*t-{HV z`Bl$opa7Xs=#9ji6G5M5-DbA1(G1MBgHhS%thO}y5T%23-keaViIrI$v^DWYtbJ?Y z@CPInq#kT%zW2Z)W^#`Ag_(uLfJA{D%D)26-uzVd_ixyq7xHfs5iv&t4>}nGXXmH) zaNpsx^xvOB{}oaBxr{I~u>7Np80lx@9DOJl1Qf=ZEk8ac3x_INUOhZMb{q&(Ouf2( zd~{>wL6b96K-ntoNG8aJOwD~?h$_b+Zyzd_$ADyl1 z_kko^`Rk?Oi!ObUS@YlcAtkHFfNdZ=JUpUVApx}R9kMw6Kb6IGuch1Jeh8PL0i{Dq zCmoGM7T$F?PzeVMG?XTQ6Gad+ZKWiQ3o?)@WMI+xfGgQ_Bx`JzSlTE>Ei zKGc<1jA_?VKewX0pbi_uG&&aiY6M7x)kp~oC_z1#x^otRnFTss5$&;j5PwLFzSDo?Ld8Na_${r-GLC+}!yq-5f(MJF#T zN~dh%?)(RMTkww`p+9~kzQ0MO6R@?lbAEaQ&&c?EfBiJQ|24*%;|~vh561dgOrBoB zJVztL^QnIX=Fc^Vfu4c+FE=w^dFE}68)6S3Ht}z&->bh}y|1A+U6pDb1c9ijj^t+d zh+*NuvwMfBHQsydb6;c2yV?3W@5iu_0ghxmXm?uD#wEoS)l*UPMSncio29qlLShlg3-@7R`Sx+%>~ywz@| zI(TlkeKWo5u61&5yyiri4Z?%5B~^DLRV~){H)JMIRwO~wR|@J@DU&tj$ZRU?V99+O z+irXcdp1uSuq`Z3@1oPA`FtL}n)zON<>|K0dfjlGL#Iv-Q|fK+yX{O=!Zy*tpubY= zvT64LUkoTX?@u2zt!4Nq_rY@e30R!mo;_H}f_bv=7bKHyv5t zXDyXxfVzlpwe)CHF$^19^ZPUCnl^fRjr+Dvf!I!Hwu+f|oNUY5+6DXR-JcPTK9uUU zU+q9CT~JF~tQS^&ocz>1GIelaa}B9Juf4H<4;s3^FEYEdf53*WX@`%+QLw+?m%n&` zCplP}oIg4Riqh2W&nJ$q*G2<=Xj@rDX6T3ae&w_Fi!VYzu!2^WEw>+pid#LM44`36 zj-p$1EFKlp_x0-4>}a`9UhwG(O&r45yFqeKhqLitC4UxK&`sp)xp=U-c5*UsQ8ION z^Oa^>@FBbf3H$-KjTAd1RSY9ZdcKCRlC+BnDZU(6$eIE!^5+MX%;MnBx znOiGHSv+aWu?sHwMZ^54ZQfDupaTS+MSHTuO?-Dw0iLSq*xAlyAi8)@H#cQi*$CcO zel0lUTL|U?sUPJO1IR=0ZH*Jl-)d{G?il5tlhavWi>01y9VSFQ^qIW0zq<Mg2shm2ACJ+_>Di2mAM4sq2J9LrM1p^N3!igZOF^H_wD36e<+gfZRK$Y@F+P-?SiGuLM(vE))}F?C zCV*|(me0@cRi6hcoH6l@;d#B@{mCX)4P=xjDyF+7}|?k=Y1)o%8x6p7fE>?PcV zXFt}1=D5HTM7e<7^;Lr-Iwh?PG%+`|b`uVPt3QS|J}97E**2uUMI=E(zAlPr%6bP5 z%O@H%vLXHHNFf$}1{|KB_YiMK5vTrE)_O~Y!s_$I6oSje{y@}8b7$!-Zdq$J;~OJy zWg@F#BmkNEjBa|A99y{=G<5u%U2gb*N@DG|q+|(wv^E9olhccfR4btqUuq$p#ffqE#q^StK-uE%(tYDNOC+0W0mhY%)$!F!s0oZ z!^=Oc19qMRF5M0!9b^vUspVirzGTe^wE3cYr7A5|_jMgy%U_H~?o!HD;^mt)sB9{dmYA%gGE?${2w;l>&>Q*DVVQMaDp^JE6wfc8w`2 zNZz3E75%DgGhpeiIP@f|w@;1OLfpBKOx>F{!{e$0hS2sP+$%Rt@t7N;XaQ#X-na`* zmI9^7*gIctu18G(qGGjFKhqf{kTnROoCTbyMm_~V+C(#!qfA3+gw%ZHMnMrtB)uS? zpER~hd4`bsxN4uusa;{rVF|;K_?Uj;i%~wF0z&E2y<~~HYDC?|Hc5_-Ij!l($YCZG z$RPwF4dfgK=qnXgX1J2pzc45n?G&e-A15E7d9VFhNJ&|yN7)|pq0%Y4_$bHOfSt2G zEs621KGAAXAUlUzMq7~F8ZD^46ox`aM{*6TKgY|lw?q^;C5+ZhVf0y#$2+SAZkMJ` z-g@EN8Els`N2%>ac%wr{7VsjULIb~)I4R_T^irT?jGvf&E2r=woXUcmr70#w1O(MY zgpT6YNk>4DKAvKesWcbDzZdlFHV1%$!P+TH5fE{f$}=2#-m~<5z&)^IRk|2o-Xlv6 zMZR%5pnD|Y>D4T~H*s)|+XZ$)-#vp=%3`i)} zC0vs&qWWQ80rDot#e`k+ z_(+@CA(dmT1h;1&BkI)8H|PN#k`*^2c(j1%J{02t6Q%>xLC70c6OqiA zQg7`Yw-A<~EeaT+WT)B7XvfJQ9xxRPfbdvyHr4br=RuTgFG3|#55V(;6C@?!AzqqR zLDWKGb*K9TwB`JIpuuFuWbeuk5(JY^fCM$SZt)GZ%V!Sx(_Sqnbpb&AtaiSafMqPYd<2 zsY@d;EvhBXIGt6?jUe)-R!w%tBZ`Y6oj)|rrdS}X5wXmpx$XlW6P@;ZqjQHh0DSk$ zU}7u;&!M=ed{#F%5k758N4I2}-lPb9nr`j+;TQ;XCMu23Q##Z<;9{YtgFFSmW%!KIy?dro_PTGMT>lx~Wd znmD7Iz@+dZgpXN# z2i>%h4?W}2y>w6$ohW%c`;Nop&}c*mOzBY7ebz_RbAIgtZTNM2kCV)o2_M2)xsBrw z$zPZAwv@A4MG$*gDAt3C-n0z0(0FLG(2x~%Xd%Vd4?8o&ECh-zL|6X!u5*|#w@xQFeD&4!_}vzRtJ}Pi`;EI zJ4h)CHv-aTpl2`uO?GaE52ubnPrtar)jsnk>EbKC!Il^6(wb&3g^>#{Hs8kTSv9B0 z#HD>%zt=t7yyl_3Qmg9Q7CBbpTEE5%=}`0MI*f};RSJ;=NO=^Eo{J|*sy zT_=9)QDde4mIvlER|)JD1YKu0&Pf%or@54iid1;4^OpzE@EGdkS6&;s;k6tu((M!X z!5sQ)IieU4y+4*L5c%A2iP#q6gt6QC#b!FVQMQHxR`4t z<$Tt|6(%sSRCa{jat_52QtsNZzZ4yzCHD@&di>i7!WIi}yb3&S!FKZNT$UD&O{3dF zSS4dG@Q-eCTWuf^dzF?rkCJ6O@LA`a-RG%}fB|E*ruL4`cd zW$hmVR^Cx8qf$46ZLhGY@StU`sV@&FrK@kPIPX$uI@B*15A|G(5VVMdj!T{Zo095q zp^ox7e{`@Ah&|dM6XQHQsv?;2B)1SD#iXBF;qZR9+p%;=I=hlE)9hn8nKHc9SLUDR zLB1`*nT3vhVo&{2T@H6pQ)@mD=ADY2Fcm+nx*k7E6iv&!Qx3lULg%!`k*&L6ZXD!G z-Ene>?zup2JK(Ob+F!n<;-(mBe;d-z(vel_9~mR`P**SqT(t$2GLx<-@!)b8VWARw zW?_4JIPsFeFi$+zHBX}2{O9-|=xFf;r zKMVDi(Z;)|95L`SBElV0HRRL=UADXNW5RI=7j~!O7BD}K2Ql|kt1{Q*D=gXTR zgo&BNq!qY8v5Kx#5`il069+%}0wLkqu_@xZeMAFE*v7MLu|{y~Wbh;u7d@Z~pY2uf zc|16!H}+Lr1!B~*^#tW;2AyvZ5HVS&-m92`A=?0QD!U`zLm9V;Y`oYUZh&eDOa@;7 zx%dRaqvSBiif|I4&teXMt=CAT6ikpotSm-c;g4aqYE57e1Y4y~)$EHK1y#$LEh!f4 zAeM;g5=Ayz0z$@>g$tEor-g2e#joENhed6-&?mRwUYSABJ|{Wx7uJmt_auqikwBUG z5{sxVE`FU9s1Z{Ii{vWMy%%o}i{vBFO()*yjRfmZ5X3$qm|>xpVL>=thiIl5%C!H6 ziLN*G*ykcOuv3ue@C~v*?n{t(a0&L}xOXt!!7HerKb2T!R+~q{-&anx#Z6W+dR2tzETvrZub zvAh3VIDfv8;FZ~$O($4t+BL=90k(LAx}MUmwC=FD8icb>x&A{@>yd077~|W4V06Eh zufi^vBmF%bAag2_0Y-|8*2N%ivr_=FAD5Xwk~y8>po9?9?$REW0daExP;Gft!YWEf zYqXS&qQ;2qL2(o8rel0q5(%Sg1lJ?|@^QCO4r5swjhGaC%kybc;a2sX=92pKB#Z}E z-*uq@(Oc zR{gP@LuFiBk?I1GZK%>Ly1hPF$)+4g?!K2SGh64@}Vx&snErmrBixn z?V;O(A|L99eh6l*I^QHeLw}}BLsN_dnQz4p6{wa(W5ut{^oXIT1f)gSN0keV73P~T zK_qod16CAOQb@pR<($DfRqlUr}mdnKMpy$6jl!&>SEkb4jkCTw&x_xMs zpooST7)*^6TE+zZz~$=M|KYUkRvk>O!mkLI#0N_|O;@|V>V%6A0Rn>=$aDhrEdrbS z<2xxd#j zyB*AF@j<;`kZQsiNWJ6A z8v3dYow*zeI#OyhAUu4d!QEZii~s@oL^a_KT|<(ainCF^?n`_aw0onkmdQSllZXW@ z3k4$Ash`}{wN0P}h!>L-R}tmImDh{5IOIUrml^}wKa6tvotr1xVhaN;n}&b;=%STh zjaP^xFk}BVEsGNFTc+J*R2tIf7RfN+539^Qtl}!cHpei6{p00nlTg6&@l6C^$|ht@ z>~61LLL;WEi(O|fXUq3Duge|r^xG$(3|>NMkDtfsVKm-Ddm+%MDlGMlvJAq!eUpg2 zxdP9l_X$$WC&HW|kS6C0$0Ja_vcqYOK4Ouc!iv9O0TFnHg!W9ta?Q#xR*qI$2ayvb z=s@^-_B3$%G=*QGMhLdmu29W)nQcO63>g_W8)c!949RU&k$O)1OsjH+nO#w>1xrY;((xurV$@XylqWO+`hXx5Oeu?EEVb2qEH#5f z#Bq2M6+d#1jO8-({#Wnhj<}Ne?!%V>DPBP`wjVX`pyO^<)5sq?uA-c)BMhXF9hJ5CbkJmE~&C%_`9` z0_SvAk&$uK92--x%$&FpD-UhErs$TK@k~bFP9gO?)a8p=fF``I%i+uvv)`p`NJRtY zmx35}IKTA1oJGau5N}L+so0x>_pLipuNix^(;UlJf7K$q|E9-)J1389JG~^`6>o7d29oq<1J?Q2NEv{pwPU8wJV~4E&o3 zKw6da#^J<1T;{gMw;WK#|Lw4dAMmk~Sji_74u$&9w zY1dKwg0F+oE}@cB;*)!I=wfkb1!qXeTb%|^>VY9YNITOYunjj#z9A&F@{a(z9bx+d z4N}NHQG+#;*F+Iqp>Y9Cszkuo9#_s75)^nUFEX5I>D zpSx4}o%h?L_?w|&DWn&8#BwbS(1>Tz)|@uXPF*V^1(Yi1{ZnNS04B-k5}b3UCvf>f zdJh+9YweYGT0`q#G11XNc1XJV_8TDv_9cVnr}eU+_koKJ0Nmn6^twRhz_2*2a&0JELM}^ucs}mw;$#@oS8ir{tO-W9MmL8Zb$ASb=_a zjnYu9VRQCOVA}CT{CH_{@bt;rDVzk(U?&+7xJTUe^K3wu(t^&~5qC$o=R1eDrp;8Z z)*REKQQn|qMc?=8WCcDwhTw7l^9|d-NWe%rZ9f~x6@rcgKO;uYxap3S^D|`VsJUy< z)hD>F*c%YILE03iKAL6WZ)tD>Izvce2BE~1&P6(0Ssh@fa)a%v@524G01qV4 zExRi8QYg+)Jsq;I3PC$+XW?ORt98aScKSkjbyppbYXpes8QIB5<7m?Aco~t5S0z+K zRUK?|YYr|=+d`e8Vq~(?V4SkzkYr%IBti=?!>y>rFkDaLHdkI}MP?=i9do0T)K{Sk z!-A-~>CbDDfYDr{$N{Jck5bUyF;gHD5p}NsGYgMul^q%5+E`7xBLOOaZbhaoy6^xH$O9BaXwS06^vp#*LB??tRIz30# zoYN}LLhTxls5W{w)S;bb#hVug)tR8%b| zEETqd?_INyvH;+9=74%wuF*Ecd95DM?w_v#TCHA2WLBgM^3L|c*aAh?E-rSLx9Uu>_aS5>_lym+Z0(5f)!cqBnvIx za6r{n`?fqF9^BB`uh)F?ZM?A5QuA0ZmvKp+jCJ%FDzXCO>XgPh<-E-LFtXg{&Jp-A zmi^VofE*^xhO<3@82DWZXL4v1pt4K$=o)3@y?zKa0vc)@lElb+(Lx#*?{)YjEA z1l+(={R;txt;rEhw~WRdo`H~BADYbc=T?Y7W+JF9?`V1JMNEaw5v0|~UBFQZ8&MsB z;0{1kw3uipHyrI4N-&jM3+J_xm|%P+t3A{kkZCuYm*FIVhQK13aJQq66Q z^vwa~MCv92&h)h8tn}OMhk&$@ct9Uf71TfhNG?-Ny*u+1p=9?O7Zco;x)v71#nmk- zkxPMgD@Ttgw>rW|EJ*UbM2dv=uv1uG3yei$lAHt7(V2|6W+kn)&*LX@N3HAwR)#1iC2}33>1{QL z@)l)#I6QNcLLpbaPni9Oqrp`J|doxFa+mx^2}`!mwHryn=_)lpqT-=7`%lDX@BgdXZ2iWH4=& zY|O1|AQfYFg%yg9o{4eoz9V2s_*l%wd2wX6snK}-k)x8@L~CFPW(OgeoKS%WbK z)8;in4?h{QKs3B7IEMl-dW+#MZ|HfRuDuH6G-Q5tL3t1;s7X2XM(If?8Z>wI@uVj# z`#Vt@uNu!6DFKB3=3pV$VsVA#UYLRWXuMKwVxj5*Gy*(@Io*ZpO5$V5=AgU|mFKs24O5yO=-|mvsEJfPD&GyQThSF0(i6M)~Fj;%5EAY9yf)Kev@dj0p2(qcZB7<5W^@E~<5{W#y%SWJ~ z5tQ0OwTg;sDliv!PEvIb-Ywzys49Gb2v7zAOv04)DKU#&7t(QH(jjg%`rEhKcBu+F zxu&iPBM>puIaf{11|;_Cg?&*DtFRd`R0T5N5{^Ly)jae!V@3t9G$z)T;N!U3(4p+3 zbz+x718N_WLUl~kK$8L?K5}LxnoYHo&}WZ%Z5=b?Tr_2FV)1mh7brbfNWIqCp&}M$UlL3K(uOtZwcD2z5@_UI(a7=q@%M|aO&`0 zLE}rQwaCSb0k(p*_gV=lR7zY+75h2nFOv(W0Aj_=M2@I-Es7za{DW;98L3GwNR_86 z7>cz8FBnnj>q~%59<3Yr70gX~RAnrUASPLeEb@xTW9vNNtYBK=XJu!xWvj}O17WXF z0}J2p_6kL?#oLEzlT#)hy#kLF??4vk9|Arf@?^ju(LH8PL#smKYj20d2B;LVQoN@k z*Pn4`AqyqlhzhdA@&GepsfSbBrLy90C$bKKa#45*C5V>fcuv);S`rnk5@`g7Y!Wlq zo$^x3u&G&GbEx!d7&NOwuwi~mZ^37j+F~$qv^l!;B58Ica%@>-li`h>Bo9nlHQwBP zM0PgHbmts+k_r~a6>wmIjW9L;S<)10Q9IG28ZggT9x&@j^V$o-T&?0x_9Cs_&jaVitCcd?3- zY8xAn7&Bj^4|l6tGnjAqHwvFcsDH-TI(>JGj)Ych+*gcNP-CCzk4;vaOc=T@f*9K~TNkfM0#ekqa{RYtR)k4inuX}d7`u??4VV4;B2N-zef!I#iSU~aXUpmjw=bi6XX0zn0+HY-+NT_f7^ zkr1h4f0UnvgOV9(Je}cfB!Etn;8xL{04En_7s-u7L<>QnAK;Z*fJ3#X7bewHB8*4? zt#bRs<_=cYw|fRt;Tse(U5e8GN*)2~WjiZ$E%qqQ)xy&gdXYK|TVNVV@?_`!5~cyB z-7Jwc1%BaacVixa?r}{V>@tlVOD{YmyqA8$DiMd3FD5BFWz`v>6zfBq{Xz&tibh@| z`s9ErQPPR(opFG&`(n@Ocy$571%)`fx@C40Sp8e^AT$zW+YKQ~529+Pl@jD)e#28} z1kaqP0d$XS!xGyuNJRFgZm5x64=fh~C*jQqL-nZ_T^8s&?^lcEZgKeK{FOnXbrG+R z49C2t4G*WZ+P{Jjp;n^c8hyBL*l?Y3dzW#dFZ(F(5GxR@IyOgxA9uymhUxb0+i}xp za9(P&hD-E{cB&A&E%T1HSI~5?#@9K4#|1EN4Df;1K5$_M7Xizzxi?QQon0rY)Y50kM-`T2(XpHUiQ~Zas4(7*Ee$sM+|VU<~?3NAiM6o z=e7wxjJOlGlg!`u)qb~s`bPeNO44tn`*=by`9*1G9{-W~5Ez2|lv``2-cepC+sOjX z!!b_Q(dqmqG`SDn*Quc6Pg^ExuYh!dsNPFD_aw!sQe$*Es2v;Lh6dv6+HO)Z`G={3 z9WwM*-}}m;zbgI7UeupvaQ_!+<=?lW+PT;oKkZ(8$|QfP{Ydor4iw<{Gfjx&IZfy< zN#oCr|F+BUKQat{k}UaK63Cui`30-LCxPsFSLqL{zaeGjzXufHc-~$5kNbTY>A&wt z|KZw?$@~34f$tta?TdZdM*BxnAtU{BQsGnm`+j4^e~gVk4*pxGei9o$NnQG-N5%PaYp=6~0pHM(X`acwqk^XNK@Q=nHqw%lI!S7JOKV10*tG`77|JXhM!|HFC z`5#Hs9L%)L>;yk*{}NE(kE#Ca{4@M7=^(?OI>_*+4l?|ygA9MCgU<*57`1;TLVmIi ze@z>nUHcWQ|6Ch>qE~;_hUbC!6{|lv?=SZC|A`Jhr;q&T>+f3){~e$J<8$Eur`7O3 zqJ|8Asv*PQso`_uk1_dIV*R(M;j=5hVDxzodqY zf2twlpK8eXry4T;ofbsNwUl|BBUr-q)Xu z|6i5=Ic4sL)!(-o{u@96#^<>Is~A3K<^AyQ|0{`_jEvvK^oLjf`-z&2EKiw_Pvv{& z<1lvX=i=ckNT zQ3ir%aiSAte#(Y@K3J56;14|E_YBMLDb_+)YNtKRoz7xtpy25!3j`seeSp z&vQ4~Sr{1qF@gN6j+UJ^TbvKu%HwGCxc&lfD_*9^ibzFmEq~Pa-XuGmc8Z})BMVZqqD(BUhkTl zj3QpW`mx*AgEm%We_X>h*()y>S-`=Mc~;qm@#aBDg0>9eR` zWxLdtS*{IT+qrHpyx2Wm@P=7SPt9mE+w%AhSsO}EANJM?Pu(w&yJogVKF(R)Ie8p3 z*S_(*`*bva>U|6Uy#Db?bW1)#*46aXHJ|x4q#1F!zq( zm2T^{XpBn572CFL+qNrCRcza~Z5tKawry3Mn^|kkbJt#H&3)J0d+u}Yk1yjHPan_s zcD!%<`p9UlH(^X9obk7?Cwa%s8eN}Y`ShG#YOC=~DW}Dk=;88l0Tm46TN=_&^Z{#L z?z1iK5@QTAsT{_A=(q^b4mubzJQ+MAl5%PTnH#xu$J29MbGjXqt84Amb#v=K55R>N zGHvFYvo(H>Or=XPdz#kxBT6?Ab7;g`;%g}4odx5j7eYFeuUrOSp}i2jVUxU|aCiCQHF!-X zi2U3%Zk9Nhg-@L8Wbf#e|3+xudSdo^u3ro@ilGho^QOMu)5$m#s-Aj%(zLxSBmr6J zaz?1WvvKcB`xse8pi_-KW13%~w_A`FrMsuYpj&2d<)DCOVOuC9vl6PRni4oFd$_>X zll|floJQ}M3?~V2@n_~I;L4r%atd8J*4hS;1T~IWwe)orar_%bR0{5;-B|z^y-!7amQDf*<+Vt42RYxUf8^Bww zfpBi0Wwt?*%tiU+R2>}l(F1&6IG4NoxkifmCLD3>)8DuIYOmq_WBt(~XaXWhWnW%| z@FnMUo+eNU+mG3wJP-B?-v~-KhYN3S4z6RLAB0dnr&^2vrlD~d=-mtaW3sn?KmPg^M+I`#JR1NR$-gkED^2r}v3z@y} z-!n!{SNqO$7dm)?YxnIIduB80H?&86srGm>67v`CjLpfg4>!dyDR@LJJjshQ`>XM- zFI01bFb+%+dx0$fv1Mtub*dvF$(Ttnn9T<7L_^~uXOmzUKu2{ z&InbiI@}UB$Mh;~>obBVcnId#A#Xsrsh|xG>lcFxU%BhnQ|=KUsugUgmIN*u&DjUktJdAAHU4^>uq!~IrcCD|+e{ZlX^h)*G*5&WHAKM&FOWsfBB%^R~ zbG|OGQMZ`ZJ9!nZNJdN}#f}LET8b$!9}fKGB!E(4OW=tA+xLJH^(s1Hs9+JIpC_nd z7a*u-tN;*;igmFvBYO`RL79co(Za_}-YN9PFOd5m|w zvFuCL33P&6OCjx?GQ6GReguUo?|JJk@|XEotW02EwqGAHl_Am&TL72!Bfee$6+v7oFe0t1y6<-kqrKe2Tq`GCw=53^5=s zPKGnN)(Z0IKPqz-)EpHq5SvKQ@9!G>n;LR%W^J`|`Vrqc|4ODXz76qvw*lVZrq&m zoyneP={RbKTNJ@{>Y?Gc@2}cDM}wmxs<#jv5gtE(PH(5M=^}# zLPU;`7Nxa6fJ3R0OTzO<-g=yyYcQCywh z>DCBP$jX~f3X(4*-jn#m5KJ4UhsRk&!tf7R5LPXuZ|ML6i zm8&l6Y|k<8aMwh<E0Xw=i$KilrHZqivExaDNC5SQPi@vC69rRzG{i{Ns9(ByX#WOejg{v|=Y} zKCv!<1W;xD+pOXFEe~uQyf!El*j}Pq{8c|LbmS`PZmgAUpZI?8$*s<3=IZ;ONKC0V zo`8&QxS4M0g9eaIUDY$zjh7sQS#wlgli=F2OxrSsv_0RUa1l4mYCy9<`-61EIE@K> z@~(a=skuLXRFSj-p;6~KY`*Y1%O^2KY`DC+ATEcT=&!submKb{cvVC(C17)xD{u&@ z8cHzB;Z;Jn^OQZK5sE37Y)g?3y3N)wb5$y-xl}yCdjya7J%kD1221+RtY5Uk_b3dj z+wi`gVw;t-ZSc;TcHsNrms_WZE-%K_`i8vt9*dZ#=`XA%LgeqzUeRG7x}GquIB+-; z=bTGMH6%K5(fcBlv5QjM`wfr z41S-pxIABkz```=jCoqFXfmmt3Bru(0sy8IWmSF4QWD#y=yJW5`rmW(hh@-);Cl*i z>OHkU-BDq*>v5o1)dx&bFxklnHMX&iS40V&2NZd`4C`SJW8?^*BjJ4!3+zL0o@nN1 zg5zl$XB0V2dM>=N4BF*9y}5dZtIj+Z5M;^nbp45UpI~+&Fj|7y3=4>UF82&)`?+On zGTRmSqT660I3WIxMI@r*5)e8d9v}XGPFS$+TB8*@8a2&|HqFPamH;`1__S|Uib>Kn zB>o|*Z00p^mp)Tn3J&nAKxvqklO4V~l}D_YY1Dvq2OLA@!B(@<4)Vj6f6^$80}(_%4Awq_HcW0iDOs9uRBZIf z=O5|wQ9tVV5-~~C%p4+-6zu6H_P&AX()e=t{J_yS7kCex?YliZu-a`MS~}FT zG+^h91l#jaQ_^#0)ZzC+3HgwaX>e&C3Dc_r~(snDGo0(eoZb*#@KsR8M^r-z;cVRoLQxuy4{ zjjD@W!`EOS`v*v`m*<+FpLup4K@1_EBYvV>UDmd~O)2XYVsbQci+A3UOumy3AVUT2 zsvcxJ300)MN??EDix~-a%_1SGNA-5w-XitRnAK%m^2&wOsp*P7FbIQwZtCp*d3_e^$U7`h~SFhGtQT6uIzkL z^cp2vOzw;}$uWxzb;;6+Y!KxzN_}vip{wEYzLQ&Gc4mEkT8$O)@Ma`b9dWENB*Jbf z;eLq^zEa=d=;2*Uzv+>w*SUC5(}U~f_=Y~WkSO=`LlF|4bsjr07Ofi2ndKx%j*O&M z|0Dj}aOK;+_7BCFpD}5IJNwPJA;7O&)zuEMY8N z<3v`*(>a9Ct>Sw!V%pAK^!2}d5nsRWShX6v0~2T7o& z_!r7|Z27s>!shq4PE%DxJ#0e@S1}~6W+A?ntiAKOKT^C+kT^jWd~q*f%Bvb|PHCZP ze4hjev&L{X(8&1o8BjUi(LK;UQUlWtt%*=F=hKy3J95U0q|?9wv2rW^Da6*sq_p{t zqp40(Z^u>CV~f%54anE<4ZREZq2@yA+OsPxbqCh1GX|;ZyGiFg?+Z=gHeV)h0Ii?+ zfUGA}qtRi_Vf|t3+5Dwl?BrwPmhRLUYdqhFrD7d;l|#&yP0_(Ga8er!rLQ_1y^Mb% znDQpFh_DT$U6r1dUl0`3 zWE7!Vg@s#A*4-1t0%wSRpd{>vkdA-d7DCS~M0=vpOY2*Iqadi-JpyD4SZL(UrtlhL ziM0!#J);|@_yt74uJqLtF(iSu7$J5Ng;k$0JYo7C=H%@} z!|fPJ&8TW}26{8G0-cznL_>LBNwp}n7Em;SFvDSHw4ryTT{_g*V4qD1lZ99WY53ND z+ct<3U~^WQcS#X?82gqu$ZZ#&Lkj!}_{!euXALh~`&nR~(N*wrkIq2jOe$4cw5@ zns|x(Q=eSR`B7KN!gC(Q!MzWKYaUyCt^iZU%EbGLFI=%9b|T&}Q-~k$U10@G>M6hO zgrW}CoIIFvw0ppTD;W3L^{}ds0are4_1WsZ2&d$#4TVvZ^6OrI>UzfKA3c>W-!?d{ z<6=xsIZHmD)I1k7o>*A>0?STdc!=5rDDkX}J`^S`Y6+FG@QMfWXUt$eE3%)(XpY2c z^+!DHJtuxq=V%Pvdd0x!vg!#iti}yST)b3}Q=X@Nh#7hmqTmB2T;OT`M@B z%bK_q@7WG}R0*qcWbrPDd9jQktUb$Pj)N12$QJ@Wge+d$@wz~p+TT&mAr&_>1E)jO z(%0`kf4AWTpF|pIO&cDN#xPFy@?~2cL+&tU&$n>_NI-iut-HGcfgX3lsTu7_5Lg2qQN^ayzr5CS0A6Urf=Voo13Ox|HeprQGmB9-9zd>_jK;3_l( z>I|)72H}@7!e6X~G>{vicorYddXde&c109*9giazg`~&bQR6SXSq)CzF7yg7n%}+>GFB%s`%-&_jT{izjj(aNJD>#4)ZIk3Ox3TsQ9OSA7rnFyvh6^ zD13O3kw~@r>uKm*qsdkLM(EMeVBxPV()xy^s|m{MS~>YiCi#)|beDob!n++@g02gd zoOo&(jhfO_d+jGb1I{y$ElaHY*yOwO1B=g}u1Gj7KMv}w)|iiZK1ZNxo)BlT#J{)h zV~~W6FVR6qm*g)yPCGl&z})>@J}Wj}aR}o5vrl;^tDa2?_-x&;op=B|0tAsFFBN_~ z=-lnHi8{-1k@LQ2Z$ct(RDxArxt9r!3#PrC_VMhr0}9XAXU|$mP_(`QpYy&v961S) z%1j)07K{WayTq>8KOb0J#5+%tXYlalTyQ#rdczKvdl;M8Hk?Yr zLEPaRK~vh7u3SqQVsoe@e1WL#yiu_S3)vPsV7DEwD9z|vYI%KU)Ru@#cjs?w%EUcz zOErpNoq&wFwWELDU<1ZIC^I+v`d^3ezTsGZd^t=;te-res9Kox!?((E<9Rmr8MNp& z;?YntLVW=JS)`tuH;bc0;WniX5Gk+ED33(%IS==P-e1uVrI2;dxX7YFQ~*c|UoN`} z;gqP=mK@^5KdMA_TB;lzEp4ORH08$hiK`SG0~-_bAb5=~X-o(gC+fy&AK?#3| zJBMWL(ZzwUWdAlnGRx+Hj0W0KjI%6=f-#t$&Qu4VA2gV&V%Kl^jXMl!Rms!Hm7q|T znq}$1)YKsaKt?c3suD*L`N;+k-0HBrW!2E&LB2vIs%vs&<$Vr03Dn5i$d+CHMuMZ~ zsvm>QR9eCZOr75Mg>bBTcTrqTQd}xYGcB*-XG#ZIO#J8Ugv94<;St>ER^IFao>GW` zBNu;J8#^e?Ww}FnLr`zll}=^h^0x2Yrf7^TYnCev6H3g7?kR(9F`y17Kn6HQ(gr`WD3{QmD%Ggswl?;iWI0l4|Ouq zaOjv|8seeeV&+=php0o(DO+)OBWLi`NSLiGeGDb?X}cDLka^;{KC|0hk4!w=x8Q&g zy3V!sy&E7#@EXv6VTBS2Uhf$}9=;|wqORKRKq6aWm;Q92Z_}Jm;dIxz_{)Veg#aYk zrhPp*CV%LsCWEH`$U_T=oqd>~jzA&bos;a)j_Je8fwX>fqmo3Tp&J@NaHIf-eermn>V1$tFpxQY4)~p zF)dvh1GQtqVhrnqpV(ux21`@)nxAVAsW-+K;z_;5p7VJrpiyp+)npS?^K9?eEb4`& zbjnK=!SnJLzSM7@uFo(=|Wi$bNjmb|s4uA^q*~cM-%Ne(Zh5p(ims?bbNb{&yO6&;n zzU^|+msMIA6r8-TVc09^w^oErntTNEMLFUhw*>d^X%59Z3zU9AzIZluN)J{Aa%2gr z#1e@%*Go6%>$Fux`w)mp<3n&^t7BtlYy{J#{tjagIOtIpFP0_%5dQ(Jr#{k zW7-PP+MdFAot*cHWeg@^?jOO_sm1+56RUZa*2{#3u0t4mOj%Ca1)_LCm}$*~{JND4 z`^eGiG;}GG#eu-}P;vR%hf((S3QwTD&s6f*Dg z(hVN*n<%_m!;pZIf)mvUN2hR)$(xhF>^aEuzM)PSnDv;6R;i?bwHvbGR69I~XF!&M z6d1i>lJ{XdUQXWCy%~Q1Pw%b`gT~ONpb|IccdApPN5xpkg`1yr?bONz(^m%iIS9UT zS2Na-wkY%q*K?T6XYqYgFK@i-(L;7!P!gnS@>HC_S3&;qf|Kd+pN2nS_HHXSD_}tE zrW^FZt4c0nZ?<3OAvSk48O=j#4|PMx`?mzM;wn_P9ehCqmx}sCfeeqp=uHk^9XCmw z&iz<7Bv2m}!a^$}35HKuriuVN2tcsLpnr%2Zxh7x{R#ILX<<0MD8*ZaHRcoR6lKmn zFKmi!l~_>Y`qO@cI@?P)24EhCxEy`iy$ce_RAgDMi&oj@r{lO=IVKq=^*ANROVTX^ zRH5|{sCZmgt#p^qZJ`)ySZo@bP5~nwFcX^?!h{_s7Dos{c@XFEAovK6Bx2JiQ~g>pAT)2)8ls@vW=A71I%PU zX%?q*4K0tNue!dZ7tL$A>#i3^^hi71Y0iTPtb~<~Cr`qMnoq;`3H^!^PYgYe8|aWO zhvm{eM;1IK!CbT8mQy105TbUABgzOdrvc6vAx0gRGxPfaC=pd&zJuBlA z<ldWB4SGZ_@wHIY*V@ehs}dKz2%Fb z*c^7<+!&o%8*BX&rQ;`|%$I5g51iJ~bi3AO-r=~5i4Go~sG;6I+eUK{ar5TO8M)H7 zA4Bx(X#KM^Xx zuQ3dr_-j%V{hr@GYt9Hd8?!lha9`vvpYg~Z+@;?16k}*{5}R%FH`oBKRg9ZX=i4~k zji4M<@FanZa=lTItzhm+bH~i>9`aydewcgSzunFC?QuOkJiNVaU(fCD9k(82wzYS9 zy52M%Y`k61>_4iY@N{^5vweU1(AR9VJ#S$b(X4m0Z>+u@$BA3GTxz*YFKeR)o304$LlZZ)j#LPpXECL0e|n$F|2=e?e|ss4^#068Tp@c3jcuT z_|KfeUlZ|XqW-~s|KwQzk(2k|AL}2?|Nl1D-{@q&kM%FAgFmmT`IYkjTOIt5AmHD) zfFDf0e}#a*8h@XSe`gK;5(NC!l|K>nuR*|HyvE<7{sk-lyN>3+qN-u|#RdH*9Ax}I zgoBKK$3e!w;~?YTaggy};oz?i|2}K~PKNwJ9sW6N_|>&P6ZJ2_!Cw>cXQKWkIQVPY z|4h{Xe5`*k|No4GzlgoRNB!%n;lHG+`IY$pjE0Q=BWTF<_x&K#ze2-bjla*yzmw~K z2^#+D%Abh(*P!7qF8A+I|AO`ZU#c1=>W^Fh{*IM}n)yTL{-1gl)BjrOe_np3|3hHN z^mj1)8{z(g*8X=e{2L$tpEiZRKK%Qn{W}Tr2gUg3!0=bs{!G;W93cJxwEh_&eoe%m ziTVfk{gY$;zp^R(rDgbitbbiGFw!zme>^|`R5AR@`~R&N{t*j*$yfd}m(CAw&cC~K z(lhBq2Ury$jDVyw%`Y&w4Hg|znhWRH|nW+J>TAh?; zZ~-Ba#E1y+v4C$Q$^z>Bt<6lg=%q?gg*lPw=vMp{4eQRvEmka#bjG_xeSYREQqgNT z0>yqeUcvUDQ_>tpO;%0ENB33+P$=-%K@%H_(a~h8uv1dMxxd`bw^(KEQI=HJknbB^ zq#BtdQc|8t2oCZRvcn|V&wx0`5WtR{deKX^D5q&Qj7c?i?(E_n%FO(9WEUoFrd`;q`ELg{?((b8c04ks-)o;JQGrXuul0LtqS%CWP zT)b;7Vh-!)0fv7GPZ%ohpH?`a!>&n<@S^wiYuc@fX zDDZtT-luMF&&T*!)$9|h{J8u&%H;%~tQy2Ta&&Q0AR**nAM{)@IYYS}UvxKm^zg&b zCjeoB^7pZs1h{x{n!4K`v0^i$TnBC&>B$$P&7h#~k$N$62`m_Cg=W$8?}rc!Z`tw9 zB0PGxk{yIc-JFO~q~W0j29Ge^1CqPt_jUVm@L}PeaTcF*pgQ?-fp^>;t zTRUCt40QLE@;!SGEs!X_Kk}i7NnlC=e<64?UzeY$m|cYH9sm!A6gM%SR^ng~Di$Qnyn&OGnKQ?@7v z2u}tl7Y3pSRP)1mW+ertwHd*GU3o!>!Jj`^-RLSLpp3-b>r-xq6_rV8Y!GU90S=lJ z(58}3KEv?z_g>sg>-#P%rfSGtu)u|m%}~+M<>wdBE1H!U(qX;0MRJ!u0le5)Z_q z-FM0Y;tR%_!Bn|n15U;`8#kT}yy64D0tFqZ>{h;I`i>$q2ySN6;;{u9P3h4%=>*q| z9kj8dr;>TE;tZ9FS`C4XDMei3SIUf8&mz?QV`k-;PG!srdslawHz_aLp1BMnQaz*kTK+iU{Z2Qjwh5Lgz49vEw^n;u-mrrv>^ z74-4c`LW$&iHEYjfrO6Z2PO>8@GRBCAbA|Dtk!}K4Kq;$$%S6w5_(=Kb11GcJiv!j zi&Hf({rJPsgw9m0_M@1sK`;)O!u*Jq5}7b${A zpaT}uzKx_F#u~5gFN*)IS(Bf&I*Ej|e;5h9D~z}-p12Iza?lSS+2(CLjnrO}zz-wi z?L8_O|N7$@^?tRuT8BY;g}j6o;^~cFzZf?TBAzB}hmyOPUFoS!uB$NIdbdPp2sr}> znJ79V=wR`+SqT&llR45x8LoWChXWOlmsA3$Ti5QkKQ}Bfo{x?XbNp`JWZ$5>B&z;= z^}a|Ho6vhuDUk$9R{PnNlA}%hq{zTzuRsFq(lFu4*>0M=2t~+Q_~I?1Yjo9zV!gU|iTs!0 zkaOKX*cYqLQOzGWu+D~KRv@Frl(!{EVukuV6%WUJK_rXbcQ&5?F=A@JDm5?L`%1({ z1%G9pv%tqRqy5_CwqHKSL>9ugoX`LdTRv`okYoS&z4!k&`g!Jm{59nNf_|Qr8JGU! z`b~ZQOF#eL6V(5A_4CXhYW|PwZ(4Yk4+f$qT3CK>Vf|?Nxc=7iA=jt>n`Hk( zX8&=V-*4AP%YRWn|N9|<|Au~^`ENb=YxMIhzin84_dcfob?e{p@Q?R7bRWOG{nL(i z&t1dXkR@in74SXKEe=dUtxX3x0HtG2l4jF?!NCCg)FVoXd}KyGo=Pcv<=tZ^6l#L4 zI98mDibeJeJh4xd9(5OQryt__k%YzC)5YH5&EaqE$B&o5jkX#GX|O-$q-d;SV^>x+ z-5!2;zOL-w1>$hNM_OO+53IoQT&CiP7^$>{I?FxJb-F+Ncz?e19YgWEd$ht~inD}X zmJ40CZZl8CY-Ag40SaSZjH|Q6u)H39wlqz>G0)khSuo+zeyDkT8S8gh=6wHpGbJ+7 z`Ep8=X}vmtW6k}l+21Sp^hJx<%@O?WsL}5nRBo%0+Oosle=6Igb4U-vbrA%mk;}TH zy+6uxSIv|Y%IXGZ_Kx8=Yrg66{-rgxj_aetEAKa3;5u#`V_j*~@2V}+pS0%6r~7u@ ze!_3PoEltk+jX=(JgDWqI|4n@6xBZ1J_fxn@Yq$m-Cxx*K6~b)d#g`cbJxjHd0yFO zY?Wkwo8ma?27N64v<{*A9M-zC@j`+l@_3XCt8g<9c>jv{ee32RgoA-7_hrchhvdxn ztDR?V$!0OAlfH*Hg3FMeaxq6hz1&oaSSNcomgb#*3`gmQzx(Gex+zV*8w^tJ7nFCN z-Izj-juEI0rn<9DD3bk!&cb7-uImhywR|gWl3HRMl3wrIJ0>qbNe~ZX={5|yyb#}> zrjlybY-cv+@7&glOBqSj!1FM2hD~}Q&NrI3$XLa*sZv*-#*Ks#pl+faD|N7S39rNk zA#E{+v4!ofXO3zeuAdk)ETo@@G@dSE-Z~c3=3^;duJ;);(=uHg?S%V=V&9AB?}H#7eUGdSq=_jawaVY|zf6&Kycx(U?}}#jcE*aj>~cdFoAf zz8&AHu|Ix(IK|j>-Le)20cccxkMnZ~Pe7H{M%X>we!5lLQm)}FdYT13qkmfOY~!FJ zL20y>F)~WCbV5pNpWX8-sgzyFbo0DE-s{YKDw29kRB1%94rT>%1%1~w*q^g^z1)c} z)Rd~X`qpOB`eqZA_7rDs7M>PZkw15_P-x`!{w7U9aJ%=GS&9C%O2L7pBlhztpySn! zuk8sE3TjY$TxB$D-JEg}sw|O#@_?aVe1!NZR>c}(BfKmD6bBvZcsymrNAr9iaK>(; zW?u^sBK9YP_eT0MdM~MG`>wZby+nG+iX+b?O(VFnpeq8{dUr#EJB5$Q4II|1*x{jN z>U}ib$_a;#b)eW_&Nl3~(3BU@f(bpS(=61MgIr6PqXQXz)ebUmYX-suVVx#izWB=y2KfRdW~2ZCk-3v!E;+1Rr%i6&LID{tFPOU z=(DNGRNQo<X{LFf%FOtsPh7FbGo7?a}Bv?3V`%DBdgQ@Ek2m?F!g*x&Bj2bG<40;-Hs0Xvk}AbYkZ+Or9QZD#t%aYE91VY@PY! z*|%e7Y%nEeM}Q{arF+FnygPHB20XL)*AQ@E##b^Kmw8v4!!v;%66$MWaD}u$6+Z{E z#R7@1B(%vXGIHYNp*$k$NI*o@Oc9rt5sDTP37)>5JDGFSu4CH-<*h{v#iYpMdrI%j zWm@>He!XzAXGdnJPwMS^IhTC(neMxUXbuYJ56vLz^YJ?23G(7GETQIEm3hT!)lArF zU9R;XevO$VfsIbx^;|&2B`FaXw!}@4yv7giETQd*LJ%#&`Wp7SDg+}YK?cE$h1P_e zjCgBSmtIL3@TELXR>gHLGalcv1^3~FFJ!Q@7gwdzs$Ip#1>pt~fr~G~V6{Ft$OHgH zCbxbQI*Z~{PVQszuGBRtE-ooVxCGp*ZXqS~KEMwVx6awA1)m0Apj&s0;p2s`KQH=t zaWd^1Du>h^`D{M2Xf}esA@w&t={>ivz`3)l;l$5%;vP2wRTx_IVR*%16wA2C zXNqH!v&i@Xh;BjYix8o8V|g(q1A`b4=a)vTBhO;323>6E&UdJ~)iJ>))O;7TpQbM& z8PJ|wBkf9h_6{m&s#;ts2Nfc=&KBd1hqYizW`7;=X6G;E<;!}hF{Kx>nm&jgl!6I` zS1Lo&Zqk158z7#3a}`+Ix0DGZ5KL*dBo6m4Xg;4?$xYsn<6gD_>_vALfDYw}?F##b z!i(2(GYXo8L>#Ik$Jr0hS9`UQ+VFVo>xsZts(^5D@LCN$@~keQA1{Jx)}t>Y%{onc zh}dy+`z?$zIIlQ9;L#GGdaRrb<1Ap2(&Z#!L^6UgWs2=aq^)tb(xc=%4U345p799D z^x5-h0=QnEg)`|FtHtYF!-h7%fU^wK%Eu}o)cF?x2;t?8x~RHZHY0GfM^~nZxr;cJ zSA=RdW<;Ewq^8D?-#v^fO>#<1d(R>xfH4*Y9bh{s0ZO$K?J5-y&g@7hcatLJ<*!OqNZZM!cgFHNI?L*+h#(O!}V()%i zpLJF>={6Bi^Hl%z`b>jpr(;y5ItjzD=JI>xWMORU=)0NYFgLq4;yr!-Ho~>Q&5%%OXkn>(-dsEqdUh&DKe7hU zBRfGvE#(aSdBhz6gy5=*k@6>zJvhtuGfUDOje{ir_B3>~yO$gHI&5_JDr~gdz{B~S zz^aDH8f>*G{UaVfMWl@&K(J>V0lt(n3nP12Wi1A_70FY@XxGyB&*3trms&$#^o%Tl z2unQ&)}n&Cngh4UCM#ijEv7%4j9!-Lt;JUNk{C&2!7ZV1H>3dKKLju*2VDL@+)~e9 zu&9;X=tQxS6@rCs`&0IQelRf7%j=$w#8 z?`u&B=M}|$k|`)~bAip()o#&^$!v{|R6Uu=^N)P?eZ{vshv8IImU6$TUW4g%geEzz zrxnk|)AmQEncqz!a1aX_iV65vq}@+??P~Z*&&Qevj|iZ=p{S!aJLj#Kl{3O)TYij*pW7|^00rxz_C2(w#o@}W334}-!JCg?TI(g zuezAl!?BK!i0!m1P_4dDk!JNi=rgkc@6=%<7O?f_ehfI-Z6^$2B>@1u1s`-~=avb% z4I#erqPhQW;Suha>!{8~3~gy&SRQjnA)ju=I`yN~f3ulh`6-vs3{T0Ig3LUB`wrAZ zKMO$?b2)8qDo5`^<^Hxax8s^_$>2oCIo$76M=hoh*MpjENeJY5gYKC=G3r9pK}=Mb zjW$R`!oU6I@a3NM?H=Ss=fF1HPvC}iKHF5NBmG5yKAkqU73yb~XojM`;B@da9in=s zqHb9Xpb%n

    Iy`sPJn0Rw&UuU(BXRl_jYA&DcR;1%-1R=2GJq_@ScRdJ>#Ymn|4^2C$*CPI-kK`3*LF@8WpfkVQ5y%YCX z$VPj+VGYkDXmsx`z~iFDsgA!+Ap+#89-)RYu$z63LRk8J)!cd1uL|A9pPMCS+FJHS zO5QfJ58%fdUQ_LG_QL7Y=IlDYr%h|9fK}wZ=~A@hq19-5kz!2DrTFigF+&B$IVQ+FH>Ts4Nv$y2+=@;h%hUq`@&2YH@ zvp6JtKO#1=KN_uzO~0Hu|1f_n>Lzw@NY;`u>L7+#xCNcaPkR6hz%I)>pdpILbnSX< zW2#3W*iVKBu&iG-G6wUSFb6@|<`$9B+>G#pd6XlaHO}P~g%w=AcX^LuD|G#wd~-Vx zRV#gqTJvJz8&~igU=iD#RNngW=cLah39${)!ThRWWW4gdY5cuvspZAy9rtZfs1|iQ zV{wI9=xYo@%N88^a;7wXZ&f_vozQeJ>uY`3DhqHCWlnl4NK+MnqU0OG<_?f-gJogb z4GivC+>VvMR+xvHH?eJDn}7pYVhS+AX#$aM;AzlO3NVo0!zVAnothyK*}QMOYRq!o z`vAo5Fo&TvFdi^Lt$S;q%nH*SDJX+WS`AM`G5}gW?|Wg@*poqfQ85E+iW7B-28cRQ z#c7liR+A|UNVk+6B#dCFre#AwQp-XL4pVVjjAo!xD?&Zo0c_}?lBUb#=!(NLU~w{0%VkKw z*RLAIGIWCIE^KuRhImYjJZVh*d;!r!!dc=2Np}3;48c z{A82mJwxGh-;GBviuB~TAzm`#uchH+Gc4H0!Z$!gW7&9V_0hX$SJGO1r)EEPn#!Gv z-O<%4kS|(uE=F z(1MB=2AI@ce!~<+Q~5fSOrzb~?FbMrk8m;9H?*)v zJ`L2P^1X-#4D=qAZvlp@Z%Dsq-<2TEySG2{AK&bJm zBb1IX9l`GNQhi@Q*?zK7j@GlY`@Kdj?R{Q1G(HbQYIS z;~9PZs`!cb5QXu{amuYfkd#Eo zsDqczqj%`g_^_h1e4v^FsO#G6ddeX`OLPP7vQHnKEpZj*;`L~WQLTp$dhs|=jMQLvD zp08WoV**HLL@e)+kx6%2yy$D^4zQ`@yb8A0r)6sA9svNIeTZ}7Fj`IALNp;21JqEb zmW`n&?KNsqZmc0PHZ3zmD*_9mXQU9KWfMYVBkF#NLQp4%pQ0xY6D+UXr68w>4-}8% zS@A2YOm_04{jqmGC~DK}N9c6-QFT{lWmRRL%0BrOui1&ONXHF* z33?b&%^8~Nk2^NPp1ETym)h-Vb2YGDUM(8JP7|9Z0a!apbtNGy1!6K* zRqFa0Sb-~ZBAO`#WbV%`RGNAmKOn17{7Xbd_wE3fWTj2T)ja|RY>An& zvk?bVyRmP?Eq2>HB5H0oKcQVyZfeeTca{ZklgAeb(%iZEj9>^by=(mIdz z?_WFE6E)}91iwJ@*$1K$NE99|WQ!@by2X?swKR4^=ZCkH=EbTho2b`c3n*P^dvu;u zUUr2Uk3%fV=V&c0N0|=L0My?=tsE3ac@+^(b|fE$Qx-*F;weTe&3wC&wRx9@sf-R* z&q8a-n`J+5mnn#fgXQJs85fBS&1!;(w2~SF{11B-SF^SyW1{k$qIjSKz9?*$n?Ns*En;ogxct=efy?(0a|uX?cqGHj30M5d%) z8*g*-9aCszLO34kkuC2`5Q9dfq^H`cJ-e?{Ybs(`L(^TYVC_CfvVk`hd6%6Y$;nCX zZWRPy4qjcW#!4p>!flu2+I$XdHSMc8Pu29pzC|)RNV3B>RE<#2QmU-cOV+E9|M6}v zW&X^bZ;t7ySQ%+KZVez4F)k7MlW(#t}rqz^w(0-!G~oO9_f3tK0v^1 z`bh9)LShdv1E6XccF={?X6=p2L_95ya2GJ=ZECO4&J~?vHK^xkJeg5r(EZQV=mt1b z!9To`g9eE!t<@#zA6$uvRiU_(DOu6{2pf`8l=O+-eL|5NeqQDxHzX8SjjdUOHTMnL zypDkJ;~4;K!q1YbqrKQa**PbM5g%wbi*?Xzoh%Sn0CkzZaUm|(@D}byzEX8M-ZwjCM&Nx7PWux)^TYs^US+|lgP>{ zeB5-rxU!b*FNPWNcXsJGSU^+bo6|Ph!3ru+fka?+>t~@$VSq< z_a*pOV`vJEA(W#)5DAo&@R%iG`IV@TN06Ib51_|juUDis%BBEaNH9Vhk{fp3Jm8*) z+rx33`mN!TexmScrhu@#w@Ypo;hf@@ZOn>JENgrmSb*DUDA*J$DtXzq zJ{lB%-kemN#Ft|)14vmIfR`x0HeIoXJ_;Z~-Wqfh?;0CdZ3XT75jLrRhlDS!E+m?5 z+O6Utg0Y6N3nENs?Lc-&mkRV95sb#OZ#aQ2jiC?nd|>HDjN5LiZZfoNfFAQ{)ioEb z>xl!puUI$AcerU%S|BZI+^N3TIe;!evsk8y0-$JX04M=CWi_kgi&29pfJWO`5+a)K z*1%&DD#w%pbI-efQjXN!ZIiK@ERFl{Il;6AqfSnZ+v@2eZbL~(Vjkf3!sL6BBE=?64o1#14ykn zR3am*{aRCC&AtW#w>3X5!w1NLG|{~F%KwD^WeNe1LytFF_-PmG-mXc7_v<4h!nodB>?{# zGQC|b1}q4G;C9uQ4~5^l;1RnmF3t*a0Q3;Ybk{GU%(w9T&St)_5sUfal}ty>guXp{ zmjJsIBOt{xHF()3Gb9(Oh(0uMR-@fOH~KAcWS=-%-245l{OCky5>W;YFjm`clZ2NE z{L*E~FG2wP?+L?|4}=+=X5M0|HSmi4E5(XB59SJw4_ADN2b`8FN5~$5hneHPl-aT> zejA)n&)QG`9m^1}=J>;Y>V%Lu7VL?n(|dg+zKkazU6N2u0MaysZw5H_dbn>WWZ%m$ zQn`7fibdZ8VK5(YFz<~!AZP$FCQDPldI)7gwnFoK7*By@TQa#Jq0MLWPD$8pI$vbO z#rvUQMS4nCE7EQ`_0b?@8Ei7TEf~5QfmXB?88zkWK{Zwzs{s?9`Aa;|p^}joNm3rc z_<%kwlDZTy;_cY6v3d&ySd|$hTN;6Rj#J;)2yN-pIzEX)1S{fXY=&oY^Fa0K09%;@ z4m8-vq>fnK=FylQb2Mig&yGu2lO(wEB+zx1cQEE9oT3GBUavF|lZ_CKNffFkjc;l%ug< z)+j>kwl)GP4}Y&6VOVrUgR+@%MFW7!zoKzS8rawGPd)r&vELsc$mxADWCHO+?BsbNG@-pNlcj{?LY=rnChS;l^r!{ z?IB5qKJ6cc&7BljMN{OwmeK`i&aS*{2apMtlThi#8d6ykv$7h5W}|hsYE;8x6{~Gh45>;>G)_uT5OPb(dGzjL7O1V zF>JJRIN22G4u`x3b8IsVi({A47xLp|L5di=9`IfV*j3|4nPSxeB;oz=PCs&zV$5}J z#U(~%#tOtz{b6%|auK>~LU3&csdafZTFHruAm|mgwC)t>k8Ew8jM(dO=`7|}s;qD{ z#5}4MOsc0cPC2Ir^!Cz#`az>DqD*z_=2bA3P~W=Q_#4P5 z4K^M!%Ge0rH&Eavw8f?j59668gB^vq>H&iuF1YI!JB5mKiNMr$Gm1K`O-=mz~xNE?AeDW2RyleO^Rq8qSk2&SPOs zMvXPh8PZP-GxM#$e_mDPZ6GNmXau_?GE#hF0rp><_~z7OnAkafrJb$mkfL`s8Yc>x zp560+bB837cHJdTrF0xt?D&CM;7lWO7S=K!I&m1^45ETqgZmXhP8DJlRmMR?0Rjxv z4IEKZU!wB_ifPte*Kso4t_X}Hj?FUf6_7Oo5rLnP;19+=Ny;gXyFo&1Uge=Z2?wJC z4Pqy#j!=LszM-&j6OdoM5v*ggV4dAU>O*klZjiH>b~Jp$lyGUosk~0#tl9~)nuqjK zV2+%F%`-qQSvpQ`KE)Q$98fcJ;&cS@>BO&o`eeSUiPEy{QuxsRwZu66!&4je4{@*9 zRSy7v_-f})DARQxaesg!NEw3rv>~Ihai3!(cNxZZnp+(K%5kH67h94zjmK0HK zro=mf!|sq%h6mfwJtWj|-#H}Zv3dg=rH4@Q^ zm?4D=(8zw*Ai>B3G=in8q zYm(5>O!G+BxglvL_DOcX1X|{i>R#KqQxe{+cQdErM*6wJt5Dssg0G`q?_MQ%fP&Bn z+Hc+=$EuLC>W=SG1m|n(fboCHxoX?(xy0E5hf-{K-Gw=MCHg`jS4OTGB=>}P$vuQJ z1*#|_W+n6*P;fndC3`2wzzxP)l9SQjAHJTng&Wt}STq!Ytwy0pCF+Q>%`(YL%@!qJ zsnMszTGlJdC&@KDoYM}pY%4z-o3AT zYx^=w?g2@pBATMG@Id}(nGmdl$6Q;~+dU(R7p@d;!3h1sr&Yl|?C1k2$y?SzN5zp> zJ1CF&PQyCphQtvRIvqG@Gjwp`Y<#UGLLMhmR1?U&)x ztmd%LQNs@$Jvoz6rK5e?7zv#~go~Dbl9SELnT2OTA}50-MQ1^lK?ozoucB3N5^p#f zNt9S58(I-ohhOZmoFMqpH=>4}vD3JwNb6~)a zQE$%8^(Lg|Y+>X*wP{Yd9SxT8JMG3jwV=W=~Fg>INmeUfyo zVic|H;@uKc?&@4(Pco5$mv6DrH8LTr3@b+)JUvT*jJxrqbc*<4FrRQRGocT}g6LE8 z78$hMCl%VcX)L^z@>64FTG1A!Ms0q zijXK+yDCl`!}_Zk=@b*%Hrt$xgEP##Z-bVwG?FBak!A7_Ykeg@wn@3nZbqd+b2_1l zo@J&piXVUufaOj z?FsE8vM>+Rp(NUdazEDeiuFi@&`SUnFM1CZq?q3c>jfZ8JLVUd=YTGJYee@Ymy@bT zPsJ?1RFX4a92#l_qsB&W)?6a?5$7njTIje6r3K6YS)+Z#1LxW)I_*GTpjcLka`Vg2 zq8J^MG?i7(9*}^35$>b*z0t9TX*bTPY9MrZOmK=>iYP+y@5O3oqyrW2q$?e2%BI*? zHLFmbHKV?fxhvN_h1@TW=ncIy7)4~`9C6XW-rf?A+#2tnLzB~;o@CMQ4Gr_o2?D*s zJ|8^47mFL;<1?80NYb96~_*St;lG<8@)$b_MxuDLp0b((Q#+>zz8(Ucf z?tD;)!t31ha2E4sJ6%m0y)`-szj?g7%AL*rTI$CbPN?&I0iC>5iAUj z^_U?`+>w=G1EN48&A0?p7&J>VzRK%`wuOlmQP{tH8w=G0hO9Y*1CvK2 zl5#&jL(!oce$sY>fNNO7p`RJB$LRY~1OnrJLpV@W(jOmcG# z6k(cr5~cX48W;EpRrK6aS@Xw+=uVl0HH`E=J;qOvdJ1TzG8g3%hJ%4a-$~W%LQF-7 zsWc;MwlJ06>h(i(DB&C*kRhZ2=xFa7>|`Z?7j_1rv&eaeD|+GNIAWzXC2*@g!l3(+ z8s85UlbEJ!$wBfaM2ta&K05KrKXDq0@;*dG}-w-?xM-= zfk^6^j$@uvmcmCy%@Priv!?9TO_+v&cSr`%Nr=Yf=bD?^QwrIpa+O>H)h4HeiP7T- zt3>BOO)D~f&+3IL(%YB}!3ruO6#N3>Q_Y)jLINsKCdCFB6G&PJwBv?KA4SWW3tAVS#hW7jg_9B>!k-du;|<8{?Xucc#jlRWH4znX|`8dE@<-T#rpSN+q^<+zz1B6rQaE6@~H9rPi96cya?Q?u0yq z_1UiP@ZwvwR*T4}rKg-uqGoiApI0}!m~0*E`)I+~muihTX3t4~6!Lc6(T>FR1;59! zM9*Aj;+E!Wg7JA*`*yW2YZEMZuLF}#FN+qW{k=Z}mmJmd)i(C}d?=qj-M=uunK_y1 zk)Mraet9{Vw+QY$cVJw{`^nhdM#krtl?jZA@7ym}g$riychcWHKXF|6xWP<@bJ(uy zuD+fZ?nD~_{3O)ikJM!ty$&QC!^^MwOK2(8I`wX{=%4(AGxyfoGtb&*-bAf7pDeQ9 zgkr?Q@0Mgsj4MTV@|(|&YPs7PUS3r;4m)ZFqMr~5YYy_0Ccv)aoEk^m*I0mI6F|9d zm97l7kAbFT(F9(Z&D507fhDXat7L4Lfl!YC+EV?PpI6=x2#aLo{v>1Ox4e-5orUms zf#cYJrDwe7dHmXb@DCh_Uxoix_^`~Lh}j+WB<^DlPDAI8S7H~(9u{$yz~;Fk35uLIbOd@{hQ{&UkAti8uEX{ zgK+#m@qirv^ne`y&IA4`{M{P=GWGssl>aRr@RuroLF(V*0e>a*{Vw%y==uK}aNMu1 z{$v0D+DiC$1^=Zp{$4-G`Aivst{Xdg(|E1gi#KQl%t$$+w|I*gq@=1Sh>)$sU{zu@r zf9#?ChpU2LdBVRd_}|i9IGKNkC9yO8-@}qPfB8&?|BwsI#mV@u(2ui^FUnf#_}?F{ z5$SCaK|#SpGl2cW5P%J9H#x9YbIi;qI|Ex->t6i## z8aPRXz$=&6hU({N z8=~2k776~2Q;L$qxUMsTp^ur`0Bv|8JQ4KzMuT*vXBme(XMloqs5^+b0h<@)!E!ma zguRdC>=@nn38MZ=_VaO3ZjWAONep-iqnge zvsX`22&e!j$c9hL`BM+n&9HzfR%Vnd1QU3QuS${wL#vDy6Hf2N6Zbm7#OykcZR<^j zNsVA9xiB)khx_F2Y$$YyxbY7$eYXN(*Ja(V0zPSt&Xlf2chqD_A0M1{-$h$$%|oV3 zEU-r>R36Oyw5rw?N!9qRA8a+ZnwQqYe6S`qr20KSUfL6nWN5K4T0n@?NFD#u zK3roKDwx|?S)Wai6nNE^KILS(^!+|RT0=N*rWeajrS4q6yvTABrf<(Y-}{VW@F;B| zDln~S->P(Ij7^f7m?2UCFDJXoHdhhS zGOU!wm-`S7(yOZ|8nz+>*B<(DwDvGdZ+FOcGcnRa%@G+7cG2MY()e7)teh4G8Eq(( z)5;m-sOLj2M4O-8igiZF^ydOXf86_2=3;UzGh@* zlGYG>N@pg!FOmbLC^sxiorIy}r*|p*zVbrA>PVT~r>J&`)3jnG%)sT|`<599|ABzz zc)SqlB5HnMAuiDs<{J-z*Up`&Tc`mx6fl>URW*xdY6!Rh`FDN`n|DZ94wJ93u1m@? zt3S+G4us>lIL*19nl}Gv-72reF_Y9K9T4VIz>}$Boj5vkb$C&7it};bjn4DhUUGnd z39g|P*9g6tM2dd3SihEv0G7eLhMaGlD$Hov6jg6@`F1Y-2dSTP5DZXpkFzxCMpc$1n-1>`7`KA!yP=%M zG)gVIvkh{W>=A&z)U4X(#a?OlIe>>qL*b;K72z@_?V1WQ7*sZwy0dyoNPl9KYTgpO zIY9;OG@>TDF|71+yJXJ`q}(mprR6fY8F;@>9Ty*s2uWYQ+`8kVp2R>F^jZm;_vY*o zXnfq9_pyJsIwb6&K&~AB>8Pc=x2SyTcEpr_IhKmQQ%z!Gblm72}AVs^m7Y zndq%XLm$#f13}{Vr5pXjm30X0A~ql*eKQL~;(5$QF+Y)0bM`{8++#QcVS3VTzfI~! zXyPmh^a$e9xfUK5gxH?M@x?*b8ZNZf`!9->paPe#;@J{zA)VtnNLkw;M82Ba43hUm z7>(8^l1{Vlb=_5RL2&3f_o@r_`+n9umdOmvq`yCWOcqU_(0FvJZ2mc;2}3Wcv#GbB zecZc$>6ik`SLW%bpRJFvfDC?}bk&kHtcmn?pUR^$GEf#tOH?4}DmJ@Pdj4oihz`Z8 z>8y8=VAP>gwvE_8U-v=h;k<79vzPnk3DSp-Z_}*EtE9P+Y$3vl#HFZUde!w4Inmw1 zYy-nwHqiTSxrZY4Sz0o9HmZ&0cIbKTeG*3QwcO6WkiZOg*a&M4)YZ#157>C?d81%8*0qW_`$2EZeJGSl-LM)M_-E9tzteuZ8pxLmuNY&<|sD1%|=)A0^Hr-Zc7!|SaybhAD-^g z3aP|$dN#gqP@3TO`oSpdPFxU3C#lG*fpep!WTa*hPL?a!JR2OHRSg(z?$y6F3l7n^ zH7(#1L^Hz3rXb73gTC1mu<3;Lb9mhhIMS-@qxCs;>lJ=Pj!_}Pf@%GckP^fm2=L~a z$CT%&+45`z=g_%_jz&C;Rlb(V;p4B~eDj&s6&YbA?TTeHVpv~7^PZB}>`D_3pHX}4 z$couw&<;zlMeHF2^E(+b{AlLxVy2I2{8+m_t+5!_guKdjz2P1#9g1I6YfPmdt)CR- zU7Sn1T$x%GHk+m$&w}OWF*myB;$A1uWkNH5gD@b^*V#tMGlF<7w!SU;im-jPCGqdW zQNJz${-(>1{|AsDP7bbrz5aNoH3s>LiYUe>gzYnFoqOG!ow-OKr%<{)@I&${W+nPX zDE}1hbU>$08NA1|+_q7Cf8hXPxs3+&{zBmz!Ejqdj2f$`<(7J|?QBs{EKKm z@rR@f?F$TLUDPhgX1~#PRpQP=^cZ9VLw!Vv6e7`Lzxk0LG0hS1;cHaGJg0Xbkj<-! zRvA@&y+Gyn5Ng^1#kfF!tpZA`Js0yaf=HLR&bExP zCpeouS%rYSANrRK>ky|Mu%giRCKZj(zLajnf@z_!WQg)VI*))YBoZFJ$1V2KF#e!W zc&cT@)%26}F*tl}{O-fYk0DPzqHmt`K zgVB$!bt2@Bmfz4B4A+H{A!C8ow_UY}C@NsV!nB0Qv1x9JA&3&2(j(sC9wCJi%Opyd zg#3+?$?s0dnf9vBoHKlvTb{~utIWybzN|QQaqkNcZ7I4u99hjQ0C3FgkjKYw>hKG3s zrge^7rm!amufQsG@sy~3N*1sgi)x_qC&G`OtjJ$ozXA1;gRAGDnm8y^nUB?4nX0r} z$n6NB!5vLWRo-{6l=Xlmleay+yseZ{vKmBvv3>jbWp?dJ~sr~IK`4k?!zRpHD4Ix229bVb?p01di9Xf3IWZwLV+g|>@h}M zb?t?|#G|}e#INiA#wb{i2>$itS$jNP-h@*dOCsnUMIN(zCsv{r=(ie=u|Y z;t;)qhJFED{}848RrtG?`u$VCEtUxXWbOF3#BYA7@)xB3J@K1g*rng4{tYYE-+8`V ze;swr`74(8hXpIwZ{d{R_4}8;_;0%ZFYbTB_0Mn(*ROER`@r}`hI)Vg6Fe*thS7?_zbC>S`Hd~zcE6%J)k zGI6wZb}%w=E9~*J~&v17vzwh4l-{bCB82_Qc--EmR#pnGUclVE~e;=CvA;ib{4!rx0cGLR#mHY-smL2_X2Myiptofq``ps**2-pow-cN&U+3|BFxR-j z_v0y;|3{N$#)psIZzPSOA*}0cL^~@svkwf+Ee9#kwHB|(eP~+2sbkcOkte6a?4M== z=fXd-KGE4{lhL!5z748o9zZKOlwu??EciIyH_nB>-RyqL)V;;UoHoIiPO6+tNTt>h z86UA_87Ig{*NgO0Ad#J@50TOmq;E4+)YqhIu?4E`LB9)seSi)G4&6y!viVR|{B2L^ zL4GsF%EeuzN)af(z+KMRivWBU_Sq3Z4ZFgURWA+)16O(UoW`d)+@%RpTYI`(gN@HQY_{%xKb!*?g`8eqG~>E z@NdQ?wyk{9OQ)+F%*+wBkuG5hzav-0TCEGx^s1In-~9IWS zqzn^kJsqWDi!D_Isy-0t9WXq=AWvn8_%*u3Epvquexje6N)L#tSbOAR1~>Aa_Cz7; z5)#F8{F*ik;_X`GOP8AcA-zHCdO{wV^H5-{D|}@SLJQ=0-JD0l3W06s#ZTdY{jv!4 zdFJPSKGX9HFQU1Vsoc*EJelFK@F7HcTpwk=SYQHp5#Ut}yeo9!U~W)+<+gP_bO>%4n%uo;lJq5uNGyMBsSL zuNdDyGa>Yx*UuFmmfGN|&N^jRTxAInM7 z;Evenm_vlTQvTKF`~q!-(xlzLuZ!~b7Y@+(D_oy+v3d-WX3!g`t$58 z_*EQ=M)hY2Si=MIL}ayTWOP2nt@ThX%@{5d+1qDl3f|VU`q!$fOpWblHMCD{eIYrB zSLy6qegihMnh=aX>M!?>Ej=m9DW5Rzrem(AM9d9O+0P;nziV&knaSzUDj0r+l`V*` zGca1_lBzkOtH89NQ5Dk6;Ae<&MLy7g@cdb0>vKD{t=~vtALm@AKIjfrS%g*@g6e3m zBnD3(5z5_-JL{sV1J`2|byG`#f=FL|#v0{_PH(HOb1vhB)mW!xD!*Nd zMX*(>TI|lnKK(TfkK6b-;h_Hff?@q0=y)+)Cr{sJ`4e_0%d-5%#R2X$-zj!Zvpx^f zGqGz~4*`iL+4L3>Q?U+tUjsI+>}xa_7iZ|VH0^NJT`Vqr3Y*aAIJaUdRvdnsXuG@~ z^9JL_q!+GmEN{gd4vbWnR4Pk+t1701g0sPqq?DIN3Wr;baI)GXY8&@~aORYIQ92iF z3?U63;qh?1i}1FFbqkO9ICW>`&xPMO?3&@7Z7DJ`t;j1f2M5!XwbbZ>a!t2Mxmk9| z;CC5H#a*&3h8;%)F^j#~e<RV=Z<%n_PWj6CV(HDP3IPIoYyVs(0oIBjXW z)=GW{Pq(5h4*$~TT(hi^`qU?vH5Cz@nB2&{G0egg<=rtNXTg?XUa8Zdxke-FN`1EX zFAq%;*HjxgYaDN15*AzRqXlBo!j$J71C2uS8pt+QoS&Q&ZJa~jEuk<8R^)UxThGYA zdwFWuSusMBCzjUSs-s!8aj=}9e%$q?90W2Ok@iyttlhy_*)0rZ(K-2Xf3B;ZX-JlH z8TzzM&BDx%v(l0@o}C;XK>sllH^niO?+YB4M*{#d4Bu|-o50Q#Q#TzNLPqDr5j{(y!k?AQlzHhbM5~Z!9nX}e4S?(~jjj6B(JEJ9R z-`KzE67-Na7Q-ygQin=gTb{9#Q*|!Nd|il5ecz&*7O-LvDi@x*s?{g`-BGSQep&_& z0~^^xGx;amjNVyAg;SEN&QJaV_BY#`x5IL6O%PfG;c_-vA=JSmoKJD`vFrtYjP4ig z#V5+f1e2sdi&c`OhC25Sr=L<`V5bAdYN!G*hAA2*z0g1Z-{1GFw4 z3yRRK{YF4SJ;-928m0`y8!w{mOpQkFuhJICia;_wCjKllTR;vN%BxzjEmeM4?o$fC8A5N@leWO$O$ftQoN0(0}7S;wV zVGeIW4%BOX+#{v1w~sYae<^>9j?!t7A1Q4S7IRM=Klk##T#Y^b{r;{51L^}K#h7Kz6{+ZCloicpEd2V z8ZS&uA(wt6PP&1_ojgt0AVBh&>7?k_N{!!l-VIS|o5^hH4ytdT0}<%BXhLH_&j`xcQTAX&?Hp ztB?24u%z1!5XJ^y7EZW3w_3S4`{1cXl@zozEfyQRt=wCi9 zNh;T+3F#KA^1+BIIf-0WA)H)weiwQv!Na+S7oJ_J@*;J3qix*)ZrW-P%?04_jA#DH ztB}WPTJ41dHx8a~z};(x;sMYzgEICTszu8zh*3l1@n6~QOV33l9!wTE0%c_M5bSIfONQuZR5snN; z8gNL;Ib$+#G^}?o10_BSQcUyJMybo)4aXa)LkaEC{fW;);^2##5^e$ys0Eqf1DsVJ z@}%dj-rq&E45?jB+JZP32JR4dRNSk6cAp3-5eHIY5e44-moh^@3JHX(_r!RP?86eeSY+ z0XM_ea!_~gfQ-y3opSlg__z#aO3L6Vc^VnD3P8LtduSS$)d8-*Hek1JN?mTY`>87nK^UQrg0mBHX!;Jm|P*`2*a`EI=L$Kr&E3WmlN zErXt9ex6etd)Lt?rs5Y0a`3wipX%rDprsX=v*vHpvCN;AN8?58USx^Vo*5(N(K?wp zB1CDonkEDr@Wf8=S1B`vcj~nfzsKc5N!eGGlW`)d2aw4vJnz;{5{ruG;j?31C2&+O z=lXo_8{N|2iCFVaMA(2gblu>3Oaw-x%F$T4qXI#hfzb?|^M=;2U6XY5Bl5k9*h1+zJL7S4YU=hMfJ#1E6h@fdwe zWce|zz*1TGIXJ=B(3CNrK#cumJzk-!1G9ASy_J?s#L}M)OpzG)J~!xHi%-tcUZsq0yD| zAZ_~_DK?iWom-$iRFezX$ZVu=$>CISmWVI7G*d#FBsU0MB3kv>0NmLi)T3{gW1)7c zM{|?3yXP0gXcDJETF9wt2XkK>X)!IAr>D!oNQ80Ouz?QY5A zF(KwY_Z2ut6bpW_h)n7>ACZDCMOqpPIfE34Oe6-WNQYEUusOa10#WZOS#$z&)iRD? z)mf2flqks_5*g%rnRw~MfVylbmBiFj?%5$DirQ4d3w9>Jo_dP{)El0;f^dQz& z_4t~Su#O~hIc%s%YKVP?DJb)T*GnmFyYyn5q69EiTvi?ZT%&tSr=#HOI8S9s{jIxf zbBO-LEK0nNvJjKS1YJM?Op7&UPjxgcXH>h|OO3kuFl1kp;7Tt)RF%r}b}Ou^8)c- zEcy=~pWl5^Z0YKpJ!I=N6VJ=ZX~}fePk8NTHpiVH`Sh7Npt{04Ub}0MGt5F1+i}Ht zo>Sj9frX{bJW!i3ot9F{56iRT%q(>YIKc3R`%Dz1C!=NW6`Xi3$EPjXCoJ7!#Xi1{ zI-TexCOeLj-$EG>eGTGjdG7EoUvss0ov%oO2QL)2llXFx4C03N+=TFsM;t<(Qe?1Z&?7u>^5El{ zMw8$6#pg*+t&Fj|m9YBCRjZ0;p<2Ud!10EcZkpp**S^)PsQR2Z0g^t01T#})IX1LH zhxg${AXu}tuWN~KnRWEOX6SR&(cXQsvc{cmMGWJ{h!-`;{y{2yOtrxq4?*=-uDRd1 ziCfx;hIwypc{eQqPA!+z_j-!Sx9$OKC}}TRhe6#Ki-ot%8`jNT6*~+q2c__>8u?Yc zzR5W32x=@l)-CAGZnM@0*HX&OGWwQ}=ALc4Wmb z8+iQgkk-{KG~V_`O~1ZwpRP`8bQ}Wh5IhfgWdf&9Jl6=+jXJ6Khxj%;X2E$^-`Mf{?sI`ls7Fd!Y28J3 z`95Wf**=+a+)a62galuC;stdZGZ@*&3{k*LY0p@{+RN~1MyM4U{D7(O9CWb!*shg_ zy^q@ZrsefS^K1sv2C5py9*^ws8T6_-xdpHB^Hsy)Pdkz^>l;m=RqDy>I@?Cc#s02V zzZC!#HI-Ir(&(27tWV}!BGMSX6-2Kn$Hil>sRbP-$Seaod2mS?YN!s*PRO1Yn&Tf# zjuRflr*gW8ZNoH&*>#KpA(vf_>`~P;{aFp;i8?;E$2n%B>v z$>a%V^PnrC5|Sx15Wu&|HkwBTlrJ&TTmPVCQ}m}cd*c_f1wfgUB7Wm#mHQtg9A`y@StTzoUIfqDItnDm7jKpKMpFv7Y(#wVQg(;%Qn zh=>EC0aZl45(L04idxwGpiLGYsa^8Z2B`)zDygw_>8l_jutC01OTXtZTCsK+fs@Y$efZ0q^LA+8I8LQD(EKbQe7^w^ zq6y%FR4g@PC8n#Ukv^NXSsns1{T9iu_iyai`0PCHdX01&KrYIZ80rd590C4m~8c=$c*zhg58jspX}Fsmo8~2O>O&V zjVON!YkWING?16U!l~_W=t5u(Ajs=B=1@2^;^89U2eXSI@@4WzXsEQ2Slg~Re*Yf7 zqsNi8Q27e_-H8kQ=SQ{oa9b%c5B*TX8nF2ihqXlPANi3EVTUmGZT5sLU( zGC@0xms@$eG)OHD|^H%!QPgh^o$VRKwfI)#;r{)8>iO-2iy2`nuSYrq-e~inQ_}^_PrPiXc4-x zA6peHE!FOvrB^sBOU<;)*nnn1`Y8K*8%rDu5oi~AvUA<{txmIq}CD6 zf(YCav9oOMJQ7o!Blz-Rse?MQ@-33cFCIADn>2u#Nf%2&6(tXSK_Iz16Clo4MeQhZ zuqNW4=_Am1Y2%^^Qs9g~BE)>exmMf;<0Y1?-hvIKbv}W|5x-RMW1ethCv`}Ma1GWe zV>=3kp1<~iB2`D1Q#H11?cBBTw_Wj`fc_-q2IuX|oN^J!{!BedB1meU^L=qXF z-m)qdQA+A1!0p**eH87LOdUcONe#)3taQdZ2<9H2%pbWlv;#v)KV*)h;n-&maUnz; z^Iw^xbgaa2fV@o}Vb{XsBLoU2MunhhcfZTNl2Ri*9bWSIXv(*sHEmX2_l1!t>L$qP zT2Q24$T)9veoDs|T%5DOc-(|C0R9knA1@l)L&@Ep^uRnMlb$xVWv9k;sAsTGR0xxq z6TT-&vq6m>`nKIprvdk;1JAjsg`sMo-o(iNd9M8@&2Ehm!{c=`@;~oR=bNj3Fb1T$Lm` zgf826To-ih*qqc5*LWAQ92!9Q5UC)TV+@4vENQW{DTYQjV4EPOVbm7%RB20WYV0WJaNhW8%18j&1M5LwinZux4oq zjy?PV;}+wb&U!>QRbvo1db0^z$HYO=m3MrrB??pcNQMDteCa1kbHV`3Ij-jfWxI}@ z`nm>4uI3yNKoquBsHl}=D)e#rm8Wc!Gb$9gl0>67H*rhtxZ#1XwOMU)@qT7f(+389 z8_jnvD=F<~tNqi)xR>z|*1cP5b)Y5%DVR>fdo6n|j|uV8ZB(f*$a2|53REf0K>p&$ zf+2i^hp$^2N=e`B0+y=PR&yxtTL6dSs@kiIL?T2Si6OJSWRBsdVF@gY?ARU4X|>{d zmy;8e^Vn4CpX#>b`}V#ltz)HHA>GEdppsiW2UWI@&RVz>Xe(NTm1r`u*_&JbFY4X` ztgd9+8pSQR1$TFMg1fuBy9al74elPC06~LWa1ZY85`qMIyXnsT{@bVfypwZt?*HDG zZ?U9`nzQDt8d-ZYMwPX(-u(1CBD58r90N7Z*eligE5f%B>_St?(y__7#htS^{dqRp zvwO7`xr?c3XAK`=Kg_|xF=5DIwsFVFURT>V9>0x`n}Nz2mCtidYLvA6W{DhI#)c<; zI7SLZ;v1i=eoOL6}{sifqJ1~|tj=0YHv@4Yd5g6?#|)SEM;RW!zm znb{U!qJGCDCMNN*v_2ORX@vdGMJo4k&*{kr=(QJUnWTCpxr zbkTqI!u_^i7T8fJBA1G?ffrc_*=@=c%HIUnorB;4~PZaHi3R`75MnBd&T*P&+^_IC8@i@-O}jhOaai@2GoqW1~S zJK?Vc3MfAj2m;I6#ZA}ma8qC;ZiRXm;>qUKAa(;KnNpFqvhd4W4B5dq^84}^&zO+o z8hCoGW9|5;$hgeA?*?wYioN?-iMUXhRGz5Sld`PUJowp+`at{hIQ9TqN^$C0rsED# zJqUxd13j^0h-;t|Z$gW)u0Nq@Np-ON=yzqdS8Fz42i{wE6ps4#w|wSJ)$5* zml?O9Ds41<_5s#@EpddR`nC?$FZAS9Q&Jp`i#EaVQag!DxE9=W@$6T)Di!P7TuL)J zmPEoZ5(ATZ$;D>HnW1}E(OkFN{frqy7YW_ZH7ykM!jiUmdJGmr(4Qdz-*pox9e!{c z$@X9Gm4NY3$_3H1WSJAh6=|uJu^jb z*ke8}J91bh-dptpgKu$#toBS%vj$Ps`9ND*ew9{@rn0(M@&3Y(KaqJJ6g2zNQYzO| zcwrRTG@n~cd2DxngF~X5z*nyFKw8eWbIxG969UG>{L2ZUs~ItHAp)=~2{bY7Og#Qf zFVTtss4sAD;oWPSF~FyAttmV5QXI^9nVTm23HzjXtQxzSKn!bfxL4Xpz_&$enJ9bQ zFmNXiA$V5YAnHOuQes$8or!OnR3@4H;2WEkgWni@btk3tv_PH4k&EPv13I8wIUy2en2mw_cH>t9TN|0sF|EOyejYBMqAyP(nz(-K&N_bXgG1)U&*k z1=7tOy(rH;b^~W;YWWg!NEe%ge|;43a?W^a1<9VP3Asnv zIC0@4D?jxI#RybCA{)j@){2j4?$kjI^6)}aT1W~)-BJez)7SDk8#2^PawCa27QL&g zIlD9P<}NsSku;#v2fdxKBQ8Z5X;he%h+v|JLP%)tg=6Im4y3E77F)g&;UaPZDLL%4%K#e>>|o55K!wXn*Xz#8r)^~= z3PB~{3nzndFq;=IRjM(8rTqL~xIewKC@r71Wz#D2}wqCIMJWMVg7 zx!kF>ZAh?kxuk=f@#5Cq?@?p7BC?_kM;=Tjh2c%NS>?}?e!E49=GyChRl#Sc!>^IBec93Yz)3PfVUGOQ6 zhAxH$nat)BYDC*b&u3_mnMg6{PiNO*p&L8=3-)H*X|$|EW0|0thxi*^%Oq4Ekbx1g zhHvy(9q-Zna3+b`(0okWQr$aA0#(}PS;2aAYcN)AM3SRwCZ;vj!;N-gaA@UX-wI}} zoWr|m-KXoV@#BqWcD;=fuBFy`n;UG}^G(0NylXgjHk7_aU_PgNQ@w-ItVauM8ui;wB>K zXfha>&HN1O1oEL~Ucp(;PLt8Zd@Yz2+H~7un50D3$f`kA2htB>8|U_w5J!E5JKyEu zj%o;t8$Iq@Xeq@8LatJg-|+R*9_>ntU~6=90bB$6mEzE_Xo@0MUbL*1{^%;&yR?oe zI4$U4XgEcz6%fiKt#61hviouc4k&p9HY8mvd0}nro#C@W@cV2=q$2{A`UMw;oX|4s z?i?P`ArlkPS~0N@D-62(Y|HG^lz?yL_ri*D;HpPhoilLq^7dS`3y^>u{S+3Y?1FjH zodl7TxUf8_U`-sCC!3K=`7(>+T=c{iE1~mifc?;I_VK_(gwCWpA*Gh5L-t-@V1d8h zgiaCF^Wg8RVdoTTo&XxQoQQ|>21yRtL>s79U$%!gZxm%Om`=VK88toE2SbNJ1adCa zx5ES)sJ8062e$i`$*n^;s-6+k-dNP=E(Cpvae6WW)J^4t7K9ocaM_x7jbTiIuY9V-jnxZmr?m_C{C77(|#Vx>GUJ1Kwa|J_w0q!2@DjS8YA7Vbaq_*| zc4vu5Hrsa>8u!cG`e>zHbvT?A(8Ph_3f2C(M$Jk-%X3e=n(o0lxoLi7O<{NCd)*Fp z?dVpH$I_0CP}iEY(Kek_WBdKX!$@IYtBMTAZp0q6NbN?WSUKwB84S2wN2{lstBb=O z5LdgWTT@fpYP-fKulvW)&`^X%U(fr^tLmi^-*&g}r;mL-tq5m1PnQ?hp#0-}UT*WN zkEa<_2F(mkncn5rqhagE5YFl%PBv1Yk!u`fe$Rr@EpsKt`|0C)*B(31?@ZPYG)A7S zx4EZwlGidN=t~Crk~8YAX7tP+mG!inCtJT9Kq4O4eA`-*IDi=WOiay8OEA1z`C&H} zxlizp6MXqRBaxG49=y{)pP5d?!;gR#NFy%4{qZ^2$A9`S`S;8Wzfmp!i`e&zR*+$4 z_+2H>%kcUOMG>M!Bn|IQu#Vf7bKj{mI{WPVTbpV-FC z4F9K?{Xa*&XJ&Ze_5x_Mzptsm%ff~clYRX^>D>SGzWz=8|J>IX+VqdU{-SyC(<{jQKIH#A z4;cR=2*~&c0y6#;0=^Xfh{nH?gI|JxFRuKA)n9{vFC6(FR)0a}{|76`Fu$bwAM*dd zgoD3p8~~z$KX8!o4;*CtD;#{O{3B}r$_V+JdH8eK@Z#FfSp6k9_!5YpvHDAJ@Fna& zWA&f+^>5<;XB>PHiTtqo>t@5hvw{pWaRh=7hTW~tG^)s|Gpmc_e}p844M9yz>w*WmqMmLz>w(=Fl72GFnp=} zBWVB10QsBA_;X36APszp58K4oTm$tCA0_Yu$oV5s87+C25s$E77I#z&2kd>8%jv1gR zrekGc`AwXz1kg9C5dc=aq8BA#p%(z$y)4KjVB~CJXG^a{uc9dN+d*OO>}>DEMNe;L z;cV_=NM~ecLvL+uYfbNBYhmhc>O@KKCxmAH4NHG`_0r`3Z}pge14_mj}?CyC!aqS6c4+S)k-mKI=U zdr@`*5-ek4Y+)d1=LT4R0Pv5Kje&rbg#qwf2w)nZjCBHZ=XXDUjUMx5>5~7_W4<)~ zV-Wnk9+Q)WlkLxPOf5Scv4qoBpvO=jWBM3P_wkD0JkV(Cvr}M*0V{U~gl{|(s3sOE zg*>RA-aOrqw>KH9xTv`%+E@Zb6edURY?RXQ$jGf={CQ0c2o6<4whGoP{NRnk?3Cq6g5c|Mp* znN7i0Rh>G#V=t#~cYE4Xd%oBi`s7x7Kb=yoI$~~)osVucpM?D(R5>ELB8V z3;mUkD{{z@Ac0+5fn6yeX*A4>V|*_4)~g;~ktGt*vdJG<3WQQR69m+x+$m9Uv2wMFx=j5u-!P z@XZmCXOr0H!Hjw(|!66 zDm;X7S^`Zu;oRqtnIWq9kU(<^->>qnj6as((p}x_(2nPNslU44&a+8;XvRb0RG4}< z?!V1=5CP`DC1qeih{#b2I(qTsj)+Ub365x~$rx7gF)z*P&dUU?&=_1$lU9X~S}fhA z|GaM*W5Fo!K}p&1JwT~cbRf|~PH_c>7GwE^Y%vQ-o+~;efITRNyfk`OY%pU^LD(-o zVuUozq6S%neuBOsg>~zs^zvc3?8A)4^|>_c%(XS5TU6)~l7JB0t}TvV=q$Pc zoY){rFhQ-jVbC0((otcy5>Q@bAqdKKvRZO%1sl_NR0Y3ydC{TSj(FV|M)uo{55(u= z@*PD!Hjw0m5zb6pRMyIyytkI)rsXwPWR0elN}E4UT<>C@%YLxaNSE9x)T~|9Lrlhd?~AZdf3MA z9WzV}>Pg`HcO|t41giGTR;%1_j>f5>K8uv?8 znx2suNwl(g%f-f&2xzDzfDlq&8ETqg2W#1#uUmM>oF z{Lbv{#6d)(c39$?^GS6Djt_NZ1BQG>eQ4qHIG$k$8VMY!X0^v{t_nm4`CzQ<)5q|6 zguRTTF``avC9K~`92d10rD^vAG54gxa`sLnO&c!_ZB`J$*De-n3uWA zrMNxM5S!>W@RlRB4y60Dw!#`e%)UX7t19j~B@{4@(c-XY!VoyKtxf~dxA_CD8b zqZt*iWJO}UQ1a?FE!}K{lK-X%4P9>OGW@lzaciROj{*Ir1?oahp`$Mb7hYB<39~rH zv#O_8tv#Wo;k0S5_(B0+2O2s^5Gqsib%AMX1>OM)P$S{<_K&-%AHGu*qBoC{ zj|iT2+zHdPy;DOuey>E9ztGn%lD92yu)1o(&J`BdtqO*TSu~2MZ86GZ(IA%JFmG*o z?H_W~Ejf?4(4_|QVA3vcU_d)Anb z#$H`(!?;;W`8D&slMut3_PmaiBjSpFF{`QwB{gzZHIMpT&Fzh2Qq2Q1%joh3SF^K= zwLY%wthLG<6W~d0b2iIP49n$zR0+J3K^vX!OOW&@3_q<)&%xztFW z%*UM&tYGXDTGnYhjBvf`NBeL0>RU5wJ>k|IyHYTg7kD6evrVYtGfx|;Oc05 z;$L~|6qPlQcx5b3uZ}$6NyD3+s%N;dotBJ8Gkr5!FY?hxhfj*71H!8ajl zjzG#gNtj9#x~;v93~4x=e{+6Y8bKA#({o`uC(+}sOVHbnp1!uGBV|%Qe0H4s9*8v7 z^ueyay`Jk*6a;eC{Q{~e5jK7E?qq8Q7V7A(fID5|cJeurTd3N~(l6B=JW~L>9>#0( zz0L1tWGz}A^ml6QXy~r&znP6pNihlb`aA^FCJ2ZY!8Y45MQg&K_@M=nSq8I zmH-vsJhh3zsN23M(T3}y6dx9a01(;PfxF2hb?LTr7s+a#babuVjpOR0{g0Ia7E+GH z1(6d8tyw7e^PDD&tYT$P-+A*xnA3XN%W`<=@G6iH4&ytiX-lG_A&=?;BaGu?1G5K& zq02$lN?R>>;z7j-N_sJ%<(ENGT~&FVd!+D<<*U?LClSV6-K`gh`P5xcvl8CeXmS=U z3`@Yz8Ya?T+K&(!UR@v?CgD^u)lP~Xi5Fi)%Lv(Q7dl<26Q_tdP3AjgIBOE~rN@38 z=~*~(JqmKbT3p!4a9Q5lb^i2Ruy|>Ar#D!GEKL!z>?Cezu~&8YcI3$jmtoa%JJ_a~ zf>J_{;py) zP)foI4Vei7oY!u`72o-b=l8Y{woCjf1@zFdUao~V-eqPs0#p^7ol>S>h;eF9RBP%E zFo)(9H+1ZRou7+(pWLD5a~dlk7!7PO1tye-1M8*5&V8aMx@*UEkwLZN&I$2@g+}Lq zi9EJDZNfv#M_ijd-`#omR6EXvoT?;`t4e%ssUDh3t(#l48PU)hZ!s7;pL?&xZOcvn za6gUq^jY(0W%}{mZGE7rY9Ftd-LRW{V$=j0eqP?)!L}bpv19%RaqjrSj^y<3rckpY zEPK09uT8@u`g4fAG6{!@ZMrz^2&d3-8d0p3zFvXNglV0lFJUTLMu?wl?S|2QB-XTw zfRf*Q1*I5$MMEqYE{kN6ErOXrlzL5RnA)XzM@eMcvjjbbp4o^pl2*24ZABgng$tq7 zi@wB_#MKRvLAWYGv@RYxc%X^Vm@Yn6tPZnq@lji0kyne3vAVdKBzQta$lAg~TDG4o z8;?P8hM&?4se(kBofmPPYtu3yj3d#L^|N7^E0e^YarzpOQCR#)jZ|^44UBW)!SK0S(+kfd=D);l3CtsE zbc5g7Ehbv*_Y}z>s-Xy=Q{H^s#J9#`!0g%R_$>~7h+_2nC|A+1oq%l+wtzFV6h z<20fmqIANjBAI4xkJS6C@)5(>QFJDBdJtdy&o__Q;a_JnRPZ@#4-Hmg%ni}F@WkRd zL1(y}Cre_9oCEq|)&)v}ZpmLe<`ppBpQ>}N3RO)-Wv)paNUk2J#E%MlQHAmE&nTjU=1WFn39shc_1zeYc zba)=p7?aKBS>nu63j5d>LibEan_2pPL9>5Nq~}!1@Ex$?Ax@FLR4ej{noD7sKx@Ls_+EaM#cv-hCtU>el@RQHYuvNxIaal$H>W3 z=fN-Ff(r?wyu(Gh7%&&ng^3H;X5UeH16EuC;LQd35*Bns>YBBU3ySmUK8IQ+_Zcv2 zz;L3T>(lMIDUzveSk}mzVZ0J}H-mE;lfWkYDgy7FBzLnsJ$biILJ=8@+>RD~!V*EA zHzCpq&RdQMm~Sy*O?U-ChU-Ujnqh*kmwB4pXc^KY)0EWbRRtlE78JgS+^Lpa!XbSF z(V`D)>{V-eP>qWAhE`@_h)DU=W0~xR3cG#FL((D|I0TBsBJV$g=-;UJHZ^7Q6IQB0 zGnMBXdKF{o7J0a0usvFL^nsfVhkYLOx8}rb zB0Sxs5kjHgU()MDNvs+o=fx`lG-3vwMZ$nl7;ctZeH?;`8F>8dJz@ULNG=p~i80Yy zXblsXpsWeeq6@Tr<$?U=rUeK~R0^T9O0o zd+#R%@A7&ULsd2F@b#3Y zIg)_=6=Qj_Zka(nYFktgF?0ST=S38uRSMYU6RH%Gd-A>V3COD;bCRk>UNF}q z&meW0Iv`moN6)nz@LFD(7K(+46#WKHj64+TuaQPtxv zuW>foB`OF|M3ffP){|t21vV>hzq&_Gfpy8+9gZ?Hyb`~@p_#xTGU0EJOp*CVNOb3W zg0LI}xg~QTszp0{;D*~K)IH7~;XQ zCyegHBHfocl5Yc}OW7uM;d-bmO@~1BT&D5|A%eAh+IP1R`AqGcXJdT+F6bl4DNVDz zZ6WNYAv|@OL7A~2(kl*!JvtPsxA+UZseRFr3s3_o z_H6pKbe`?>Dkcfe5`6^za+w5u-)k*YT}lW8yiR3mI`?T#h)Rq{l+zR{mt&{1 z7$AAR!PpfI>f8aPNWfBIf0^WQ6(N|DtDzA?8zFvIqn_m^-_uK0oYR)$z$<_{xBS7m zl8Fi4OT#niy+rqJEEC!{GdaKS}5dIo4JpuSY zDQmS?AAV)GXyQNni4xsCt610cqm5vidG6N8M;o{xq%g{rYatvC$^KG!#6zkS%7Ehj ziPK7dy!SzUhp>>WaUp$?$d>mW0fWec)z-$`z#S%eTt^OcL2JVhfd%~+@~nW>ZeU1_ z;`)3TgzMMsAt0gY36t{MW(#(pf<}hI2;#JwX0|nfNloMzm-TKQfU!+1_TBbm<+&-P z!m+q!bAic+6=RTx)j=$Fh!zXQL~M2Qzd%M$CW-U)&rc8M*%z3^)wlB34saYnjWfCj z%YSv8(Hc_N7hHsa+q9|F#BBJg2wh+@p>07wAd&+~g8g+udb6R0^rDfkFX1+$=#9!5 ztSp=GwoS38xcJ$KxE!(_bQu9%RmF*1&5`WLG+i@_xIh*Lq|fOt-fEC*t&^rndrgzJ z?L8n^89wPA3Rm(l^s2^fcCE!Wvw;r#edCDK9PwMay(qN~y6grpU-~O4IAbic2PpNM z*+R#`36M$7M8SAJOd&y+(G#x{WbB8*QA4((w(xg$w^%^v5gO(vau+7FU0wzwD-!Hm zt8nyZv+loPM`GY|ESHgj{&;O`4YuT^M#kE3S)BwMF?9KfWgNdoA$eeoG8EKN8JkiI zkza@c-*mT==o&4i+aveQ~wO99U4YfQCyWk)b&Ht4Tz> z2w@9>954%cJ=m5wvcT5|C+7K=mIp<6qQkj`V~d23%-oD|<*CDzrwg5{Ir?ha6IcC;%adhhFd!FrdVV8_M2^iz&jHCihIgMK@6PzR1%yspQlUdAk zH^uW74tUUsUlUW5A^ARJIB;N4+N*YBIb*={*ne=c1S&vif4hOyuV}8%J^1cTOWoZ= z1jXzW{w!y#fKM}737E$^P<-8bim^3Oa%m(3^DaBT+IScp2(*vu#iPb5PILUhBjt^P zD@MTW%D}{p*|VBpR&QM--aSE9mR@DfY#lXpGGf-leP_xsWxUr8!XCTTGF-pR+{Q8b zjXHBlD{-8;%2UYq5_ki%YhJ z+}H3z-D_iWds^8(m!e#pEDdWBPF@k547l#GXxnPs_tsg(%uFwQdW2KE@a6cvBPz1` zosQTod$oihZPLy+Pf--wDVC0hlz(njLFLHSj~7oxp?@a!Mt?u4iNJGU23e@XNqWV8 zy-q zwY}yONM9`v?#E=z=fksSZZeQ>@8&C+S@UTtLK?89MLC9=F6uzvg~9c?88AFtIjPSn zcLn^-(}!NFD~ARn*iWB!(evW}SKvdb6*`_vsx)#Lm~7l&ONA z+P&hzg2$x+-}C}+Omuj}cdWAzD0LW%YY>Al`qEY?X{P8+@lioF5by;%sOpbK?FlMy zJo$GV2ie#J_^tIw+2T-2V=`_N*Qd^b7dfFInwhlPY88!X)PhuC>W)Q_?tS4dl3TcI z{GjD}$SC1llkYOtI5``W-`pxP<>JVNC$IF3h7eFmWmvdQ-8e9O@LHeZWZrb(d%*X1 z{lMs7r-Pb&KU<{AS)Lm?ojYiwfnm%ugKN+UzQx_zR>({`bi+49phg_YJAu->P#2jJ zY>{<-qQ|FS!`XRI@PgH8?yNn_id(3A7wPD{L&Osinn(RQ5*9EBzqqEDF4+u;r`*Bi+j)Rn$caPDU-F9z1Hrgc845f6rJL+y#Us&Zw*i&p-A!0&L}p~ zIZ{VTw9ux(FxPSMc|bWy@no%!FKT0xi{D}dPsgkEoIH0z z@1a)8TD2k@-Z;RlmUcKYVoj8j*iDd;B3 zHs4Ayv8e4uit@6=30iAS>U8ss7nR;h(n(pND})}bJ7-6uc1`s-pS45EzAR9maz&YH z6Yf&@jvSfO1@Z`Iw%_7JX5X%<^%3D{s^l^W=F+!UxHyPUyvw#^y|Ix;bZDU=szyM& zSCZ359lz0M%T2?U5Nv9lz2S0tI=T@VsSqNRk0?~x(oHq*qQN5v(;x0+uv&^+LP7kQ z6rMfa-KbijV@5ixDZVjM+=N`V>yuSahJPc^9 zUay3Aj=%#tH_a-{QA9abL#e##qXRLm6CgyBnP49IQFBMg)4@0HGPUCq-xSGXez+ns zSr_J#)r9DXFCTErLhY*6o)-I56CAVky~nS|0*A|R%Mm?D)M~-)_)W8^Ft=b{cVmpZ z(FCF{qDHfHm`P;DLLPA$L4Ce^sdlt7S3VSx*0er(q9xsyAJP;DiAP(Ew3+a$z}+R2>x5Uq0;7 zk`N5&7FMAJi~$U)tG#Tna}`fE3Le4?q9Lpp6+0D0*hU3W8rdv1f`^yz8i+`|0&xKv z-+p?mE_2GNH#-NNtjS>-)_=e%LZGjQyI+M|nI{L#@0fsLSlK4*EED6@F7;hg_zZm5 z_eGL(lpM;wr5W0zwlLj*X~{0neDFICT?2Km*x{?lrly+iSn|!VoIT#JpSbHC+C*w_ zhlj;XLvt#t9OK`unf8soLSn*`(?=mOT-yR$?>xvjDY`GM?sN!apMdh}rmD$(r;T?- zLa-~uDtw*Ux9TBGmVV`55~t9$t7KAlB%L=ZLIWM4S_O>JmxzVdmDHuG$-AwQ`BrP7AMI*vDLJmZMmm4TEuA{F3^ zGC^vOUG!$J?UG56We@>obPdOhxNwT{1vDGM^)YO?p0eOZxJYj^M7&~a8y za}23GwvW_ zQi8nEIr%VFtLVUSWkeqn(1?Sn&_;;0e@Vf_1Uws^J)W-n3goac^6f4gGm#lFDC=IJ z?pp;WM6)+t+6GK`wdfKLkao(3MnSBu+eVLX%HLneG6~>%L^gs5CkgcwS z9o3=yj=$5)n`5uUB?kc!+%AcLi_!|>G9lwcMH8KMsNO-wA+Dj|@=V5zih_@O)DnHh zxJ{!t;=;kWWi(<+Vg_Q(`un}6-SDDqdx3IP$DLJ~<0=NBl4@fBPlk53?>)l=i_g6a zP^ITLq_KO-wB(Pvs%Eg?b(06#*ED0IwXdehtQv@mik7xrIoeSag|`*d!fLUrB)Vqr zFbM*o4+l|7*1+&6b2-E)th*;aGjgN2r4nTmzbflYYz+37_y64#PtJC~CX4^xj-it5 zTdNf(4rO7hGEs0Rjsb;L1w0(8v4!odSjXs=^v)~CDPA=cHx5BEwQ@24J$K};H~3L7 zkOj_U=bhH3L6SW5jRE$<`Z(^(p*Y=&VtB4h0orA~SIhqVrGTHjwL#&%LSY{|tNbK+ z#`PS$^+@q*)i6{6ZXkGpxud%2j(vUYBY35Wh?DHZ5H8ltAU3RzJkFd`TOZzApxvaQ znkB)LFc;YNrVv>Uk8-cXFM)6#!2uu2H!}E_IkeURNnSq=zoo@vdsyf zJit!+y^~mEnMAiff@fd=jtKU_F)tn0FR;N_@Mxua7nnGGZ2J;RZDa;j_PD1AeJP} z&g_H~1av}&bC}{H6ED-5>;k8QA$B$i@7?>M!w5+pWsh~7f?i^)@Gr*s5Cbl?r~)4Lg9BVRq*2oq3*QUCftJIXdkqx>rYybb zzeM4x9StK^I*j5kmUfG-%<2z`(s&`FiRKm)&D+kN0T=JUTqZ+;myH+K*J!T1P8q5X zg0RdcTG}a8M6IWW3!a@gk>UTEBTCwczWo~&N70g95>3nd*t$oEMo77G$GDTM6LwC) zs~%oi_3ImVP7JE08{r%o$~VpJYb$t0%k$9RvKo8H4Z@KZDV3N_p~De3T=G=Ezv3`S zEE>fZ-P?(yN$_fof6~z{a8KU39y5|N1*}sZ(b^+t8tI6@XLl=1%l7dWoZWNsG8x~c z`#YPu#`6ZOwo>DpIx`m^9ktJIzpo8mJ(+qxG>H06^|WZ&L^harR5RU=DfTA!EPg3C z^SMjRo?6)~Jzpa8*nt2>T_{=IePeg&-dQm^=9MVZoK3B z$=>kZTITcDr`?Cs$Iy(uYTvsH_|j@Vucp?v$CYXd14@)(q?jNB#1M%P*d_=;u2gc> zo$GS&uZ`O8UB8WR@(Enm@jl*Evp{55zj}NnvU)c2dY>YmH#0&kg>7E$RF3vd%T}=ze*d&{6f11u)|(hXn)c_y%heX$-hw|{Sq$p ziz`22_1AErU)X*>to{P}=f9_odr9&?FhBp6h5JjlrD0}%p*sWUjV~lc!1*`!y&~Yu z^xOH*W8-H@y1#@5{o>k>Ed49w>t8~HehI|SSp6k5=$EknjMabM*T0GXpDEujMAsiy zf6+Ynd2Jl?3j_a8Tw|912m-SFfq*Q3g@7-GKcexkjJIEcfG@86gwg;%BV>O}jtY*WbeaGgkk3U;ifl|IpXp=<+|T{<_)lZ)xLR2L7LC1M7bT z4O#y{L)O1S!aRh=7bV9JtG^)s{{wB@OQ!!J{r^i~$odBu z0u&`LFN44Dh6Yfc{1q6!RQ?gPe`SFD&1C#JfOv83XRQ8nfcP8G`ZGYh1mb6`{!P0- z+1LLoVE7_d`q9^4HyQpNZ5-=gy&1m9pnmxG{~2u@+y5zFRkm=pHldKWb2V|4x3d83 zRVeRh;sj7;8T=-|qa>ggaB()bbEJ^5Gk3Ieq#=+pa5i^z|E)mC(d4C$Fklly3SllL z1||juHU>sURu)zk4q66was~!+Ks6aVf3ssbVak_TM70+|4w|8`{oWc$Z+Q^1`gp!^?yivWljhz*GKhns*7{3ja!O*Hmj zVl(#t7S-QGkaC8WO3pU)vIMLQ4D_NFj!w=5EF3JHfJo>?d`6U--90z^u(YL0cgzptd5#lt-q^@QC4(a$WAbRK2Ovj zI;%!P&j>8-!T8zk_cFQh;KM^YRcw&iP-bsgl`O8tmdu{H1{Ix#s`r@#LMmkqZue#H z?;J|g_0OFzji@VKX75{=Sl0%oH$B&sRoUGRhmgiIhA_R(YfHuZhHCxwsdsET z)ruAK2=zIWApEMZX6u18fdoK1_pw0vXH}-z;B)ks7cWiJSd06Hhk&5K_y;uG3it<% zYBc?2JAGRefN+4F8|T&)0&6J;Tm@&@()IkMY-E4}N$6dKeEQ`E2XCP{gKCdLgqSei z8|I+;CHFwwI7{Up8)$7oR!rlNEKPdJ8TC8An!}jtQnp+Qfbb0<4H@Wl(5|eC`f!Wa zfdTuDg=*T710!Q&#Nz7`hBR#n&ciNws@>~JpaS7-%QI$Yz_BlkY;GnU>5;iDcCur}>8HMI*jdE>qn^!+Z=Mp|H1!o9j!jN_6}>RGS67I zHZ;|9ttQlMTqpskbg0poqv?z?5bMNUaaNwuu9NR%X((_K&GUfvK$tetnSa5dFcspA zQ;{Ma0f+Jpc#j{r$SYQ{2nlGw4+|KW#Ns^bJpUGcV<0~*EboBT0LNUA0#__e%zO@h zYEWP13CR(lomWY;B(DP?fVS9j!5$52<9V5rGOG}{8o0=Ky67(5^vvfJd;k;MK4^El6ZE2H8z$Qe}wn9 zuc~n0VW$zoy!%I}P{dB56| zRm*zsg`M@G~jMoA=Xn>Rj*>S1v-ZLW3xQ zW4KN_lPy3ny+9^%n2j^?a*(Et9CKRd{InqnKuW6`kAw1yz|A6Zn15_9~gO-7rjnw89svoY`I1^32Sf3o|)WaM~{?gn-#YS6~C=1*c_)<%EiI@YfU5ra1@g!QKrkq`kw9hbE}|Cz5qX8QUQ;5ZNiG3X(Yb z*brfX1Y7qZ4)Wl{+#NDi+F(7RT5(Nr<%OWOTxU>Wahhr%Qyt`U@xgn=D+qZ#Ob_{T zJH-)CxGbTefDFfxP>~J2zI=c?WFf_rC+KD4=&0J!%#Fba`EC5_iolBfHh%FrB|HI( zQ<@|2JQ5heS5Slrc&=j>a4v$%@kw!pPDf){_lUxN+#rHjvPTNExee=L17QSIE}#NiC!|cNhF#7f{z+0QW>7muA|E zIG+^a1h|_FfKy_yACUkJLm`0<=e5LOsPp(YM}4poWL#DC!!qQc`)D4&t`0PL*hi{g zNgP6m%2=~rCev#l1o)L2fG@qAH`c^s&1TpRW4udYPNxf^nNIthEQE#_f=Hk?r5t+` zApCz^INg#O+S68UW{F>DU|G!b#1IoZY{3fLbeiYBj#rSBOJ3nnZQ!km56T#9O6PY3}_0YX@7 z9c_ugwapMa`-JMFK)bzZbN7x}Hew2n_LC!l_C8R6%;60vP(CYNiq5%O^K1g zOw4Hs=n8WiDvQa#EniP)4?5_C=Oz(@q)brCpD))<)QjS2z6y)v9A#@Ggjfoz^K-X^ zYZ!4#>6IVmaf4b2@)GI;cIfLq0S#)Ui9mq}VxsOWXbxu@7)RDyE700C6=nb}0$U$qAA)!vupT5&APifz zMyp1x4Gv5cVuD?tTEA3TG`sgv1pD<~_IH6~oVp}3so4GZ<%nAJt^6~#hkQiBW26n( zM1Ju~LcGMra&Ucx_9EC#Il{UMLcCi7a&XsQp|i0L-y>;b3MBJKA~B{wc#Ye~hS$l# z!G*z;uRZKuhBmFKsPXBxw|hO-v~)ZiObfZ*d31QUE&4hwt?2l8H~4C=Jk5A7cihGf ze2;O^dK7WF8+fide11>o+w{K4?Aw-|@An&D@|h(0?;FgDofcc@OR-z24f-Ey8#7|J zZe_66Ok%dGeOujJCUqVHa+V@T`#`go4)N*UzE*AZRsb$P#Krw z{mOl*GK{8LQST!eWf+TT`QHb8j?3di=TS>wl(&&mAYT4|gi%TPNYd&&!7H`O`40#fAceV`lqt4pf+1Q&RwBv8|_zPpLD;e}b`htnw`b|$)t{F#i{|{wv0Tju$ zt?S}6?(Xi|c;nW%ySr1kyEG0B6zoZ8C6pbv0RD6_VA)E}C~tZ_unbSiTY2*oOdPoq zBi!07AUAtg+Pbp3lV7T>+y1+f!wu=@OJ&Q3jvG_x5-ukWuUwiTBT~L6<+2gw$Rn!m zWva0qG-V84wYTs_taP^ZEugcd^Ejsr+5PpjRQCbSc?a`#?RTjzIV4)+i&P5>0L9~( zV^k?lS(VA4M7~yKFJk66$9qSdCwqEwg zReBwiJ+uH22(1?LQs6D6Ve$ID{no$J|JJY1SQ>c}IoD4ppbzRxVgJK)@ZBiVt@6?v zVs@+6O(V%>$0?dZ$}R8(^%8p=_L0t8ZR_l37wH_Vysu;EWTx?;lCf)7}B9bKV5NE~rJ)~{Z3JC#jxrSB*o0(}!G56Mk|&qa%yt;Rt;{SMzV zCi~Bj$=G1rlI2TUg-nF~9?q}sU76Knds5kNmN&xgIn}b{vXtkYywDmnYurO1^5HjF(4GPR3#--+SlJ zd26DItrEThLz4Lt_E#7N*8|(zbZ$v2^=hJ`iVe}P_n6tMF?G``e>#xdsLbNumcd$1 zG#4rJ!fY@r0Y4Oaeh8Q|a>ax$w$eqJh5G5@TV9~t_tXwPHrx2>zS`{ud=X8*+&ybk zwdtpAFv8GAS>&+AFyEPR?V&x^xiJo@roi{&$h~6sCc3(4GHU6yy0!W-blB_p!H-+H zf{@fIzo5}su0Hpm@gxBDF-YY5QS-COV7BF_`i1%f=bDSR>4R@QB=w)ND3UF;+QF(5 zax-f*0DC$r=QlTSYqW{FOwKwj_zJ)-&AnVvk*}J~t*BtJ?f&B_b3^dUD5WzkM>|B+ z7wKDFys)u*r>j_B@{ClD8(vJ+SvVC%`HEGfdC-Y+1j@ZJ6Fjp;7VMg(uf%&bl4hm7^3Z3DONT$u`QqjK(8KW->h+%d56M`TVKDz4ZX4 zY#W~S$r-9|hPO^!3XDsIIyJJ*Ch3mD4SVsnpHfPj2Ct5&rQmVb8P@%zkrZu8;*3qxEA+)~R$0tm>7fm8*VDjvrei6f23+4chh% z`hZ&9X0~QW%j>=A-O*pe$izY!@S;PLOSSL-TeG^Pumr=fy5sO+iSvsL~R7`4akM-BFLoRD*P*#+} zd3GqA(T-A*{K*5Ax;a$Z&bE0PNuT`rTIxG8O`%qyFy9P5fHV4&!@rU_EiU;XMh>Ww zX3sAmAvLbP?lzZaoFS;M46qN>jq?`O+0zoJPCAy#W@n^vd=y>SpWFdnadI@ht2YpsYYDX)kxVdN!d(TMfRLz zm8J+)oGpF6tK`n!{Jg6;n2){oyYraw8BXH5WV*S^b689wRMup1Fu{1C3ogvn_MRZ8 zp~Cp7B}*i=D8~{^YKk#1b4u$as=X6D9Tk%=eSX;RVz~Q~BwB)k(V!Z$kG?bN(ps3! z?1??Mb&?yJ`;yLZrPlIov7*v#ZhuYa>@ST$n{ZpC{uRLW?q1pDJO(w{J1Voi(6;;$ zmdc?`8+lM3e-B$7d623Se{UuNtHEu~S2?=M-h$i^P>bimiUEUnvXAveZkv)yY73M4 zN4f?z@4@CiGX{)yy6nkOe*thbP6Q36_GJIZVmR^(BHmI2364a$dKhtTD6hs6iP~ z@H+?(4qx!wPSnIy1;g!W3xr!MdNjc`++(g@D8ey57WW}!^R7Z_d%>O06+>cK&jzNm zTXoOj)^7r$wd|Y48JcwSF?D_`!ksUCrfpQVL!v9F#vehqu78%}xCWXPPRW&j5Y2O5 zl&zd6pEtCeyPl+bK_wjIifQ_AE;<-S_z0o0engNLJJ-q+vy zA`0JThMa~@-L7x1Px+we((dI;AdrLQuo>@1*5t^Z>bE`W;^4Y!Omm`FF%lE2&(#O2 zJ50T`x|1n#8v>aKr5{Fo=ZP`=$xDiWa!jU7Dm~2( zWO|MN99ojv^aFoA`5i91?o#safaDE3?HhX^+FC1gDA`eyMP7hSaZ8RhO`S^>PAd+>Cn?1mJq3&snEBE! ze#kB?UsD!ElqI}?gc3T1!H>I@kMiFJt*v|v{^jsCJ1>cuOa{ZP6?5ZKqp?ElT84qy z{**%O8BF6ABN_cDoB@{fRhP>dRLuThIr$C5rI|5-r8CYIjlMk3!-j);$_Tudg&Qff ze&-M=?9k}9wbA1hk|X(!*LZHu!Pl-GoG#$&l?Y(#`2Jp!B4YbJF3#toDrz@oXhq(q zEVP3~Gt=_u;U$IBIZJ}inJicQX$@OrTA3exe=};k^_$EEFy^jaGKK&&+4W?NA5fLs zIw*TCis~y+JOfCH3QimX z2&fS+63FN1sdc^tTimQ2U%7tl!Hm+No}>u7N(7vZ;@PyKW%6>Tko3=0*L3f^`TBU> z4?i=91h;z*V6SJZyHOX`d9I$Z#B(wM7TivJ>QC;``!)8VuAB3M36Iy>bcLWcysv7W z&)0xf2zx!uDIqVicQG`*t)X+OoYdQ%#Y=Z|ORZ)l?Jp-AStgf$VMi}7Dc!?ubLo*x z@{T<6wQ6ok`UYrD`oEE$veol3AUKMebBEDCiu%EfRQ)>yyV9AA#Lp+?U)=QMa&b zH~c)+)>Uktw=;a5;BK0o%5<_X89w4P@tX0@DtbS0uF|!VG8t?#r`?&@XC`c@XA3x;|Nlc9YYvMsyJ66zGx~Y#Yx7dYEO-zGQyAP}| z(bj6_GNlMw86m7rga^>M`IN8TRcjp%6V@co>e4+Zm#@lK7dRID1vf>tY}ihqjYSSt5on}WWh@0Fx(Hd za~QtY;BiG3OxD)>QFur!cG_4ORX1Gidi!Bs4+I7sWbHV?xe_r^ts2I{7b3C4QYdp7 z%8w3|qJ>;7)qL#tpU=F_Ilu(lD``oj1})(kr~&*nm-^~z!YxhFW=1Fr6J=$r`_MC)5gZ(AmvWOt7cYKoec)SW{--FXZ- zw~tQ`8yB`_Mub@jh4|=6>;Rm0>W$`zTRE6rx@4=*8{;d=z#KH!@>ndG{JW?~y-Dh< zTK;*-d6?0pwacAe7iUWP0k^`OZ@5_+rx(NUr=_6KA=+ldo<$qTezI{&#>Y|Tr z%Uq1`M-$O;X=*TEN+*=%-q`3!YI5RNCOUIHwTZU!3=d82n;Ny%WF4tmM<}yB_^}b3 zl*F&hCvtMg*(X_Yzcu;$`7lG;UR!G3!lxct4K%dieZLlj6E+NNzCJ^DL zTm!*@%tqYb$zdY*vK@%rB^>3k$vI{A9~~Z#5OVWqZbl2olCndVu6LNVeF@F5@WT?b z!^po!a&oU0vzNdAbP~U< zFKp>=kL;{W_TIP8A`k$FZVt0HF%6JnpT9WBnnbckN&w0F?PcC<0_YwEZZe*i~+$iMcfMCUGQ65>S zT!}k32Xt}?IgCw9ER_{Enx)n{M|yOL)l0rfYc|2{(z^6h!?^|M{5pp<-C~R_o9$1f z|62?T{9BBr{ExUp@^7)O*U;bM6}W%Iio}15C!TBn-o+~4KREdhGIRfph=6~vfc;AK^2<@E&@X#~1&+pDUS8=Av;N6tIiqiFJwB z+3%4&KQK2b{Lo2AUrct(%_$JwzNO7pYmw65b{-{g&8Om;BjS7EX{X|wEV<=R&7Q!M zv$xU-zp>0bK%kP_x*pC{Ka$QEd-806|B<{Z&(xgVA(u@s&a$>}Bx~q5^?tzYvkJR6 zECUjDgV`#%e{>JaU_e>{it?@D#1e%-2<`c*G^D*#4eA@Vd!JL!R5m zp;CvIqlmKLs4)55z)_`uo`n>vR3+I{_-U}FIQvm8%f_zSZ!S}no*1(kDV5J7pMUP% z(9JXeVsQL2HOi8;thl2v%|xRbdaBzmy``8)ee;esXM*ABkc}MR?VN5ok+$Ncn&`N( zBq+TlP##OIC#V%iRHyuDx{V6G|8$nwTG&V8$=_USPwWtLzOrP(z%;C?7hl#!uKwg2 z0I0LP*;zqdlUQ7~(E_@FWXs6(JU84}k$RZxF)09Dg=L+?)5y#1I4Yi6e8NW|^d{l5 zO}3xHW!f}jj$jPwO3*W#Ok(A&8vy#-jDgvc7gzBcJ9mK+7NksrKSZJ&p4fe*-jK}e}B~|7HWse#c^k;P5$aKM^4|mosYTttN>T(Qw;gQ zKN}cHWYs;6RKIx~s%8Hm!qD{khyH(<$r=5|utIBVh7Es7q+mFie8xofroSdqL$$HQ zzaoNjNHDsGd_;vNO@mox8`Tes3UuvSz2!Jk7qy5GO&VOYUlu8ed4~T6jt{u<-B>p# zCg9_eS8P;fi*vmXdRcrv%?9B`DGpmfX#7KBg1mb^EE{qW`P`ZoJTx);edXUIpWJGC z9OJJq`d0sL=h^?&M&SB<9W{|BTq;xamz>gjW)mD0&4-*Y(_e8l44B|Y6%aBbEPw5i z@w#V87Xivc19eEwG9qb|0PLft z3Rjdv&Cehg=a!6ejK|*luL*Q$XVTwtyK~`fJaOt&*T)4KIY6TTx!r~ESDrY1s_PR` z-^`)kXiqMrov7bc*5T!0po4$vJs$q}zlU|X@(JB@#Bqfwn_~B6I2TZo{lx(u;W8jv zG@Ps#fY%(~X{eUrpn$$#vF*)TD=?30kkmNt<4xFpjtO$%VeZ!~v&Qnwch~Jq ze4(RXv-jl=T`c%sKdU?S*Yr2;caq4*f1PDi_?{_Q+387`4J!$Arnk=?T}n%y-LHwv z)o3haYZ#yOdM@}(EOt~>)m4Pf0Tzk^YJaRz)%jpqC-#LKXE8J;Pe}Rj0iI5(#uMt; z#coRYIVnGo?1wTe{`>A0-2Yl;ApdE!?Bx7+kaMv$GIg{Bm@~=Q0Zjk3GJ);i2GaNs zCI4E*^&d&t|IG>fKaz0#n~nZ|B;n-#*E*yBNW#VOuRQ|(GYQwfzWu-5qyL*)4t7>H z_W!Y#<16dGy9!t6uKgtAbj&^gc>)L9^%o~Zg^BPas$T!2%x3 z5FIMJQPMu2ek(e9N#RMReB@60MBr6EUAg%|lhGg(x|55ZAp z2Slf%@A@hwh2(;W5xCKF!$-A|$*BZ!ZtG!QYp58mN6D!^XYMW2e4dpZ(-}0OQn>=T zGJ2E&=cHwsV5%MS$)g!IzyjJPg`aT{dLDr-%$C)P1=6~3SV>1Nz$UP@Rz66d$ zU_!LR6HrNuk>4iI5!}IiS>P5Mku!mm6Jp>{SinP$3xf!Z>f$0j%&v7`q9MJkA>&45 zQXb?AP0l>aI7pUs-29xoMrX;7{g7{2c^DK|bik4?zHQ_Wn;_`LS)sWEs2_qU6(3l+ zuXqHxNhCG8Q&E#%%1xPM1>hbkAB2;Vy7I@qqiQxGnKXpUtPcvxQkGp8+vN!e%M-*0 z9tDz$NLNz@22&iTwit1q_;AoLHMwxJmPK{!=%LwO-WDbYt zeRl(*^$YqIZ*uM~!lLUWM}<&7;$eJ8#}~_`wTFAO{K=?Lpq2-Anxbha6+_d21_HVA zJ(!C+yaj6`&srQ-s=63tm9EC5A3-Ju-fFT$G`|n!z&xDxB6~he0aHlG>fQyHdn*cSuT^LC`a{yJkOZwpdbrUWUA? zkt&pu$aqb}VYw#h&(X4Dkx-uNY7wF%-(`Df!xudUH}E)6$h$)sf@TAWS@?@~5rTo3 z5V17Hhu!@VY5<KTibg&KREX%sj;klt8x=1_5kN+lp8$zWzjl`T;m z!Z#qIdf-~ws5YrTew409)-1o#2AfV?wJP6}EQ^u@)j2=N8d=g0jk;z|!7}*6MjCqx zenqe^)(ipIw>dyMR2#mF7&z0^wM<6cbs0)((8O3RAh_ns#?KY5Zd1#wh(xWhs%H1x zw=Y$ICy5l_8rUW5F*y)Q>a!UmAB(rN=#P*kty~-iO-%#k-$kpytQg4G;Ioa8Rv#os z+!&$+<0AnlRqf<2in1Ert(_$J!E#L#ML`aGto+DSpvWlhdCte7$ciQ5-$2>}1<6dz zS_ARSggZ%|h{=!)g%&-*Qi8$})pOX1JtYq-C|La^15pr}#gz`8LNpH)RxoxE9lawj zj<-r5jk|>ffm<}M$SXlEA79#@xEaHO7(uU?$sMh?63l{pKo@cy9SsQZX+ZE@~LTc!{7tG0JC&OVDv}W%} z7mT@3>el*>DLzNi7L*6JSZ>tBPK+k-Yq}0jM#-g)Xycce72(oQQ+Sj;`XfwSTy`E? zAET%e3S}?X+(Ok>oLG=a8D+m^KHsOL_-ksxbeJRjQd*5YnnEeg?>2;0!KXyUYNB^R!-J`jN0u$x-9j<5UPH^EsBt z;^0k9f`}Er3Y(B}-~LHqcW*!)g9{UN12&^@&j;`Z@&f53*G0%lNorU~gZ5dl#31H_ zTzP2>ImYCf0~A4sNWp6rEd#*8YbayOUBHkD&|Ad7X~7E6B!5Bn&M*XGkp6XY0@+_@ z15Efp)To8UQ2b#XJ zP%YRQ2~xD$xYZ#=R3nQtKT9)dVo>7G3_~cr{m0=H7OsZy8WhK-$n7d|4iv)lSjaRO z=a(;NWPb(j#l9}FE9eJg&#!l@B8SFl0DIzmX1HBKFn{t8SodFYXhxEGeX`xf5n0^vI<^fnkYd5w}u)1c(6F#;?I<*aiBdmoZ2Cy5*%0j4nd^LJqUU7V2?Z*`ov!2Se_ir6pKvg-B#r z$(9+QRI*IG^rxf({Z5Rs#LR@~l`)G)lvt$v zfMtei@(NIJ^j3#-*KG*uipJ|0JJL%P%;4uL&5diM8O4_cL4pJ=l?9^U1wGaEtd3#J z93Sagwe1}S=V_7#NZtm5!(^~&z~_XaZ!so;mlO2rR&kFO^y7*caFn(n;6L5FbJqMa zQXHUU3=|9~lwc;a!ivLA4DU;Ws0xyZ-;~)cB45y)Bv%cc0!o&~+^iDUn5RK-Rf_bK zLap#51@f9j-U>tDq(tjoe&Lr$=aX;;Vb72lPb(-C^s@NTKN8k`DOS->3PYc$C0(#H zL0U|ViU`xLP+Y{_^W8FFA-cFw^coKFmsr!6oK%TL#QFvi=qos=QKm1rCH2J39f;N} z#y&8#VY|=q@~O=8$@EcR=#@-Dke`>BV?4Bh=XibK4H41dE#!*<6F2E7={W)9TaJ_HXUyNEbTpV%-ihz~sb z=fowPhQakttA_LT7bI+H^2uMDH|Q8Q$_WFCEXz{lWe~-2T;*RcpMnKrC1@G0!bJ2@ zEt|Cck;0AT!Dl6SHQ6-yrSoZee*)EC6Cxz?T@4g>(>h+x?DTyGt$+GFyp=8d*4tp> z`0e}XEzsuav&ct}fN8J;8@#wv)Zt|2wmH* zS_X8y*LdA6e2{v*U7XLkU4V4QKDTbX?Fj5D=`ikXeAs%uJZZA@;esa#cNR^i2+TZ6 zJ`)J2zRy0*?8>JL=)T`R0rxPDck*Wsn3&WD@ z{MvmU08L_U<&C5LEicU8i|71q+Rl?x42?wRL}8}2?F222*?vGeW4Od{%Z8sy=RpN{ z&fhp%3gWb!bix3-W#oq(qt_KR?@qNXFRw)lnmQI34>wGlC+TzJE9qfoaeX}3AL*!Mt99VXhq{%;} z-2T z6eRa(*{jCX>|kTHn2!D&%Yw3=m}Y9W(+HZiH&Llgx_i_uCH6VDu2pqz008RgZUhdX zCP=M@hxhb1hMf3(5MYjLuqolX58QN${Mtf8*-KU`R#mXt|9U7}rZZ-7~;I3|PG@euALZkCjJ@o@ zoUhUCz|weRq6+SMsj1J6_j#IFG8C_X0M(Vp)QvK5R8vqlIpudE?Y6*L_@^!#8=XcO9BC^q>qC+eipkv!#wFZLMbwv`tq;!p-S+24Ck&a z-iv^mI#dJ5ufOkjTM?EPMIx(1!`(@Lj7IY%Ox0@5H_uCVX)>IfxOf8}x@!(mgMWDR zPI~>Mx!@~YG$KJsR(FdEd(rkvyy3h$mP3ZTi9e=5!kz;V1Jff9LwT?xrB~!CdZ5bAz9kKuIm1%LkapK%IALS#VU$bK2AeN1!$`c2 zVwFwp&&Vs}sF;D&uK6yOSk5aBERoKa#Q> z;W1<9*y_pXajYJJW%diGx(Mw*>sVQj9^KKj0o`Uo_R`_oFh)DHv5BC_kgbKLmVwqWTFF~qr-RR8Mz9#B8nA83y9fH>^1hA&YhNlx}s$tTFCq@%k_Wy8p@w7*9s@&^VIsX9xehQ^h7tr#?QeQ z{>EUFzc5kmeeN$jGSDaZ3n!-j!9E}XI!Tc}r2nz6fnGo$v09dDIKW%5qdPcRJ zdx`B@4h(RQ1M_7S3?{ioOCz441$>>Q!tJqc*{s|xX-ut^-J%-uQuszlN-ep_l4%DP zCllY#e@mw7eGyOZnaRggb4DJ!m}B1BKORs8H&JHv;lCeo1a4#`4lvY!PqObTW=|ZW zjZa?bdrS0_W*!`)wW%AXzR6Fz`)iv=XmVfT%Ef9hhqcJ@;%r1~P=>|-O9NwAja)m(!fR z;FJcJ@||N?Sb1ZPf68$1Vt>fy9Q7ux)LfkGBtAPFPd0u=nT|L=+=waBp0HhZD6#-LbSv@8mfBY_|3mamdi_^L}=}pY@A4+e*yO_OZS7YTUSEHs)&`;_JJvZ^!m* zk@mZeW73>&f<2jW3|;+_dO16^LlkiXVc$%kyB0DM_lm5q-l~ISX;lu!GtK?6u6Abo z5T|q?Ry*a&QPR+<##tN^>uFN&n_&~d4Nu7VFzLxcd~?6|&s6f$(4D}0Bb{kI+ZJ^X zvg+nMq~bnZeh9Sme#N)JG5^}WsfWn%Gfx|LJxE)|L*Dvoc(3~Fb?-Q_#OeyQ+14|m z_}7ssH98wV?DKCyeH|af_2>K*_q|SU{KF^h9d$4LzVdGg%E|ZHCAUZ!UEO?P_(;!v zZmpnufQGfTUPM1Bj1g)N-9b#0Vl6$@Jolw7I@hmx!C;EWOe4%hkk5I7`sjy|U_Oe0 zQPe`>3HBtiJ)1TdvupYV^sms(ny|o$9-& z-1OIBo6vDW?(d-waK~rhMTMkO#OWO?wgSAvTkRbLL?Kj~Z3ua>q#yTN=YYkt{;Oh^ zcy&$Dh9@UjZ&e)*a+Xf=eYb{G-)76qOg(qK#${uAwKPr9H7iA`r+l&53QX55L9MArPPvjkqvC&wKm}bH-xj&eN7dgOxqMaV6Oz;$aqk_`wnebHn zT!2-18Xj;c30Hf1EQz~%<$07_liF5@+I!1_M-P|{B!SdQhI zrTrIrEa|Zl66Ri>89nWA6Q-dO>S;fq63&mJjeUb*1vI7yL=zc~9KC@wPobXDunR=c zQN&Uf2_!fQyOZ=)&7lLv)2q{P7ZNLVkG~iVh%J_3y8N?n1;Bn%Qu1$%C4`r)D!)MI z(KGP_d9y?SE^@70 z#3`%7<`nW9_W&l79mUWo<&Zbka2pN5)|18NbXM)_`$0e4z5M458xw1IRD{iOSxTg# zC4v(RRD~hqKmz;No>&*`zEPWcuTF;DJTn%R2m6?qN%2HJ|8O${&#f~fyZ+m%)y*-z z&oi!W6K1*_1ILAy&3gS}n{~Cvg3(Q7|GsYIc9&4wE&++!UFQFf0xp#_eJF`AusVapOSNYg*rr|m0^F~jY40`V3?<{ zX-&jdk^gUDQ(y-wW9K;WXZHxLMGs+sLw~N;y2rBz3fRxqCp0Uh?U|a|iXL=@u@iw} z$+rc@p3K9}FT-;^6L7A*mes*Nmx?_Hh`DMayKx?-v*c+xi>-s9E zJFI|uEj<9)i$rCT+Iv2 zziT9KDmrFgeyD7p{J^-GiR}93x54?5HC|jjd^YG2Eqh*8=KQn=T^-gq*8^*LEVe0I z>qc~G_iW-Ln>N(eLFv=tNp>Zw-w&FjwW!nJi4sWXiko=KwtWzthR(US)Y7@l6Fe=U zz;ya6;Z6k?;<%si;V7%9ya*NI#Mk#lWHhES=~iKlQx@3~7@^wY9T+RsE7)F@R&YlJ z(>Dp1p3P#h^e}7QJfnVSZ#9rJw@0E(ivWMuvk#n?rmmRtF3!Jsn|Z}EW++uwMv1BH z$a)fW`Nfi_w`eo(3G@5zHF5kk-CENLcr)fPF8kd64GMWNQ0ZyCE%~4owcb;c0oE~3 zG=`Fnd#-5vDjXjp)>u{QzN{`o=T)*jYzRt10w0OX~~Ic@kgP;OplK={6Pah&t(T!WH_Xl9xt{MkW%T z9_(7_E%oa(*^}&mX>s6PvOG}T0V~wW=_w?ZmJocw%=5cM`aJ=Ot(IG`%9FPTh8s{I zN&cI<1Lu?R)2$KOlx^5Fkql-^&+PC%wCdoZLjBg8uA952I8K%p#aKCCih$g0XKtJbf}%da-LW@0Tn z*r+n|#4zXVFU_90sU-WoNo59yan3pH6StN6jdfgBWgFCZIaL2zxgSl@<+Io^z0*K+ zLe{vFvP|arQM3ViuLqt2%Z$32S!DPJjuhjVf}>$bJlxj=ou9e8p70b4#?;M(BExss zQk8=Wj(j2UCoKP(f)^P!#U7E=lygFS!>l< zs1ns5i`8Ha*EE#uMW8T1m(^hn8@Vh;ns7@N>^XNNVJI>@a9MEQ=j|;I1agOp#&RSzqg{z02ogc}ke)h3gS%ZFhQhLW&1xEXyk7H$&K2)vz$-g@Q zRIih5Jo~rp-Mpw({$Bgpd+?i|aUQAnDU)siu1&w(_2p_%qw}pd`{Vj-CS5j5x^i(+ z!YD$LUg17cb^|Uqm`iuiOKt8ylZ^|be}NPmRF zu2fr|{QG;eo?b|FW9!+rYx5$}Jv>tS78+>xnN??>ypfaK1)N4;#`;~$O4d4>Qcu6N zwg%XMr-#5tkARfGs`n;c?ER!~8X*=r0%z-a_G9ZuZsR)tlf$#!6Dg(6FShs9*9Xpx zwb%FIHTp?e-}hBu96cAA*Oa)UaP)34?l26y)d+)(0Pw(k;yNe0tk(5T&lg)Kk-1H8 zmIy|$4B6S0S#~fS;_3~tk(yLo%#5$?Xy+>ol zY}++px)x}T-Sap*nB>9so z=I+=&as#UZ-^6;y`p@K9Q)G@HCXdifMaG5boYIk@Y_p3Aogc&g6___yVv#wi)Pe60 z(>21)=Hz;R)GV%Sn%sdSLW3w>WK7+m3z8zJ$lbZ_)5C5p4`^-fpp?%8=;D`^Qp6ZO z7;#3S1#jJ-oQE=iujPV-H8Gg|Sz~eyox3({U%cXO`YnOr_Of+3CylkMqu-j$H1zp# z5`tyvILzd<8$$PA8q_Sd;kA7DVB2qqX7&2JbK)(OjA ziG4l(p;gCP6KB|ykVeFLA_c?{fQ+Wp$qKrNh%>LRGSX|N6VEl|$JiL^h=Bo?P1e%F zDFd0&7$@r-`oA=4Eoxp~MC1eN+iKe!)ggW{XJ;6jl#LCkrG^6|9d4epGfDJupGq!cH)rKz7sDqtRD;?mEI_ zuOkdl-GgCbw;(YcEoobcOr^UOK~#7z7F>a;Hb(a#mQvPqw;uu48&+dh_PD z#=!UAJ798sj&A*bESdhJ0t4=UGo<;CBrMDv|5}vrpJ`bB%?##0)39>?tHacPruoYJ z|9)&=IsO`x`hT0F{D=GD{@4BdUkfw7vUB|pg&CY|TrB^!FvHf4e9&S4voHf1hIuGd z5Dc;9sPP!kWxu(INxvuuks0VcwAV}6+@@{OJZhhuG$F7q$H%VROVOWow1x?<+`pyb z;^s1^JjcXtt`noXfs^cnMfNV8nF~ScJl{I;xb9j>g8UBSmhZguBc$AE^HP-F?C439U$u^0RW04}28mUBN4_ z?no0~@x!LZxRNjw2dcA0fElew#L2gGF!FCROHnaigYiy%VfZICZik??++@r40W_Zl z3obF=z-d`Od*hWj$$4(d9zfYXW|fd%>`frN-SFy)e7N7QP6@mR;XM!5?vd^o{k zqkc9b5hws;RQ6Rg%0yk=1f*m)-7^y6Y+2sl)z~H&g*o7$TVv6_8|G6o)lTMpVfvc* z4VDm}7gW?9)j5MYFbE+g@5hg7M{4dd?iY5`)YP$xJqYd5%1AyK(-w=xrYKX7KRrLd zc^YS} z5W!xY&-=qQ`S%Bm_eUi{WsHPme}HW0Tz& zHVLvBZ_cL&Nb$pd4xXqYMO5 z576CUPRl*Pju_5^=&C#WqsoG^Vh&HJRUM@Fr=Y$UF2zZx5hT=@4jqPmHDVkRyW%d7 zf3e5k6LbO)`VLjcaf^j${J$DG=ArMxVy=fw8Ho(K>tZ=5uL#WU=6%FdDCV1XVCWM( z9-xQcZn0!ray!G2{8zh$$%7G~V7%NQR$=|ou=@LQxsnZFL1`T=-Mb}s zfkm361mxu5Hb!M&(5S^=Jrw!GS%@HfZ6}b?dnNu=#iFF?QKnxsP4*zWlk~F80?rWRc@VLt|sA`e}v1 zK!SIw&0HnqwZ2edju@y$zX1M$8Q0zjZjO2&Z*7ahScDO7Gdu7 z$T#_d8kc8)v-LfpJDwap!Gs=}QYgZJon^@|SY|(9mq_v&7tFtaP+7vh#iWgQk4Umy z3;wI*D!IR~BQqr!H93S5<kWKFC?HK@n2g;NTSiIh0NiJWu)BPrJ5tv7Dm^Ob0+nAx|O~CFR|P(a(f0)k3A9cW!bw#>@9K2vHjD%tN75 z4iJ1x^ANU1ww$sIaiQ|y(TcRUk+P6QwM{A8OU2-*OETqHQlUQ)U2I~f<0f!)p)_58 zghn;sHKiA7iY=0e%%qN(v#h{+88;BwG(+uyY8R`5-+qH&Aj>(^;cdV)G|rBxbi6d9#dE zFCprb03`|MNEi++*I4iA7Hd>-*Zv_1XKyQ+cFG=`()wQI4XKQt1Tm0zZ$94SfW;u1 ztvKy-Q^7cb&_zho`k1{1w^#~sA@5+c(^L9_Vz|$6#S$WZps<#QOgW?!EC?VHBfW4M z@i6D2f@0^A%o~Qelvkh%aiQ;E=+~bDNVr%x@S!cpr1a`f3o-ZfI54N;L!l8HF!UNh z&!`$O(bGtfSo~0|DJ@j(&`BpkIGD?)jDXE@FFnS4a#VMMMLc(sX(gz{tu+v(b=jIa z^h9#{wd!f0Ogsj|ek!rdbkq?4WEt5X9fOmzm= z?kXjuk=o8;O%2|CM+~QDRhjUh$Tgt&MkQTlLc)na9uQ1I2@Qjz9A=6YS$~bb0%VKpMiGZf}6-ZII5;t&wyyGPj>JctRA{h2|O{Q!B=55 zVMc@gVH_nSr53cQ$U3Ml<}P}y_(eHEYZ!$n0$g!P97ymlOkJj0*t97L zM`j@1zOaiL+A?6#*gh$B(*!*skwKGWmgY)IU-$Q@nQ#b1Ac2~nztVUyq##USd=%-Z zfl{4_g=`yg%AQ5aPS`v|UlmzTB+f7`nbs)lSQtW9H#oBhJadmYF|f*T<9kvkcEipi zi49HxHdkjq0anG=bLQ|2VoR`z_C?a#z3JMmmO~jz! zs$BLZ|6ZfWaja$?FsrJdKm?5*MsSWz#8&u4ep$q#3^4w;F;chsRS->%AYP2jZt04#r$G<3{?_R&AVL zoD=;#0W@}Mx?QZl0nlwC0$;>@49{qudjPIM9Rot>UZ@ib$+npH2Y0=9JG}?zt#*DZ zx)&GcgrA-la}Ko|(F{#Sa2>)91rd?95pcB;qX%>ooNzCI_7-mEmz8tZTHT)j9Ff|^ zAYLDD4WbsL?AAXY-;Y!hvXTyZZg+IB^Mh^E6SyzC98XRq!^eBxKc7CBj1xTcmh-Cb zFG|5Ao@jgU(@{_l;LKc?LuA6cV9e~(_7X?v@@%Lop#aN%SfZ~0_<75*tsj>>-A-eW zPVN@PQL zF)`RrL*em)_eu-(?kO&0IVO{*YIF-HcbzarrI~HwlBdUzyAW_Exi&(RD}Rdrx)o-9 z$X2-3`(7mMcfguMg`;oB&-$gEk~9%a5yOdQRfQ!vPZGlkrnD?Y`ioA#PT&gezD`dK z?LMl6Dob)sPDJzWuId;AW|B5NJ=P7z?=cLG=ilH{dq$3%Z16rWJX}xB#R>-p#-ZBI ze~fOL`}bw_;Wyu-*w45S3YkKzh%omJ8LLY&L)F^YeIm)yI{MLMH?|`R?XJXT8oSz$ zv@h3h83yd0DNi^VV(nz$K254jtP6+dWq=i_l-gfq8_d#d#@UwAENj?TOspG*=WT!$ zOO)ERpNVvvagL=l+y9-=0ai3qYCFj`?0+iQm(r|j*cVK!D~IR*N!)xUJ{6oxX?8X2 z`l}QZwSy?~^mZR%-@HE`HG4(*Eq*N8BCf4HH??RIsIP6^oIlU>`ZhkDJQuE<4vfp} zL|T_0CuNw2YS7By@o2h<-K=hJS@M`+A$k0NoV^89Tus_7 z9K3_OTObMU?hrJ%ySp|P+$Cs$;O+!>cb5Rc-QC@#k-ziiow+mLoqN}vf33}?_ES~o zobIkur&c}BsRCNnj*?bqK<^|IRTMl|yBr-EL4cgjh7Ka$71HRYpra!=t{7&&i^d2A04Rl z%)L_X*c`j3T&*-H`=JE6?Qp!Lzt!^BZ$kWvK%T8#%v6DY^`B{)`dgr+Dv|q4cIFoe z>SX2W)2JLV1JWGHNNSA+S%pkSbZV|*;}sQAn{J81?ZTK?$=P(ZxDVh;veaBf#w+Qf zqQ5ipVrnEu9kp1pD^}if=plu$1X1xMEq{nS; z475CN6f(5EI*1Gzb|l`N2LPRdUyNb3SAYk0~@%IEIHBYjEuub}s~;oBaXw^|4w9`G`h79p1QzlHaU2^R(j@-zs+iwy<19tw>070jHP^8D&?MSL6{6|%&>dkMGSYSfq< zc($)-$sFlM9=P$^^L_U^DCu=?@2LK&-C|^AqFrcn;_bwFPFt0o9noV3d@gTU5kS0x zGv9pOv_-M1ZCP?@h(ftnX&FE5ADbF>R$DoKsa|_A@d|bv9|82lUbat8*UU?o3QjZ0 z!n84?T@Q&cs=h4tpB!oqTNqkvSomGr9BM(0VZ; zYBi@%UhG|1I7@=8wL~U|^GC3&E{`pEl8ZQ2FrWKP!(3er8p2&&of|6d?qZZAnlF5! zj-V#rGleb%Kd18sUs$)0*o#Y^#yEVcBf*P|LTNee**ZCS+$cE(C0^{iN6{uW^_w$o z|J*_iQ`h1PoHF*R0*_J07TmGPl*nPzqJ2PrG{y+*u;3G9~&}1TDm$mxVP! zo2vZ!5K?SNzX9%I(cV7csih8t!E6B5TjNFa1`=soT=BCm{+m}5gq?Z^o^=FI zS2oPIGAwvvIZpHopIGh>8hA3HA=|GSd;+>8F*cwUO^Sw({3%Ql=0=+y<(8e)I|;cv z!QksQ9p;wZhaen|6J)p@YKHmlfEKfj`Sz*?)yL18X4D3te2^0Yb%7|4XImM(Y1ybb zZy_{GS=-1Nfz`s|&W|>!t&6|;<)HPu)4)Pmtnt{KihZ^=-7yCRLiWWs{pEhTZfWLW z>Zz@i@1{7^;nY%&w?vaLSTUlVCD^ytTrzz-Zgs8pTzo?`^kY*F$tL3%m01+}??yxy z;h)Dc&VQ|acGYM>c5>mxilxDvD9hR8H1D(jc?jP$f_0fjBQJblAHY~P)BJMX2+HPJ zh=}-*)I(8~@I}12MEHuLR*E{;d(^Wi`>|0-y{Mfp!8eh5DDv}gtet%@WBc_&&7Cfq#QGmjq~hA3H~EROpnfF_s8K{=-DZyHcI?! zbo24yRU>BJ9ap7GW|%FiJZU@jhizlS*nIqk4NVSVrQvc1AlK(XItr>LOo8Ggty??C z;m?xv23lB-;P^QpSHn-bJ?y5w?)>6&geEO(>*B8V?=5{p`S{0RdHlDW{7e4!irUmi zwhYdOaj5@sk|NdpCGW`oNKoZp5(nXrQp#PE^Lcdh4`RS_ zhiyh*@8@sdW4P{bSLE!Msv*{uQs%xAl{ibeo*S`}1Yyj((U{YzHt#uK(K>QAkCt@URHsxbQY{Oaad6H8rzGB-WfSzL6e`0$ z3b{JVgGJb!Gv+uh)|XH5cqws)PoLI{MWrDm#(izwyc8NNnVkAYI z@3ZEys&%o4!O8-8kMCwsio7Lmgo{ZOpTGZYDqH*r#td#zzXY3DF_U6Mw5m-k7>D1L zgY&rG&%CpkVF#l~)nc;vPY$r;cahaFGH|^WdvY+h!HQ_s$u#4MgW1BtG(K048uz(= zJ-X`g=UfNn>_?xXypHGR!#lO55~#4Qdmq~epR*40a{i@@vw^D9xTh!a33==#7xpe!t;Qe+1*6b8 zd9yDL+*uEiH1(C9#UwY%0*B>x)1OAnWnG?p=smdyy}Vj@_<8nT@jPzKvemey*J$Cf z)EUN#kdIbIvb9dODM5{i#oa={_LZVe{N{2e}5Iz^BavyovN4!XWmoGLi*ePf^yM z+|p>Poj*4tJIFt`lzE*?>aYq za!sNNfi~Qy^#g{(Lk3%Wb6{bl$x++tS!Brwu<&jL5yvU0zfUFu>X$+BQvBrwQpUG4 zft5RBT90{N7#Jnkadk8-R^}rnn_98AC4u1Ka(0ynf@7|3eu!og!O6kmr9Wfr?!3-P zdg{?<8PSmP1egZYS0YQ0t?s~d*H}~qOjKX<84lJOO`Feot*F8`)YYy#~pXWN+s8%@5yZ zc`^9BR}XKNRR=70d9t%) z0J7i!C+R{1djC$CwKNyeJH7jDPu}P4ryuNRW^-&oceLxdAJ^hq?eLjP8j0L3g0 z&z#D2Vl%(_(!XcibH2zSi7_>MW@@CzL+m|?mabjR_E=Q$0sH#;oJZ5d9z ztf@==UwzuG2?ay>()AiJ!#EY`O4Z?s!x&=|XcBUWpL`)gJVy!E5_z_-N=jqJ8(hiO3WhTA4ckOg%%#q<_*J!TR zD4vtW$;9ul9L}Y_VJdmpW4{`!8BZJI@S*-4$Fz6I0BKd%>Okd+w&L)WW}hy)uz^}} zZ5Q5c6id;koksqx0XS4is;davu}H6eYnn%8;n-p!W@z%5U-poo-Po?Fa#O%b7jl)A zNNUW2wz%Jv!#k0UJrRw<(VLAp{=8mSJsYzXy0u8Jyi4lw$Cj}v8c;#6hT`%d#)q&O z^UdK`x_JLYQcjh$017@hxvRbP;jq-JV`VSxFFB2h?+bBM_26~G5Q{d)DG#z0s%0SF zjbvQB_wW`B>GPGRE8Qje$Pg5CsG1=sh8AlhS~xM)lvm(5$o@lmaI~Lr={zD}_$yt4 z{{Wff(vC94LAFVXK&`gwigl2ZSLy9g0}?$>y*j*Z?@<^_OP-lC_|eUR!^Yw9DRKE> zt@U#ssRCVTGiEDbS!w1W4fZn|XN#it?0V~*0H|PP>;AcYjzc5=LuJOm;g6|8HR;?{ zDw5a@s)5v)L@HdlcP9QdgA^1PsjM@NNOR42DVomlO5jJiCD-TAbRLaGCMX8SkmcrOA!-iw)K=8C1K+Nh`ge%i~Zl}KLz zRf@}9y%pT_TJ7So;WL%WwNOUOsHUDSj3UIsTv`<~R%X2n-86VIO9Z|=X`Va{1E-}* zO45B|USCHiaH9U>0tHNLjK8;~`tSU@Y#hJ0_4)7Wa4`P&erk^YeklYd4vzocPtC>n zud6ityPul%uT=y7+6;n&gZ2O7r)Fa0V&nXGNA-!9j;%3A%#RGXB%ewI?PK8{Sj0Ib zZw%aDISnll?CvX8jlBE8xBSMRHOfUw;@A|2oKK=M7eX5C6^mkAW0?#5`V1HM755LB zC16nkNP@pfIU!H+Dx^;-OL$)jFoiP8(f5W*(#28;DC|YWSeQoK-@ zFevW22&0ASQ4Q;f)+N@C$^bJylwHy8(4xzXihA`YyBCb1OQ}fqcQk*`kuY(fI`D9E zSQz%mP>L%&w77Kq&fB_$&hvTWd0Xc@CciX4v=@Uce@wnitekGf#c=!g-Tt$gxP;B1 zh0u0JIX}~h0eJt+9|@(b_oYqoRM2!-IF!AGXMI%f(FyFpgEOH##;-XKuw#nxnA+#^ zY%pp9qq>r)&zR}4wM^2{CGQo}43mB++h-VR4=ENTnNHv!NNG%6)|6t)J!KdVqRUt~ zU#ig?lNF3T#c9X$DkjCuIfuJRO7S&5E76jD@J@h+VT-3SDHyx25VabOh{^k|oxr=J zanv6_${N1gB)f3H^1{|qOm&|uclj+mHDHz*Ln@3WNO^ZCbh{u8la;G7Pkq>aXhG_; zqv2Wu41P!TVmz!8^3_sK9UMYX( zfp)2Gj*CL-ZxdXj!7$Q%m87jwJo)sE`9#{=knliHgM_0)B27H(Q8#J6`%!I_9LKG{ z!y)|x-Gv^FN^~P`P3s6*XwnT;s=wZ(+5&cs)~bopNQSZsW^}~Py)A7JDFZ-ucmJA- zz*x2>tS01BIq)+PYkFBxz>!R(C{U*^h>tBi!Pd;Jp=p58*<>SP30?p ze%&SuVLq^kOiv0a{q>-c8J4TPD3ylV!ZbrOn4{E|V_k*p^3J(<5$qk*nT18I*iWbr z%dw`4yys!qHH=0$`0R+$aPryemsj%iMb*zHH)k)h;s$4G2RE$~Q+N zj>%1Hw;Q8K581X#O9!ma8G7d&{y}jThV8{&|oTmwSrjxPeQTwcwyu+ zsv^q5p;`uJn4;4Q;f=EB4T`vmWRlgC>tq*2@AleFDqs( z{elotxS^}|MYMI-m$3<=pH8vOB`UTDleVbrMqT2w^nLOIzo^F)=AKu%sl4pY&I*;F+)RLQaQTP3Qho!G{-2GMKqmP8cgjE&D*G1Cv0s5jVY9W&;vf6Mw9#uiUbUy>*Md>BRos~S`oRYaqui~VxC6oO% zgCQ83IIUts38LT~DUgWRs$wk%esE4UPpH`_(HkSCq+)W56;(MWCnXrCX-%UId=?08 zD*pLuXC#>lKc73X*J!xMv3JpU_8^#VVPU12&CNEy%aF8?xc%T=zS!c#BUnZ*rKAfZK~O`6 zR>OZ&t%s_TOxtUMwbMHW>PxO6!^XOu9M3TBIy8RXF%iUWpco8OR%It{r8m&nUdC4Y zRivS8MqYxNu}{xnH-TrMDWTdgKjFe<1Qf?A2|1)oZ`GR7a!BCKj=9~-sVpQ;-J77b z6n0-gUw<7$sFquYD#F=~xv15Mw|7glV1WwnlV1FkQyTJ96iuOQYrDu9ZOr*XD8W{i zn=;&AEqTcvc@m<@m)0`zw)#iaudKFHW7<9gxb976^pI*&yEuV|tSidd!5n(%Hf!g= z;{=6(zMQ)6NHOwZksX)H&Zw?=NsXq*j&_d~}qvMD~v1!e&F(A>!+aTU4&BX8Z&uix)~9*@nU zAX3Wa#oHMikBz3*Du%SeRL||ihnc0U6!T+)FICV-S}T7fO`jaU?!u2?xhKf3UeaxIuPEO;TqxC8 zXf|J0OkCT(dS3*C_XlVRKk4gzZQFeHzCYeP3U9!fy4Jf#0fi5TBfX~410Ui?`mm%x z;V3x4;VXR}FR!l~W@Kj!!2H)CpQWnegUuixf&Ekt90AYrZo1du6qkS$(XB5RyPLW< z9cy_)q;Y7?y!nX$W#d5MSZZS$!uBv-YbwUFH>=pC)mP35bdjbh%m7_ar*dbD)trS!^?0Phl)T3X7kLx+z*H0+O5 zV%usSJ5tVqdiKMfS0DlJ=Loh0!`GhM1g+!e^v2?mk*K9mrRo~t;6-gMwiID;7q;?( z4uh|(t7ZJ&zvmjgdA#P2fp{XRJv%LO)E(DfhO15tJsh>GTULlqZEbPe5<%ea&&9ET z>(o`7^H%ocV-(S^bZ0XC-O5;s+n2G1mH^%tdqX|v77tv^!V^K^fkNNt1rq=aWP+V{ zkRU2!f-B+;@sV5<(^@9Ih)*9Q##{_&asfY4oP|y2O9TA@_M%UneZATkX!=JcNd^Hp zM7AcKEJ}qohGy@S_x7=g&qO7yjGyt0Sr9C@)L zR!(qoOj4h{KKj^C-4Rr^+N4|)&pVnfHot&A#ReAvR2loyD1LQ*$6IgXJ^gl_rQ-0H z@oPunT%z~+Kw_QQtz@^sQr z;Ag{%Sw-;{=C(WN_Pl2=$Ykx-yeRj|@MGx8O4z?3MKw6#YQ@bd_`uJOb18iy(`yRD zg$DG(q&c_yHF0T04ux;=`>0ZVaB{+p9*y>tUx918U!|7r)Z^2B+KAGg@#c)J=+foU z)fD_27-c_|t*4;u;`~L`lQ(=W+C^Enq{5{+Bfp5}0(PBEp~V8LQ{J*{TUQJ3cNRziQvbkrk61m&cu)6=y z70|T(I^$+8N0Ze_UT=O6(S|i+6aH#iZ+<<5LkU`mrR$yEFLT#8`jsPe<}tXeTH(P) z_ao7678|Khu;zW6RSBHdP7sNkU-g>zy~WA&0C1W19O} z0(%B_jh>A*!G+wZ=%x)n-K;TWg{1Z%rAY)Fvyt{RWpyBGN+t88^ZZ(wt6Q7qy&;I> z*|ed0;MTTz6zDaaA6R77Tzxg2v)ad60`&q!e6fnl(w*?EdPrTZ(^@dP;vlkWXjg(g zN2^m!Ssiiu0uxofK7sY5u6ACc)78;R6PF)+q8j8?H|%o9;iFYQF6}Zk@_KKU?zi_f8l2AkzY)s`DY!`eiw^e$V^x+=fp zMAex)Jw>_!)F} z$HCrJO_)N@nP!4}ec6Hs#G=|mYFMl>Ix`r@=wTE02b!$ea12{yH?&xBAyAy1$85<0 zeu(GF+C(B=99PLlysb=Pj6?$1*Od$;q5$f+NWo_aN6dE_MysP>uhmWw7IvYJBFH_6 zH5!n|V44<@PTri4vxp#=t3+e~#_x8HmmN=SO;QXv(naMbMiMEd2Nua{>gZ>%>1d{J z`JyHqAu%O?xyR_d1LS*RMz;gcV*|sfWf8gFHfo@g6=;yGz?toqVTFuX3I^0U-O3_gi$dU)(7+k6(xQk-VYrM~O#0M@ zoyxtwMIjJM!wM*pA%ARS?Goo^7+G*k`p2D%c|sX7ow}pnyi)MAI9Z%i&GLpM;-jg_ zRm6(F(f#-~l8wmsPzaYQ3I_T1Aj)s zx|V#TT<;4)1}P~hixojlf$!{yc@a<+Q-X`U-|K59+XJOuMsg|1$M*f#fZyacj9EN? zV3!1pQ-Py5$GM?z&`PHO^A^x2tdK7FT|5zQGQ?2jWPXA>X~U-v(nq$$1B$N2v>D0c zAMno@3y$|H94CZdyr8du&{yT5;MbND_fot_(=Xf(1UmPAC}i%5xn~m`d@WxDKyv1*XYLO}~CY-E$#xsmz_ z?o-_v1S&RCz4Np*uK4|Io0A=0vS1CoL_S0$TVt*5?7*>kPHi~Rc+?7QrVuEC z+YjP~9;S!_KTsG9es*%oALdu-9(qK<93Yr0izZ?0*095YGlwJUS4 z_9}>+)rrc=(?VJSIr2Zek{mSTH$?RXr{eQ(4%%EPf`@ZxrFJ$%KY4u^=FXbl0 z-1E#R>DI}?k@dfX)vbs#9qL?iS-nG}SlHBTSDIhuMs}PRwe_@WJwpR+H3?VYidIvWV>+KjFxhZWUgKq`fB7-R4%PIt;p@@yMqG-UhE|^{%~s zp}Dw#J8~W#L4Dp`AOLBu%e9P9=p_kRXsyn=*aZ?3X6x+$#@PMA?8Q-bcaBxIi#FhG zGUmm6wM_ts>Ij)RgpKNmy6LqU+=IT*Xqz41;wcnIIh?3aj3Pp+s(;;XS?J!vDJ*nr zuhv`Wcn0;|;hbI5)V(^`=jx3-yI#!nS-m``_;@(ldsJ<_`fRW5Oto>~s69F@M|f-9 zNp{4v%v9`pyIh>~ogHg6x;(AzZi>&iue)ro?OlCm*adQpT-L4ZtfcQ&PucDhOAqkG zlU(z9TByA^nCUI96ek2%JOP!bgo}B+K-hXGU$L|T>^iXUqU2}s13~e8#~+qB2yGh> zcW8{q@twWkXpXz!J+zrF=D*$?dG2V~Rb2I=7+fdTfN0tSRJl=6< z25J~zSpz%DJ6gA>a(NE6axUBK zom#*!jL8UEz1BkPgku2-NlT0z>1Quv|EJDK`Kzpo0^~i!%g(LGSUAwJ+QME9#^;Wq zR}hezgJfcskiEr;kHeZiV6`R_iU$wiHX4HCq}_KjyQMzKYk15fVTuy7(2_maqPLeo zf^1%lqcLj8-U?99cmG0+x{Mz`FaDwFMk!S%(IMzoc5$GrMUexp=#*w4Q(M3`!{pWs zNbDitt*bPrFX*VFkUG0K z(03TqE*UZ!MCn^7{@nnclRqNM8#4=T;+gUjN_UT9ATjhyE`TpYQxXic4$9Kir?S|!=*APfQ_(pLmvswic`I(3mm$&o?b-Co4>ZT@b)T@e$nj|AT`Mi|@_z19LW*mNM0E+M?^p^D zA(^CQmsll;xXJt+|w7Vv3oa=(W`{>;}! zvXE+Pi=#0q`0}{x?o#9R3?z3-fa_>8KypW&_3q$1UInU;uf22|b)`PP*j9LVbhP@6 zeVwFmIjdg^h`T&LOdZ)ddD*^OyC~kay9Awq-?Dg)qCnBrJ1eiJIHP|!w-Q(duBHmt zHorW)Jg?|2yeIZ!ogWWEVZWK|_NET%C*>WOTvR}8S7-Y?p78^%8g~UhWCR#KPaV{t z!E?ECo^wz!Trv4G+}l?0HpnO{+~_gp-8=!Xt!tLRvdwQ+ixA${#lLl*S9UgQ5SF^r1Jua^QL#=)WDK4@9 z1}QG`|0D&XuB1qPmn45W1^+HW;0IjYoQ)6oze)2=j5$HyKEcE}Mp=N!QpdNT0&~|e zY}faT`SP`V+2(Cj_^UY^becb-&81GwX->@%PR(7v1r3;YiRI8C=Qi9S22X4??HDzN zvxkTi^yF^PSzu_|I}dh7BVrB;{s+S6NoJAU{T9QOyM3O4HThQ&t#)TS zPCSVizxT;ClDnlbejoA7!IXR@x!W4!M~N5mFT_=0JW7LC7~@Bs%p$e>E5?r&@8=)- zld~V=N561P_aU$05*HBb$M|f9KKPBdm6S;jyoE*QTm6Bf~Zdae-kQ z`TtJ@&wofCAkV~z2_;k|#xT3zhu@~2i(_?d7@=D_5zglR4`b34^=_?XpX2ZVew${l znxpPani?CbGL#S-DrtU}SH4&ES-0FhtS8%^3KHfF?3iUhfft!~b<~{Py}Z|NGU64Rg1c#)q|2_J6HaT96!yZ$+I!4Kq>tP`pouyI&d!` ziVFqWT*Olh+DL>}EYPraEJ_a08^ZYw&>WH!4yX*d54*+MeLI1LCW#h}`Su?eh};wokJbglYmz;YN^Ch*&+$Z^w_b z7ldnqO;JXOQ~3Te5N2@+I{NOu8G%TFVudqD9{N7t2uTS1zvDhW@SiY`5BwjPj|==KB;o@92O47o{{fLsDu@E;G_q+LG&{*o zs?BNiKdA2y^_4u1#<>0Y)?+)6&!L-`%OEPPQe1YPoZ_G6GW%{ZP6P6^N;KJd0L8I0 zhI7l{C(qC~BO$y}o$nxmp{LO{e{FswMw1VwaT5_FI^9KsFbc%2ZY&cA$447A^n6&?Euioex2`7IiVr{JL69xm)fcUr2n6l zAvT;DJGFUa-=NB3M?*E+kH+&peLyCPhHJKe4B-{CC(I%H1C+CnDcC?GDloSFfGA%) z^zmo`elYJUNw*;kP;da;fxLo9Dbc)wClHo)dC-w@ozXFHzxeb|AskVXJUW-5c8MA9 z-qgVr^8GG>BJbh52sKTdb^8yK0SdrF5VsTw;U_D}Yap#J=kaYU+-9c`>h0s()aSqz zukKrZ1D{NoyzZo9G)DrDr$8AmY~DcVF5UiqegtTUqBrlYZ)<#&-=N>VY5-8b!wW*f z36esn3%>QEJcoof^o5DT3xQM-C@>-2za7{9hGXcYKWfAn=YU`D4VsyPCroDeBB;1~Fu-~2cXG~dqp z3o43;+$L;Ho3en*35_ZJjLy$3+P6ViVVh=cZvCH;<0E6Ab`mEkX_jG6=w zsUoya4KRlBDP9(@8A$>OR??H+o0_S%?Do+Gm9w6w9I*$z5CbZ50fQCbmKt4-) zqxqeXHf@x%%LG*d5i(o|&lkqF3sM)BS4ao75dqRoNS^_44dd5Aii!&j8z5+a%83YR zA*4?SsD<%sAVtN2-t`y!-%IxU3u>TpBK+r)8bbOsfG;q96{M)x&_Vu!3aFg$ke`M0 zKLBc>{cK55k)eb91es7dVIYe-5x(R>hA{Jz0N}8{!)KtRLHT_ltwg3Z1|$@BUST1& zX5p(#pl0EXL#}i}vO-C~R8slMKv=`;L(TdL@>GHw{o5n}Bvu#X^_%zl6oBjB4PoB5 zic}sOdemP~6IC4n(ne^V4$uqZ*F-9h1AXc*sEay}0O=*P&H#Xh_3O%l9CP6%Q+-ng zX^tgW@B%F9y&u%DbXM)K9Pz7N&$Y@@6O4f4FfyBp%$V+a&&_) zs10Tx#`Y7b>|eKq9ThbC*Ul7Fa|}q^?(RV}Xh^#5Pl4Y_lT_tNp^*ax-BHz1AWym= z!+Zxxl2Ea;;MB|Q{IKXzS>O8MyhA02HVp*dfDS^DsPib0&qC`00BC?8gaL#JEG!M9 zx}VTEIkflh5}VYhcO4^nm}W3MlL|3n(BWGW+** z-lBt8Zx5!DGUSYuU-m>1F|rMIEUIeM6jiSfUV1mRm8AZkR-a;K{O_-gnTg}S zKMynKfBkuwnYsS!C}n0A)_;xE7PGMaV&cFcW^M4rM8w3%*4TtW+5~9!#hjFti|yaT zkiW(WMWr&Lem#4FCus=A`WAX3680UvT7v6~%S>%SF&w09*fj3;cVX~EliRVg@DEHX zA(D&>50;?Ns}s|s6y4s6_A=*soZ$YM)r_U^iqiQdqb;(}IlyA?_b(N)COCcX++6tX zZfl)_*){1!pXg{aJbbWFN}5-e7!;m1t&Rv>Fi`|9b=FsY9KFoPI<-mU&|=i_1xBc|tJeU6|-e?9xMwRFs8S68;&2<5NGC z#;4@f##y#f@Iuhh)`j4+w30K&07&#MZ+FS1;SAN+(4Hd0xs z!s}LYP3#FB+!%Sf(TMpNQ>5ugNcTd12^|eN;pXfzbzyE|@1qnTgA}Q|%5HYA8(ekC zgjgJYXMlWuyaB=J`4~|{vQ=c&^3LkM4Z;Ic65>DJr{K2puN(3GpW6;QGb0!4-wlb0 zl!=Xv^?&Y)ckBhHuRJq*y14f%FY)sSV+qsl@A4l}xw9pLSOM8E%#3g8-o0fLq!h}- zL+GPSusWM>^XJBshh*P2R6BkycD>Ln&r+|z34B`Pw_<;f$?)>(G3K(C8t-}K<#AFs zw_ujpcDImA`99>QKMIBl=JswPy|<4pnuQKB+J_I=+mS;p9)^!{c?%Wq;frv2-j&>y zwRn(#gUrW^b1c10R!)V-yg##Yr2c#onyLPAOXPHRL*C4Su`^&cJCf3|YvlDtL5Ei% z&r$WnjV0GZGBgC^Z;8$jpL&3sTBE6hf{U%qO zeZ<0+C621f>8SQhXzwVMh!@IOg|_aTm%EtH^q5$ym-oozPy_HupV*@3lYF_3JXc4) z{auDN`kH%)`-f>d?dsY-$s;=fC+2pby^PLsY2-<$YKvV{#pg&-`P=(xip0}QV!mw( zONJ>_zT2DP#19=;)&=F_$RsuZK2(1zMM{g_AZMX3-+D7DWx&=|O!v;8(?g54(HYw+ zU*I&%xem-NvS~HtFqcT?X5>p!(>5iUl|mlG97*z|O=k6So$$YY$F`Ye9m%uG_nOW% z!Ew^GxPHU#?!O|t|B8lj_wJsZu17XUboO9rMB!4FY8BHdhhtNP_mt5tn_2iKnUA8GYS` z(Y5h<&wlT3xjfQ1iYHVL37O^$$WPAn(*u-cNH=?4eoN*c*)?`bz+BtmwNpNv`R>BC zPVx)l9uwPy^z-`XEyjc}*`@q%+_g=+V$IK`6)tg;{y~$zN>2<*KF>KynI-(emrHSL z`BY{eWn0fjQXGdYg->6cx)6$Z_sIqA_zY!b})Yz2I>k7!k@cy}ZXCwK4PBH!E|{Fokdl;?2+0xD~^3R}JE_@El~ zrYc$l+=~w{p;wo#&3qh-D6k?>mf@&xU#0RI&dpNM z7S&6+vU9%nBKFYSi)`H@Zh*M?2&*=)OnU86QR*$$Oc-@B=9Si}aFeQZuI7|TuQxEO z(>$1c)67n82n#g$ZpqwgFyBWic@h6Gr;I$6-Pd94cJ)OG=_;jnRiGIM0zthKvOj-? zwmWW?!~*m(OTC}9QIU@ob3s&+I{OynL*XvXGIg`-^dUdYu2= z64kSozYD-__KqcU7LrExqGjJuzpo>lr=pJby@7?O@s-ramv>lvapF39x#87$wWB9^=iP zTP{p1r1tLZ)kIoZa_wmT{%T&vk`E5n?#B3K%@*aRg+TuFa{-0g4Ee)#Aiv>#2aN)` z{NcW~PAOx4?~#X)vVnFDxt&E1!h#j)*jiZX-c|j{EsGd5HM>KXT3;Ai{lC6#?8-mM z`PZe79Y*+1Utc|byn5spzm}aPi+S7=|qKI3c@vEbY9D2Q;{X22DP z*VpUEf4i*z(=mUaKYdlx`DKC+Ft3q(Dh~Iu6v?XLKk4%^_8u{kX`75?u)~EihFei> zD8DOC_$zBne{|}{3HU`3lu6aackUKYip)yxdDoH$naq?X5|ZV7)4-~RhJtyPRrQ-XH6P=sJINxDrSMsCg;A>J-Yo>`&NoSZBl}8*(1@Gd({QitK4QYzhZ~3%Tc&flx_*hILhJL z{@P*i_1X2=4aqgh&FB4wP(#Xum~%{KMlucDSWoIK!*AkTGQudP2r?oF;S$(IIpMez zLw(9j;?rSYb98AjO8P!*D%vMEDlWe#Mr-Mv+SIs6b6vdiOQ8iuKWEE1_Q}!|u@P`U;7E%PSX%F2Gbc zA!5_-`;<#669HXR<*VtQ0({~S+dn4WAu)1KuASmvrR8!0WyVDeg=rivXN<~^N!yCu~hLB!&&eDN~} ztp@?&^2#B&$ss6%#rQnSZvpD@ZuoG5uSQDDokCn(OVRV&2X1CwTGscRKdmR(gpy$* zyWv|na=WdD?7rbn=p+!u+bP#OD7&8gf2@5~P#r+CChqP|(BSS8Bm_CQySqzpcR0b_ z-QC>|?(TkYC%A+FH@gpe|9#lnms`~jBM&oG)6><{-Sd4;xUDg_8;sFxKAY0`l$NvkK;JB@Eu;^JY zmKTv3s(KCpLa?tZ&CMKxQeKiQj!L<_Yh&hlRBv)=8(AD4+y*Ol1%$XWkv39pM>uhs zbrMCAJ^2mJJ-j1lSLb8T?)tadSPmT{S!ZLcHv**68bTj0u%WBj4$f=D0hF#g&NrgE ziY2?)$qui;;^a0@-*lxqt^&tXifdM8`KO}=3;b!({!OL*y~Z31Ex*GFmsGD7B<&{I zW$Jcw`H)Kfo$i4IT5@%eiGnCXvi9!ke9$)4^?d8vN#epf3se&wtwZze zkp?3nlcgyqD{^^6Zc|kk4>;ilgsR!!<>!~3#Kfz2+GX+%WNM_2JQ3?R{&_P~cn)u# zFb_KL%zF{Ayd;vEM00DwUpj0%H_9ru2!MZISJ+jQj}7}Au)?GyKS5A0DKkA9D5Hqk zOH^bZzbR^- zwN9HWM{SZLh>s42F~6$gSJ*^cYMPq5VV|A3Fz9B5QcQWJrFxADOp@#nDoABsT0T#e zV*HY3tQi~BVIFcnp{sTUhiR^9t#2)`6*Y_>*;F%@rwb!o6wWKFZJ(xt2~D^KM0T1s zI>ikHQ-A$%wq{YWPToAkAO>XWnGA-y%%UMD_;ayal>U%5WS&GN6l zoi)@Cm1KLEoHUpys(M|Gm>!$x8R2jfy&uke-m~?L2#>Nq_nZU z_{?p1mo^tySZB3hwA{ga0%~M(Z}`Et2fgqXKUR6T0U)v{FL*hR~bl1*`0xpKeVG zV9-L&gkMmCbQ64-k0?M$p4(by8Gm!Nhe& z(0FWU7;a2h%w438`WZRTXG%A6w!MmpK7t$>bz<9rJ%ME%U6Ur*iUA;GZLLu)S?@Zvq(Q;;%`5fYzG{(4GOWKYp)8y!1k!R%zjyb7|9rh!A#-9>p-UZz7o zRb|X30H%+uNxp$3?fO-3pV-XR!cZ}~#1anbebAgM;h3!GlSOZn zwT;ss#JJn8x$%AqNsR})?omO7FoFx2Uq2M;R#=Er?v96FT;ev$T($K>_uPy%w9Re1 zG#MXX$m8aFrFmK^;YX&y>^l@6k8XeMi*~-NRue38{<(FTi;$noUCkX%Gs=3(&J!uL zyMlJH{T;Fk+jnx}0dfr4h3Z4}hyE+@LVuG4{tAjaU~HCoi~j>v66mj|4($Q$2x*ZS zAs0L_Z=lrOnspasV?^(SY9x7?RNh5Q2)o`A)fPqzOz_Odr*|&E!l#?hA9@8;E1R@n zRV8ax3?b3XsG4+OoiH@W0ljHb85$&jKLAVg4N5>*z?5d82G9#IrEU_7)s&K6G9*W1 zlU%M*2mlfS)EJZthFob%>DkqR^Z+@HLOmb{jSxUi#iS6+fd-J0o~e-V6yP66iFYj0qK?sz3*Tt9oG+R!+*85~V&+0l=4hS}u`^ z4q#B$D!?4a`e^Q3jGH zzZd|2C-W5#wE=vKhQw&l=x$rEK9%+vX?oT6k!ikZ?*9h(mJa#Te3TBA0WMYdv1z_( z>{HQvR1QS|eDjCi0NrInU4ZVQp-n({$xs%cyI|-7&|NxI1-Mk-$E7jQ*r%Z}P}_&4 z$<^HFr^!{{C!~3zyTzw*XS{93>Pq$m0KcYqiUUVeJmr8nfJ>!9ZQvC^sB(x3V5e$Q zh4qalGx>($wjK+e{x%;=H+ijk2paHL8~B#et_bu=X_o?arL@Ze1ykC;1LXl*wL|uR zt;!*0z*hCpzi#=zP?cWYkQ+d+YKR-4S2L8C+ztRTrnHL#CsWqShs*(bl@di*X><)@ zz(+u}s!0vj`5M(ygH*X`oYE*+L9DPisZXfD6N1|%=Kp7cNkcp*AXMNJ!R-U{{4dqg zt5o@YoYDz-!HmfN|8R;i4%KtWJE$30Zw3c+my^Sqhrt`#s}t$!&? zYBQLzo|XyG883_fq5>a8)b9Fik;u>n#)>;lgT2N~r{6V~)ttbpgOvu|m1U8m)Vo%9 zqJ%~+YQ1ADEfKlWRA;O?@cS|6GRO428y&>7dFCKjmNnj@oMD8o5^7pyMrFDMn>vR& zo4jS_$W^I}HPWJ4sY@vbE&3vLDNxR#RHYPA?jQq@KEfVjPBh4sX^o@Gz9^m7f}l3U zHDF1KNHn8%+JqN^}pn3@|lIK3Vc;Xh1B7tkWDws2{RWCH`DWprLYRlvKb@5Np?;;i5w}v8&Oiu=6bqH zhDy2sn|4N`-?*vhRYs!kxT)AxI@Wpy2xoSNuT-m)ptgzaM-%5S`b7HRi-C-RbnyN18bH`{d2Uuv0=YrHY|fFHXxT~e0e&MrLcrpO`OKQo#^X1X!n98!@% z)%NpDxl}5>mEHc4<~Y!|cW6AHmX>vKP!644c&s_hnsX7lSK@MYe>VHbzx1vB+Mv`Y z{o0~4>P(?jFr9~eduH6>fT@jRd%%sCb9=~@Y7un{?9HM->dL&xt>?jQ`HH=$uIE9s zh%1-IX>ZmUO~rboLbXUEN1VRGS#wXL7QFHY%Oa4)qH`>b4cnDs5w;hGx4b_sa9S(- zx9lmp&R}z9R^7fkV}`~HR;Ad9^Zk+B2?w)jrKQe9bEqoo@o`l8xi6x6ept30q|}UM zrFrHkSDH0iTh=sJln+l*)+(F&Pcz&_lu|$`Qt6r8;!A0n%WNr|{Bah`$&$G*#Xoht zzLw~X&uU8MkT)WlRojp92pW0aKf)dPvw!qJb`aHvz_o$DOUm`@`>zbiKl{!s7(eug z`nj@m!g^o0^Pvp?dMov^e)uZovi3qIUqms(pqjd{AF-i64)4FS%^LplC(^3_N4^Mj~yn8)KWgI)qJQ(-@L7kdl#NAU6XD^iA> zg2GBn1>@I?fu#)k5c+L=Dya(Wq>!KPhuPNIQ#0Cjaur)FVHMCxDfAf?J+mCR$%jpH z&@K1cr6E`uSn2y2`{^bfVko@z^_-dX|67l#2KhOna|+}rgu?5{|V`SJ8 zs%X5!<@dE*Q?9IBTdp)usX)eH!<%=gU6G)?pu9lxy-&y&GPm6VUxXLD7dSqIPe`!s z(mV7~T_9@^Yv62=#rg*s_vRB}J^2gq4ZZIhC=Y@hn!Mnz=1l|$9`pwCNw~NBE&0^g z1MM%^xbmc77JelUMm=4n%4uB<)C}q=4}k4WqdM6%2=>SP1ABTxb+VBgm2K+7>lBVF4>NcR+X3 z`{rJ`pKj(m!a4-?5_UL<*AgdOz`fMM*Fx9A)d|{h<=NypaXMh_@e0XQLw0~##4bQe&E0nwKa-(xCQwPDlMu)_jUhmN0 zw05(5uE*%u_gT-?N#Tj<`Et5xi?gX}n>(v&TYa5#D`g$``fOcwdyz-cO>vyXLvcs% znq~5-{VRpPiF?InPsfrlMc0t9|4*Qg-4nqx!Hbk7ewUOzi+kJ@*&fS=(15T|*PMP} zRvB6<+7nvm4|gLu_=nJH%C@p)^KDCEGwy_C+imk@)1oB%ah=uECrU>c#VLXrXEUBJ7;o0E4UrriFZ!w zF$v-I3;yexlI#=0<2kPXZGDk?=*WB|Pv&N1Qbf%pdDUS{n!aOrCmE3}uh&`?;Go^m zCT5d1ImfJmEk2RGn7YL3?5%gAG}W8_o!xstHC2;Y$F(^!)&aPpH-GBun#>GHB4<|a zWUO2b95|KMEv6OGp?s9kq!zmS5Ri&yWR6B_3$wk>zG>LYWEA>ZH;22^W4TBF+@Zen z)o{JTmFQb3v-UZczcK^%W8-PE!E2Wp{ls0_4qehcZ_J?RVj3uw;bdXg;W3~KDAEc2 z`8pDiXsH$D&aQ5Y4~Y+j4}l7L1rZr>*CS~mRzu7{r$V1VO$K@!(bvOh!7M={2f>1T z^YPN*r(lmEnu4@Ja`|xOaFv0CqBKg7a`2^)szLN1vwRXdXdw`_5hWN75QGTw%!eU> zkA@wAC<;;*WoN^pgOY?83=+?W7=ynH)H5OgL%9SNfcQZeMr>>d1PEr((19PKBKfRy z@TaH`P~K45qUia8@rZmu4x*a*$a92a|8Q{>0-;Y4iXq)WaK%vcqVOOjYUryV>U`38 z1Zs#|Q4$Q~Aqc!LM3GRGK?q++5TM0_Fuo8VKurZ&YaxGg6NQEmRbvKYj|LWj>Oh!a zwi{M3J~$6F3sMBdfzUzEAP*1)SQktT{^A7jLi7TEgL^}B1AD`N;{oP061m|8s~VBt zAl|UwVBTngzk{K{eqdp+KA0424`v4cvqNAvFgIA;i1h~PhT{h7hVlmLM(Bp*hUo^S z9{vTnp4gLU4OR$N2vG=92u%p?62={38srXE24fl#*W=zm$_L_m5_lqbVyr>g!TpDI zf!;#hg55&gg0zEjfOmj)fOUX$fOCLyfN_BM52FI32djsu2dRf<2g3r*0!tJm8;A<> zHX^N0t&HGCs168vWUc>zUFie;!PokPx-c|)NB(~g;z6%L!#qI*35yz`WBiXc|IgZf zK`pHHPp~V!pbdPjHz->}qbua~#*3(oTrrK#Mao`vH`TL!t$(hU3~|ANw010mqPcxzvkLh zF@`Z)NF4Y*FAeh7NtP#mas!-c-w+j}N{+IUe0NE@I)`1V`RP&zqFWO-h;Kf=#I}-* zFybGjel%9sM8xvw_~QF+Hblg2dn^znEAh*h(sQLSlw&=!e(E;y6J&U)$=+KBDeDAw zYQJOy7hENbeb^s>r3$(8Enb5WgqtJ-0VuU&0nQ+HeTy^I`Ae}`(@{rNS#u6r;~AyG$Iv^ zP!*1tm$HYu5rn9>4@T)crjP#KIw)#^duI7vGapbkxz|^(9%cUFoScA9k=S(eGhSOG zj|)q4pG+&Sor*=@cIy+-@k05PxsE7KS0`p_T2iJmsmF3*b-qR`$OEhzTdC$D-qg80 zSM%OR8new71=Dm2#RZm2uCCi6Aw|%0@^07Yjk6c;JFGe3aEav^ z|D_V^g6soF4`4RGdG*DYtYq~R)jMeiH$kaxQS(IzP?sT6L`M`}_bJFe(>BMI`PAp) zA-Fy@o-CTB<8RI1c7*Q-GmTTiC+g6zx*?giNBkCyoiHG2T-v8BWN-TqoD1sOTi<&}2q}b;si7&{?Y?P2T zG<59ekF4ggtMPIE*sJ$oo^$pe(}i!X+29zyoD-7wm(!iAs@XC}zm2N0W!yZHjvhSuH~!Yur^OJSm2-Cf#vaCIp3xFm$Lb%6#Uz$@1}8gA zCk9^^_ya4_|K(tE1t%(EtXYBDtAY9raeL&L0fbvHneJ~{x=Eh)b8A+Xht5hfzC6Bpp_2y-1c6Rn!5Oh;xsX~l3 zFB=?3Y21;~@APjy-bg2EpPRd9&7CR<9F*^)8!P;}Co(F)*fM8IWHJM96O;JcA%P2P z@8s*#^o03D|HlseUrpwoU||X5Aa}YLgMG~_A(8Bf7l-w{Bvxq0ZwzLQk0qG{tOO2RXEG* zp)@O-^+yCQ^N0SQ8^5G)3To#4W4T^VYB1pfI=J;3Eu4QiAnoGThJE$;Xwa+!D~yO- zu%2x)4P*t&(w`W`HkiQus;*cOn-73~(RCsfgSy7~PlpUL3_Q1BmDe5!D(jq-d$ z#D?3|hJqPWPidt`;ZLnLe^sPT62Z)D{cj6xUmblvpG+4bW*OIgJnO%*erMvhTzM(j z#C40Auw~V~fa*u#KI11p_Uw0Y9 zF;*D%H@)Q66`V6F3&6?wP=+CoFGBc`j4fOg`8D&k<=dKhk_29fa`@Q8usoYEZKtxx zIlVVmn8MKU{fZc3!tkKEmnFJnH9d2MwwZm6eAJ{BUdE=)0qp}Pca(#!?OWJW^5;SY z9y7a>Pd1iI=ZF++%Laci)?Y#3mMPZ0DmTB5%Xuax*H7tg+dmV9+^qWlTs*hBygcEy z?l~fp6?uw13gwPIZp-&M=uXs~K{qxR&iN~eW10W%;Y>9Ds$|9c=e_u1eoOG@>}%f7kCQFIUr5|0 zm`mv#M%#iPF}&oA-9bjU?<;4lF;m=^o_|3eXutjbcB0(gw9WXNs=>|;ttyajeDgQ~ z@i+K=_CXd0W+eIaPgOO11$U!&?Gq{NZ&d%c+=r$wUz7ymKqUQ~KCoM97YP@^9yI*5 z*FbO;8s#{W!1ux@muBbavg&5EA#>OdGYv=KZDIwX02$eol;1WF95Ny)*n^P5@$SFm zvCm$DkkB*p+{A(M)%?grp-s$MJBXj;j?MABNnmlB&q zgq_s+-pFZX4y2&AnR96nN~W8|x;pvPZIFL&Tod>t6T{ymnz(Tj-Qcm~On$mW8?8SO zM8jHyiM56r7NyetXL{siT6fm`C6rFxazbypfm4((ZDo8jY z3uGU)^*K#~Su&6v!F})^+Ckfe9E_J#Ten5LYKQivkOP-+@xT8VW{Gz{k~1x0h`%pn@R)~p5OBT(qg_crCQ ztt|5=%6zY3>N*xN;|p8?M=jlVLvDopUxI{ilUI!mU|irE7)LWzsjVw{$EEsc`O)s7 z1pzMyt&OXVViYW1L1VFzc*bRxp7e#H{gB{p;7e2>#t20aj2ad?s|#JE?7sCFLxR&8 zjP0W!sThz3$vUygtA-~pg$l`!57eWICN0}r2f#DP>tq4|8zW~Y2A3hEJHD0|tdNvt&utFOg_*$&`z9nO zSxe)|%Lg^KzWQ;|k78>MboHg`nJ*Lw2VG3pVXK48G~1N^n0 zbk}TF%XzP#e|a{wv8%Hld~uiYFB3L0eyjfSzSmvBcbhdes{4nWi0^qdkvO26(#%Qi z&+KQp@%gXu!<)7UAUTvf`T%(#PmsL+m};4um{G9vr7S~Sn>e911y@b{A1#WOf9Zkh2;SppzWlEE+Y#!nGa#Tu0Dd4zedUaU-ufBq# z_2Yi^%G=Wv_I569hIEJPc1?;C zqdfT5>wboYx_mzJf|iPf4K?fO?*AqBm0U0h-yg&wy$c|Epd5#O8lC{O z$x0EW#r@Wr@33!Qw*fTQR`=WA$jQ_aJOQ5DQKqC+d$Od!Ch#WyD)wwcdFwQOaYlynh1TCva=ZQt z^d<<2?NkQ)AD64XHCN`1PQ@4AwkYm7S0i$Q8<@VrW!{uBjUMo0UVE{FG?Lb+H*_gt zu3Rq|MlV6l+?(*{6H#pF($utJ-4OkKpmnBmd}M`!3cPCxH3{gkU$Kl62|8 ziF}(~Ux2)cNxAwz2fdg-T-IHYhqugI+ zUdzsOGgJc+&O4Dx zrIslLwz&oph6L2JRrG9W-!J@(qK0qC#?q&bs?n-$XozM>ZMk&MoZl8h%mw6$?6E6{ zwM9NlCslD^R6nJU4;*l@j6i)v4qShVd_cir3NaK11qS;$+p~Xy>HQ6N24}ra8d! zui^PZj3bKv?W4AFPHK_~JAbiI++JvmiDgNpTuJsLks?fsji9CRP=Z1bKv*0(O3Z`F z)SI(^)ILj?%1k2bpZ$W?TeFOIB76VmFZrX8?718n@hg+1iFEc7E({!~)Rah|o|Iz2@ZIM#UvWu7yGOzxZ!?<7Pd@Xv!vxIGNC4 z|7ofzEqgRP=E%Wq#q(IydPX)J54pOTf~o?c@dpK z&z4ZnnlUwuE0p~)+K-U*UyQw1ca79<^$%SC>U%?`dEWhKNsUyZAJz*{ReHVAALf&X zW-Oi*eZ{N{r~S2edSBPikAW9%N_WGn+77il;^c%1(~Q2i3)7Hy4!2;j^bfLd)5JM* zeL#`0p9ZK9MDH7dnQdi(zeBYH!-;HKH9zG_nVm5_=)H_s$F!P2Sy8Rbl3&L-}NcHp+;F0!V;tvgK`ru!$&wn=&vhAnrR%TUE*a+dihhLV$)y=x7fja&#hmM zC~N2j3NmP&Yr34G%u45wSU^%S))!#;zGUf{D{(=Iw;8 zVY}tgjUVe)<9=t0yau#a+##Np{}`(O8mLUIBYvwr>{QkL>r4wV$r2&=Gmki z-E0OlOm;9B?zaT#vyAdodJxjqrq@(7uquAWTS3d^QdOsua$=}zQ(4rdBRUsaN;2(< z)vzKumMG8|C;tdn@lR~M)@9k{^GBq!GD8m2A%X9I;;hDT_ohTEs;&P`DRG%U{SW5 zwxgU8Q3ujA8VvUTEJdmrI6b6=nM>AhfjJ*!lO7;(}7zC8NLqpqACTYfKJR zPlNUEl;F}xl@eC-@#cPW2BSR6N=CD3Sw&V2#k3U#Zue)g7S%3!-a%=|2u>s8$boIO z_JQMu)A0d{;0Bm%g}yN0_^&jq5>y=)4Q zlKYMFOdDz&2rxCm>XX1|kvc~87Uipc7&Ks{cBS7D;d`kMF7>)Upw7K7kCZF-k(J;Z78uzJmSJoRtKwd@E2!?0$D}MiMn1Fb<>af{_W8FnoCkCBEgnbggrsNp@Kz9{L6+`l(eOs}dmt=5WV~t)2asA$j(ODJ}pMC&}WSq4$zX!7~PwG^N|5 zeBPC=8th{mDE#V4jlYIzNuD%YnNE1QPPBW!aq7Bn;GEIGrWjqSDYE#+8ByFrL~;f% zg5>Fl?F^uVQwH10nMCpSPU%7}xbQ z(rB!wMr)M#@OW6Xp{pUemt|m^i$T5SOkOnpSlp5DvG1l_?!o-yg$~jb1ZhIpN`8Yb zg2;~Dm`;W;xm12`-UHNJdhFL&8n?u`46<6_K1_?Z#QA@k7p?u9+Vj(phL<3^9_z3*#K*DohHN^}$8p#}5?8}- z0c2X^=H_cd7Qkt1bwt|8PK0(};5CqoYu;;JYE#OBGUjNj z@tO33J+2dMpg1-V|9iiWN*fU{EHRg<29xXgyE~SbH6ea)>)?>UGE^$&&(B?r&E)Yb zhy@#%On(Sz#+dE$LN0NpQ9Ui#AahiDOMxRO)+v~PK&fNpAf=%fQKpgAsUgpC5vr6$ zFH4Rb*Kr641n7+730l~cjWiFsujY7za!WvIqnjIkNc%Lss5>XVEXBP$NJ8XPtZ zR{Ti_H6|lahRS{zVU(FDK696>VGvBl2@bWKrU!HB%RK_r! zD_#lmb-+BJn%q8vC)H}Bp#WV3B&fsc&;eN?`9_Og3~IKdj?sNQtv*((R>3W-TN5@) zF+-bHBF1T+EL8zR^57|(!7!n9u@3_0Umo#(@KgF9wPyI^XQZe?ap>G2?qs-j--R!r zlHPEyzmD42cyU?m*`r_vcp>KJ?yiYSnl(r?Xst=-ab@7j&|9u zmND7Tj!r^Vpj&Y((XQYyCqdYe>c`&NCA1rRRz@d1e(hFh478kdwR;Sr%<)EZ^HA0v zIV!IWOe9!X$(S{FElEm2n7GX&-j*$q1&?=W`8U+vjowv*K8D3sFLUA9JMUl49x;yg zYk$YN8|nf|eL5!t*3V~1?Kx%_xn}ErpT;~nTuoh(L_YCU8yv&UXX0jCl&#`E0hVo_ zYxwud&;Ic+U7W)S2*9{LaBj0yBiRnd7A(Vo3saeAdtD%FjS z#xJij&(1A|DDG3|xGhy^Pq6i5Dk04AtyHRb&Iheg-|NnMqB1b3Hm)KiKbVBwcFj!8 z9UZe<+U+&s!r)$6q<5I@hK&LOhEuz~GUV2CM_WQ)t>=0^@8DitWM;qE@_uu(tRt|k z?M_v%N~w16Vy_VVjF>nwHeM#(?P)l9I^&U5*Q|81vMY2u#~Wi=fhL3>!>>1Ai^F)n z%}=p$W883TX0|G`+_R{ZwR7|{y=`4VuB+PAM`W>;%XCV*=MGpgx_p*8b~e!<`tqrT zu4()@;@{3CO1^?F_LMPJ)V45khrRI>!}Qyux3nBu+9~wl7nhRahjLKJRKM7RQ!>>r zf9a9}t6}g0aj|uZ>p~PB3RGyUb1Pb{tb@Omh4xA9_cR*G&a`pw%zA?9+m)8-E=N?0 zuezKEq=;nKO@CMmRk;L;3}l~Di+Fb)A9C^faTDbe?SO9~xcvu;a$5~7m@=W-<5hr^ zkCbcLOUX;>iH%l!q7I~we_qQX!g0pvEJ@5LuXf(RtPzvZrQS*leZubi>b%gF+|p4a zeH0ols{I;;^fteBt;quEop3kRTtq z%Jps{=Luwd>E5YNOEM>uDGjF=!;Faq9}Zswn&++ZB&;P7z%Zo;#RNn?x#9Ri zfjMfyM)n01YfqDGDtHBMnYh7Whbs3I4h#)gMk(v^E@A9zs4H@5Pb{|`Z^cMqPGOE| zDGM=u2My2vWQ-WfW{FtXav^cG>iDF$uWP^2IOi27q%O2NjH&HzY?#$MtD~;RD}V(< z#eCE7(Sbau3&rOvwbB$Fzcs;eE$0rARag5&TxP{FhLjnEOv*5=7Hbn^$~qm$k$oTs zu$+F5Y<>&=xNV6ZCQQr;1ZPZzeZ3{^hO}p)BrGEPl_AUmHniWB5knD99 zk@^Xmpb{lN^edzH*g{Q zA!SI02jLlRkLa?popZXuuA?jHzqt_;eVRxa6zsT4+ap%TIgV#IW_6f;3VzhRd4|Fq zr{RFE!IpPWtZcJ+m7XwjPW>*0in(%U1eM>E9iIJwgb_tPuOBj8kMiUm#TJQ0%r{ub~x7TQ(pAf&#P+HA>4rVB) z*M+&hxsL8J^9s3H8qOjGGhS~B??MyIr1q>|Z(2eeXe*-^3nCW#Ah3PFtOWMN7VD#2$`~z2N&;Ruc&kWnfC?)Bk8Pj z7Fu3^lkNqpV;bHEC}j|FW>B$3eb!QsUz56et29>QNVM^w!b_i2^qO5p4I77(mWR(s zz3mAcz=L~AvN{WX#K-S7sCVPo*CXIN%L4161D&ExeEzSFX&~2r)R`*1TLYQ{2VZS$ zDinB9$+X~sQXTI3(A@mUB_L9m>QvW@BKD6lHR?H>C9qBexiG# zU3l&f5hM%q&fV;?UIdw|*0+4$m`?zI;h#f?fpZUBHL%OQF?*jxyWD-V`lNdG@%B)- znkGN})N=5-Zv4pAgSV7ca-%ITJ*FUi;WL0&xo+rgLWdN)e_)_gIiGORF^!J=Vm8;+ znIO68kLB6?nJaaN)V&bY<7Ix0aXNw0DN<}*nL37jim zV1@614~PgATskC>_n67pC3XZ1}7IQAb4ugfdaH{vhZWA1@f&iAZz={!~~?vvcRLy};(qX^r0Y{wZo#Goj4Il>t)x_zErLXBc?xS4#WX(Q5>$_HwkhyAX`X|<8J493_ zM9z=iu1UKuQ}yOvAqhW4&Nr4#f=hXMknK*yTFyD&#uXJvv%`0MPS&*E=3t`1s^|*b zUs!i$+7yFUXJD$;z2AKt*1YOJd;fmyoc$L7O+d20iJkd9{a58Cb3^?OS0)$D)dAHU zcMXWWeR*BcdBbIZ_jBDBJ8s{>{{8pm6*%~woq1*P@Gdw$@Ye7Oy1u-kD|u_}B4xgM zscZiGWuX6WJ~~JgF5!~#1DKMqwdyX?!#&tj#I=|K^8-Jyb|e7NWNWOe9_b&-xAq09 z^NnU*V}C5?M8<#eMIm3r$f?w?-#U5f-ytIGBmJ#Ic?F!`Se;kQ*WRS-VjfS_snx|&I#jx)DqW@fzNMDbQs0MLYIj>* z>eFu9xE2$j zB$I(}W)l)VZNg+{AP4fkqe@2~wt;oKr7o%JRlWc9{{Q#?zu!aS-pIPX+Xa?kMh;hR z7~3JtRV(F8pszY2bWF;`u8(`buwIl&F3-Zfi1ZSh+)@}cRRV3@aJVYM)+%#V5wbh!{uB9%96@%#N9Irhr>CjEYqezFgNjj7V#`&gD=%yPAkjD;T z{ZPC0!#6-S3p=aRV}+rrAoNZG8?89yoWZuYAhFlf(SbR95tv$_SIcL!wDNXErM3CbuGv&1?{ZORjG0Y`^Fzw>zqS3&sdWYkB*@!g;J~3IFo*F z#>;*XE&S78b{yWE1s7>_9LJg6oJym$WGWSU@IC2d?O{nLKSw-=bn>2i7CyTZ>fgQ) z;MGCEt4-jbke!8$3Vd#1cwxA3pX;5; z3aLiCBiTZzpc3mY%AP6YUXy!@+C2KHst8*>vJcru+-@&y7>t#$X=kcAvXRUt)Ya7) z%B;aA)2b!b??;tas_eT4sCtgJh!gSApfWiDACc-oQQ#9;6lfI?1=b#Pc{4utJ!rQy zKrWYvmsaQUUnT`vP*^hy1x2Y+0!itjH@;_8$G#TxI`EMXVmyO0qocVY;qS;l5?crx z6(f{H1bADyeL2;6N_^Us13z2B=c(C}$GI@KrMJ3!OxRLg5T+-Ggw&)mqW4YeD@0kY zK&ekC)g3G3R=rhty-7@V_f|#F%)*wcxUwNfqa}H^skO))=Qrw6E%||bXL=K46U=Kk zLQY4Bhk1=}k&Ak*+Jdqttg`kAnVFPYrEBkM=dbKRcf}f^r3}xnvbGcumN%4;b#Gl?;~{3ljc9d4}c#* z3)vebW+#RFbj+tJca{CsTgLVYH&y4#gE^r(E3BXFnv4pjdRSB+u8Tz}G-`O!fYq&j zG+tN`mzuuCUln2Fb2n8**!-+e6}|9Ah>*!*E!!_I2bY%9d(zEQ}{KoB|^|c6A^6<+=s1 zO_Yn4stC@OCos_Yj(VWn5+lFgkeL3;2hdy;`#L{>UZO<-*xLRx(9PR0rbkJ$d2@Tt z@7E!^o!yno4em$t61VMahlKQ&OM|xnLf$yGQ@FM|S#A#xR?9;7WD`Xj%2y{Vo|FK@ zHA!A_;Fj`@fWWYo$!n`3Y^OX}6`NXev})OVA^7{(E9ReN<)N|i*gl~00EQ)+f?DVQ zN9|Z;5)sc-{7&%=lw)`p`NO4gI1QoS6Alxdd-mwda~%YQ1 zOw}_sPUx$=phOA70(OUsMLb#pK3;%qG|LMxPwhkvxWsx3o=jDQZSeZ4h+1(-48+Z( zTdNyaVNG+=MrO_xQ_SSv6-k=0{o~QHPp_hBrJ7M&VNRzbz><%uVbw{%to{`QWH7V8 z?iTo{$EE@&DKwlfnd8THm%Ma#jWiA>e{G($kQ8;uc ze`9s)*t8&2d&>8^jqz$;2u@lj?}yQgN-koqdSg|DZTEnI z4z-?-S4G^u>V6d2sOpt(g+|CKuWD4h#|U*b9=|u%D}Sxw{D4{I-6h}Ud|H^Ri;wz% zQnsY_$m5YQn8XVkpnPbQDytWL0ptTN1%)lZdIQGH{R+v4708EDacwh0FR!b!s|w4u zh57rpC5=YuV{ifb_=dG*2x1g;f%x`8`g_C>F-`0wd1Rq&=FAwZ6BtwL!ApR%kdtE_ z#q*cmfr-_kR3QM~UwayURaDJ@KT8JIOis?Y6>K}*p;EzoHixpX-z^!yADIr7+pX%z zC}8Hc)Cp6!bMO{(0S;U?mx`8j;62K=tLWbGpXoQhVYBV(rGGOhqyp<-?H#}N)xeCj zw7H1tTdS|&G-SHtB23xZffPdA858)ByO;+5)A9>)y->UQ@t=u$n{D$OB5Zt+{*4Hm z?_K|@*gGD$_Ej-3BQJz5;zC#M`Fc_k2#D-)$Kpy5iSc#`tw;{kdSJ~;caL1T!rMWF zMs~*P#jg-{Ar_AtY3Yc5(8Asw_7vtnKHYPJ+iV~1esg4THru)Ht4sSIyS|MJ<^%a$ zA?lBG?znq0Ip!xhlc`qQGv7Xzv+ub!FXZf-cW(MkAnDNDck9HSAvb+E?2l~AP2aw` z&C8p!zHpY(Qo(iG2Zk26=A)(Uox!2r4ySXnZQZU|bbkHR9a}OQb+GoBq?{pMQA85m|JbfJo!Mq!IjbJ#@hnb@Yu%p7J;}ZZJ9Y1q*B^T3-Z4{j zAh}?BhS=u)Yqp$XGe!Z-!v zt=)%#JQ1WPLi0q$COY_?7Grq&QYgK1E;Ak}*AqHGgX$=cxG=d719UnwD37BN>!)L9 z)O>|>&GHF-`fplh)bAX{W1>YRS0aynzVe~Vr&F5CkL3&rMSSAYpU6#2j<69y;%gY6 zGdvSM7^Zotg1!ZN$b#NqTZU1DF^Qg~mIx1F!_>4%SJ|X*Z8F(yfRs;b{SY-9ZWwyT z$yE>yU#X_0Ivc6R2}1}w;Ynvjd=?n@xy6KKUSZNxFEgT;n0Q!Fd zP`yHIC9cC#`YmFdT0Ffmu>)?>evNMk&Cc6gqL6EF!YJ8-oO-W;b{>z5$;>nC_GGe5x-@4Jb>;k^~J;29Q zdSD<$jlx%WgpG#DYGRF6qsa}7h8-D=>!aNf)4+`nOi>*&?ce>E{+%NcQ?l4j9hN_< zOMv`KP?vsOdzjn}_!A-8i6>F5@JsLLP;zOX~^gye6nv zEcG7gP`q*WS~GHeg#d`8(Hv&fWYcOTu7R%;C=vjM_K zGuDhyq5u0z1tLg9w@pBUB6wxwDLw4!)1arZmhMRs3UP(bo+K5ecl;?v8<#~13Ya{7He0G~TorHTILzr@L1Sq#SOO{t+oMY_CCM@*d*8K#ZkB7*x zz5>QMMu^7<>=?NVa0ct#Mu_OQacCvfJZ^RcB>l7GJHTP#Nag8?@d&E&9v&YmXL>7{ zNoSL^Gy~1{Nob)UGL+!`px1%+O?97|fR4u*abkQJwPqAso0G*!LW8)RE3%evVv|-m zN&?5cgk2P1wcAFseTPtm2%MAQ+eWf|hwB~-gu~#yoNDrOmEP^6?ObNl#8_n8trdU6 zcM10`^IcYc(il%~+dAXQ4JY!WDGP9FCT0H*aJ@jBM_gwy*Wrug3NGjD8YuU0(V9=k zak58<;NF{%-pO~)Nlqd1^0b-plrvJ1*#)Dj2KG1C!Xfu5>=^3ArfYDD;)?ueCI8y4 zVV%$?rmo-`wh$8#aToLq;X1Tm0xCk(krxmhJ;fxMFq26W88eVEJsDGzF%_9YV1}!p zC%?8TdNK@dd^s(tZGvSEw8An!Lunye>p7N~S_JHL!rjW*@$h+)lB#Cm+BAu*xjNYL zy!;Qto{(omb{xWwd3@U0rlWY$y4%5 z$zZouk~OU)Yg*Y3Y9;z;Q5a&wizLxz zt+wJfa%lIF)KJnkdEb*qHr%~?&;nw++JD+SH@N#w6h9wF@iV0EIc(VhdHuRHRMWIp zFE21f<~R-4WTc1=EQn1z@BCKUc}L!nN8WiyThiL172UeDU%_P|eO%+ca7_j`1>Kj` z*JQ{`GUV%ASyX%L8s~b^k_>ch%aV)+^uyK1*czzmeygfpB3-!Q*v>?G!&nOL6=hv5 zq^_NjDN9~V_Jvt_ZANPp{qJ~j1}=H6N72;R(0XiKce>a`##l*gS&3MzEWWHH*DMrY za}%U3lw-mLq&iyCq{m~dEl{yd64FqHT%U9!UOtsZCu+sUnGR``TDCf1HG7BzQ%QY| zQfky{yEkHUws-Z1ms`W=aDSiI5R7>B3Yw&^=Y1xPMx(Z7CwtyKxzZlq+dUd*X|-0X zF+x19O_!?Fi-0B-lEZ#uZlX9bGx4#BCnuOD(CYt?z^aj;4a4QTmStep2*jHFMahqm zs}W{36k7?n8ZNPbmeKtT`5$QW1TAzS^(Ew1g2NbSuc&`gPwBJ&)T4dVw9&N7v}mF+ zK=scdiu5@D1~v$FXley=Z)&vQj+ZuW-BgMvXOGTCg#M&W zMboGrBGg^UrAkR#qBOg8b16<5H;6z=>`rUMZvjreJK#2lyQ8trgg=xXx@NHJx=Neg zY~%DS&q4f3-pO0S?Vfm7A`nUqZXqx?nf=T`;vwP>7*(s6`~|Ux*a2uhOe~VGorxqZ zM?V3BwSLyg?jPPWY++f;@E*m~-Ne*U!CyU6?%lDsJn`o2#@Su7i?eigHaokme1j)t9kh!2Pa(nP_pgi3uZjiYQ`o0+%F;g6$aPnJSrUVS4DiI z*$KdzS#CBk3pj)BwYTF15!|i3?F|v!&&gIrr=WQd1ydSn%xhf-m^NXbuX$1Ss;m8| z<@~Z;mcLDFM%S4BXZBMHmENB~*&5$b)(nxg?{H?uCP}P<2X&IYrL7smlu)D z#(hK1w3)ZJ@3?o%?44Uuzk)d3a)L-o0GN1_PRGhAfacYJ`Es%}?g{u-k&gb$%U1t& z8)8bOLx`GZ?N@x0=qDaT_A^PC!WqdXWF%XWk!(dq8iX>EUCp56 zpWTqDh6S&oY8Ud2lu`8}G@KpMO2CU3(9(R+bX5fB+DmprRkRDLd~*sY?cy#LTd;dq zqC_yw*{F%9( z*i9TqKGN*)aA9``?7VZ@6Dtsf5co07P45;K7L*;a>FS)&1GAS}VXAG?Bk)z_nB?@p zJQ_@I0m@vEvS$~i6%HtE#!5}hO;^P^0k>ihaWhK3Dk{fh4+y5qU`x=ZwI;fx;dqVQ z_2rk*%0#C7!@~YbXaufBLut-*JIZ3cfODJwAP-lybQ4+E<^g@Ku~ZGEt<1VL7Pd7f zIfJG(nFNtb0axhXF*zzmew`o)6&0oa8g2}VKK})f=wV!~aXL55!RS5h^QBp6CG~Vy zdRXAn)qySn&geW9ozmb%tQUdpBXg+q7S2i{ju^nHUFoVgQ0fw*R^%k1^O2K;DxOh| zR`V`1S5M`y`*?YAdM$UAlytlPa^k(yOOoOrDt-!vj;~5|F#@`rPEXIFI%UIpg9rQ& zUFj{vv@k3T31vl?hKd@U+R>!oP?j&u<8<%^n7Xg85tm5k#LW-_Cr~X4L z^(9s(Ir;bUgdJKbZZ`h9d#s+>zt$V%j9Q86P0TP^O$LKuEq0Kkyum>2((3j`wF}sr z&k`>pPF;U^1afNO?l?@otv|A!b3vBHV)w*0t?!EQe5`BzCWg3q;pl;*2X59@9~iq^ zI4q3BT?^HlArgq<^xPCIBR)GgRbTgt`B%VMA;39kk?{E5l+u?u=|T-ZpskAarf*)T ziZ=^$V~CY=Q-iSmr$y-o822HKQ~E^-UixZQ&aSRHSFKP@?5}74Rl5|7Myax)JiMQ+ zue~(~ab~}*iJK}@Fusa(sJY-3WrKVfRs=+cLxwA_-d#g!uTqzrg$J2cl=hb_qH;5` zh-a`xWGLApDu;pQ@5S@Tr~N}S$Pn%=>^1J4pWkbw-P5oNYkeNhbDoa6H^XtqzI#fT z91`+EIvwb3@1=TY2zNCqD4+?jNv@(~0E&35hn5q`5E33JIj44`rhHh@16TsVUWo^%`7(g*RZ$^7E>c$JF4JJ?8@o602 z`Xx`!FL`p{{T&n$!#|!vn*~cDvR{h({AjhAB~yv(7-uDNo@5$e4;Zjm3dvq>n$&fd zG|~me2%1=EKs+o%(yk*sEKALGH!^ed7gc7j&F(cRryfO9gL1mfo)g-Kj&4v{{V>_4 zsm~bRwsm^&rU$R5LiIe_|DD-65}n&h9cg4WNg{OVPxMhh#WvC)p@v_2r(}l{?0&S9 zOVm&L@E<;sm#ATr-mQ&kcl2(q@1S+*_oW`V{g#Q0naDUtCKxi503WOik&zG?gnt%; zWF$xi(5C@15+LI&d25giLU;m=$tDB?AYKMXZ% zbdxgNAEG^xo<}o_Y5bpr^NW}}a4(s(`z#S8IR@QG2X0hv1LFdwfx1b?SykVLuxX}8 zMnLpY^ih(csoF(^73oVjjf&cf3I^_nXZMCJ8bwV(|C!QSf^NIdq@us9&}j9lzx+3d z4XIEYwe&Wlbf^q`O|J?Z+5{Uq zqPE2SzOYTF_~}m+I$Ox&jhaXe>8QP}CN1%RH*D1^F21DDn*45W)J$n=e{3^a^b8F# zHuuyXhw#3P-eM%rkzX@f3<_GQRn<?^%MFWk;d5O-y%5B zQy%Vv7(=@u=D<0fno$j8L>**w{vxoLfVromPI$^$5uC%saT;dg zJtBgTjc=9u~k8u`-#-O1TI+I=nKdT)fzp4iDKpBFHdV(P%&pSD4 zZ!ii0o0Uf16fipMF7Dk&Rh*lkh*9b}s>Ha74AD>g74q3sw*K?vHD?G|X?>pDUh=Zh zM*;!2?N}g7wr5LODx1~1A4x3rJghxTACl(((dNu1gjL^MdKV2u9}&UDS=%v@$a2}= zXK6hRu1~lh5fe+=o`*%$hLi`8&_5e{--Omg{%lOHH3NTACATl=iq7}9O?3Mc6XM7g zgTEsd9nARD2D5Qs_qx&fKG!E_69X}Gp{=bLq5eg$*BRQQNxrR^%5KQ;VRy=7FxyOF zkHzY9c)O=^AJ_8%J|2(60j)$pt1l@nM2zSmb|6}5{q4_^+h9tN{8Y(ASo~UJ+sV+9 zbHDLW$0_EpoPg@yMDUF!{NOmI!70D5(Y(gf^S7sd`gt#NieOU{8%CPO}0 z;!OViP7R$n$uAWi*Eh*}A4V~kxd~6=6DLJ}Nndze)He&Xl-N?XIHH$@8UsRnhcjs6 zS!M3J!S&bnxdOvGi}~4vibWh%K9CqoL_l<@_Z4E1N|yRp%)#MYJ~#8>!SbP*bSy?@ z8MT52V;n5yxS9SFR;#OA2}J3ev!Dv>X?~HHs`ya|P+9!F7S;Ltu`$7jJ73f(GUnb--n*9Wy7_=VQe}P^(oNL+N}^;E_k-lX2QsyC+AA0R4k-^wEaNn1b-rR&~wOm5Y1my@xqa}3K#H#18ckbQL+6% zDs)0j@!%Jqi6$bNQ|k4(QGT^`RNalW1mrW6LP;yrb{Op)(HjlA+muGP)drY7tpoOH z+MXXT*z-0GLoxqoG;1k?-ktLHIy_!Stq54Jfcz)_hr{b}bob15xzrl9!Aj6XihP^e z19Wzfm?W+xeuXnSo5)cjL72%|Fr;lDH=oO=!B5n^?mW5qw2GL5#>`D@A+ISVqKe~v z2NRo)muwqtlubCsW>s`IaBKAXKJpX_5S&+E znXmRqkgqTQ0wQ~2sL$6CVt6Uvr#LQx8`w4-7YQ2&ZlTu;$3%3O*U(+MOUFcbA0Scz z(m*%e$bkY02|*6rD$PAVpYE$;@m#)!zI#OpJ@{3vETLB_r6X#OK{H`C28XWc^7opx zhFIX?>}01WTsb%)+&JuOi+ciL7w-%WZR_ylY)|WS-|6phC*6krPEXox$adxK4?8AC z)BRyq@pC6{NjtJa!DY~E?VQ;`DJfg5H>ig7PU__Ga@~MOwa?nHQfH<>+$CR)ePoI)8Q6+W>OD|9U40MhFU4-Bw zt}{+<5%uerjU9eqfwb#s(OE)Yf$q3ogmPh7@$5ZKm7o94ur8bY#5Nbp7((sYU?|%j zY&r7{9RASyo=Vy zm|LO!M9+BvKWMD&JoJpkSjoG{SRWG^B-hg8Hj>BPIJeln`h$~%IU{uQSspx}POa3i zHd8dBV%-+l8GR_13x>0~AoW7~bhl5bQt2(6$4)bJrjP1e(HksH*8yqjB(}r84C{zI z`DwVj86Y?hI(8O#iCXeJ+~W*ZVlI02n`F6?#0u|nhU%-E&uFUEth+Y!D&o}BsMS&&*Y^GvbPP#TXIEe(p=4bao10Y?xpQYNIZw=~i>#aNl_c>fpjMl-M^?ntuxm!bV>}sOir52-GJ$Yq zA+&0UG{(5`q}X&uQyt-DsL|*>v}i{mLoJ}I-M!(lgiHiyIOr_w>KK^Kw^*|Z?9%ZS zT|%7!mTUv?xt;*Mu>h}2+MCr%J_z?K174;^>wJ`53V#7d3r%~ovd%|=)69gwAcqH6 zF4ya(o=DEzBFrpSLh;EX)8hv!(a*5ab=lOqgcZI|Z>8TJS)9$pC-;wy9Ng5FoY-GU zjCJ`uonvjOa;J9zKCMW;LwyN6EsS!m$FunDT1b8yigK)k4Td<{_MBhAxEHx2^*{&K zaaEfG9UezHjZ>Rr9e8$gkCvC#)p@Zp)+sKmbG4-sc3A)!BhQ6g9oG#uVjUrqC?_Hv zFxDAcnn`OcUTf`L2JT{|1b$Q%swZGkzCAay55+p^&hKQhFxCN40qFl7QvXrZf7+|- z!BIhue4u2p=z8K_g)zLCI_X&IeB5}LIf4U&A}Tbj2L`-+o+EWqbSxP=9~aTNI5sG@ z78c5j!Iu@i@ix`+-~byMpn`K?XQnOdQL?-#=)~bc(5M4(FANV@xzD5; z1a?w{fFeIkf^W3O2T1eCAENnyGFs;mauF;H3-m7Sh4@m4wfPophZ^;2Z(YEM?S{Da zg4p=}Dyr15y5X&{$sdB?j#2Bt@Wldeif^_oj#MmcF$Q@Rh(>;7au`*NlF>O5-rCn& z^=-_bu&Whnr4jD+vz~g6tb$$?F_!PQh(2=5*+5&MP47HUZY_BUed^e2`QPWMeD_1n zJ|?=PJ@#GGOD4+1Kg1kv-f?Wcb<43*G@m*q;?1L@-4BWAT9ft|Di8zK(!}r&NfM4h zj?uauZ>zswv`47fVICUp12h)$Ku#LKiAQ3LS_1uq9 zjBoQFw8T${PD{z6J0V&Qs?v{1AwDX!k#X8kc(*+IX^D_vun#{YpHymjUy$9kWm>1x z>8F%Z?E7hOr2BLr<%%g4N`|62-T|B*#f}9s2BV*kGHOPl052a!vC;p@7g&YPjAZ&b z>Mq6#d^Eg6vnGty*$YP#Q&>YM7wwCIll2h4zC8in{z^JQA2`_*+}GWi*4Vx&$;7wh zn5es)VYA)9wD0K{*p%m$z+bjHIb}~@P)N!{O+A<|prM1DRAS{-*1k7Z$z@1UJ_ddA zQ$GOFot217`<9u5!0~sXEhC{?69wJF+&gFnndwt)s`uBK% zlpuhB^Ci#f)3I3Vq(0@hx>+@?c!t(mLmp4qrlp@{7!BvK@*cC2eu$zUpwyh3u>wC! z&l+k*HAF8^>(u1G=}meyL!sX6v{=X|RB9y+`q4)Hfc~GLA0x!=Xq`+TnCEjjYX)?t zqzhWj16H+a-8yR#&P1A2*6zjJIxFo?EV>WLJ;U3&*0(T2+m^|Dn8E$b5D%AM*Sx>$ z@=nLs?tjyt^j$zCRkY$OwAvDMxq~(hb%3Py&>AZ|vTEu7te{mUm(}husi==o)UBk7 zbJ}c9qmuqOMcqQGO_)qN*4RX*J$ilZ@dnY1oUTqZy`Fpm3xHZ#o7YK4PmjMkd01N;zno&G%eS7&%#PNzLjo&&LAAgoI+hINd0(NuS$5E-9q z5|Vn%jy(G2^P7?rBrgr%aTU%@boN-{F^Oz;$)#7RsErKC1zZlFMNQrQQCb5$dsl!{ ze)GR5NjBhedQEEj@F7~wx@|5$ODVrhy{XZw6ckYB?|&bdIaHQS>T7BP_)w{0zVmkq zILD$d){|d|>VvJ7B_<#Y=!Byv>G$ECiAOwtUP&^1{By!V8Icfdz-;2T52SNnSQi zHk*J0Sff`}-J{Dgn74VqXIazJ-BqpXs;|E9|JC=^_xbZVzzA?=TCLXFvf$3seydMo zHD|%yX`R;U+Go)i_h}AL*DAnqaXLSl%WD^ulIBbgO7h?8YHHr{vLIvIYARZ6a~lEO z^QHCCOt13~AJJv|6t{L0XMw9ZjlqT<#DFKCjZkDaDz10!yw} zz@HfdoY%dI#R0r=#$$#*L?8=(n&vcq;mh_ZWAGv702EZu!T*aAnWYVWyYS@-#3;Ob#lygtf=aem#xfAHwv=4MK#}+^PI{fVIOP;;`3eWB; zsjVrF*49LGrC@kq2rXY>a z(Za%8FE0deRCsPn>0*cuz6Z5fi^V0X1faGAUCy9IZP3^wUT=}4(MUyJZ^W(vhw{4; zsAqJdj$t9Y{HWF+b#k0D>i5U&9A}To`_td2Hoz8E5-a8PTlDtgX@|pDRK#S#k53wn z$t<}eZA^tiUX6L0VG^3QI^<*lP9$*RmA76-b{Z6V0+e2;K>r2!iL=h4DvWtLgDa)$ zdJ?GG$xK4dqT;|canOpq5nrWE3oRZ|%>u{?L~HYK zt;3fdYpL6iF4C$vHO-o9`^GAFKeRoOdmeU@#}`4N`yxIM>@?~Z8}Has#r)A|M7CXE zvAIpEqPF4k+R^r~-6O&IW|izFuS0Cybd8L7WKn1y()ycC!U`-*HGhk9_G}e&Y z6}D+Kwy?_`vT8sQWptt72RWcNeAIT_{(oJ*kipCrn<>tm**`8B)g)wKB9dk0adUu+EGSF$VdZ4lczu9rA>Ox1@qVisN+}qsJkZ4IH}kyQTdIh~nJO!9n7egp+V%+{D>j z1pgX}-!jdm;dh7}9Oo2sZ2ncNO>;wf5b77gos8aOHwiW^ppS7<$YBrJxO=^o@tFN( z#L6*InvOVqXjRCEG|JpBz~`w4p#EPf)7y!V`B`!e)G!J1rqfz)G{G2&(sQsGnBGvF z*3;-B6w^|v16^N6^B0zH;O7x%@kJ14iTDs_Vf8lQ4n{B9Mcm+9b+(XQf(@p8z7jMs zN_>GBI>u&t@GJaC4Tk{>TVd`M;5$4!dr^6ZLZV#GL2S;FH=Wgbg7$942)X*X)H&>$ zho%dXueg!|R`TVuaiqd%7GQWt%sYBo(b3LSSr$XZ$@ZA*>@5sM5_>>gR zguNA|G3jNM4*S$-faw;8Ph?rqhde@ysaN6Oe4S*eAz}p5%N^8pR2;XVnh0V$8ym2e z!*;;VLG;ZA(&!X+3G5Nnnnh8L@~Cl{tJuZkQO*{Wq=1!!)Xjg6(Lw6wty;k3Y|GrR za5UXqQAQk-AN53)*c#E0=p{!43FQG`@r0i7E8LHcIEzC?&TVj(`A10P<47 z=eW`hI5KxoufdT(BeR(pf#*@`HTYKOd=!opAVBWdl$QCgP#W6soZ%S)ZxKOXBz_3wUce|x#vg~ zb$}p<jj6#lL6xx_bVRs49```TOt2=3yrC2T29&Us!{55Q$pJEa^Ij7fIMeu7^OmZPo~0pijXbO~_o8 z2M-f0;eelz@siX@*r5d4GVsUqZ=c6zGk7i&O};(tzdckK-T8zZAqzIWl15b<&Ep8rc951&IUXV&Zq4PWxc*a)JQbKZ&JvlsjKzFWM z*-cYMx6SO})nsu?Tmo+oPe1i-*cjM-u#MjU11FvW8z0davtxQ;#2-!7=<;kp3K1eR~y!VPBz^1?BUOz zy|x-1k7lo}J{ju1eqGPOHIY#Fb?bVrSrZ{mdtUs~;My;|cqoI8_q}+_hA-|*H|)M| z!-fa_x4A6TOz%^krv9}&XN;n34^K8)F{c?I_*lf_63>-DylZ70#Nm`nCv_oOl_(vT2*Y1V0J}|4L%jqytOvDu{TinlQN%mABO%t&!$(*%DdH+z{S(pX3)BK2b z3q3+8L77o-K}e2B*+8@EtXz;sCC_A(qLLDISqUu227_}@GC4<$`PfhbLfb*$r^`*^ zkJM=Mhu@@{$#SBeMd<**Ul_AL<_(s43_mhjbGyh~47}bEC^(Ojm@{gf>O3Sco7wAf z@l>-0K@+u6bLpNSRd6XM@2e-MN!kZzU`D2}YVecOk@85n&Y2}UP7*p_77){Qbxc(j z)SoOFw9XTi;%v9T1a+bgl|G$;#Y<56Q<;)M#=4M*@*Z{t%kNO1^Pm+J7=)}Ze#MoC znr#2fcXrl}_a)6*25m7fXxY)3+Px+g=(>KuTUr=0xg@Wf^k@tmV>aa){3}CKU!SNr zz59`=8ly!r7y5ZSuaR6fU)#j0)JT(uVrWMJ$@_e2lgku}puwW`(bHk&13%i?3 z1vOxG8g00Cb;ah}2cndtW%b6yEopwl1b0SkUJ^MJ;vVay`=2ZM!Usq>=S;V^u34zUdt+8)rU zHMzvX0# znc(>5x>{-z;lJ=MF`11JHdq?8_=FRmT`??}OJ2<)fptE*E z1pDj7rO2lpBcFDRd~7oEu`P>y zYda*3?ox?{T-(Hm1^PR&-9^X;)wA47!*SV|NEj8eyhZHjk^4aqOLO%cF z@WwArrRpZXuu(Zex_lB2_*dy7$i4eyx-23cX`|o~IQT0*Gpy2aEDPB#Ff2kyP<@u5YX4ic zLBq76s1;B|nM8-6A^#0SC7U1#9Q#}zDqXr+BRJ5EgW4YTcW_kdWbYS*mv)~i4Z*ou=(cdv} zKF%Z!7Nd&N8g)SG94zCrc^v{dhyR6r3Rc0x>aAMAsTc}!GBcBb{K*M(Q7!>d<^)06>9 zu1ScA>VzLMDte!yaJb|p+y^?wP8?rXZ{ ziQUcnH`W=|EM?GZlIy10TDG?ZqU&zxxe4}{iscNN{TL8c_9g4ay34g_R}{sl40Y=d zwhrDg1Od@ted|5`fgHbLSvZ8t zp-5e}(;2xWWd9YP8VynZY8LNPoBTMx`OJ@?63?QTG~fbl_QkyZgx7c2TFG+II>25L8a=q;$caotW_^-J&L?V4 zr7iB5SlPO-FJ9Pp^FZUqFos5rNEb1SId^bXV(QVU+LOD!HdQCuCB0s72%=L^+dXz~ z%dXC*(Ue!W7#cBTa_lZh3%j9Q_#Q+?$j0}A5(|0uxwOql_H5|{(L*UPo&v2YP?-Xu z6i8*s*0fpYa_MeNg2^Q4N`ks1h$aE#R;_3D5x@sK9c@H4%Hc-O!p9H^9nfWGKTK=k ziLP!okzm3y%xb#~W<^I{i$+IA-o`shG2+7C%F&iR2)q(?vl&?36wU*$wyOYM?9~9T z?^R|V-QRcfaAN^4#@Af;=-z_vbcq3~CP2k$xNucZ15V1>248}q`^e)8__}jT-Dpb)fv=-aPQ(&@J7A}6 zgLCpJNU83{*vkU3*Uo9e!iWfaNy}m{NyOMo`dI8Ww-8A^g`roRG1@|oKv*DI@Y|U$ zm`o6O{l^u6S0oh1u!|zrLTY&iqpoVA11S4htI#HU0d!ZP-S%j~FPT-YB=c7)^H(bK zS8ymmVLl|_Dzrw3h;9`~~0j;5`7>nScA*dpdI5DH|VLs6_tyqV$CyTD?!Y@HjT)3o{XQHHY-P7Im%IWpF4#X zHCxc$TrGM=O9DQ$CEw_le4|^GMz^5JBWTeMRc*47i9sORz6gjaF$Z3gH@Po}6NeA{ zf~b;cJ`hpmAB3y$arPL7tL>i_uIAkUFUPCTH0*zL_x8WvTNmlt+ukq?@#;2=S4+?a zz>cY|!Z+MC>ynu%h*%qTWQqgrJKIvDjoxFo9k~^BufKILUcC1D9tE-X4Og{Yw-F-N zy{U@L>sR?OVjV4ROV|-+ZK(BD9-jGnysNpg0G#MGWgGReAqPwYd-CnFtO>)Lj-1`0d2 z)}tdS9ew$Dck^vOyyxi4_jadm`_cW^d}T|dZt}rlcr2=)d=Ldro&6BJNf#1+qHb=r z&NS`Et98xOt^klgZ@(SPF0@+r0-DH3ke<|`#k%HG8GHxL;uo&4R+rzm0N&J^eUi;< zQU5RAqGKpptNws9*_@)}6;zU1Hk_%cEl0Ja-fa_Xf{y+#Dvj*m1=}|ak!O@HCmM*U zJn|fyU#a_Sex3cv?e3hK#O!qh(UKMi*&WiFVaR)Z!SYfusTH_V#Q&A1tDo)aBp=$M5a^N#-iyY0@3_S+k z6onOnL)5Sk1z9|HoBoxysljf4puR+M1{^BEW!9SoJ@0W_O1ie!R&GCd=Z3Erq8-?f z-U=`hy)}9jQuML&Qr^c{pE$E_d86!`HNG;LQv=m!XZzJmeiIAi@FPo9g> zC-_gJ&w{$lLu1vOSCtu|Ak%SNamz?!-KGYQuWe$*0i>$XTD>|`xgixW6}OaDY;2FA zuzjdQ`D)mqtFP-T@+7-s4O?15pm^oAYh%34fr6cxeRc>e9HlLh;`X?mHQ3Ff-M}~! z?M1%oBFPbOFb2EHXyNrnkINiv-P%;%-x||WOmWL5IIF|6m#AZO2tuxevKp808?#{c zR3XCW%!TM`UxI5u3$31%%P zI840Fpc1TtuVzhb%OJ&2k{A4=%c;hyu}PEr9iRY+uGt4^V(;wlsa=p8l@o39y5)(Y z`YdVAug{s&CPG*Bt9abRc)jV+MbmiteTC&K3G$XK(b?ivzse}ZO`(e!(>RlUe@+T0 zb*_4mF1!dT?keAQ@9?Vq>*|8M5M42G@2ID_F3PJ(R;$+f%UX(8Pjv)IOLc2!;l^9m z7Ctey)7@BKY4KF|RaUMpm%#e&+eWMXslk1>cXs^EXZNj3s5m2Umymcb&N2RkLQijDCv01^0jR+$OkmF8E+qqrKU_^8eVnlRv zIgD7j7$X``c*NWx+F%XZ{e=Pp{$b{#Ac#D99GQfvpNTHJO?T-#95#*f95oRRh0q#2 zTAAyDV>toG5@EzH;*h*Dmmo>JGDj|0nX7r0v_N6rLRujzs%dFVCL$~oece9Z@wzg& z`AsfMm*>p+()Q@<8T&ZX@%r4@F4^jG8_HRZJp7Amf@OcXG^`y)v#g>_C7{BaAZze_00g z&{sq~Yl?^OpKRZ|zA-Em4c)(c*WV5neaBJNSC#556`jex>QrA`AjO+rI=%)fJU4t{ z;N~A3U(WC4DBi zqeys9!4k+``8;Om$Yg0s2T_T%HUrDB!h3V*VS(v#Zmq83ifeTxjD}-po?$qYq>y8N z@s428(P(}GH3PLVQSylF_f_z2vuHP}R7Sff+IbcE{%1A7=(Pz}BgZ^K(G=RUs=9O# zGVZDp6 zO10>JLaNEDDjAdUsK(^CNnVpGm*RM|uV2H0n_1CQ;ze0oj_NJBESY-@41r?5{}mOd zqG|Z?3S)FhX18S3ky|-IX7e)>KC)}(b+lp!b)I4s`&PSp_{sW-!R8ItJ)71JFEJV)vL>_U_DB; z5KIm2zjamXUAG?SuTblZoK|nrnH)xz({r_3kF2y8m-b)N8(&rFDztCCeQl_qp$AE@ z&2W}9FSanH6$?|7Vqr>7JC>DT4q1XZK30M)urM{lmtk@T89fwXfkK`I- zl7cL`fuz!cy1b#fv^d04g|FMjQ|GBqkYf4Lw!+slws9(TK0_^$V(hX)jQw~!Q_lbI>d4z~>_2aY9N4Q;EzZ&E!yX-`FI)E3^Fvjl6)6AoKx zd3~U(uQ%^m!}yVqCpzWcvXG^)W4vVPNkZ$$0_{oTx)3~^U1%5m4}bTT&%oo!;x*TFZoIzJU(|cBYs2+je)8x8 zfByF9z~dLc_N9wYjSf6<@nPNZmyXu=-1)uz$}#Uo51Nf*IK3@ zZOS-@bNV7%f{sh$i+E?oU&X0Lr=eHe&rv#$O~g($5G?}~c0KP?1ADfKxGArx56x@p zN6vCWO|7h~$FZ`$4k_!A6YTnh%KB1!(OJ0?QrY8*%DxV(>=CH4uV19HFK0Y5U4?56 zvL4msU28tvI8iOQAO^16va(bQ-E`24?mw>)mFy7I4X~FMSdl6( z)7=LbSdlnvMfwxc!Pi;gsX~@lYenK@E7G5^62A^vk+?a+{cG#-H`MO`*8b*w8)^kL zL+K4%W$)eZm&Ik%dz&I-Mv$D5KHrkaQDoOi1MlYMvXtT%gn2EEy14LZF` zy=siAKzd^}@~R0G`O#)ry-hTV{*beD{WTqp6MeND$&{~`y=p$BX~v8Z&-gsz(dQA* zN7GhpL4qBvAept={wso>Y1xXzrMb+4(#zqV1rczs=vzaZ#h27(Y)w+n{A$j(<|W4L zk9&e;Uc*Z!6Y{MY0*`^C5pV7nNHWl9FNL4w-8PfgyLGbe@41W5MEom`5eQNx}&g=8o z&n7jwrqqDkO(9p&Mp}1te2k}!;Pn+q$kzruvip&#THa>kbS9UXxASV7OA543b~bHp z^wQYf#_0EHL~K)fkR$;}ZoLDdW5wTwWU1!}B?>Ba81~x&x!4X#uig#K3|STPR zScy}Xyafv+FNz9^tqLd@DF61z!W?PXw6GwXmoqP#3qDCSL&;1pkjymoV(FfzkKXv$ zcr=lD`p8Z2__V*nm{Ou2q)P-6%A2>$jy+ELZjsqPYg;2-?LT(UpfuIwF93W%|C=jR(6yebkQEb!% z&en|vBmoH2`9zVD?W0KP@**XMbdl2KMN071qAZmE29Ja)NgVp*1jQl*kJv&aC>Egw z*Pla=z+sa$u*Smz5Rl57nhvfl;!4jGVP(Nfpx)(XHDE=!Lo4ka_L|;?|q8?D> zMp2B@*}Ne+4XkK3Sh1ze>cg>Z$=Do-kw>KV8S)l_le>^ZF^f5r@$ag^4KkQ1_$F^4 z!_kTV-$R`xjxrx@n5Y$9$==Gwu~lU{q*0TMTB_f)yIzJmw@;lTD?T3TL;`BjV=-EJ zgC!J{Fx0u}-lq;=sDt(I3)FoG>wpfKn-0Q`cA<_Q1Z8U2m1T(Sl;PekL%m%F*=$;i zz|d;R1bWga8h02L38T=@bNEQj=QCg7nod;HsZ0k5o?@qQ@L>XH-i6%7ppd;mnMnpT z+$K19hI*8AwLXo5*46_&EtKgY+etqK1j8YVan$M8<}9KZk)hO^V(q*&6a_J=2ORZj z0or_&@c%EKJ|wF))%8!M7kc`D1Doz0ueP|o2JGtN@)bag)?Z$|F}(nzZK{qS&*uns z^)W(FY=V>$Uw=r!XsTo{vY;lQ-2E3BqXE1pBVC6=jmJP?2?!N}P#6@rK-dX_n0*#V zpuh&gRuHy;uo*z%35g`cfDjF$PJnr-DD%>oRf1ovK3>U5F6Scu28QHJrw|l1*8Z{*wpY$Tte((vrGA`vL$tIW2*+BQl+4f>f?YD`1cxXfw<)*7 z*S2%cZE7CkDsEE{B`SVX5GB5Z5$-lP)1JZz_djHwEI=Ns75t!E;AM>P8-)pCxFIMh z5a97InBuR&DHix_z825RA^veKUa5s|b0~|$As$UnR!QLC&r^jTEO1kK1G}eK6jWmQ zKkR)8lpMvGW@cn%W!9NT-FJ1}N7r4|_fdVH>Tao9Qny+{LQ?B2)JO>3Na)})0}M7W zHh7n@AI6wN8cE0q1el{S_JC}_*j%0kd;IXRmH`_Z!=TbeWMx%Xw+`&xdHZ(Wc7IZ3 zWMpJjxBVwL%(7jA?;)jR`Td7~=&mffMArFh(ZEbigzt6I{s7h^_Y%u*pngJOhHMG(9V`h~qMB|Ck`1v`Dm zke$BgnVs(E$vx-U>3crM>?GaK=isFx5%-=?#Ai&Afw%n&u35Bj_mD3z9~}q%Uopqp zE0Y}|jV0b5PPT{nCzsx_p*FMj&gDz)+)!7w_Rfjn$u^t6dtD1WT7BK?1iM}5;`r5| zr}+Na<5&fL_)%ek>?E!^Zrwrd7T-TBOp=N3pJ^V=371OmpOKf^({eSP|H; zM{5<-6NCctAgxIyIZ4S#*k8&huyu$6a`du~N={DV5`d$(6k!hg&Q*c6P^MLd8OX<> zL_a8#x< z7u__i2Wyq;d`N+oNNJV)Ysp5JJX`Ork2q~t8j5TQWS}9{-^%mZwQ#)%0BJPfb>)Iy-d(thi){Ma?94PKkiQung zRh>Pa&Kgf=r>BNcn@-|?%9=5CZz!ubcZZI=Vt>yL>~=E#Y|Bplb7F-ORCl=c0y|VJ zz4p+N9A;U43OGms1gWg7@Afq%}2943C;#D%GL94c=yTaY8SwPql zS=a)-=i=Z5XrHxWhd)%DzdC?c?2OT8MXsLAk9C5-V_HUCB<#9WIZNpz@*%!Q0LzVpM7pHTfbJc(e^b7d_Kmc)5`J3q!f&! zpvnz6@q6;Z$hia7ni0?p8O74#F6jNm{U`XVSq4+Po#pk>RU4-7OiPQX(Oc{aVTtY>xR`qDF1FE9Fa@ zc}AnhuY$}pEPC?8-Jbt>)x%IOu{s2F37d##I$?#vN*!*W>dHT3T&T z+nkRA+)=*}&Ekwh8Qcw9P87=)W-E{St54$p@~GTsl*dluN3sS?j-}Z&_%My94~Ige z>ltom!}rRhv263KmAO#*a1JKqT+igVon*uJE+UVjnvi_)${e`c)0^ua*_5>??dicQ z`wX#=Q%NJ8N1HrxTg_m~h&z|mcP?)*-==n^eUW)~ji=h{&$yUq_42Mj^{N|(;uDuI zZ1Iw`QmNN*Iu%LNl(%JZLTmG6#~NK3x5lXLSW;!s!;v?RC1D#_0c(b%VD%bV0Q+NM zM+yEh@^((}Z_5>mDOCC@8fbN$*yZ#@`t5W?XE~y#5}1;}yshf7hxY7mU?ZRQ6@;=0r!R z`EpP2a=a~C84v2+PCHwZf6C-{yEGc7$8Sg{VyLyR$REL1fu9yF2_QQMdj&fO*CIOy z_(R8a4#gp?d1~6w!7&tJY@RCEokLdai)Qv_8ah}}U)NA0!4*awqhn;iopE!#CtZQl zDg(!xlmvKe+h+$3{>M%<(g6}u(zJK~!S?n;`}Z{i*mFZZ4s$mObGKB;UAk!JV6R~3 z02U{*b8zi(#w;&n4;eZTc+W8uQ^=op+R{O~x(anK<(3YX^6rd_kTH6--l!nzYisKP zP#QIyUM7fIkZA0$qVCMpw_w7Ff^+e6NkH7@DYnqp3 zc`as_vnJehgxTkSb?FFs0M;K^eV{&g_Vs5)UUnqM>?7Sra`58cy3-g;xH>m>_pEDm zRJ3pG8QhpP{{vf5<#nekSPguRK+2)QyXNg3OGX!8Gt{$ZNxEu$S5M92I-9j-xTbwP zqqEl!Ly0ENeTE+dUxvJ_rpU{VWaR=e``o&Z+&_brW#x#LeV!98T+${$iIW|qIg1WC z_(7aZg2V3SltiY|u{sk&a>yhuKvw(yIzSOJEeGS}l1(_kaaaKmCu@Mp*9_%Zg^d(B z+3Vj$oGgM!nIy}~SOac264GaM6;1gZ9CTHH{w7B1LRy&SixRT--ZvNH{%;~Cj5 zcRE^W@+VmhA{d-j%W8S0CD|UVXl|Pq84}o81lAoEKI?q!_98nwo>g_v^K{pFy1PAf z1hNSH)2tIyW->uGj>jKta|H3=0}h7KIvix%K8F_9J|tEzfjz|7Gn?VR;+oN{-Nf)) z+Tsu99Ki>2#cv?B4;8Cm;R|NHge1Ad3|G}m;Z4tPLAJ&Qy4N(hn$uoJDwEOHP(#R_ za4@{Tsj`)l7fd6!boHidZRwy*DuI*^CrG)cs?ArkpxUBwr|f~op#JelZ`vkPbF9g1 z;TcZNnkqOrDpVogSd2y?Nk!Z-R>Me?8nuE^(sB)_GKCwxwq(#oOUwabPn`DLXP^<> zBCxjA)2!|KB5T{1<@J7xBZ@~K6xrBb`Tl9*w)uj4tscykrTrV3o2HDp5s0}pDiwXA zYYHm=OSGVfODO5@3AMp(wE9i*KDp}m;Kd)FG+Q^L)mda(b+ZI&Csxn1a(sh?Dj3hs zZ#Cc@`PVR^cm4=yzztYU!RPbiSOVX7!c*y~R9a5Lnq|b4sLgyE(sUNqkHf)h9C}>; zU`ah8(z1CLhqI@@orCFvvBSCGHKhK*lD@j+d;Go`6iz>b!D)hh>$9Pjs1+&!QYw`h ze032=+NbaCiPjo8Mx(_?6e^`M|EVU%HY{s#y_z1bbI4RGxxt9cHz^sV$`Z0A(=?;S z%^J>VvsrG&aZ|h-6Xq{J0uk7vP`@w8;HWo+D7sv)m&3w-2+BZ?8q)oN0LeOW=eMDX zuEA@*b0Ha&cv~QTe-5T+oZm(|yXHF=kwH;!UzoS~%4u7~)w;Z)eMO_q-m`058}`|h zXz?A5bAskXTS$XD1|p3MQihvj4UQHoV~<#EVT<~wk>O@$T;y)gst42xbq0RwBq^bTPBCn8__SyqYZNS4R>!^UeAsLI32 zK7+H7M*eV4xHf~c5k-qw<<_bg0BG_PLZLx#?Xp0Elld1FDwP61AviW1R%tn2o>wCE zFOex}{G8clHxddRWNThnW=U9PXy;U0(MoU#KZI#94}J*k|E(zO|4p94A3{5)@Q1U! zJW@@ve%${p(>_NF-g19g0bYcm!5d8eZ{!dO0afT0bZ1&A-5AVw*FRuq_cH( zd+X&{r_NX7w#Qw(rT((Emhl>McSFbU*P{(}iA;mmYhgLom2w#Z4o+rG23j)@2Rf2g zeK;Gk#zQ`qarmP)S99EE4A;5S12)G3T;dJ|-HwP!t}|Kk|EjT@&1$95;@~;EK^;DK z4yyq>zyxUnlmr>Hy#dA~z<&H7X@W62F{T0B0L(%fy%-aLcdnsd(x zFS%*Nx8O81nm=aod9+}14UdHdx07Xt*Gl%O@S+uC7pc}-wR($!pov4IT4&MeEJ}i= zWipDOkx6+Ot&kFwTC0FWe+{-oGAr%(b2eIcHngK(jPdwga3Q(K$X7ce=mPopJ3bb@64j0AKb1? zI+gT`M~!$ih{xh}YS#Je#9c|$xhGQjH&Us@yYK}6 zCm2|SzHK-Bz@JLSGH1JC=+1aN4V+@4JOu;4LASk?j;EtA09%U{oCdFxzXekIxG+C` zD|iLmOa2O;pAc3CmVg()cgYX2RG|kuh8caxpCPLnOe{~ln;;-&w_3`UGo?k#xuV}Oz0hg9 z;ApAb`B=5!e+PUQl9f>NQ02h2E^9-MVWey+82@GhFbNP?Xu__gN&hGa?g^aJ}Hg>w@8E8ZxaKP6uOG4U?4i$^~roZkZP|BZ0|tZ=>r z{D82Ke}L!Di|Jnidj*-iAe=9T=R1Y-?+Y^Og!QM6`~d4HcyoFjc9^3lJPA)iWj=}D zkX2zSCdH%-wMQGwWDr#%gXkouusfo#kMWGqe+rhj3(14EM|0v0CrCrdRZd|eMES-= zpD+o4>X4lfXL$Y7eX3BgiuI~6^8rT-b)lY#`i5mzH7_F!3bRt-@W*ZS-3DLE;_0sU zcw3iLSu#~>r`I5H+$UM#zM$ew9+^ht~VhW+wm27VMD`xh1 zP%;!8CH#oQ6?UW1%1R}z`S%+D%~_2`hXN;YIgr7Dw%*Rk!N{5)fxoC(8Nf+Odi*G| zYe~?Y3j7--lK>!*OYhGA7&0wU(_nTi495t9rS9Wq%$Ir^UyLa*2YwUmuPdRdQDX|T zZ|apKo=no-DF)>{HAQckUJ5`f0cT!60~;16xUk^ON#9fsOX7M0-okjNa_FWLbP8&l zE$1x*2>w=zt!wtX5O9R-p9)yjJHjDqrRuq5MlV+wk|qz6bM(cG3~G z7(1b;m}0LYp@2}YiN^#HewT%uL5xMcB>Y{?wErM+y4NRK12+`hpJg7EvNqx8`CwZ@ zAE+=$XbKJjIa+Sjs&y)g{8=$CXMdEKU+bXdN>Z*h`z$m`Drtk?QRw4^{t&A#tg;=A zm{0|9$K^sjaA#1ZjT#h$J%B>?tET-tB40aDF(uhENkM9v3#FXlU8f|;l2;vfnIFYM z>osUnXzoUMa^hX&C`V&aQ@_P!uoAO+wG96`j#I2rhrA@&HJt{hiIx74_zA`9Onk3g zqm+SvhffXvU|D3(K1rahCK9OxhMzCR|IDm|U%{RIBT(^XMoKC<6?!wWzyz?y!d&Q4 zMd~EJ=vZSwdkS9&m3R%T(M;6jLN(h|?kFdACi%K3g7eapJI5*Lse&&dKk`h=o&W51 z!jST%r>h$HjSTkEKHg$9GOw~84K4wc8rFTInzC~>T8mDr_y%J&n>g4NF=ic$+QjGR zC18qk81Wlz?Um{If&6&U2{d@qsFSKqn*6I)Sdk*BHV6K2QFlPiEOH+>hI1D+Gb?K5 zn~&2vr17pl?wBh1CoF1aouJ$xQ*lh4@1d~hT=%?pS}lW)X7v1&LBIUck&bExkzTBFTV7>6V0a!&3&?n92`I78Ev6V_vo?!L+ABBMLe?6ju60mC7Um&)kco9CjGVW|t#w$9s!kdCeV1Qp zH+3zjNH`hrZlOBL=_jmlwm1@1 zc#&a3##mSM`T#Yqk(2UMw&{uh$M;ibPYb>WZ2VNtR#y1Q3zU3mq$c#i3y{h6(UUa7LCSStHD!$as;II3k!8;2#985K6>=Da1~zxKUxhqhk%& zJoE&ouxgky*#Do1yA{MNq5#!?vyl6Xq%jr~O+N|j63Sez6@6=^N)FdjB`;gV5<;P| z>2-FE0{kO50OY*QU~s^bS0p%NH<@gZDBMll2Bd1JC5?;}e4V(9AQ`&>*{CPL`!YFN z8$0T;7vs9fkZ}~eG$|kdpu~9)7ck51M(G0&@8eAKm8}X=^!}Pu#Anx5x3=W zKUgNYQdpl9>>PoYj^`NCeG0sUTozZH!m2P>~QSX zRkl>7T6~PYvK1b^YH)h_J(GBC$VTdDy?d~ z3e-q)u=_GRsxnvzMIj@Ekdb68g>q7Ign(Yz7aXD&CLV;f6Hud4Ju6qf zM4L?&`nwb=e3eP7HJNo_#X6cwk8pJY4amX6~r;nE0Fa1oL6 zACHBR1;XiHpRqiM&WjG93(JF`Po>VAIE`9+U#2t2_4c_N6K2!>%wminS!tt{&s}tH zIqMZL@yYbXt?8gAGQ6!f*j3}DUu()kD`fC# zgLU1(9lK1G&ECXdl?@nc#M4%yZn#E1c-jWjVs8S12ewbrO|RtZcCj*P^91 zVWuV{78+ivDX`sEfML^+-DWfApHVQh3|G;hXe9>zYolLd{4> zqm|P$95a8se^ADo@BytZk@y7wuExbsh9=nF7s2)(hO9Ao)SnP??)Q_NL&!Nt*3NAC zGjm>8120YgU2^ZyB6rsneIuK)CZ#Tg z`f%-1w;pPXNOOP(wY97I!gcFzTbSB7G1}_mDVjrWIplPF{>FN}*P9(}^45p+cEez< z)oPD7qtdY8Auu9Yi5W26F+vNe-Al)ETCqM9s=SmETp-T0UZ@8nGI{`C!%1Rhm-g&BPAP0Ge#mMxTC~U-jTU!-rBHqrEP$o4P*!G=`TdoZL{WZ3 zvZymkC7%*jkxE|ts(rpo?eg*Yg)K8^{b=c3cieR6>eXY5D%#d|cC2i+EgD<1`qtig z1KpgjvGNw<=!(Sy3zv?L0BUmE+O>TfBL0c?%Dx(hE!i1pTN8>-;N$7$rYd8|>(ONM zM{DK_9ar^*|ua;SJuwKrVlPyp57{-w#-*#6@ z2@taoPN9CltNk0h0fRojQOzj0n<{Hv>c%E}Dqy2Ise*FF>no~;GA1_MTel5w)c-of zn$1?`rbt&qLLY17bX<_r%A`D}w8XQXifqql?G_=Y4*@-~5o^Oz$FgQPZhPswIkTCJ z)rFoGa#<%7pcXX%1`f_)38unAmSf@Czd`j^U0CxA9?XH>+p)6ARUflMKFLtlaGiHx z{^;fHGDanD=^RyAE1iz&FuX{^$8*OUDps$V*SugEzCGGkWm7O59O4gxwcP%YW;}8Y|-W4QK%{Uue8%2s2I_|g#Y0Ke4FnkS}vz=Vyq^+8vl}70ZGI# zz9)Z3@PU9m2(-jfERMyFM$Ll8j7CXMupn^{%**)_zktTSq{2_4s;l*Ou4?qxhs-1; zRZ~V^jelT#_;N?e$f$V+Z`0}WA8O;qoxAYcsu$MVWhx~>Y79=LTG`g$+(I!r99HpQ z=n1eMweWV>!Yg144`Q`g^r$x?sYV0=itz*?jV=#@%p$j_(9K>aq7Be0rIK@*!&bd-mn5pQI znlEa32Sw-4NzHb@n#Q^O1F#C=_pxdst+#S#-%`stHK++|^&HagS=hdOP--U3gUUNA zlNsFZK!9M2Qe&ai2rMdUrIOUnQ}WX~e#Rz&nHt`=ch{CZ!y~s_V0qTg&lqoCIJ~_v z+uYR9bG5c-u)C$Jucs4VzjWoIg-wfnEje#<)NBmZxvpsOEySIb>2zhV-kHw-&sekD z(GpA5R3j;2=eA4!2=lMRxT8vN3Va`CR0*EJkRsP*P<;~?^CbVy$Yj37D<;x6ebdC5 z3C3yRF#OyDOvW5Q8FqeLf_sra_to zTya7C>U>D=r{piP(Ipom{w2)bt7&x+8m)<0K- zAzFYDyC{yCmQBOF+_KTS!@9%f7xDi2ucr}#O2FteQp2gglp}rTg^_88 z_)FEb1yrw{M$KIX)CNTSsrKA#)ZJXaqG3fWF9HTHd@zpa02UU1$& z{9nft^L{gjIsBhvpjN;;1q@~&-Wq&sNHcWx&<}?GG`|nxI*425f3RS|f+Y)9UJzei z@LUOhSn$b(Fx)--&n0{?k`>TBGIR-i^FIR%v4sm3{&LZOTdZ09-$xUpH;g$Tt{MB? zlH9m={O+aUr5`M7y=*7M8_WI6kFHp~;@>Xcv=U!AvhvfFpRO8N^}SUuuKMxnj@55W ztekjk%^<{)HJ`7&ZC&@e=hiP;f7$v+*FU-b$qm^KAQmmTMJ(mT1G=AG`Hl{+&#n;`bh;lDO^z5IoEeb>)-FWJ3v z_lDh1LQL&`dH3rAe!lxxyMMp?V~Ef9e0k5^d%g+r;2h>KhdIn)4*$R68X3f$5by6D z74X_V`M#wBesS$eh?Ccz+;89i`~l5@tq1-*hdIn)4s)2p9Om#}83&0u%wZ05n8O_A zFo!w(C!z1)^#|WNGuKK zVpv)hPhs^YgBYeUo&Fv%EW=pS0x>Mdhf3)cSlF~u3@b66=@v1pq6pJB#jqNya()Zu z1V>(&08d^ih6{2ogbQ*mgbQ*mge7HZ3UV%lrDgF2ITykOITykOITykOITykOITykO zITykzwbA*G7%s^9JD3woVsR{i)xgj^Y(2IL+k#DESHb^lu&pqz4TiR0R|=n(!}QwN*FVNZHFla;kQk|?-;>$ z3Lz)f55IXQ{NAho7kg&{7gf3c|Fg_KgW^Sv%;lt(m8 z|NsAcy*wx2Jj?k$-|zdoJm)mcy>vwfu8OH90)AYWQ;B<(HH3i9A~TYFPdCsS!$i~d zEs)^GTmkD^unv)7WfsVDr=V4WUNTw+{k#Et^3ZRmHaIS*PZ8xHFq6m*?DRe-v9?F8)Ysc?mnUo~eM{SbUbW)3~R8uEi#Y_E^ zPQ6lu9ukjbl%SW&waLD)Q=23fUK$l%NVUlEN<8M4`;}EnA;&@axMeM!=qaVEN;18a zR8Rx5qMO(#hl zSv^CJ93}EcL&^x87mjovU9*rzB3WfYzz4z1{B@lymgCwTm`x&A^0*W8Z8UcN@6=nv z<%;{irs8Hm6Dt!%vuvn5E-;g6RJf>SKAdhub}Y_vddAQy+c|P#49GmU92KE;+D*AS<@z8=vB_z}J?4||kmASl ztDd5O4bhiW38Y=2kTUhjyN{Hg(z?-sb-G$0RncOle za*wpLL}C8WP7!>5>5>lrDcjKekal-L-k*KcbJkA#WLFD(dRkCw zr6=&@QAqb*C#@rv3tB40o~2V%N%g=b|CX$huFGh?NcE|Z?qhbjwn%cwOb4z07miV> ziaTY`1f?q(Vh?$$D53SvE^{VTJ&e{%n{0#fgrQLG9M#d(GZuPX$SBWOUF%%eOo-}I z7i_dnm(wQ-JM}*Css(+7P%*7X(pao4b6%I)IaJQxV6_N74Jg|Dp55sZcLSks7a19f z4542jn&+cW@{R4lfz0uSJc~&FmZHEWuO(G&r|kD= zYO_b)eU#@4>A9#__OlYlQe3&^=N)MU7k%!wP&<`Ku`t11zH2@E1AGx6WTCbbZg#oW z7s|P6m7lepRM*fw(@vkpy)-su9WU%#T<3P)=i%+4A&d$`-Vj3b;gaPs!M7S^_W7$@ zbeHO;uQhXTAC~s%vElg?KLL$UX-@hC*-bzAtSV`*w zs*6qDsmcPr46P-S4`XGIduf(90y-#p-8q&nooeU~FIgG7({zsMAc=CyvGhOmqp};6 z(YH&9j4eciLi(Gm8RRk_Gpr$dhwmcwSgQPm)BN3)nz zv}R(r%%MH$=o3&yb1*Uktz`5jqh|#%fwyMJ4Cv? zCrX;;pwH0X>Z#L%T()AOm&62j6)P_`yOSYs?4amwdvaQlQ!e@p&LVQH@%fK15Dv>fsquK=d zCE;&2G$-;WV>FNM5Lvi0nOd4l>7~fbh_xBC6|_q7R1&p@P$dkKaXk$`lLPE>=x<4v zoDgZ9+0CTm!C8`hiSq9xN-m4GByS|qRvz^g8I6{GnM3X9y6Q|Ci79keBDFX-5Fx2F zdL`|Z=#xkY;gOUDgHYo#HBS3Tb79Ub0SJiQOwWLgf^>Ji;V)M^gPzqChwo=^bECc|L)8_HU2vQnD|Grcw+72p-z9v8epw1sqs!&NSn0Ipy!bz7`H!R{1%gf%q7 zEwIsvwOmC)fxVatmKO5aDtx%vzQ7hE$Yu`l3Z)ijrC=?COC`MtJ|`f1EU?96_Y$Hu zOQ}%iCd)u#F?ziA#hC7M!JtLNB#QtCluB!n;IWoiJW$N$iOI1QmpLq+KwOPemL5mq zVblyDA}$ymGh#$1^gfTJ&{k^kEFgAJmj)uU7{uMA+vYdo1pf%2=zf$mR5T<*AM`tHnz@$h_cU-ZHn_ zVMpMQu^3^ts|>^{g))SYj|3~}CLpYEt`c!hp+aQA9Jr5Ml=@-hG>;-lo zsZU{fP`SuOvW2KDqmC8|EMDm9at11e;)EzU7HrO#a{B_i+g50|#JD`gv7{A?ne*hD zG8FEmVL@|%h&g`|JYVs8Q=XAQX1ql(&v(Hlf(E(dKw+b}JC{j)e{!GpoCo_0!KDYS+-Kx39DcJgwaPWU=DrhqcKys{Yuc6cS)hS<&9Fw_jRh zQLT`#<1mE~PU_>k&Z8GkEq+OsSS+o9lKW(A#9q36^XXzv-2dl@VQg~{HRBiD#JH{N zczGUaLnB7y+eOJR*Op^tv5wp_q3|AiM+e z5ijo#)O@;F^B&A_ep@=);jtM>>0Cfo8ZpUn>^i49PytObC6XwH(u$d6|-%Kq|B z)0&@~jtl#ko4%6vkDxt`n za_7Usv{PNZqe{&;wmwjidC`txQlcdtq+X9BaFV6~EG#-CCj%bJm7$Ukx#3amDyTSG z;Mq@@z6fflh}^<*mY4Pig(a{l7Ze(*DvKLGG+2=HIB0KdVR>rc6l_6Cl)3_^l`U{* zW%`7qJodHH`XUP|!0klr`9@~)`hREB_FL#CMU3S=ncv5!C%~S=(3}kAiF`Exg;Rdd z#o_gUQIYiWw3Aj$*-Gp~9{Nqi6(S#v(3Z%Qc#=~4z3CHDwyrUwzFYq`8Z)!+*K@vk z{g=NRF^_+L3Hvhpx}SGl_96{?r3iKZe0o2yjRs)$^_y%=H+cD&e;I;@m^DIbs{2B= zc>$0A`sy8Pq}U1VpHC65)t`C#cgr3HV1wnevEPiw>{_C#M2#82vpkx@E5sNH-nC|N z@wq)Q?X-F=9798+xty_0;q8(@V>2N1F1@7q1Tg5&F&k>8Rdwz}6iV1XC3*KuQ-p>Y zraYSxW?uCjI&VE)6NK6ni@{xmd{QDZ+zLH*&)|evpO7}fA0@IHI@wEGdRkB9lzu$+ z^eea?t_f`^;&>d@G=F~GhLbP4BP%8Ray(cJle<&zbhXw!qu3OZSD#&`)e9Qa-@Bp`5>{=QJ((Ph86m zFIaBr_S?L7_8yqN>0R-!(Ot=}*`&ic>yUq{%o)qscKUnDla+RU=|fZBXxM}gK7p?n zFuZ|ECH}Ri1mHsj3DvG>aiog*Y58VO8gjycmAe_?!52LhSm0#81#Q3*6zJA}!Nl-} zF_puALC5gseX2JCNT}|bXoT}CHQ*2)q-4m62nHJ_BY?O9wa8PMVCtz1)nP##hU!Ql zo$d-$uxsGLwBalaNTT})8t_N}Uc6Ijhot-p1A^{mBmfTtXwj!`2P`ldsw0ad@d2la zP>oz+Q}v)h6g`aS;JWT39DvR{P?BL20r&+D6cex@)N@3ZO8mZFq34Kw(*!@+r21X` zXG2aXFm=EJJ0JrE1W5oDRNifanJVxOq!6G*@unTqb3_Pu{H}*4*r-d{7DWga?2md^ zA7;o23$}X?{Z2MlP`@CGN97`lGb@CM~{9p8tpdOhVl?g{k5 z`vSAEN0~F^+iUI*NxbPDn7+pbVh#HS771Vp9>n6maF~^b-xLC9!vN=X+aRYF1}r!O zC}IY-&Z`i?oPGg;pRjUUK3sN(50cgsnGF?7YfKbMc}LT2M3OLdbmg;=z1Vnt|B>Y zM85~CPV{ZA+kRSfsZ4P7E{098VAlZ1`n$dZufo=|bRQuf{=KS*IP6V!=$#kK#0MAo zFUT2kl7evp7pwqFFmC|b0lY}wGDqtVn_ zkb zan^pqBq700YA-00E(W<&ewsPqwfId`+V z+h+$`(qOtUmaE)viw**LlgNm#d5Oz`?3{vrCWq>t<1f#*Yh9Mp57!M!N^&uXW$$Oc ze;{j}6t|T_(PQtLwa58oed%H)UTm3Q#y}tm6&X3E$)t*;0#*TYZD2hG803RE7S;8{ zAlkx6GuEM+QQq%Rr6rZ4w{29Ku?>%8 zEn;k$amZX}#Ju{3P(_%d&o7Iu53Q|>@X4J}w>h}?;^P@TT!_9fM5#|DZA@5D z!MgCGj9aNP#basyaDY_ZJHx3ApDd#^6Qd6XuPWy$0SR2>D z5sC=Bt=IlAAKJppmJ;&L#Z`|Ot?uCHh&MldvfQQh?5jWH;VlK7S{V*>2!~Vx5eR4- zkK}mw%#Vie;^M-NN=uK36!pZ5*da&auiHap@zaIZBb^pxKG*a;M8^M%u{ORGVES(? zfvqajhPx$kU80T{N-0-{8fCYCiC8P3cLrCU38s{jJWVqp^GqmH+f&`#=8Ws^w7&&` z#=(9UYr2ltTf`9?m62qq>{UEGjXB=flcRj@LDWf>VjGWm0yaJpd26mI{ylcSM0^&@ z2$m79Bv1`7dPT1u!=7*{J6`wjuIg1iFOHzLty3m%vd&P@x5$z?-ZI^=$Z{;f@`pCi z$4bfP_;idzV3?yL)7q0~?h1E|gCayNNmn#_vMJt1xoN-9MzjW|9wo}KX%d}ksUSGh z*5W_SxX2Htg!l-i84akToJU%4>9!z_aXi)Q)QL|9>wgaMJM;&y`lLS6u#H%J-Q%dx zmw;&|H(a{XZlIRUo0Ur!)_!QRibqz%qd1@0NnvB2x5`%@l4Uz4x}Sj*2_Gu!-Od-Y zH@q(LB;o_}feH6pNQ(6b4^U9}Tx~S#LEFEv8#s)n936#G@;<4teqd`mWw@ zwJm#{V6E(O5#|(HjwSPjJ|;Y{2L@A0bnC}#w?)+Oj_IX~kK;|ZN&$Oh&(RTq=#Xj< zZPFTne0<4FwDn9WL-6^PJ*OebgEk<77Y=@Z*$%e6O=nM#|M!5znr za;(N)pKOx&#k?)P|M(BNf5FZ4u&FV8)+!Z&<#Gr-#*(2aiqZA`^*ybx3tg=rDafJ*q*(FqMGGTr=cW;i3?D_9S|^ z$wHmR+@p5#1?Iv~D+H0n5~C01O5F}8 z#h3J5)P=GY6{ktvK8sEFhcZO1keRERxQLXZVu`@T!BoxxTd7q^jdBv;<@W?NyF2Pf zO&=NTgnX-Cv-wje&hy=Z32-oRVjoAA%JeL?G|?no2B%iOXt5V>MV1n1nDKH!cwI4x zhdde`3pgfdVwd_AIMahSK+rZPF)J}EHX}AuG*c88->TfKtWlV+eE96bZS637SMH#` zuFK3cpHk~uh7)FSYi7E%mw=$BXeqOu7k;3xgu*9Ayl(f0Z6G!PeYHP%Lgb4}WdMk7 zeF|SJu?mSRBx9TF>mRhO5UG*$E9IgDiO6R!zK)g&CDg&6s%+`J(W@D;vQ{6&XU+^o zYC=C6><6e7SV`fotEnkpqkR3a$%p*o`Zx z%&RB~t%B*p7*=i(Bt%<=*>1oC2Bij8*Au1mZ(td4A!c-kbGQM9UQPQQ-q}WR;N4aU zE&on%CmG%M@8Nk0PbG{46Z;>0w$E`b|Jnm&AXg0LaZYXS(kl$FU`>pDq*!(o0 zlc!6crw)ZJyjh2@`wRZyCMX-oHgq~qvr#h*(qW4;1FT$R+_E|1pz>AmbP*b2^y#pJ zlkpOg6Y&UYFs!~eDY1a&exG|pH!+m;Xo;1{)g zM<+C?S_E8-52uS>cRvhIvHVG|F|0J|W%t*n&`Z!ZVPm5<;VL3@!WP(ka(H=qFq^*` z?zEq(r55#4QJH?ZDH<3UC~{38A%@0l7Ro6_4h(8NZv}xk=GrkbwskqJl z$^Sf^YpH9dXRl}QNB0l^gT+$?0(P8nf#jNZGP@@`9zs!U?RYf0Mr^d7cTJL@Evc>qUKrJfLuS7M)XE7L0eFr=8>%}kx z6ad#DCu70vV)6j&=sD6Z3Y8M0cDds~=Gi|g?9-6hS}4dD+MdVKtqr6(qRfZl#a8MM zr~X@7ma*88Q5I7+CpRr}RAR4IWu${tSo-aR9`5qzPRp4u(MC zm<4(1)Mh9SwU-T2-aeOc-go4P@%ts{k+eyuYTU{*$vgJiQ=%1~1 z)z$7e`HB<&l;VjwOs7x$Np_~*_d(#?e)-SWP4hytj%d5owAW{{9RnA&I1l5b1F_n0 z-=ERDn$Zz>2-t*Iwz?agmT08;h+LsY?`>-h*o=A95^}1hzI>f_+Wgc$=>4U58>bj)WX0fmcW+nd1Nd#gMz`=zkB@@c2nz?m?1 z=5_qpC%ta$L4&H{HSLN=>w!#X27srb4*?^u=+oLJ5!OcRE4iI$H*?hYh)0s3y z@zjw$t|Rz?53;S&*+q77{jQ;b!pXOF<>_g+9=Ac(Y~$#FB^Vjc*r95{!)t54wXnL& z^?LFt%N_^SeRufD;9v~7>$>~L_+zF=<7274IT4AEo`}cfT_DAw-M6paNe=xL?LEf= z3TVh&mG!pxtGgke5$wl(&W;x%)(pG2IsZnBYFy8colE9FmT3xvmh}4wFIH!iHJ9z4 zZ%j__9t&n%PUWk_a7#RwgO;lj2Q5Eb6BdMHc|CfehTA_@XR0=@7u@fP2??>WBi=UutfG3sc!)eCDjG% zf6A-9F>ibD*TP5?PS5$zddyq5WqUVQ25VqDy-rFqhrLPFI-7aDCrTECzOCFmi!wds zwd|B8JlI!eKl|hyVk&?ixj)Yr_}sD>$TO~7mTtFIk~*&~JeJkkqD16$dGA)dz&|cs zmk-8BmprXcF83)h0M|xeEly+NV%(mH8uKpJOmNUVkADPKzg})J0p%6E?WCu7xpWPR z32;5GC2~4FU)6hmlll_8G|!_9+zLhRDc#Jtm~`nq&ux9T_b%qqf0ET&?0mi*6}*XY zGT7m~^!;bLnG+X0q>Z%}0=`z5s|J1Yczst+1wG?hVjOn$#jWU8Mu`9@?)xK<6_!^_9#Er%DV%`UAdtJxFc z<&qAQjIKJ3$Hx2P?DV~y&MteW(UsEiWjBk}^mzvglLz~fJImp*_i9Hf5op&Q(Y{oW zLt&e;@3m%5ucSo>g?FQ;`{%mYDKFgr_>rn+U{ulw-kVon(@-=;J&^Te?GpGJYlQ2| z+Pkl#I+X8EWA1r-mX}U`-wO-?IT0f+~-nDzQ)bJU- z9v9;KTN^O?>RIZkaBLz$LVBaVI-dXXeDEtO=lYbp_G=I1?}1on@)_au>Ps?DYO+I) z>!0%lqb|n#!=7=w2h(jdi6$4)ua9d&m7RI(LkvrgweUVtUI63Gp;3+EC<`x#vktJ3 z{*brXcc;C~?P83~m(!c9P=HURDuqxeW8Cjij)G+v&K+c8!XZ_`0$pNWWyNRDm-%Ym zaD(>y(ZEL%k0Ms%h*Z&FuM)+GlN`pPzVBh;mexvgBfrPOB5);Qi+GLaL*w%BxQtUd zK1Z_Pnbjl5km`IC;`iwr3R*{?kch$4PX*9d4Y3*bmE3)@F2+dRVdT%Q^eI1BXBddY zzf;{8txj}s)B6dN-7M0wtgj7=&Kzp;g%v8b@09<-`X@K&XR5GFE>!`HJnnf7hBB0j zkJI}?M150)9E(rk5;Zm<;l0q#yUsi1+{u$yBxkE&+tU;;cf~|&ld-mr+<9C zZp!d!oX{3&%GVZr_xc8pb#|9IswG5JJ9w$`%yi5~oy20=n@_Fn9&*jANLAxBA2gSm z!EzZ*RmV#eU595)ox>Yr@Sdc1yIVX$$Pot3&(T^Y1i(i@!Ly_@jTrUsr$jRk^WA}N z$S;o#p#C(C1dhSdb8DLl z2~n-_#tNo+Q3{&2lbynyf8(G_hut^Jc{vBCfw8W~ACJ~J%f#CcuCFEr0y&OiPM>!! ziZ32yJH1Ds3$Mq>pTWjHM7%)iGq(cxNMGoL1epUHeEc z<$yaFdcHCA6Fpx%+(cU9GWDjSx*(;*y;%`^um)6!|y9cMBh})-ztDzm5&Yzx_Q%zhH zoznsdI_a0D(#JRKpx;}0B%;AM0uA@m&Fmw^%U(?nJtPTU2jxu~Ihpzn(JQkoI?eG2 zxGD;Qmp!?EydOh$Z6SLv#Ef-4E)4{Ux;Dm$aZXBojJy`dpSDSltf7l{Tr$DHILg2GgJW^)$P2c`;yll#RFci>I(CR6^fteU|TLp#8bnYc2)nC~KJbH`b@R z=FYD9r*p7(%=qH$ZNAS@nZ&JHroP)P-Y z4zu-X63Ptg|3=ozi`V-l%R$=@z{Zvfja89M<{iwX)X5AUqRaZ>be8S8cH8dTo^!c- z&7b4tk-gh`a{AnMI_@ImA^CET(ko%rUx_$|PDZZ9fz=_?c_7CLwV7QNoZ+NmiX6h# zJ(07h^!Q5&Z|0-L$Q?R(Zu%oS2)b``S=N3u8KFE;MQ*XoP6H3KkyUYXa|K?+bPd|} zmZC#XkGo0u3v8=?_|DjI`=L{8RTb;QzOzVMRBu}VMcR9=E5=qKan`B`2`_pDgE5WS zjW9otCgTF=^#c4olg7t!**%IJY7@JzPHloRJ(C${Reg6IZs%g#110l{Fbk$E`T8qc zI<^-ALMlQXTDZG*AdYKbBi8R>*VM?;=#?A}TaCjL(Lrq+6;`#cZX|B0%p^k$7Sm`| z6!hOOY&=PsM}}x-7#3Da`6H!@g1nXy^FpzJRi|0#s4tFi#EOl54VF?#_SdBhvC>#K zP$qvEGRNlHM2hlH_(5W~_DiyZ(V-W`c>?M>v!$Y)n|j zjMtyWTrI)ll@V#axrlkQ znv=eAjr{4&d0b1S#~&lTz%)<9i>?_p0~QP7hvru+Ge0{eB*$vlgk}?jlRFvAE*WoD zbF1c}$XJetjM?-RpIej$ejWOwJhABJmwx0cI>ear0O%YOtd*1IpahHr#-j}2~hJDEUNjboEbBsSOc}pL4r#~oD z75M>XW3IGYyw9WYQhe!l4lQdn>nU_DuQbCd4?UfOwBaM2lcF4q8|`6UkI(2`N<#q? z^mq08g$8X#9X|-zA_qQ;?(u7urd_3I(Zb=W&U_-@5vQpMR?Y!%g^>a76rXuHgZ%kx(XYOPe zJ(~V12l9a3jH%553r$c^BKmIOO|h_l1ZY>jL$;90H!Bdv$=&u9W~4c7t)@Knk-3RB z)){<}6us4#8-Q&1f0hir_mRxqL0uHH3;8QYNK0L2#J)`@sU%;(~QXcNpyM|P(ibiizgNHEY)D}wXVux^B_+zB? z#vZr=#}}PQn8wE@)h|V>XQ>*8cCw}o{4$h9+)V5zM@I>^h85Di_&**ssvR!374DfH z$m@sbgXx0;LGIkuzOhbrHyJ+{LUy$@XZ*+ccn=zi1F=RV=o5>0eT)!Px5ynGnP34E zXoLw~f1@nU*VCx4idH0v$=0-gOKTR!_Y;O8DRTFeqqh)d?HfZ>TJqo2Fb z-%8fB%eam775IZ6hKHG&JgWLmDnHsa-Qgu$O4)L(e$t3&x3wHzZ0GG+w1`bJx-Zy! zZJX>O;^GBaU1qL_)kw~7-QBd)I$Tx@ID2%}G-F5Ps8HwDZocr*3qq^t;_Rk^YlNR* zWzGEqXMDsf@`_ppI#8=L`WA$;1_NI#SMLI>SvN~55BMgH2QZqc!aFfgyZ}kOj zO5*h0*?vFGt6yL?8eYg$ZX)KLAOE>eF0Cix@)hv9C93KoXBiO1$n!|YgsFK#pU1>C zu{aWPO_#D@5d?b^hu6wyGMBV4xKG`%z zyf)l)%@MZ4rbt7ze!dfJZ-d?re- zabw{skV6NVbY);i#LJaOMm9VRKGGvvExFe$;7_!0Hs_1Cay$VRJ1^0`T!YmHMil}+ z6yo>Ol0ws+(bPQPcThU8dKTXP%8C}YE6{x;_jaa<7Z}~#DTTJ?#DQ6fK0>^^vzZ?A z`_pFBRtjuYe@6HQt2xO&yAtt%*e(C%9sD@)7vTOfuGSo!#a&B`!5tq7+Kz~~1)D!@ zbpkor#Oc&0e|Rsf`vo%?N= z74YVt{mq7zl@sEO2LKV--XgI7%ax6b72<%6gN*%O5^UUT|C(<~+z@?sp0{+_cpxGW z@HPh^@U54N^-W}lh!E3%A`c{a;G1tCC&VEe@GW~F=f9NxDR92c?mr4#5U{v-c>ddz z`(Fb8K;(gp{Fg%>$Xwsj01{?pp0}v%tZyRl%_%$Z4TgUSu(SU|l#Pt*AJXg` zZ;^pO2q6EU<@{&#7*2~TDwrU&zGB!@`|6CkwWSsv1;^ugp569blxH%z+ zu(NW%U2jdUHz>H@mV^5ZDSUT0{rWH%Nr8=jWKWO{BME&jsKsS{>Lu_28b^RXaAe6Z#;ct=o^n9BM@Hx$3N>^ z3-I4eeB_mq_-IVA=!UEdAK3jLt=5U{%b>+@x~d5 zFOD~s{5L%yV-N@-cK^kMHwwJ1{l7LlWag07{_kamj6v4-e}KsW$>-mV%FoXtZfWCe z=Ex#$W8`f1#mvOs)Qm;e%+B1|f{cxeTTl=M@xO0$_e{OXQL7-pM|jAy=BL49Bi4he zHl$AtcCcvAxlL%f!xqpS!FXt$-ZYza-{G))z>$%!U(CEKqi*;;-4blNFZBtx4YP4* z{M`T+y~aIB$uHENBeji;zt%xn0+EgTG|9XC58WF_E5F@htPcMAi`mQjPlNupRo8vy zif9x91f1*Ee@EJ~94xMqTdJU66E5NOFfQ_AKiDDljg>2@hWH{q{$U3AmGHQyRjeUzbSX520xP zx9`F?ivB-94B%vA|1Z2*$+)<=fSmsma!5IJcb%E6Qy#a33U33(C5<|Dow{vrgRPYd z9KNPd;=!=v`RIh$`Gw5bj~Gx3$cYUT7#7g7tP_@S^!UGn6hRj0biw5E=G4{RqDZ|H zHywi+4|0|=1~injPp3(yd}AWNue_^|i*vkPZk)z_1zw#JS;Xok$GY zKnx=8SXPg7_V{?OlLMp+=!;LpK|`X4G_L6kzAHX3KROVDLK7%937aB)9bbKd2EKh@ zU0eFPt49&cAX@yf^^x#{*%~V^KcEjw-xF%*0hS*&UyXi^|A6RGWJTfNG<<3Nh z{~P$h_GvWBB14@FXy$ye7n1ov<=Np&x6fqF}`1ia-t?+U}3^PN-(H2T&#?`EjP;R z-ypC#I8j{9D#$xr{vdr>841ksalY1@B|lof^ve>cEtL9%|Chm_-URjSw!l4FU0 z(Woo8F+NEl8G1v8AxC#~JyNN|hPN(YnE=;Y!zf!-9#JRzZanlWv`B;{;_`f47;S9d`epM6_ zuYO%OfA)pnS|<5k7VQ*U3??%m1!=E>dr@uiS`k#XoQd*1AeWo^*hwE-Kh z!^xtD6g1+p1@iM<<8_|_*{;haTvc_O66t|s0qNy0^V{8rZqJ$n0uC)w(10P z+1r##g^%pofvIWrW3LJ?-{2#g@8;37#V3Wco3&NU+E zD|ot)##+`u1|9fZV?!x1-w9;3>$w|*^v8R<9c!`qCAH4%jhK+2Cg(S54_Eku;3X$@&*K@zV zk=K?CXagC4pXJ6%B5Sd31u!4Fe`uozS0S7c=J6R5TZTFx#S zF0SU_`)#e~4r=S*I_ntK?IaK|@Uk@^^c`%%OHo*rCQpzw}(^>+%BlEh{Ud&MM3K_R7CU+LK9J>(P!N zJ;Oivhei<*^0o%*AG#}=R7N}dAp&Uvqq0_>V)trti>N~%<|HIwJ6sxu4e#plO*FTD0r0haV2CglVH&hfjXQkuBL1Ks^(&x@9naHbASX9$XYy z?v)xY`%vWluDbrreoNxL?TF_fuTvt@k%$wQ(I+o%%g^6iNCYcp%qDW*Ih>2h=gpYy z*x88G+MOj^pfS6R@+*=Zb42N44x?0BvTJ!VqAwdu744;4cq#rgWrkH{tf0Hd&R}Mm z%Ft0vZ?XI7It*2vmKeF7eZ`7?SOzZfwbW*b27{W47%Ply4qC z=6_Z!kXmypAXbiMB`kwy_kxCycc8I=sIT47VrBd_Bc5MvXCK~|YZ+DL5OHG)a&zNf z+*1!8K1j&?1o;!aayC?v+E>cmIQ zKlWZzVX4e{_D!F_M5mePZ+JJ#5(%H4IMk)e-uO43gcjR!~ zux}2*#09TXNc77ARxK~oL5XBmO!ekF47Nxp-=4YIv=Huv!yg;o0Ta9))9?cufGOS;^iqSp>6i$M*0VdX&hud$ z{!DS5>+8J?o0&j>*v{u-ev99tLVh6uV=Y6-Cyt z=Sa`yQR^G^(}0ZmElJt#^kiimdkv%t^_){F%eUDKmDgiM;G<(nj5l3Fb{49wao~@Y zqh^tid0f$sBPW%mdGq3%x^Y>7Awd`VH17*l*2hMbn3|Xz!2t)$jl-(pAcS9-?Llj##!jE5GVOZa%RZ{3c^)R`wB-b** zH^u7Yg3M2Ex}&F;pgWJBmj;_jm*s6~2B4Na2cb5L2cb-mL0mm~oSZfvBC}UN;#uM$ zBIl4GR@mbqBeP$@e|vV9>pc)Y-Jj_!>Zadwmk`w;``XGdu`q$I>Cs9t(etOCfnX8J zwjj&a)&jL5(+(B#_zCw^Ob8tnlwoU8g7(LR%UTAzc2*TSLXwfvU@&uIGB=J!F{SN3 zlB~Usht}p4Wy9pN7tQVdp(e3uG^~=Li@n>2O_nQJntW9QZ@I4Uq`_Dq&aDEtuR;CwL@rA=7aYRD=Lw4rt?9=79@>Kdh16V=l zJPg(E+KRXnIKqb$AZN713^Jk68<9mMnpnNPk7b`n_fRcJKkN9x!%xilq1os92F1O{ zYk5D$)@HP&qS8LEtvUzf6O_%^Xu3NX_*LmS;aqTPTJGb?N`)Y${R7)DFJL24X zTEzw=iZ{s4Kn+|NKp#+e?76j zSYU(sGVEM=1nbT8^l|v@g~7xZ94~HLq6@Dt>7-2iBX;+to)W z-LXS8aL>3`z1D)JrP4)=xcCPcf`RmOsV%-sL`Nql`p_r-5Dqo*GZzoo@7Shx@`#_U zj5)Aj3_8D>`5m=i=xpfgsJW3;PKTR*N|*j)RadV2`{c8^S+bTeqD4FoqV9K&HlZ)rc)kIhOJcH=>2`yKj9 zMLn_ZW64Z$p=0DapF`$zQ#?@?(vS}i457>35ep~1OX}22(+TTH*>Oy4K|B%F(mfd+ zRoelXEtw!|XxBz=Im+9pF{@tGNt0i3;|BWutVsTW-=bd#7{W8orVL_hR4s}_JpYCf z?8yPhbS){1iZ+eG_9KB5672y)j-5`4?5($)I)C*3{>~t%>>ExDXRl?dT#~04K7Of3o>pSIP~s2FgyHBPTS{Vo zwz#=49oXp4m}}7SxHVLCzg5PBX|Gm(i-)9gb`O0e1qJ43c9|LjVD*YCV za}XCa&icWnZ(c9ss=aEO@)P&Bvcud+o2#$2xl;dX3Qbc7K*@Dt)?uyH<+j>clkep4 zw5w}coRawGS%1@ji9N~B$5Vv{4k7_xo6&?eE67>U3mqr>VFv>p3QmGB+(*y?(&Bbo zXQjm83byS}%DmsX)i_MaTdH_}F0U=}t=nQI-2feGz) z$LNC;lH}K(S<$*11a5%3FOAYHxbYo`=AWJJ`=B^3MyyW}3W~AVA)ERanJ)C>{@Nqo zXI+mA=y^%}2%B?Orp4B~xw}-^xlB>sEi*8PCoCCGmZ{)Tt1=+G?vofhV4Qw}9`?A- zJhedZmHqbNVKNf&o77BwG`~bPqX*7#-70;RJ3~vbYfR{>I^k+wz79t0;HfnK5B<}; zRRC*8Sdb%<8E8Sh|05|53W@EA)^sLP9NuTZryoNqYX-@fB{iN;-p}4X3bVv~R(cF& z^`DMKsC(2_({`)IInC}K;U1pwxl4^IsN`&=JXG(sm8 z9un#zb|_KDmr}B2=~@`e1z#Dq>eg zc+}NiAT+neX=CR{S+6eDcMPi@B<^2p=~(7_9LQ=Bm2|o z$MCF8TgSgkG1vkIUrcilUZ+x6;yGLYh*^mg$g5(Cw7ma~@X-Fzk(Vb$pRKY&+Lw7I zoj)np%CMTlx26H^M-q+2yEZ@tWRz ze3{TL0@8kixMVTkOM(6LM3iX7l|c#?shueYrd$Q^i?{lnBsBFRraOrYtRS zw@;o8J3{QQl26=A-lMXuRNWoqY+$KI=M46uFbI-2ak+rLpD(hh$X@A~Ycnr6y39pe zZ`7@6I>0K{+I!C`ecyYEug)y0VQ`@>x;>qubg23akE(K29nA7a-U{MAYDcX9;mCeC zQ@~8HP|A5wI^oJKZ6|`4?|Kq9v~*Fo?q-LFM3N{on~;X{U`E91t(y+)}=< zEj0t#a-1H6Zk;l?+F)GhZVV+sj=G)}Kez47x}U57?HVY7eXwybhTfX~5lC&%(Xy%C z(ZJn@#5UdF<8!@f*8Y0a$tjG8P@JR$@ZZbrKE%5sQ_G8i{#0Fni6^TIuue(nAW>)$ zi7jtpqmdLQ(4=J|SUxhD9&+brfl~^1ej^T`jeu*A+Q%LtVCa_8oy}1Z^FE@Q%Uy(= zLV({?UkmrXODBO*vrrY2nxTSHP9;}=jnAO~NGwW-mODqn zQ}eUaorJlDf%<`lIrUu$7`4n>5U8l#C&CTQW*7capkcY5P#3QYJsK6jX6ow10-u(Tv2o|-2~>+?pD0H#iwRqx>#o2 z@v-27TL$>-Rf`~<)BT&9dB&zzigl3l&{-OCPPEdWDfOJ23s}wV5mVL}OF# z?b-UQPy^i96le+*$aOuZ^)elj<3$~cF}dT=*cmctZ#Ax~1O zGeMK#Lo>YOw*tt_1qmPNBA|HbhMo+4jvj^xm`yqL6swbtZK3e?&x{?Vh)d^Tq5|Ae zFX4yxY9`Q3O$#(?w}EtTxY~KY1HOhKsfj~7A5F-r&vYF8R^+Wz+}?W2AF>=&TAf$Q z9U++Vaj1(ra!@!mpfw=pOU+1*IjnJZ%4y0;k~Oqx2&{_xK_lJ3<*RxhPUR#dBFXz3 ztIWvcIBlwI1MQI&C!`xoN()0@!wJ1b!fFC^b@!(cAs@r&9v*zsf?%$e^Em2^ZrYa+ z$pP{RcVROdJod^(1%iAuauj(FD-$gVPmT7%k$-;bTB+{~?~Bbn>S^1_S|_frt!QOo zE-!S`{tgyT-Hp_2;v^gT%Db*gugAd*^Sw z`1l!laaw!keihThxwX!`R`0SCRqfw&uWDkOqAVbbX2x~QEI{4|EhoJ}O7$>*Tw;F(52;`zkRedLZMEW*!%l4J6x5b*#aq&|I!B|dqYu+9t$x?R zYjsmT`$DtxmWs5@)pT99$MxmHA{LbOI0fFB4f+|Yu!844PF?m{tvYYF_eoVaR^_X# z&9&FPNk5_wgXa#3T~Up1)aRdvUvb zrK@mSSejo(10h)$J=mt5)6`gH)Z?|iWJRvBtY2VmY)Ok^U8<-O8~idnS`mbm(DX6n zAtI$bO{&q};c+o;Eqs}VZqvYB)tGqsdI{O|Nu?Pg;t)^Y-BaaEcR$E+l1n0YTin=> zww&;?h7QG~NaucK+vt;olVP1@&m;|G6P^Yp?a^~SmUbOTh&;eNDyOBAi7DhzhZHuj zmxANe_9f@fT2uCVuv|J@?;CiK!Jt4UgXn*!QNe$|>bI;m;l;dB#0J-P8>?a+%-%`rhkZM~;R(uF6p2=xLdHwc-=715ijL*7W-Bu*UaJH4q%AMW0 zzL@8GTLs$wqKFppO5~o1W=~7?f(ENfiP2dzM-1(@kI$RMfavCWSOJ5IXw+GwI5h&K z5F<#Y{xTY&%i2E;6w&$2n|x9OO{n&tsTxIm(aZbg7-!9-VqO42Cb7f)a~N-_dxb3_ z67p>-M2h@Sk)~p$d1}IfTq6Yi35b&zB?6Ls1!E1&SSoen0{ehiL`ufO@=v3a_{}VC zlHlP-VAFlew@C+u`^SUH8#3yq4-xQC&5*yxD8q3$C#jgfx@UraL=Nu4WI9QlFqhAf zjutF2Tz({Pbaka~+a~8FdH9(?f`5WV)8U`@L;6SGEBp8DBw>{u7`1WIjiTKOZ%d-M zC>~ZQ_^(q?*tiCdyQsdPT$apwbeW89rv3`qp!;iJ2!L||bmKG;m<_PElBgCYPI2fD zpbOwGP#L6Mv>QRDupU9OS1f6{n)RHZ&?!)LpjhTIDbUG(`3K4;(amX+K)0;53eH-X zBfe*QxS(0wslG@`PDH3j2pKkwb0=J>0?k+&AcDJ3?%1F=i`}%dx{Ub59rffMC(U?w z2oD`MQR_f%h_MzYp$o_QF-|JXd5j`H>cz&SowJ(!`iwrj!eW0w@Hy~#rN$NE_cT(F zj~A6xBTF33M;w_*4g#h3n)(`wa0;it<#@Cg8Y+%Lb+TGJY&;dlF$C%Uk~V9gOR7*q zSdFg}Z!D<-#Dz081eDgv58IrkV_H~Sm0X*?ZP+--t02hN1E&<8ds#?eIhDvGDA^x^ zt2}LL^{3qWZZq9DJ5ANzyT?y0>S#G=o?`6qBoSFSjV|-RwRY?a=D-fe9dX9hKW{UY^?i`Y3OQvVw)3f9h&*vf|CS%)+_jvWS_KU%F`Oxce(=bR2m?7{~76A!m zox6Q9O?+s=`UBLlJO`_Z^-vj@@$2kn}ntfX@b>=N|YBGiHv z3C1{#8ZCuoU5$;5Dm)y=w^`RN3;)b)^s&_0x5)grAc^`*rXn2}jy?!;(w;dzOK1>|c33zBHXPq^>F6*d#EK}M zdV{pBQ86XT7$I75xnckOjb`(`LH#;yd_@O)v!H}X1`Q*@m*KrRGW0vyoNz@6=P-I7 z$BcRFF?I@dJ@0WP)D^F`CYA0R`3U*a*lBe?W970i#EDL)gccu1ja;9+Fn`>M<6$wX z8VchQ;mi7j7wk6iYpw`&_7v;9f`>Jg{Gt|H3X*IwM4m%;dk1S0>B#y;g|w#lyj<8~ zHRx3%(Ujuhq;&BN28smIAqt~h@>?U5LV}BY*RYaRoQTI<$%Av518f)&?iIHBE zcp3KyDO9@b;0BSE_m9)?K@UygNMT~+;S7uv^}rxmHcXLHi`TI@76t2yN`<7zY%@QW zJf1a(O^F=lQzJxzhvB5tL5(>rCr{Ve#z`UBkO0jO(p|z=(2ueJ&7z{%X$pile|dVD zD>b*X{LV{0<>vjx6J7EekrDbh!%_vJ!hG?skxiFwR^^6>t@27ovW+w6CMIbB84YK> zIo2^D4Khdo(>LNl9UI@64I!MEP0j$N5jc#vLfHOy)Wev(He>jgFfs^&F~ouUWDBdg zM&R=&&sXQx7aCq%x>FYSWCt0ET@67_jUB-`GD?N!`w6*prTwoE%JhbkpL` zk2x8j`@0+!RsarZSQb3D*e%nN_OUg|9*yHsqv4T|Lq@aJ_fL&qf&-#etgPb&QYOj< z<>k&CX6n}I;R+#!aN~aXX-q=0i4+rfGClKK5jTbEF&VeBsJLAQu7Tqv5$JRj(|bjf z^Ix_-?G5*SWRkmQV7mBXd$;}vG)?1Qfq}2zbl+1I1jLva0{A|%Vz0BETF2-p1H-D0 zhf)^K?%+`yib-%=K_D5(ouM1tu*ts3lJ7bpVQ18DFr&jJ#GFi;n>+(xu{>ps!x-3) z{zH2E#xYPtAPtdRoem^0VolE8X+8bd>v@}>3Lh6Kc;8`$9oDBm#|>Cc-Z`P0k@?26 z86Q;$+ZSjc7?d2Lr*8L6&(+V>Z-igp39Y@1th(esphZxH&OBPh7!_{gK4y-#5z;jg zn*~FzsuCU-l0p#QmQYAhT*3(JB>P7O*o|mzzuXpa9$~ggB9A5VA0b1s*EK$NH6eNB zCFjXmDM@_Cpy}PBWK7Zfwq}}VKk*Obr{t{M985BOUS?$FmKldPr|w};$($hTTY)%g zpl+tnH}j?xSNJ)mj?-yFZtwRQ(qq}0ZX-yvwE_^ltd}ZupAB?HEkKxnclX4*8~117{z#x* zLSVZfCdq zMu>a20EKK;6tt#sNV@Xz!8dd)ARL79BPVE22wHtcHbe!Cc{6*U>Rx; zBa$SiiTv-?!33IHo9mtHCh(R2esFU3-{;K~diIdd%1c}u0RXuZS96#&WLPBpW7vt$ z(-bB{O5qM8+dN1~aX7@8>@d?Qa~Coroi*<`eSIj%tWj1`Uk zgaEI%{@TqsHeO^I*##r`iW3*XJl!y6R2 z5cMY_5JnZMeJ~HnWotNCg+E`?$g#}9C||UCrcVm}1SUvkUA8H!x$Z9-ffYiEHrY9R zd72DApDI5c949)uPsg4pHrf@R5B~C6g6A)6;O2*ir~Mk$>Xo~dN!(8^f_OUC&an;_ zQ&qBek8s#N8?!mT<%H{(a`+5hkNiCiOR0 zznno*M1Ifb&#M@(wCDEaXhHyC!P7}!MjTDihgONvg;DhkuWbOy^|p zaQ3#CPhH)2AUCYkjdaO8f^9SkZIFn0!5x@RviaIa@ZhG*YT z;9ie(^&Xjecy>GB^>(cmE>+p4wq8C&J3%loVMuK)fB9`~jP#ds2^}H?;MGS`mbcrV zP{EsGD_@1|cFu*|#bwXIzF*=V5)cyl zFPQIw?SHtce%(*>!%SY;o`^RhnUdcyaeVB}SBnY`3cf;I?p;37^4@L;>4<4{y72tD zCo}KvD+wyin+xv3a|fB)Sn8Wgnu4}{fV@HRe`af%IQfl?*OQ{xL1pBr|JG^ZF+>j! z?`und+|PFjVZ&bU5v)gu>5Q5pM7OMZ^5%GkQ=2kQQs==6?g$@Dz;cJ^YV;!tjn95o zi`AVJAR`%D8J;9ooJr+CF7qsLR9pEv;w0Eb<3Fyr0L&|8qY*ER>3Z!UMOf=^-lQYOBk8jo`hfGEB_B$Nlr!?}3&P z9Pq(Tmn*Y&cbvh@K$=s$<&*sUUh8hWBVN0EXM{LjGh@b@QjRWc&G<$1A`)m7JBOF3LWDuL*kJu8X#UD{HZPc_cp$O%$C&-Rv-J#iZ>j z#Q+R~to3xeDq(O3kigM!w}*!>hGw*rm<6wTEGq(1y(VMt=PL-zZhfQ%gplu@46sQe z{)ClDQ^i)thtc<*IBKiiYGv(T7+47C*xmfCI?s!~kA~}d^_!EKrJwFt2eqfQYr^#x z=cFq^4}liT-(H2;5+#{@T_YY%cR=G^ufTf|hQlXe`m_6A=54%yh$yheAe?(sy%VIK z9hvI@Zrf+)-;oYAIL#DMe@$Rm&9_r7^oV|1!~ZPG<@G^|0y~P0+(v||4h36E_C{8M z?;Z!&)8uScZo#Pr_c|ro#sRsMfvKS(vIZ#NL!v+my|&h9680X_)6d)^;l z;B4Dp00>79;i*dX>rQWWU+tpTY|P#~s_0Mk@YWt>i{5 zcX~P>^ScoK$+wh7&Y1y{baVGV94*KU*0KU;x&|K=x`|<5EQJ$w(x92^%;8aP;<_tt>EkWtF-=EU$li65`Vx=7_3=U)U)Hix1OKPF140% z4ZS*2&z;WBPnOb179;F6S>FL^y4Ju0Lf73oh;}zfpr&;%yYpP@ZMs@dbIf85-73B2 zYvXXSINg}N_CM9k{jV?G|A2|OF<7snIO*tUR=X$+}`+5vEF1pbJX*Jw8;9 zGANNnE|5w*Fbnf`F@CCT7;)$i2c+$aA>@YX(w_9_YliKuVTKj6XqHS0LL3ddMjcfF z{}ZYoeh7=Pe5yc>zu-vZhsrO&J2cRR>mItZR!C(*!!4^}Rz+_WBOdq=J#%zH8O@Y@ z8m8q%Fr$28;`^0O^;JR+8Z4TCDw%g;+|&wsHHh@vQKL|h7`7qJMg6Mg3_F~Mmdb9R zm~(!nij7ge%P?T8FU9^@54R5WvRkp}aAu!w2W{r-YOn&M;W`G|sD>y1eMlJ|bQs(S zR_hFFTQv|1(~FpFtyKHN5R-4{EbgN6Dm-N|4cHRsB@JpU*mg_e)gg>hSvQi_uFLB%M421f3o!WSmPJS+D9h*P^V`Z&^vcv9fNi z=x|oEYIv2tVz@aAU!9-z)XOp^*&3kMG=(DWw6j5Ao0jO(vuZSICaHzElI8ue$0@;r z&hh$X%6$;G%WmcRjDuw+Rs}r_bFB=9RrGtux!joQ^TXkqCC9c_n$@W0v=6^?`;^Sn zP0B}{;eH7dB?d&|mR*`}HY}G#7tJ$dMl|Q{DTcf(>4ng+bQL*?DCS(0o9^fCwLw)` z9WCn+<~N9V;LteX+z-I-*Zm2X4x(q$=bmS@v#J7=xtwCivg+&B{8nLE^H7VhmS$iN zP7FM(2~_vH-9ra!1I^E-VOljI0ewV3CTNATS9gd*WRur>l?d!aw;TmVZbP8>0gK+h ze8geGLLUdOn7!doqvS`Q(>?=|2VuzEh*oPPTCRajBzF3UqT&HRmSa->lW36%r(p3c z8a7Z5Q1r0r{~#<{Igl7pURI%@z>pbkgt~(+d=@UXSo@?w)EJUEgBl?`agP!nR9fT3 ztR|4!MmZj|Fqy#^8qnhV$TjzqXn_SoNLH!j7+qdJLnB!&{|sDO)hKUH9eA|}vpO@L z%@?<@4opM=wh}}m{qAJ+KKpWp*)g4@`?Vhq<8f!%9QRXkWAzBYu@ocKSgSWi!D$pR ziCBLWJn7-obO%HXNduk1S<$d=G}cF%9&*n{M1e~|ap4{crz1R6(W{%FCryT@dd1N61X4%HtE#tKuqj73T@FIW!C+o>{k(0Uiz0lKL;~C>bM0N?C4sifjcH-^U-Q1 z6RTqVJz--G=~88QT81sOoNf4b8f%n4{i=*gRgObL;>tn2`_rDYo=TO+JuQrSPy%Djt#;SDU+7wAd=Sxu&_h&=dF z+w92Wk#)JZkN`9h=R~@1-wg$+Ov2V)YJz+kX*ouX zVZ&>L2R9QPZ>DpouF~s|Gm_J&jmm~gr$*S5SgMS@=8Hv_s999%ql8gIPH^?5@=WIZ zs_L3+Fv68c!}6BJ1ts-_6Uw3Rgi@(n8qI|Rq;((i;I0Wai3S$@V)Ny(q->?~8e=<=0Hh10G+d%>QvNjjC8rZT-fZ~Gx7uD*Pnv>W520{5Vd z=#1YJ&{qk#+LhOsFwCIXaISqa21o}N`h;K1A2s>O@gYM=NVOwqmQK6l9bpl$3V+34mdw-0?!$ZdfoiIAS;K;-I#y#)L4Hgu7&ZN<@YLDl`; zt>CG?3*dB6b2k!O6wfFsLkq$QbI|ZO7D!Hel0M{$bQD2ZxM%1Tb;#{ReUD&oAzr*D zNh^;-#&n75$X_P@jA(3>z1&P3zj(&$=wJ9MN`3VjSwY^)e)xpw<-a&tB`cMf`Xr%;^&naZqVDLlDb*L) z*D#5diGjOL(*~;gNW0j-%iIw*^)1S~B>d_0M4`#@cbytYr5fN~#cXL-^|1~x5e}z0 zP>Y;YqN*X2xS@nhY+gSkcv*HgzpU64pCfPB^DhjtvYZNAC>t5-hlm*9;Ek@i_jBXd zNYcx|?;N#`#|yw1IQ&}FHhcRphMMF(eu4bJ76^X=7s<6lNf9Y9i5tKy7|~-<(j3;(vU3PvRcj(p;Fe58*oY)*Xl#fT>5|_EyApxZ^kgMDOQmq zl1H2XX85g@@)M3LabN`Tw1$ec2w0YrK%=-7@T-I8eZVxh)mG~mi#SqRFqcHr$h!D5NEjMj(U)nHSHa28dXqt|3oHj4PWlE0eKE{vDjhRh)HN6Dq-6gl z1BN#88PPiqmN5O1!lx9l3FDj0v|!XAP&C651B)q1V_sQRYqmhVJzvK3ia8BU-B9dg zTc86hSe^L7mOoiZ_X~_6t;)Wt3%2bZTcgz#?DSp<1eaT;DbFBsilSoV#f8Vt;Ofo zD`DNEGS(?+Rw*Ul{Lomk;d1DhDPH*doqI;l-Y*B=&KmQuEsMI}VuoIm4??0jPpD8p zkJXcAzJMu>nXKH4$m~JMmtzqpV_5T}wUm^)Zh_UuygIwb$XbaADi_=nk~+6kRYPc) zUZk&q&@m$6hgoTkN?v)xUO7;(#l3_zH;4e|6Bze0HGiDgQ1FI*_cfc%Ni=ZPPv~SC z(ui1RSSp}%1Qo4~8ZbmAKG1fg0~DlN{e&F;DrRyeoQCg7^*~v`>|pqyHqU58#2_h| z5s@lNlr1B0n4!d>+Jki>VRCzLHOoq7)#Pe#i+?1R!43lu#==X5jS_lc+A;4N>-Cuh zfcKy+I-271i3VOv8U5#gc)2zXKc9F~c!Anz&ahSpj>5*U?z%~qx zg!JVAvp*Rv!ugEKJ6#S$TQzJ-s!l#2$R`W@b)+&ZAsLO{Qb2-Ye;mpu;Tmu;E1}+^BqXt4Yo8zO;YyJ$dg@Ol zL66v6DI{UYUrNfU29lWl*==vmN$08{j<8KBind0F>dqw&Iq<+BTu`afjVn<88w(4k z2t86UyenICh+=TdC3GiHH!?2#D~$}J{sFrPBXSOA$UJb5DbIn$CEd)CYo^*_$gXsU zZNWC(;{7SZqIt&Ko&T4$H=hUZJ1^>8$ftyF#q6nNvU7Zj&6A*Yh`2ncOE+nTj?f`X zNS!Cxf-TO=?OoYsAn6U1KqAH z?MxXlcYQG*agDuypI4(yMJ2z_0Zy2MUn((CiSO`L=klE!za`XLkJuPDK;V_qCdDJe z67zR(;K6CS#9C-SO8gqrQsWZNc9(2&`fOHu>Hyrs&{dV9<2G`A3$oN>7`z+Y%%)o< zOR`)fD-9y;?{0U4W3?3m2W|3gkSGv+$4N4bm6x~?CqOCe(>Xj^#n)QqV|frtNIvooBm@)-GF_zp2DuH&gS@ zh+y#+DB&;k9q_{wS?aX-iMS|XlrRrR9*&?sR$o0+crN}-=f{;2xOdSe{YkPJ>8C|T zoq)}MAA-(fwDBaEVr|tvxf?p>Wg)59ifb7OE9HYGY#vbZmS}9&t8uiay@Qc3pLKQv zGcKRYorXdeB&sE3A!I8+Pn7e^(88y6g ztw|2F%bFOb70o|lcY+*+y-&h_^kyYXMcEu*ZTJ=0?i9qb#jcjrQm(g1H)I)RLtZ); zI^k!ysrInJYIHNo~u~iSOKVZ{9c5d2gwEQ+D-u{&uFick~!%l~rTb;&xMR z{)Jm6@o8b}{ITQ8eiQ^#iQMs6y0sl;Q$>%b{dRk6xHL>Kpu<%f>9R}>+-k!4S`WDvWo3&SLLWXKoBk8PebyZ;AG$=*h z=7?un)A6XWCy0vm)U+Q8LKzA73i0NekV?HYzuouiH#G;M?7&DlRYFE;stAzL^hkM! zr1~f7XR_-g(cu>pmb9mxx!$5GB>5cbv$PqC*ALb_ae>v4&G+@LEPpk?_RQU>3$sz?^#=*AF#fHcyIt{dYNb@ zREvI*Zmj8@{W&YXTNJBzHx8=$B1 z`W-W0ui0j}NzCFU=VhDAV|VOdw(@$@Wz$8y{be*Evd2#}HG4veT7adgPiwEgLYQEf zR0)?+>b5h7z|Ol_wZeVA{CQESIHJf{%rt)h z89O^mLCH)bo+53mWL%0eg2AqgF@j-!vNE!AqHaD?dOkjHA*`WcBGOPR2t_?#)=9O8 z1MuR^wp=QnXiWe=TXDe60LDwrfe~Sx^xdZ29@-w*?jH6I%4;$zwPZ+$P|WV71%=Dk zt20OfF7TG=kGs8U&?Y^rTHA*u#Pp1{Fmy1$+PkgiBe(|b0)TE*K+0`t&PX3%#K zw(S|C8FS=(xcTZt!K?X#Y6r1C-G&jq*2W6CRWu8 zN)Z5;Dfhn4>uc2OrSthda0hU(h-3Ed*!S3)g4CcW1^8+{j}k8OlRa2h#+iPwf4N)< z!CE|5YqdTvascmOHn}F)@+~!tQN4Qnj>YIonU!oPFVh!NzoK!iil;r4{nA_C-r_Ay z$$hraAKUz3gu{TEyg;7ntlfEfh`+oQTN#yf5WVu=G6rZqZ7rs@aad#Dn@@*abWQNE zk=(yN$CS@L36LhU;^1K}w5@G-Gb(ylu25;l+&Z1#XHH?aq{lp2EJvzth_XJlwm#kM zlW0AT1vFJI)o53YgkyX(MDSV7vGe>ljzJ==_~ZD$<^IC&co767Zy{U-*trnE^4quLCraNi+vyw`9w~aLkT$4(f7v>x8%TdqfQe$L)MYe^uRyfl%E=M^8 zn$uf}kF4;bl&Q@>{7L0*KrBid@c{YMmu-3FN_UAdI5NiJd9uL_tBYfUDL1}~-x!Wr z>VQ_x^WL+ZqB%h3v6Y@vwyH70qa*+|dYFYfcjR&K=;Vjyp9yU!?Pcn|_i!Rg>tg!h z@#&Xh)BZt+lG&wl>C)=>P!5fzi|K0rUU_Az>mrfT-*Rt}luW*d*R|C#&8B;9H}}$R zC#olYHs)kg$gY{R-!3254)}W%rQGNC)+9ydp5+23)AnI!(KS=T9Vhkv!Hem>nYB;K zR<|a7bK5Z02==bT@bTiAB0fd^ab1Knb@LXPHK4&Yv%xc>D26Bai$ zl}E(u(c!AmICx9` zs>_D%dRlZn`xDwLF{5Kd0juyPYprVKtK)MB=})%z z22jteyHed9dTENPF%4*D6?WbiWY}aA5K`eMO$l2>?(ttKkJ!IfT|{C5wJ{kY#)s3w zM9?mZ6!RZLS8Ay%@QZLPr8ri_IKLdqsy66dgF=@)D!toQ+Sm-&9-hENX-~0c3od6B zjR63sQMkj=M((a5?svzfR_`~j$*#l>_s+T34|X#3-ucdgIY=rSPCGOWCCf9Vp=)eN4ahA>K+U@<3`#}2Z zLPHns?8mD0GU6B3%i5$#+2%|c16C59%J)1YS9hxwfG)d7HuiG!6zZGYA@DFzSiJE| z*M}%TT;^>gId1&*A^M#9mW`&-l!w=W#AO*$7kZoB`F=E=n|1QdJ^Y-6E<|T1vGH(# zYAaclG+sMEvZ}FuFO7|x)k=$%%6+|N@g{dm0;~fw1*45jw92j1V+~8k(l=L^&Cw9c z^j4BJNSBlF%XaXbg40<L%`WeQl_h1Hf!UzrM208sPaM*3Ifp-$*@L{ zEH~cU{mt-a$G$tO>SN@N7AJ>^+33hIAVSvt`!>z8V;Ec09**|94`Gik2;}9wk$T3weRmRUZv@*;ffRyV_-6(6I=W=xP^%%-VrhiIqif3!$ zB?l`7U{z-+*OM&qpiu2_eSXV=7(5I_HY&L;Th;s(TDLL_^LnB;7RzDjYFM}2n%((v zejcN7)fLp@s30i5>v8vG`YjfJ4OD<-iF-YQd}xTH#XsLr+cTSdZR;jEwyj1{cjO}} zTGjq?2nDYIHzG5UpIaD>^&X_b(e7sVGtSuiBnQ5qMu-Na?4rb9*A)K6Wx(mfuW;$>Wss0PYXs`ism@04*w5Zp#{{h*k z#;Jdnw}B?(20vdx;_5ac*0Eq^eGUYVR5ta5sp3(AWXj*K4H}5ZQ_g7-aL+@e&5#lkn_1)YF*Zf z1YJk1a-&D+L0%FCnYI;GX*@6V5^b8??E2e5J!V|mVh*j(hNMdE?YLLk4V56^6qv@n z;v8bi$aQkOy%=LIlgXQX`fEk{Z4>Vd+Fg3Y1Tl&Sdx0B`8xU4RvP54gIJEfP_~g)fCX#On0y;Iw;lFV{4tWwmCdCmt^ne4l!;O-PN7#>bQ4a@K||ZUpPcdqrucwDshrD zr1Y}TwUL0;Ir&z@>)!TsFIpu@mA~1}kflzs^YOK6au7aidglFc;x(~xO*!ZGf#=sm z&ExrWm2^ldTi?OPIzHv?E!aHU3&l?~Q2G?9g>~Uw#a2iBxZNAMvJ;^1gl#g_^S6@kx*7HbCfZmSHXZ>}9mYpk6OcNZ>1(ZJz$jbSSiWmrPsjG!(T5Kz z4jrYR%s&iV0R~$R{UGq0b(LDlQm!YZPH-EV1ZDcvB#~4kkcsS~WAwtxq4)bUtPyK< zHx8BTx|$-1W&ONGd!}8~`gn{_=!@#NtsxSZM(NSy=0-nkxAL+(d-X5ljnr1B%t_E| z(IWjc)Tt@cjmqh*@OSk_qjkeFsy=@q9tk}=%1ebwM5$C^A0wtYnWXU=XaH=*fKsz2 zbWNxs&8~kuK9vgYw3I!PpCh#?Dz&>z^$$fd6RtSq_e#QFJS^2rZL@hkuQhGOUF0{G z^VD5w4pP54xU)HbZX|zL2F)gnUEWq55~oLICg3lV6Dw(CqM{vqq#q=$<>OR+VOOWr z=d!g)I%>u$Z@*+O+?A%bxem-5yfvbFr zU2}bdiCFqbN#(Pn@!8BDVbw&j--NIfQp1^}`D9|(gCPY)vJY%AR0O@QyK8EET4HKq zQbNR$MM9O_4)e=euGS@@vK4_cdE_Y=Wi`44&lr4WC3?>5P}hePT+leZqE|csX}3xz zSmI}y@9qHa6HrOFP_etEy*czQNG&x-7rW%D=-0XksFPB=6dfdEfd}1^%fn)$Q=iQ23f~wRaQoCoupL})a^=Kw%P;lg|QafF1g$uylk?5 z`Hjya*I8FN<1$7c>eyS&fPIfZyiaj-IHQ4lhU&S5M!Ut{xv5MnzGsD@qyJD8JHGor zVTb>Sm;ZbC_1paXmwElyg8eUD^&hO1f%P9e^k1R$zc^K<@7izE^jjBY_}5VV*G2tL z@jou=KOpA6j_QA0)PEcO$3^|O_FsvW>0kZyUla9T<@G-%D#O20>VHhsf7<@XL}mIH zG5yyw{b#lRMr3{~4*#P|EZS8i~_6G5stsr>_WULs;3kT7GZKpQt(yXOgmc=(wG{F%JULV-S@} zt$TLJzXw^{Q!x0^&ga=@JXd9U-`RN@gXNMw-?0Y4f+U$xSEl_-OY$|$G7B^U_CHTJ znZqB)Y@K;|g4-%jTbrMr#t*JG1XmK&Z{u;|@!5UJ+N&r6lS@3jet^FN$o>zfCiL>I{zEnRLdq*IdT+Y>sv1ig9^pu{(|hz}cbpi*VIXg;5_AO4Q% zNkBu|vNk_~gm!dtAi!-094@=`K%IhtnNR+V{^bRWGXU2g15E{ixHFv}V@p(>M9{&> z6b!vz$d?5|3>&bW1Mc-|R1jMr5z0OI9b0w}s|Nf9{`cCU`77;@V^rT{)k)KvEIrv1@C7+m9_G%YWk!Pgvvr)9im*)BSstfS&g#Z4I=lZ%gf?*mvq}G z=o-s|)q1tphOp(5R zSvVdm%l6d6!5?E;=ft&ylpQHqNokC|YDt!|0V@Vfp$?gCb_qR$ys8pym7f=sc-I*& zI=EDC9M!pd<)O~XgQbNIu1YQ1hH+B|O3@AjTXvLoGRE<7V)iIX2h#Cap0#exJ!GOI zx$G5UTkG4PVcHQY~gU1u{87>JA*xt zRucCt0;E87ax9?Et7Yq?i7YIXxmjbnqG-adlRwFDp|?;__S8ORyS4yahjvvG&LC>o z&-v4v_@%qY!MP**w%TA@|Z((><^@h^E?} zrpbj&+!>`u*g!Xw8ez#$)Fo6a4=gPmSQ;^3lo>BkTey0th3%v-?Pj1Z-eIq}sf8KS^32``7o@Z{r2!Y!P(Z6rycQcX zd-{I>h(LG0b{|^ufOgnZRWUI7W0TUxGWjI<|o+dTiA zQoOxvv@JoYW9nNu7fn#q#c8c*l#551S)tXe@_zi$X-2&~su$5JwHhKfsZ#rS@ib)g zdcS3-XzV2GrDbQjCB#%Kg6Ebi&ncfHo;kvyz$c!~9mS zT(80cJehv{iN+3=F$}p>1yxqh7|{OWRf*y`jIIP7LSWXJH?avFT_n=vlba&#k&b9A zyv!#{zQ{iMQ~J}$`o{JV!->WQZ*-|mH-%7F-J~LKqK~yiBFlS$^W!v%tCwg*Y5k2M zbyhIsQ-CZA=kzM%`gx05_elWq_48Z%D53T3b)nu| zY-`;o<5i*6QdY|Gki!#E^W1SeU3&**~CAdXwkyy(h;Ee-XV;p+1FuzqXmMRrWrBY36C~AaN z^UY)uO>Cb)r(`!)vBW;+*_%*_Gu%{O>l;hlhuVs>risWcGf^RUwq*U&a7a; zqSpRu!11*dIqDcNPm4mFeTFp0#O!=wu+>Z>&GXO#S);McWjw1{W*t@HQQb-Li8{u`i(I(Cm&Dd45tA020 zA)~WhHZt2roQ)BYk?GXVubmC0SIGq|)qXVbd#8XeK0n^l@kt|#Ba6@;y=oSlI3WiX zt?rOE0!p$|lZ0rOQzym*E7)0*nUg~|>6_k5X9l&LXnAD1uW>nYFvLr@1UJhoFAp&; zA3HS6@c$_jIg5#1B--E+sdA)55~cX@ji@WnEq0FY)E_F>P zILgw&)*QjJUBM1`h1Pj0!*;xOrmEr24ipjZn`xasteB1!a_h3V2H8a&VUbg32XzR$tAG40g zJnRc>+gz$1{n?6+E+2EQDzo>v_L+=4N(b{=Rd7WjKC~ee(cgs3IDA0W1*WkKf7SmpvbNP7E-axErm<282Z}Qq{EQL%Yd-;XZLqgZ?>|$c{j3YdnMG|> zEr?KOkFe;qA0##eRhnLgjcoB`?jr48z>Q+c6a3Rqy$V{Di}l7+G(T}|#QRTUU0ov^ z7Vm~Np^*y!?Y-AeuAhCjCtR#Ln&)2NLdE9w&fivR;wG|(=0FyBxF9zHsxY6>r_1t0 zxpDmgr|0Oi+;}*KNV)NS!7R@{ZY-D8SLsoroUh-JT4<}y<79Nf`C2Mtx(@Ma!~=*t zEakGXUc&S$#4zI1hzAglBH{}frmR&)T!**|@fd62vN(62?2&56av9h+1D9aq(%}SR z43R@Q({a{xn1#3$@m9oLh&qjr)vQCj3h@BqNv%cU();cxLfO;%Zq%~=)oTj1Vn@8V zxJ@heFAT-=8FS)!!?ZYCMVxJNalCp`T|6Huzq%nE|-`G6#V>%RX zp+o&vd*QCy>GWNA8u0)kN8iQg9r_MjMUOF^V_Ji_3-JKr5yTUSI(iJBxAA$4z6Cb= zd&ozuL0pQs3-JKr2}C{pJ*E=<9Sicbl+iVal>QD=iT)Os^;=AB^i8DRq;H~VulALe z7w*%DV19z|_!C6BD?vEY3ZwLweP4`6iwfXQqeboKa-bTDxST%!WHdi#U)3s4l)l$5 z2R#qe7SPu~MPzU7ui?zEfs8mCu^n+eq7JFok$N4%h_@j=fT$v(m0>C&%JfCVml0ow z0z?IIHlj$6_Te;9dZaH<=c!Gjuh5@EI%@nd{e_mlOrO{Cm*~&5`~}RjF@KRh-?xL zpVkCpLgyz0pM}WN=kUqF$4k9RmgJU#(s|siZeq(Ov$A8^v=q|P*jF37Xpa)h-p?$b ze`Wzgt%2S`Z)L8V-j>L3?fU{PWMCuoIe>$?B)`d{JV=7eB#QNAPBa=J|DMS+F^(kIhv} zgUzab_64oIgxnIsT9sapuP*ajqa?do83pF|xL%Lc05B4c$ANeae+1Rj;{Dozq z)_q(x`D^1`R?k&8ghI_x(xUN!O#Xs=XgYt<_(Z&=86U`EcAW3tIG-Q;BVR7dkk4QM zzsBb`7&JbfFuvZN+{T99++2+(U4|Zwrzc%Lo99J8^7#E4Pg)pW)OcP@3o{=gEkXK&hWV7JeFf4JeFg~6a2Q&RTm7B{>o72 zV)hxU-PhQK*sk7qakrC2HO-4zbEU7L z7Zx|pZ|z;IbT#x9Kzh@WLbla+m|Of)&Ktn|7D$(}@$gw=Lm9RxRR?%52^##IgRpB@CP z+O(M|Je~%7{uy9fqpEZ?^lSoXRwvABR%`I(w^y&n+IF@iwPK{sWNM7Y4#q1dVPyrY z3`!Jt16R z4+)Z{PZJP8Njr{pB|dK&h@0I;a40q%p#;ZE3&H{~L% zxdGJ*u9wC@;K%TDI7)ovCv=@)777JLxHR?}q`+iYfYLsSJ^w&HqgUc{ z756;f6srR(>g9H(ZTK0yLo!J|nMGRYc)E_>&us(|XPk`Ba#)4>-if2Xg~qs-n&}bl zA^r*eD_z#`v6vNiBmm!o|Ae0q3$9cqJ>DcX7V7} zLmndoILlG;F8K&Aw!e|DC|=uCM_m|8v53k?H)7CwC%umz!RILbGyQ@~=W_66w}h+W zLfkr(YCCruKF@IP@R|G(K8AWO6m|%^ggwF&!cT>hIY>vUPQNt_!`Bb* z815S$h`j@;xQBQL;X7OvO5cIcYTWM~XmL-&tHg}D%On#>H8~$Ovy`kR8^{*aYG*{T7tE81=CYU1P4xV3_$_ZF>12pL%FRYA_z_<% zw1Str51!&SkSpLB+6Z9$O1uf}cqVxq{la`wNIvIc_(D7rt*DfH53Yd=>91i3&++we zH(AcFgj=A9Tn-<=BX}yt3m54o=u*iGbQK?=W5@td{xSBq&q*%fgcP`zv~l<9PSD@L zW;ntd;Vte5DE$%o6gPuEDa<9^cml3~>tI9dYS=2Y^1mc2fg>&8=Z~RxyPPZJy_jEx zUS%=*w7qz`2hk7Iax<{TiI#Id+QkC&5clG<6a55_7PJb__(JqzufTwAK8?ak!Aj8U z0Prsj&xJ*?N8p~=O1LO?2jrnY*dDtaN8AJN!>zD~Y#Y7|*5ljhZ}4oNFEr63LQ^b{ zM(A(oJi6oD{Y7p2i4*>a&r?{a7WPAge*@-0P3)%Fuh6>3p@+Q(mSOaJ99Q`<&Tt-g z5Q>Ip(%x7Tw;tE~Hq41V8uJh%bjQ}fEO-_k)(g;~58@uH-152 zk1_@X{x}ClJ%5})hNu&cQ|?(bG6TAtN#G1hpH`i&nkju!HRE&@)F3UL!enxR*Y36Z zF~J}oPRZQCQ;Gmzfy^Id{}Mn&UEGRym;eUYp>7Md_Ug1R{1k*JeOfVzRXU@gg0Iq5 z5R!lV^l_*;{f8P?uUl&kU@HZk(ex5$s1VBdDkwufu8LBb5b~1IXu8_FdnX3!C{

    6L^YCF@!xFBaUZSnR1pmR-fNG3=-k;+y7NW7w zm7Ku%Vw28@+k-EGf}k^fY!F3`qo5b7j5b5qKn)n_D5(}(vEePk@l}*47JD(tFmHI& z$wUt_jY_A3ZB?3LnXsp;Fd=qFc{#($1tb`BDQ3d!jUedoH7mXV=}>yPwg8l!E7H>2 z3J>S~e)8c0uAiizJURSPJY_p^KkoV>T!js?$V|2h#UXo0z39%$ri9w1Y#WH_wNh*t z%wQ%8v_N|7uef4Ug8mfiEksc~E$IQ-Ks<`YLKhfhNy5}FII0X7~mW#ceN zwPjZ>W(FCLqEK&`RwasULo767_F1EuXZ==X3g^<}3#x#xsJt%2ndziDlhJIn7&%>P zT1wiOG*0K@(!IoC#l$JPy(GiG6HadMP_h`#_9+s+gFvM61P47tm%l`vmDKSj9RK^60l7d+4g%F z6%dG7Had!JS|XS^AXXEIBB4gMa5kDoXoKvhE$pA=&=$o4+C-omF(z$TAZ#t4iHU{$k4(>voH1 zqJmg%nPQzqo4E$PBF?bXS&cjCJ=_lc4sj3ns9xuwHmkKjpecetMYF|HAc#nc=DD`H zL?M)lqQPh~Su9qo#P+G(5q40=Al-u}aB`m@i%~N98M6T$VEl9$jqyWeXfeu)`6?4J z4dSw`#Du&wikXc-tr3s#$Z-bkIO8$@VGGE%^%9YywB?7g&@O}p4t?*QetTue8O%Vx zik{VZnjNp9%nWG=OPOaCkGC;DRHgZ>ktb6c8rm*Q3U0sR(Dq4AmQ60eH$M};`DNpq z-;cl?`wA`gb)d1=%gV|^q**m%+qgN4R9depvlpR3TVkL0T8*qxdkM6}Ufb&}w@&ny zTcSvomRk!;HS(D}tj#M=yiIM~&;}dYNLvVB2XVhFCJZkH_JNApw9{IoJvH$YOVZM( zloGGqXU7)-`%a8XE+|OLD8YDA*gxFz^l+;%_|;!;KX3N;xKm#?@h^Q&kAv^hc`%?xoW9iOLNixYrYIc{zN0pBZO&$a71J8 z_B%3*F+bU#Gq%{y%2{KJr9^H^k1&f1k}NcgSO-P7fHrp4~H?u~{m z)~&W}#_Mf&TOPATZ68`cut|8tWVjCr8NIdXY8jb=;y&TxL?oe}qgpwo`}*IgaH%02O0 zcjWSz5x+g!wCf||xHEBoFjHnXh@%`G1m4e8<@x9th&{c0yLD1fxI#j+CT-_@Bw&uB z4GmT?ii)k=CRNxS70fQl2F;4CvA2}WjB#n$^?&+_Na+&sL2OuqJ}GL!!F zX0E7o>(uV+(At|~A98P_wc8xw|VrFXs%^aS_Lh! zG}A^)Q_3{=LSu_%rR9Cy2WelDPplG2<*X*j1}?WrZwF(ln=?7liG2{!sPNX&C zLklS)uz$q<2Jj`pZbREV!)l9tGHR!`5k4ii#pai!qjngg*{*!nhrA zzvit&iz>kG@7fOV8=EnOu4&vu%AglR&RsW_q*cWiqv! zG9$Zq_F1Ogwjmgt!HoIzSti=H0X;Y77;aV_XLde>mmZ>>1)Aum+X(Zau3j!R%H=D@ zHJpi9tza~{usE>Bq_=5-XRxNoQLcqP-V&WJClDwpc1$TMOiRZ|nxv!^6>7$pqvOiD z4qf%D&8uI#rhP|#|7rOLn=gKN&t+S7Uw8jaUp=&oaFIE+)cR!;b-euIPo97C(uQS+1gJ`gfu^TMFiNxgA>4^@EhHBf7Vb=-%!pz(R7VXiD)(_MakTI10z`^@VNTNZ6y;*>(dz;!YAV z5Zi5FHj!qtjRq5<=2~VtrXD&Im1-NHMW)B4wof+Vn@vs4kR1d1@^US(2kC|FR_$(V z+lcYy55Y)u22N*+a#Q-&==AKQ7ZxTitD@8?sNQrv)2UvUYTx!?^{Sc+mQ>f(RW3=% z<^#JooLBMa*r_$`8&AK+bhwrrrQP_hZ-RpO`wB!)m?Y3>b{Gi1?@1L0m zh6hR0nLgsT=p({sgW_3zq;Jtjf)j&Mq7VCRKk+HyRVaqlO2f9vnU7#`+=3G(9OS5 zw6NCiXPsAH-R}BTvgi;UR!NL+M33AWw1-&^_CVDR?`wDxNj$UisTUQ@CU;m z4l9G7tiE$euyFZZokRSF;Xe)^AAWE6t^13*cdfZ-=aTV>w<94RB}s+d%Ho|VWNQj( zO(D}#NQ%SZ=eZP~bMV_ucbn+NCbH5*T1=!7<6?`&FX&PPokfTUWUD~RZ0FhNW_}w_ zdC4a5dM?#ZX}V7D2ZM|+?pz8xkP`+3o`^=aNr!5R4i&Ps6yjoriZCKzTO+|wwhKWF z2}s*$(K%Y4T;TQDeHdk8%Hqzn^r9&kdKL-!Pw9q#aO3b({8L0n94Ri5nzGYL#t?Vo zsm zxc92H7Dugvy6e{eIp-!JD7Agx|F7Tsjr5e{Ip;j*+2amMN8;Sv0fpr*bXF-PAw|p+ zz=vYpbz=$k?*D&KE`B~}EA_+7<*$c1?Ut z#Z-%O(=ZLwFb&f%4bw0U(=ZLwFb&f%4bw0U)A+wTEIa(oA^Kwcq#stjOv5xx!!%68 zG)&|F&cHCN2lW6({xw&O+7A6^Wi<4c6Rgn;!c@$jx(w6*)z>KMH#(SZgky%ap&r)3 z<^cak9c+P#4LFt6`c{_m1f>>P`9Fag980W?<5C@Lg>jrMbg-QT)j1CS6FE1%fTm`WOUTB6tRZt$?5%sK4o8(uO2V+K_}v8>P&Quz!HxiKpJIe*2SzNIrGr%omJ3#Rh`Ivs)$@r1)bjxC96$#Lbd`Xn6cYhh z1Yi}|Qyci;2I&bb5I|iFvtW#YWDo)hDu9avzCyqU(hymz>w;6@NV9Q1<9E{d3#HGcp8u1ALz*vD!R!Qigxmw#J#D zk54s|6*|u}QrBmQRA*&1s*O^V!&;qh;kAms zOJtyImLVC+8njR* zz|imc>u9EZRwaFO+1pqxGzN?5rTJI-(BG{4xa(&$;`B6dH$ya!0BfY5^id;0JyWLZ z7md;VwO*!=%0KUC&~u4~%+VNK8Ib-aqd=nLm5E|mq4PTf>8#fE8M5}k${MKR(^w`Z;H^yhxO>g_pPeZGb z_0(8zy&J*Qs(0zBH~S zq!0ad9@n5Okr`yr=XKLq`ukQj?(n3R#+{~VOgCC07uiw%Z~9T+4dB(FLT4F?kpyD& z33)c!Wdy(w8TAhBSL!iY`Nc>BeaGw3q>2jxMJ1{@&0k-W*MIsh(`X-kCu_8kaZPD@ zN`q8o^|-p%)1;h%U^ZO~qWqEvyXlUYP|UgM7b??!;Wzr|u4w9CVy!7LSYOFmlR zf;D*E$_G0sU={^dqri$6z!2#6y^yC;Q6Gv0c*&^8ku><=GXcy8q8b;C;b;zL{Q!Of zpasuzumOk%2T&6rOg^F=4_4#AH%EsDPeg*1WH5(t461@8zXWiKyj$SvdLk)l5?IF# z8a9=Ri#X~fiw844pdYK-Wdqt=L=Vaj`9~u-!60X}PEt0q5u^-hMgocA&BoTGB-9~z;A9liRDi6<(b)`Y8;53%T9KZLMA|^BLK{Q@I3E1O7;MK!pQJ2&BWg`; zmx}fqu}J;cy3a^tIUbsk-iSoAB;+f&%h36fkM#JQYbuIF4#Hw1jRl4XiAK>&O0SPj zk_XR75|SVEt+6cnsK&pNBP135ezMNfU)lv)osDb);Fs5m&tyHOC;%u1==SwrQv>Kc*(MZ&zBw{9RXBR6;S4(nno=T!ff@k7{ zdCD9uE>mVoMYu?*%2UHLI3!O5;$7fZFazfcWvVPZR;UmuMcH7ruQE%4$L5GNkW*5Y zRD;Wm>0~I?c!V@vCKU-~xQ-*hQ39s8Mwz1)Nx)ZzHdm;Y;5iDhM2%~q9^526PAZZp zG?JdUMk2u_@^pz2N=suS}OJ@C2zSODPj-7#IA;2Avwpjd7e(9B4(T)XN};%^^@OCUK!^c#lz}O^A~vk*S(m6r_(LPlpa20tf>8 zWJ%MdKxU?$9Sp1trA($oA)qrg15X!ffUHV|A)NJ2@Xpd|RXzOuB?@M)G+U~Yh^0cN zQl04!XZ`>)O&9n+z`ZCeG*B=|ruj^5F2m02FybJ_1!(3GN}v-ojbxNW2C@&CyJ;>% zQ#Q$FJG(^a91V(opf@mq1f0rL3xQ?C3_L>(G806PC`+i$1Uf?-16u))g5$U{9b}{e z8byezi9XK$z7|j*p+=(wnFHfPtQ6(Qfk%X7(o1E)GTxA46K!#UuAE%(K@y22upW?p zXciC8m1?u#vQfA(bm0c2*Kf)}pQOSe{EXkzNSsDqEo28V0MMszhIcfzE zMuG^4m7we*Sw=`iT77^u#xV$fu@og<4>H7s>B>8 zy%=~ZHz}5voP+}mK06_4AkK@%*$D&je%yp82F@9f$ma+IIFFBW;}he!9I(bsh>S~) z;wHr45#U?`4>TVx$V5OiiHAdmI#OJY01}Jm@FQcvlpVp1<0cJc;L+Tq1V}R)5M|?u zY9LVR!#3tdfytpV1Sd8ER$=DHb9Fi4KDl(4Ejc4Fd?09wz2c6;pDtv^dlP)!u zgO&h4HuyI(iOWlXYDDr9lK5cC09x{s3`bMB0uBRb^SJ_OkZ3*+kcZ|3CwPbmIG4a7 zsX$ZXO*{fn;5=EtX_O_3!;S;A1n{siG80_aDA7FL1?iWaqYsD0G`|Zq;>4?37n_6bouup<5wX`i4rOuxtn` zb2DFi8nvAnMhMNVea$I!N?|aSLN5~#{RxKAitDNl`L0+mG1drQ)MQB&nOdcSQ00b!K~n* zkRa9|Fby%95(QQNh-9{eHQcvxq_KICtS&?+a^{wlC4>)IlLP{sBS`2G85lwM5z*b* ze%)B$pU#9QsgG;3`UuFE2Sg#Iqp^J{W*DuI(gFkP*3?1@1zUDAEPObx*d~A2A5Uf| zKSs7%%I?3jWEM-~X4~#^;H}f4dro1Ks%ddS&zFU4JnGmpsIIVD($n{GU1r^^3A<13 zTJz0<_kzmbN1mLRd~p}k{>9|{2!|5MIR-ZQZA?t!>mc(L9&PJaBq_LY0gv{~IpWd7 z8re>ZZ=orh<{4kL&3te?H#YRWZERNaYLb?a{T zo>@v^_RgsO6tN21vBh#}b>@+M?^n6pF8P>COIy&(r~i-jv-_1Lbrni`J8?9q`v#F(#+2**#1tvz{;F&#T=mzz z?$P)g+z7phoCso)2~dau=9VBW&CDz)6nY%dmx$HR3F?$yItID9xy><PXToKTAg%j1uo|N!gVxKFY7}!{_sbnCx|ZzPJz(qU>u2a|ODbb`_*j;HFt^-s@W#;>DQQ2Q{V8eP1V?@k z@3))1dJTOLB?;76*L}*l`y%Ey`_x{uGe=FBKVoa}yiroOQ0|smpG&(2wjW%dIcLb) z%N6Ygm9!p4%ch6zpRRR&)VA}scN#9g`liG&ql~un{q_w{4y>x3eXagT3+ay&hjhD_ z^5EU?JdHH}L-esRn?8D75=PoCY*#SUniY23;zmPRtMHTQ)FbT5i<0C0<7VGJv@AvK z8Tn%KnZm)m#D}x_4y&x?;R~_q1wJewjVX*=TFuIYotZ_^AULO5%_S-GP zr-kkR-hFHO;)Au5s^)FIT;Y2m>0Qu|r$#RLwZo@JBePFTupFiR#i4)f#iv+ZY}{&# zpt}R@K8#yd_v@hBlV4sgZQ(v=-JMCTt1lm4=|(yFA$Gmvg22Tdu>r3ScUpyQJ0Lhd zQQT~f=*w}I6ho%cjLOxC%juKO3=e(qDf94< z^L2YS?W%EHLh#{sOFF1UYhnlv%Z{)iI}1IYm&j*zBLd*m%+WU>kRVtAzM>GKTY8X0 z=+`YYy_;XRfWQ#HkihN%eqzx4GK2vELESP$O-f*_LVP#T>_XwUt%HL*?v!sho0^%4%F$iE7ziIHpj|lc7LQn}58kN9g0$i&Yl|as4GSFASudvV( zwoq_K3gr`>N?@O*Gp&$H!OUIUucsbPtn=ip>OcBQ!|M;H_Fp>m^iBJe20@)P#_ZCO z6OV4aFB!Zjt!0S!Av2ESFQs|Y_GN6kzV{(D*>hKz=V-Qk>+7f3prVo)t|zS)oh@~Z zA~vpdIl3oi@JrurGgri!>Ox<4jY4>Z0%S#x6`_UnK!*$?_{`54`+T# zrNvh(CYA*}+_Bw1F=eQEmGi7RH_BRExSLFD|$(KIW>|Wg6>SLHw^%AFzho+o-Tv)v+xl?4Dow1WgPdWSM z{PNyyuQ^S>H+yAPr)gO|Hy+JT=yK1}BTn>T*+S>|z?~_>c~|;=A2R0?^F~$Ln#k;9 zqt8_B&7M0^HdX!YgSGEg-e`9@^u73)Jlyi`xQSI;_O0GC_DpfgnlS@Tw2DbT@A3G3 zuOlp*SN`GRwZY2aiM@A4&Eu8X%slYTfIp6AP8D8XvE)eItP{$Z`a{g3hN?d*3HhTD z+zt1ON7e1OJo2&U%dMJV^KB_-+FsoIvgr6!*Jt@7D7^357ihL$7~HXUj{$9NPWwIc z2zR~zPn~Cm4Lkd&Thx5Fee-Qc6^1`Pa@DV#PMs6`=J5^c8QLn)1S~)kcuY0{YoSY4 zH&pvwKfi!YL*>-kYC)Ij3!gE>l(sH35RI(1MC&GtR)$~&VdzWN%}$MVldn{Q;t4`m znjsYlwGy12qs>yPrP@4L+=*bK8xhC~2nr-ZL2(aYp=lrir~kPt=)bD(E6Zh7H?PMo z=sGT&*|vWFt=qLF{XG-6oW9W}p;L?B&#gZfw?#|fEgxE3N?PR1EovXJVC#}0M3*12 z?BB-je>B~q#VdRIk|)zox}ON_GD-PPvIr=ZST++??%jRvuE_)BYsIMD>{s3vvN-^z1ef(!g3*Xr<+~X`&Uc1QBHU4m-Ok4SgsEpeIwFDx9H5+ado z3S0v8p|Z#__LSa#|GL<)tp*%nDMt+<$*3X3646AIQ9}s%SFVYn9+4!~zj-|jZ0)dk zM$HgfRQDVAcWlYMembvzJf(`M9XVKT>$vIkfw6OUGcUGUHA|kpJC%AO0e4I+xiO~C zt<=3014>=%-6&Hw?;ZVY=GjL*DZk%3FxT3wZg%XgCj#diyiE)4+?ze(QhxQXMbFIr zC)4iF@7k%O>fImj?~E>C+P$*4rP|jfVfmbFYxSbt<)OK>@VK25fEym{DKIOoZg-K`%c%$!ggY@g$UFFLn=$D= zk5?`A>1LkWF8x?o2f2IU6Pu%qy=Nk~-+l9F{Lb5JHfV!(C)AGgY}I9yO)vhek%OWm zo%U_tUKyWRw=&{Wex67E3TGnYw}@85+SRS-=y5jkzVH3LFJezJE?y1DkL%Jkwo}@m z2Pu!&{amvAcn{_N0xzw3%il+N99&vh?UnRh)rhd^<)ef<6y=WV4t^W+q?Pi+i~!lT zk2m|*&GI~!v46SS)K+3@m|w-fxx4Rp+}&AuT(o0!lG#NzGjVfK<=WAkwwD#>w7a%o zs$)(^|9}main778Iv*^1Jn6W{<%b=3$4Y3{r=rI7k%@1VloK?S`7?H-xfN= zciZ#qy8#|6BieV(zCS4O+ui0N?daS+6KZTb+z8J8q19EJCn1MR%q#0cFHxLW5f`T0 z<%y?$Q#8DjtYQUs+5N0x=Wi|*Y_n$6R9xHWyLF6J#g*cL$A`BwyPq-Yw*Y>ZR{r-k zStg#@9<^)e)gw$=&ZeyAC*;q23@LYc5xwV^5b-K8OU zYq>u6><*MU<{l^td;IRU?;r>F_!RF|W7NO2>amME>}o?p!srTw+=s#!yS zo6eim?peA2z&rDL`fUvyP`fMqW59*&w6Lme6$?VXIlnxg@iHN0zDH2!noxy!f?&_G z7F*hSPCD^o4{gfqS7}eq@(<1`n!4|hhqiNC8}IKnB4DPXnBkKp4@$Hc1Mll+^McnkNHJ?uhY$g(IvHy54)ybpE&zbJeOF%Y0k|@ zgI89(zfqZSt9o(4*oMmu{qAvn);oHyTR%QC|JNDmqtmwePr8!2Y{?xrQy85Nk3LjYyI#jqiBWem$tH zd6C~cnI+UeSbvmt&U1P)BB$E`fh_nt#R555t@tuh;4$XZY?fdilo@e(5Jyvwydbh%>;p~m)_7x@Gib`i&%^#n?e&$kM>Q#Sg?c%e)4;8U*54Kn~HQYSc{G54Jj=R$Vp>XGupMGdhKX8+M z)ZY8|pC$f}7XMK`r1&a!xjcIRz~_tCTJ?){D9Lv}i}}=6EnVB2liMNahx4metsFC^ z%c@%mu)u6~PeJ8<8W_lq7^S8Tm z);V$9rmFHIFPZ%9%u!_1tdNY7IsH_#n;IKhyTaF1HhA=f;R^ z`@8Oxs@K+kmoj>8!?GK#Lq*lcinks3lyAd6v$Wv^WwK|%g(sb_7kzW+gU7*MPG%QR z#V-D?&C{_%gT76@HDKe6h9|dvJ5&Arg1N2xt*?!m`@9W6sPdrnq&1Fi5mP#kGb~^2`zRqx^2l>c7tf;amPMCI&b^zmI=m+sQs@MqV{1M zes(`;cKWI3^x6uqppiGbzc}zmaPD2Lf6Cao?RQ++@bGBrP1~9r>uXBAY}Rb2#3sa!nmDOgekf{I(f$b&`j|1|hnf|g3XP|qnl+ubaQnK! zhn5$=yR`0@PkQ&_d44}F>|+*to*O`qNqqaR@!L4u6#eLV${0j*rh(2}Aol_)`cPg#PilF)^Wa#^vS&U@% zOis<{+Vbqitt}%{6T7w?V)ya&$-=9@srNpf6CC(>-$4nFk4^9rTXu`-lGiCOJu4l> zMCPTZWsK{T*R^F_VrG1>6aKA|qgzH+ndFY`+S2N1bO`v2mN9j@pSWBT`*4E%}NXxOisEhoXu}ie7z%>>6N#r#t zw2FxdzbMZdpOaRlJ24?Tm=;VAX5@}TEsd&LCMI;sP0dXU##G(9?rFJK|B;req;^V+ z%Si6pGOt5oFez?)T5ijjU+W&(FDEz-mx@z&>1r+0Qe&?Eqv|rzzg`;uN#cc@RFPQ5Y{#;b00U$2<)<5LqMrF)>euea5q zd-tBP9sEAi>)^TF-3|fI>k;dkPQWpGRLQKKR|!SReHr6&<1!M0|H;*^$xFmX{xAGj z=YOG$$lnY3O?UbC+x7eH`b~G>HyQulyMDi2zv&MACgb0G*KfOBSFw*ZKh~bVI1Mqs zIvv3bcp4MnQv7$Gkadl7(<^mcR$AO+SlDuK|%>QhG5@J)+a)UY1v1wRg_6jCsCI)}@!u+mnT56#H#KcLO64Tr57c1K$2me1GpOM5Ww#`bBw z=bDdwu`Kz6(=B3N^Pc#8&za4ij6A-%-WUGV0T(lWymau#twW!`EA;5Vog4CZmFF*t zt~lG{FGoW5t|)!8$IQ#d;t${JFuTL*PRmw4Ga=>JJ0C6`QhxG}A+0ta|GM>wzZ8db ztsD@y@T{2~Ti!eR>2npu4{WT+FWz0Nr1cYTwQJsC(}w6pVIxmZf4|O(%sH*=Jbz~Y zn{Cc7tt^Y0I4tkL#;t1_zS?Hf=BA%^*fj2rnD(1Cw!5p*T@$ul%%3y3%|vsd@7vAA zVYh`Gcqn>e=jD}4FE6m)^F49e>m~(H+nx-xcZT`S&v1$Jk?Cg$xj}+{tp>Z!`IY~Ju#>F zqbEZ?S-tJ%OM}*ZH?4SlpR7NYKYsk{3zqNgte#CDSij2dTe7UU&*qTAvK-XOHe_9J{GqMG!ty7NKk{FB?_^C4o0xWF$9I>Xe=4fN zdhKxGz88yo=eK#b!u;%u;)7rJpOU)B`}n{v-`+3QKDoc~kj7im@(&mrD?j=uf5ot3 zu71^FA_-|>9@6ZIygSENT|9@*Tr zeCP2A{+JF&kJq?!{0p`2U*ER=ODThzx0$^8tLGBFJ2>+EyOn#k<~QwIQTOeNPh!r$ zQ#ow??6y|u%0Gu~yZeVHZ{Brq_@6)f)4sOjx(6TaHgEmw=8m=NcUOE>ys2@M*~dek zp7-#oN7|enlK$y!Z;kn=aYXsr9=kK5TEBgG%tP~@A2{OmkX?UWvi8%E;M+?>Ubye= z!=GB6bBizM@BQR%Z`Mb%FP{9i$J_7bEn8VyGw-br*IT>B5Bv9fjnj^Nb@PKA)7CWH zwdUbt8@k3sr6nA3){bAZd~BPv@kc(53hOt1b|I%|8Cu zOyZ}=25 zX6fYFu}xq5sHm&&FTU+-Z(oa6IF*?n_r=2D?|<}st>WpOKkGNI`>?KSoPJ}h*qPU@ zZ&Ck5_bvy03+kr+?Uq|_`1X^o!!LF$c{*y&m{Do7lNK**^T_By1Ja0cE`dicdsbLjOC#J6cb%SoAe>&j*??r1x-X}kD4S7+uO zX)r5uW6i-k-WeJi5$N*us)y@Q!rv>=SNH4Jyhd~g|)sb{`63-i}Bsg%&CzT zS+A`7sp&nlqUx2DyVu`yGOF3+!u=sTPP&(BwLX#eO>B#AdUyL~#~s*|%s(li3HdI(@M1&u&O^!VMo3t#|8xQE~n7X*08ojz#wlpRqg3dw1Ilc?~oD zbuV9et@XR5_PRPlDq@d5b2_VQ*=T=8an#X*BU$ar;+tg@9a|Ujenq{b(=TR4e_51w zFuAl=cDLr+3iEELUl{Y(uG3prHfg@&FL^bulnl%ZZ+PzdS4(c_GG|DoIdc00^Pl_V zT-%W)!}21|o*409*%Ns!TAb)P@wqwYZi*>Qoz{uAe7)lpwJuG2tEwhx#c6wrhVLn9 zalYZx<(ZczKW!X8@^QE4R>!8adTs8$lrcB7&uJX&+ir2nAb;ENTW8IP?OnLA|Ng}( zceH!t*69VkVq3OaH233s6YnrLG`)2eeikn1zkgc%n;kEN*#!rCA8k3l!Is5wtBr`B z=N9J8Et(d8xaawn6B}$vNmy-|P0DA!-22MH?72G^#~t)dYHAlai#i{Dp;x0;Ga77} z7JrAiu*vj-sMwmVX3X6=Exwnrut|UX#9!5g)L(!4*`Rb@Rz4@w|+aR&c9ae-+L-4*V@qZ*n^uFRNc4FCmdfEXLuL(e|7tq zIsaAI^qrI$FO1zc@w?My$sMQk-hX|E{dGcKw;ONoF|^6d(GRzuT)OIp7q<@>8lJxL zmciR|hBj~adWXR!{~B6jY_URW~`kYo*PeeX5|t zhc}H_dFR^Vj}O%xv8uz`lJ8DU`SQ?p$tMdAr%rsgPetm{8#^c5bLP3O7_+4rhiW~M z@X$BgM!GlG8a?XF)XUjX^}0XQ?77L=cg)@v=Qi7MBBfdW$WybfoEX|{>XuWJE3&We zyZ!EOdOlru+s$>$?rZYgz$|l0e3QOAQr%livKxO;R{Q12d99S0bkVc%|}=@Zup0vRZt7szH|-7s6WXXqb0nbVZvk zvyv;@SB=^*UxVnAcW#;Q}~mT6`q0*4cRT(~{)#jar;+HL>8Ke$k%{ zsT#cd&fU0Wd;Rmx58yQ+?_#}f6=8d}tvDZQ^?&H&_It{@>?v(_>Dg}Q!hS64c0TOu z&_)RhmsO0Zou24ZqpVl#4veVx?;VloGn+ImI9b|g!ENDFPNshQMw8N+i_jsL_uq7C zl)rYRJd^e!S|k7`o=sp5OS?CP14&xusQt8{tFj0D7`mP^%2$60)-3qZ(STW z$Umv+4+R+*<*)TGSsYi@gYeguxX4~L1ks_YelFO*xAv%mzQY*NmA%m+8*HgP>bYn& zA}f2>Y&CIi(Y=Y!MHh#uE92+YDWN#5d}erT1o>GTwY{b3HJ7^P%E2W=N|xXM%Lu?s zQGGlImwb|zb$Lr<(%G+~TkY$5FfYB%sKk}aYL5N$pC;{0{)hDy^h}J6D4aKU-_GRb zWAX|!!;jyqYJ6Z?>UMv8xG`%!{)(SFlehc-7M@-(Aoh_~uQfPu@7RAV!&NX*t$cN7 zS#0%SV5Z2QRUBLO^WMZ8tY7}xU{CF&gT4zx8b7tB z2m@%@D=*))+kUlKQNd{3+PUgZ-(f=W(iK-)&Yin6Wz0PD{-)EHtvKIuu4A&y4^;Z9~^LL#Y`RKCmqEepyJY_~w%B&kdyk+^ysK!N;hSm;A zXw!ar=}Py-ZSM`O`9NXlVZ4Go^|Mvx;3BYfX+r3->1$Rx9~W&HTK{xHtF&pMD-EY; zQOcZ057oWO)wOpgU*&2-L~bd#dZaL7Lqe@H(_UF=JzDhcq0o&9_0LRiekkOLgcfIJ zUS8R)-nM5?O`n|Qt5@`fTX$;3&}PpjWcg=r?|yvlXEgoIBs9)WZAKdzH$Gd$?{^{08KG_~I zk=-`#Gn3m;Pc$ndGBvuk8*@6#lIff1gN6;c-0KBF{! z-E9+2WE{=!oL#@H0z>y=c8liQ>s}fVQTM>dY0c`+Ik7dbHfVcS^BskGb@!b}?K1sJ zd7Dkod~vQl#_gs_FBxjwK3IG@@75LN>6`Kcm3MtwlDKK|g#Kp_O;ymfCe8qKw`}{O zYTW+weE6u7?IzASTN%2gbl>@#x}CV^(u~KKQm$^jxo0ai1bW4SfNEA(BLc`%Kd<6W zjR=g;R&&+xxwg;WHl$?M1FFv}4z*N$K4ti7V*s!;Ahu26ykPbx@poVTF7oi)V-X!U zz4~nkMqdB+r?(z%f2PxniLo)Q=Kp~}vR{6pk{EdZ>fdP3dk2@kv82I)_bQjZ@wB=?BMb_=plVR4 zi3i<2Y+%~Mo%=ko<($85+CkqfO{e3pp{@QpcW3QUiB&&)VT>)GyHhPAFlV$MTN3_d z)0VI8FS}s+)Od9;OQ^+2!aOrRT>WgLelCuiXR5!dJAwhaGg;N`2aKrDtAqQwX!UbG zCMyiiY4MlWE;+a1BaG73#{EsN4qBxMgZ4t08s!+YYOJZBs*SGwxh(oyHNe$`cXdQ# z`cUIt%~I-0XtAq5zZ|tIe8rT}FODCczG!*!KLc~#S^UxizTFpAk&=E2*( z8ro=PTz30s?_AaB#n_1UlY^_84K8hqb@QrQ2ABQU&>BnQ#^+`ndf{~VgkX(jbFPl* z#ow=}aX6u8+6|@c+LeYch__akG(Pp*-u(@RRUga+udHk(2J@WDE2DoJ%u)4H@|em5C0r7Dg%dY?Q+BS&wChgIeCpJk+nW?L%)X^& z`Jk64z0qXK^Vv1~?#Oz1N&`1S4dXdqXVv?lv~*qVuoFFoPkA8wrr48XhUf1t-|^>Q z|DhF+51%?Vt3z!0U7)eiI3df5tr&cC%8~3^Uv3|IIK5m=pL zFi)OfG(B7XCwI$2O6QTg$?NT)q$?>w77HYD+3x&u(yIH;6JZCJIt{;_z4zC(@|6>rm-4Yzo*Ee2tI_TOE2hkUF*tfw(&FTP zR~F7%a;1E7cx)hH#@^X$9!o8{Yu3|`-2M}%)Z$Jp60t14x(r=1bG2I0HaM(S#MM@` z{*G$hSvR&~;k<8yqi?Xn!%An?bvj;B-5lvJzA!A*#-D2z&gx&X?&&sTH-!)Dp+HuJ zn!{LADz@l&Y*4y$wCc7$ttPKsNdL6Vu3GLsc)9hfGgNu8jSBPnZ;BeJW~0)XCwre; zI0a*TWTWV$%LQZB-F(;{S-Ti^QL#RZ!P4RTD>qbxk7_*wi-i<5mhFNWsuEX!&cFg$ z{hW)%C{_oDn;gT>h{7pzi*_cjHm(l4(29&j%g&8SNqDoP8e8GG?)(OOjvmX}lQCx9 zXPFx+?+Nwq-rnQD3n6=Vy=FRNe|$9V%G)8eh7WdEUb;Q)FRy-fw&sFaP3L@d&$+!{ zKDqk7=daY*cjoe^bi*$%Czupk2ges#wvGS4`k&=Ek&bO!cBJ~-PyZP>F-iRwnej`S zp-w{5(o7G>ENN+uPox>9iH*kUc7_>nM4I0((;QdS#R&LuJdk!h>>!FX&nMDMUqD>X zG;E>Gv^<&DCV5?THdM8`n05enu6mkfy8-N;RMk@v+4$opcsvfV6FKff=a_i@{A;^Z@}EzzxH8ut8k zyMWIx`gp*xMIUklVyxPpW$R@)XiB{wIHu#%ujiNnOK)?eIWo-?W6Qy<7iCzcrQdtU z!h2;{10-ac zE9>VtVqS9sh8U|3ZoQ~;z}5S`;loQbTKghRoxE2~8`3^%Km89u*HzxPNpbOs>JbVXhKI0#WM*Kd2GemKNX zyU)0utB(()d3rxJe1L!`!!}K=@50XVi~DE$9dQqATi$y+An&v7x}sdir;lI5=NP8Y z=9scB4kkeTUO82#HeNeEa6K3byMTJ~soA-*jRNRJx*htxxGw=qUmqC0fM1_qkY>xe zU``Zu3AnO-1D+%7T)(5&#l+f7rdc|-Fp>C0KLsvDzc#B2S6?^bdY-4vIjD)D z&pE1gmOjprq_0b0hgFSs?@iOu`3Ady)MmO;o8>d~et?>YxyiIleeGkQdivT7X|}8j z7J&LVGBIz+G|v!i-CHE6$Xc;uYr*!)(1$l_4PQ?#5|5Pm(<0Rb+LV7{xfY; z=e~ipvbgttGa%+zOn>@1*6_pq(C0+Ig<+%J7r$kSe($&Z`Z~-22KBWH(j33M9tNQ( zL#%-fKiWti3w|uU^*O>1!FnI}`z@&rLr`yXcue~G59PYL|AsUylJq*OMZZWBejY|X z;3(|;^1c9#vJ6Lz5A;5Lt%ou^Pt?URWE(kRjcNF?m=bN|_+?!jN2~?>0Zh=kHlR`T zE7vd9Ab!`=*ExpmRhU*HdlG5|esUvT^MzJcY2?r$0BX`+qbf5Sviuze{J_S+q zeFhL2pwaGM0RF13Env#EBYK}$TY&J6eh;vu5$zGM4AD0NmMQuM+yK$<13tm^`nE45)eN9Q?~T(;gf;BARDW&jhkzP1Qp`08UT;CEzQFu{oXjJa6cKg`9V z3|smE0U%1;7uzqm0ICWu1nhv|Lcn$f7Xp}wM1R3R*BKRXnH7B8xfEgDCH66o%&)du~W` zNnND>;ROtNpIuC{`n@9j^{`~n$ExSb^#;%=81G@bM3y1;Zs3s^hPe0Ox#(9OHgEJg z!*!NvV*kp-uC4Afnb;W6>xVR~E~=M-Gz>zKCiKG2bu}IEtgt(x=f!@kxSrgPRbH9C zhC>>5o5l5{UciZx_!zk2rnAl>}+Y&ya?hnGXx5SviDnj4?gPo(V&5#z**O*8X&x(-d zNo`o5>ir7%(ARa?X7KBC4$^F$6-aYLKfuhMY@G#NpKYnu*6{ z`q)=4woDVg5XunG1CZwFYZXkAK79>`G(%sjAWiIrBMncps`m}}PXM8KE@@#t(4SRW z&>_;mP?=_sG?S!RB+XCKY?3CQrD8!&?1&EZ15szp<1&rtz*v=bLo(X3|cpGq%#i{G$35#+J-WbYT6g+bKU(rb!)Gh!H!f z!;d{nc|F-4eylB|ozwvc5Idp+>=D<)UW-iA>a6amK9A#`V&jqIB|5ODD$5``uvc2m z4&~y3yqF{WCeeX4aJ9Uue%RxdX_~ewJE8+~yUa^;V4p_Xk@mnkOxVHSmuW-?d?9H^ zbl|y~v?Ds;pG!NU1N)fLj_APtuCT*%dznUbfS1ya=)fL;v?DsOwv~272mBsk2Y*_o z5gmvQBJGF{@KV|l9oYAlc0>o(0@9A?zcE3FX-9Z&6Q0}H_ZD@=Ggp~Lbifyqc0>pKb7@C(=!^${ZNhW) zj9y%i@Z82bhSHAcAUwBCq67Or)vl-3Ai6(ORcF#3gy%N)+^Stq2qw39TVgYX<1wk%_OC3L_~&^0m)Mbh zMR@KI{~R9|$m|;kkOgCGCg~;-5Q&=jwd{aXrFwytqW{NP7^TJH$W7g9vgxqJ#M7c)45VB|KNp z+JrX3bM@}Dv?J|7{Bs0XBzcJr!gKXbxhR+L+#x)72+!4fZt{9W2jRIxc&@&ClGh{S zgYeuTJa-7s9l~>m@LavOBjzATsUT)odA>PL94-iwrWLU$6IYOFX6fRu1M5{@EmWt5j&!T@EkGTNM53Y@LYZ8CF)0bj*ldW9T^{l=juIk zSqAA>gy(omPUa;%NBm!6NBR}nZ&&Xwi24zpt8X%;9nnE}9w0o&TYU0*gy-s8RZ%YC zxq2s6+L3-mcpf18?dqLMc|Fn|gy;BJM&>0v$BPfdj_4pf$46`=FKG|5->%+A650sQ z)%S1GPOG!>t-yFpzaZ(a-*V%vN|Kkf2jRJTpIVkn+Jo@iB|LYDf3Cja)yq)t7|JxM z!zDa-$@6o(utKgUb-0A*F5$ULc&^^<7Ih{(SKn_*JJ}vC;ko+OL*^wq2+v)@bG(o# zuSa<95}vz+=j!`-Q9r_Sm+;)x_u$cuUBYvh@Z2RlSKmB}x)A@|B|KN}Nz1%)oV$eQ zF5$UL{BxJ^+$B7B3D4DcxT1c9=Pu#7ddE=aCH;!<+$H`wUg(n7BRqEr&t1ZE^*xEG zAK^J(2qtz!2jRI(cX3eeW!^5uW3Pd16P#xz27d0AG^{2B`1p zM7f0LF5$U)pIYW6Itb6zx1cgF(;+#i-fI?l3C~@^bM<|m%u93-p1XwS>KlA{J)(o~ z+$B7B3C~^PpS#3AcXeM}ea?xwPUpT$c&^?x6*`E2j*s1hokw`?5uSU5=j!_fy+2+#2`fXqvH?h&4Qgy$ad&ppC(kMLZ@o)9_+&ppC(kMJBH zHOT7`o_mDn_^O8FmGdsXsv&kn2jRK;{!Qp5JogCC@udyPE62I|9#)n?bP%3g>2+!5`5TY)G=N{p?`j$-QB{~StJ;HN* zBulPG`jyUZH79}r`kd$yo_mDn9^twA4nt@oJXhbVNjsv0_~#z+&pn;-5QVR+WEq6# z9^tu1c)fyGMBL5uSU5=N{oXz7mslAw2g8&++9L$xCz)o_mDn z_?k>!kMLZ5_b&Pd@y}Jf3~5LD72&x@c#aRo_3IhvMkGz>fSuq6(u5AAiS|I6(1A3e z18Jf?kS5v#X+($QIlez9Iwa3wM|4P@SAEZ}wFlCO4#{)a5gn4}_-2o^hvYfzNP9@0 z!;Z9vn!4#{)a5gn4}_`s3qkUWPS(II&bJEBAK9Ck#9n!4#{(Tlu2|*p2Lpl zkUWPS(II&bJEBAGx5JL;kUUpWo#Z%|Jck|8A$blvqC@f=c0`BdIqZlI>7T0)Vxn!4#{)a5gn4}up>Go z&tXS&5S{~$LI>bO(xeVdio{Opz+feIQU^K_v6DI=PTCQktJpw#8>x6VGL7gUJXhcP z%DhAe;kk;tD)SN@gy$+oqsU8mju2wRjzqc}aT^o*RVc_!?ebkMP_eJU0l> z4Z?FA$V|#0Itb74)jY{d+Jo@iAUwyH^zwRy=LoY!?1&D+a}|q0^aH|ke626-2+vha zV{tvga}|$F+7TUu=LX@qim4;7N7{q%+#ozhD5dJxQw+s?uQL?EpQ_s-)P{;LCF(+W zZV;Xugy$+Ag}feV55jX5S54$4JXhcJOFN>2@Z2CgM*t3TJ)(o~+#ozR2+viF8Bu4# zbA#|)#f6o5$v7uGSKlJbyrew{&k+Pn<|RBws2yTQ+Jo@iAUrn+&ke$JgrgDVB1j5J zlR9wTHnEdBOyZxLgy$yl&k;zSlp)*0Bs@om8kv{y9MMUM9nnE}j_9N$uk2TdSW4`O z4&tAigy$;$pWY``j8&ONbP)d>p-D(yqJ#M72(3Z#l72;au3~$MIuo9ogy)EWO7apN z#6QR5N|Kl8ApW_EhbQVxc&_5&NITLVgy+By$xGUU@EnnGNM6z&gy)EsBl8lTBU%oz zBke(WjtDv=FKG|rpCf{f%u9HVpuEJ6v{v&bgy$+| ztLQ_7=LjxG?1&D+a}{eymO*q7|6IkO6L|^G5v7ONk@g@wHwn)X&4*l%=pZ~d3C~sB zbD@LqT*W<-c0`Bni(^2m=Xv^i(j+`L3C~T!b9F9?tc%p4;(+URDt?$ulR7NopCdpa z$t&AK#kiDZ$bO|_omR6`_X_u3zgGylT-^?1MxC2cEpJsC(LwgxEy8n)@EpMxNf|^3 z@y{*7bBplYB0N`T1qf}#KSzL8Vn^D8_~!^BN%G2mrD7F}x)7eLn2*wq=pZ~-=QPN? zLu+2&69a5}vEG2SgdfKeq_aaaan;OZpYzxr&=C%O&kWc#hzoGB4q| zI&(vmLH66#86MJ(=pg<%4jv(Si4MYZi}2jieRehW3C|JGSC&isb86YCWs+BGBh?QG&k@X4<|RB=@3IRW#6MT(YDqhygYcZhheOy}qJ#M7Bt9Gh zbIN{2;={pC_A3$}4t7!pi4O-m*&ZZ59PDJj@)Mq`^AhxXiUWp78qq;`uFfEn*OTJ| zhv5)A*&cqva}pm8Te(CB;W-XUlI0SftJt4H2jMvmc_4N~2jMw_lFPh==PFjVs0-ma zPQM^_LLBsqU?;~1i4O-mse{Ca!=pgi9wa^->@;myuc>nY^|rJL&(&E5 z(vIjLJXdGA$h>l#llX83i4TV~qJ!`p2U5v$3C~GgB3D0ffpWB4zBt9I@ZYAwOcy1G(<8TwHgFHXC3D0f9a|Di; z*CRY9@!<@c@SMblgB|Htgy-rkNzq1x=Qi=ralnhrOL$J=!{M<#X%FI`+l1%p%neaz zvfobP!=YR<&I!*+d^o&TAlHc`K3vtAgra_g=Qwyy>d@I;b@YIIUr(K3Da#=3L3mE$ z!x0)lK61QE5|vB4+lG`gT#k}ooo+vriG{r;W>#9hogyz4#IO1 z9}anCdpKmjoy3PjUZR8e=OjKHUiTsGL3mE$!yzwe55jYt#3t`4;kiS2j?>v>Ucz${ z9}cf5k@g_|ISw==WsvqD{<%6+RP-0Za}pm8uS>~wB8d+NJEDW|+#x(C@!@bi(jJ88 z4&k{vqh9DGJSXwtaGWdALHu(P9}am*dk~&Ggy#<7If)O4*V0IP5dWOSheKXD?>dC% zBt9JSlJ+3}xkGqP;=|!pI--N{+#&usi4TYCk?}!zPU6EMFBu<%=OjKHUjHLHh<{Gv z!yzwe55jYGzMAOIgy$qa9LgZ`F5x+e4`-10a7ZKLgYcZhheKYXgZSqpJ{sMvfobP!@-V>55jX29}am*zal&*@!@cCE@=}0=kb;e`9BJtsnCdUVf4+lG`gT#l!ClazfNPIZh z$@Xw{#;ZCLp1XwSBt9JS%6{b%p5w4a(KkqZIDG0s+JpG#Bt9JS%JD(s!@-WU2jMx1 z4~M)&2l3BId^jBEO4@_)oWzGiUO7J0*_ryBL*m0BFX>l==OjKHKB*z?L3mE$!yzwe z53=7*;=>^?=~u)*C-LF%$qyMHa}pm8d5I3f za}pm8pNNt6AUr4W;gFZK2jMx14~M*@UlE?Gv#`Y+L3mE$!x#92RqUpgy$qa9F7E*?Lp$h!A|NR@!?=6`xS`~2RqpwBt9JMWP6BMa5&ys#iG&u zZjbPs#D~MDhq7Oh_;9e3{2=k+U?!xIf)O4yhI1#xkvnSkND>#J{&&l zl^i7T;b2E}5T29xaL7w^5T29xaQMVnuCGXZIM@*#gy$qa9P*NJPIyk@!yzvj=Y;1Z zJ{&%kCj1~gC-LEsm$V1rIf)O4yrew{&ppC(5+4qqcM~1NKPU0wkeBEn{<%kZ?vdx` zBt9I<5c3s2KNtN9X+j6mM886sn6Ho~bRbRiE2N2e7ipqBkS6*S(ufZIU3N9EVXSKN znu#wz^|cw&hz`kf*byC)=ddF>B+p?-bja`3@aaEk59y!7j_8p7IqZlI$#d8d9g^p; zBRV9{O`qgB(nxzqp2LpxE9sxZj`SbjD-wlK5~)BmGM5x5JL~E6H=%k$xrj+hIrgmE<|>NWYRiM<@%TL-HJUM2F-# z?1&EOpTmymkUWPS(II(`@Eb&jn!4!PeBJEBAK9Ck#9n!4!PfM`XtYhM#hJH zehxb_J|xd!N5+TbIqb;zkUU376Ee;v&tXS&NdFvmM2F-#?1&D@bJ!6bgy$qa9740m zeuY4vf`cSJ9P&yXBt9JMqz)1v4t7!pi4SKQa=#sELE4;gCk!gYcZhheKY{uL#fa zQLyZ%gy$qa9LgZ=L3mE$!NU;#+%1{vps z=OjKHx+$5j2+vL8pPPi|Bt9I#92fS&0O=%-MC-LE6C)Zab zJ{;_b4#IO19}aotd`05JA!IAjLHu)z@Z2IiC-LD>2GK!yPU6EMFVR7GPU6FPhfdYXC{ zwt5=A?$MvUBl3iLf33P5zPP~pK%`w;Ms7}AYFaQSv`&{U(F0Q-4@P43Wk(Ol%*>5M zjUxv{-(blm?(2^C^?Sy4?~dsZJB;1kfampy#k-m~;=D)9|79c4(bD@9h0hY-pXUiKuyKo&Qhui|&^bd^GZEwW5a( Y2qq28O~+68S^=% Date: Fri, 5 Jun 2020 13:16:54 -0700 Subject: [PATCH 121/168] Fix strlcpy() My "set -Wall by default" commit (feced6a) checked in a broken strlcpy() that wasn't NULL terminating. Reproducer: dst[] = "unifyfs" strlcpy(dst, "client", 50) // dst[] is now "clients" instead of "client". This fixes the bug --- common/src/unifyfs_misc.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/src/unifyfs_misc.c b/common/src/unifyfs_misc.c index 99ddd5422..345daa172 100644 --- a/common/src/unifyfs_misc.c +++ b/common/src/unifyfs_misc.c @@ -28,12 +28,17 @@ */ size_t strlcpy(char* dest, const char* src, size_t size) { - size_t src_len = strnlen(src, size); + size_t src_len; - memcpy(dest, src, src_len); + src_len = strnlen(src, size); if (src_len == size) { - dest[size - 1] = '\0'; + /* Our string is too long, have to truncate */ + src_len = size - 1; } + + memcpy(dest, src, src_len); + dest[src_len] = '\0'; + return strlen(dest); } From 08c0a3f9ff14e7d609b52636b545b9ff448e31cd Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 7 May 2020 12:01:19 -0700 Subject: [PATCH 122/168] client: add spath to normalize paths This adds the VeloC spath component to normalize paths similar to realpath(). The intercept_path function is updated to return a normalized path for intercepted paths, and wrappers then use the normalized path instead of the original path provided by the user. This helps ensure we always refer to the same underlying file, regardless of the various strings one might use to name it. --- client/src/Makefile.am | 6 ++++++ client/src/unifyfs.c | 11 ++++++++++- configure.ac | 3 +++ m4/spath.m4 | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 m4/spath.m4 diff --git a/client/src/Makefile.am b/client/src/Makefile.am index 0325df935..06fd0b63e 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -43,6 +43,12 @@ CLIENT_COMMON_LIBADD = \ $(FLATCC_LIBS) \ -lrt -lpthread +if HAVE_AM_SPATH +CLIENT_COMMON_CFLAGS += $(SPATH_CFLAGS) +CLIENT_COMMON_LDFLAGS += $(SPATH_LDFLAGS) +CLIENT_COMMON_LIBADD += $(SPATH_LIBS) +endif + CLIENT_COMMON_SOURCES = \ margo_client.c \ margo_client.h \ diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index ab9966a4a..668389337 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -53,6 +53,10 @@ #include "seg_tree.h" #include "ucr_read_builder.h" +#ifdef HAVE_SPATH +#include "spath.h" +#endif /* HAVE_SPATH */ + /* avoid duplicate mounts (for now) */ static int unifyfs_mounted = -1; @@ -278,8 +282,13 @@ static void unifyfs_normalize_path(const char* path, char* normalized) snprintf(normalized, UNIFYFS_MAX_FILENAME, "%s", path); } - /* TODO: normalize path to handle '.', '..', +#ifdef HAVE_SPATH + /* normalize path to handle '.', '..', * and extra or trailing '/' characters */ + char* str = spath_strdup_reduce_str(normalized); + snprintf(normalized, UNIFYFS_MAX_FILENAME, "%s", str); + free(str); +#endif /* HAVE_SPATH */ } /* sets flag if the path is a special path */ diff --git a/configure.ac b/configure.ac index ebf9f3439..84dc7b03c 100755 --- a/configure.ac +++ b/configure.ac @@ -162,6 +162,9 @@ UNIFYFS_AC_LEVELDB # look for gotcha library, sets GOTCHA_INCLUDE, GOTCHA_LIB UNIFYFS_AC_GOTCHA +# look for spath library, sets SPATH_CFLAGS, SPATH_LDFLAGS, SPATH_LIBS +UNIFYFS_AC_SPATH + UNIFYFS_AC_MARGO UNIFYFS_AC_FLATCC diff --git a/m4/spath.m4 b/m4/spath.m4 new file mode 100644 index 000000000..c7c2ce3d3 --- /dev/null +++ b/m4/spath.m4 @@ -0,0 +1,35 @@ +AC_DEFUN([UNIFYFS_AC_SPATH], [ + # preserve state of flags + SPATH_OLD_CFLAGS=$CFLAGS + SPATH_OLD_CXXFLAGS=$CXXFLAGS + SPATH_OLD_LDFLAGS=$LDFLAGS + + AC_ARG_WITH([spath], [AC_HELP_STRING([--with-spath=PATH], + [path to installed libspath [default=/usr/local]])], [ + SPATH_CFLAGS="-I${withval}/include" + SPATH_LDFLAGS="-L${withval}/lib64 -L${withval}/lib" + SPATH_LIBS="-lspath" + CFLAGS="$CFLAGS ${SPATH_CFLAGS}" + CXXFLAGS="$CXXFLAGS ${SPATH_CFLAGS}" + LDFLAGS="$LDFLAGS ${SPATH_LDFLAGS}" + ], []) + + AC_CHECK_LIB([spath], [spath_strdup_reduce_str], + [ + AC_SUBST(SPATH_CFLAGS) + AC_SUBST(SPATH_LDFLAGS) + AC_SUBST(SPATH_LIBS) + AC_DEFINE([HAVE_SPATH], [1], [Defined if you have spath]) + AM_CONDITIONAL([HAVE_AM_SPATH], [true]) + ],[ + AC_MSG_WARN([couldn't find a suitable libspath, use --with-spath=PATH]) + AM_CONDITIONAL([HAVE_AM_SPATH], [false]) + ], + [] + ) + + # restore flags + CFLAGS=$SPATH_OLD_CFLAGS + CXXFLAGS=$SPATH_OLD_CXXFLAGS + LDFLAGS=$SPATH_OLD_LDFLAGS +]) From ea3ca6a51bda59e5f8cc6f8a1f3614d6ea5e2fda Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 7 May 2020 12:03:51 -0700 Subject: [PATCH 123/168] client: add chdir and getcwd wrappers Adds wrappers for chdir, fchdir, __getcwd_chk, getcwd, getwd, and get_current_dir_name. When unifyfs is initialized, we attempt to track the current working directory for all directory changes. The starting current working dir is taken as UNIFYFS_CLIENT_CWD if specified, and the actual current working directory (getcwd) otherwise. We then return the working directory in calls like getcwd. TEST_CHECKPATCH_ALLOW_FAILURE=yes --- .travis.yml | 6 + client/src/gotcha_map_unifyfs_list.c | 31 ++ client/src/unifyfs-sysio.c | 406 +++++++++++++++++++++++++++ client/src/unifyfs-sysio.h | 6 + client/src/unifyfs.c | 13 +- configure.ac | 6 + docs/configuration.rst | 7 +- examples/src/Makefile.am | 6 +- m4/spath.m4 | 26 +- t/Makefile.am | 6 +- t/sys/chdir.c | 361 ++++++++++++++++++++++++ t/sys/sysio_suite.c | 2 + t/sys/sysio_suite.h | 2 + 13 files changed, 844 insertions(+), 34 deletions(-) create mode 100644 t/sys/chdir.c diff --git a/.travis.yml b/.travis.yml index b8f972809..4f8a490ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ addons: - libtool-bin - m4 - openmpi-bin + - numactl before_install: # The default environment variable $CC is known to interfere with @@ -49,6 +50,10 @@ before_install: buildable: False paths: m4@4.17: /usr + numactl: + buildable: False + paths: + numactl: /usr EOF install: @@ -57,6 +62,7 @@ install: - spack install gotcha@1.0.3 && spack load gotcha@1.0.3 - spack install flatcc && spack load flatcc - spack install margo^mercury+bmi~boostsys && spack load argobots && spack load mercury && spack load margo + - spack install spath && spack load spath # prepare build environment - eval $(./scripts/git_log_test_env.sh) - export TEST_CHECKPATCH_SKIP_FILES diff --git a/client/src/gotcha_map_unifyfs_list.c b/client/src/gotcha_map_unifyfs_list.c index 8f0fe8605..01e503f05 100644 --- a/client/src/gotcha_map_unifyfs_list.c +++ b/client/src/gotcha_map_unifyfs_list.c @@ -32,6 +32,21 @@ UNIFYFS_DEF(mkdir, int, UNIFYFS_DEF(rmdir, int, (const char* path), (path)) +UNIFYFS_DEF(chdir, int, + (const char* path), + (path)) +UNIFYFS_DEF(__getcwd_chk, char*, + (char* path, size_t size, size_t buflen), + (path, size, buflen)) +UNIFYFS_DEF(getcwd, char*, + (char* path, size_t size), + (path, size)) +UNIFYFS_DEF(getwd, char*, + (char* path), + (path)) +UNIFYFS_DEF(get_current_dir_name, char*, + (void), + ()) UNIFYFS_DEF(rename, int, (const char* oldpath, const char* newpath), (oldpath, newpath)) @@ -83,9 +98,12 @@ UNIFYFS_DEF(__open_2, int, (const char* path, int flags, ...), (path, flags)) +#ifdef HAVE_LIO_LISTIO UNIFYFS_DEF(lio_listio, int, (int m, struct aiocb* const cblist[], int n, struct sigevent* sep), (m, cblist, n, sep)) +#endif + UNIFYFS_DEF(lseek, off_t, (int fd, off_t offset, int whence), (fd, offset, whence)) @@ -124,6 +142,9 @@ UNIFYFS_DEF(pwrite64, ssize_t, UNIFYFS_DEF(close, int, (int fd), (fd)) +UNIFYFS_DEF(fchdir, int, + (int fd), + (fd)) UNIFYFS_DEF(ftruncate, int, (int fd, off_t length), (fd, length)) @@ -315,6 +336,13 @@ struct gotcha_binding_t unifyfs_wrappers[] = { { "chmod", UNIFYFS_WRAP(chmod), &wrappee_handle_chmod }, { "mkdir", UNIFYFS_WRAP(mkdir), &wrappee_handle_mkdir }, { "rmdir", UNIFYFS_WRAP(rmdir), &wrappee_handle_rmdir }, + { "chdir", UNIFYFS_WRAP(chdir), &wrappee_handle_chdir }, + { "__getcwd_chk", UNIFYFS_WRAP(__getcwd_chk), + &wrappee_handle___getcwd_chk }, + { "getcwd", UNIFYFS_WRAP(getcwd), &wrappee_handle_getcwd }, + { "getwd", UNIFYFS_WRAP(getwd), &wrappee_handle_getwd }, + { "get_current_dir_name", UNIFYFS_WRAP(get_current_dir_name), + &wrappee_handle_get_current_dir_name }, { "rename", UNIFYFS_WRAP(rename), &wrappee_handle_rename }, { "truncate", UNIFYFS_WRAP(truncate), &wrappee_handle_truncate }, { "unlink", UNIFYFS_WRAP(unlink), &wrappee_handle_unlink }, @@ -331,7 +359,9 @@ struct gotcha_binding_t unifyfs_wrappers[] = { { "open", UNIFYFS_WRAP(open), &wrappee_handle_open }, { "open64", UNIFYFS_WRAP(open64), &wrappee_handle_open64 }, { "__open_2", UNIFYFS_WRAP(__open_2), &wrappee_handle___open_2 }, +#ifdef HAVE_LIO_LISTIO { "lio_listio", UNIFYFS_WRAP(lio_listio), &wrappee_handle_lio_listio }, +#endif { "lseek", UNIFYFS_WRAP(lseek), &wrappee_handle_lseek }, { "lseek64", UNIFYFS_WRAP(lseek64), &wrappee_handle_lseek64 }, { "posix_fadvise", UNIFYFS_WRAP(posix_fadvise), &wrappee_handle_posix_fadvise }, @@ -343,6 +373,7 @@ struct gotcha_binding_t unifyfs_wrappers[] = { { "pread64", UNIFYFS_WRAP(pread64), &wrappee_handle_pread64 }, { "pwrite", UNIFYFS_WRAP(pwrite), &wrappee_handle_pwrite }, { "pwrite64", UNIFYFS_WRAP(pwrite64), &wrappee_handle_pwrite64 }, + { "fchdir", UNIFYFS_WRAP(fchdir), &wrappee_handle_fchdir }, { "ftruncate", UNIFYFS_WRAP(ftruncate), &wrappee_handle_ftruncate }, { "fsync", UNIFYFS_WRAP(fsync), &wrappee_handle_fsync }, { "fdatasync", UNIFYFS_WRAP(fdatasync), &wrappee_handle_fdatasync }, diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 94c80c7ce..917299c76 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -155,6 +155,353 @@ int UNIFYFS_WRAP(rmdir)(const char* path) } } +int UNIFYFS_WRAP(chdir)(const char* path) +{ + /* determine whether we should intercept this path */ + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { + /* TODO: check that path is not a file? */ + /* we're happy to change into any directory in unifyfs */ + if (unifyfs_cwd != NULL) { + free(unifyfs_cwd); + } + unifyfs_cwd = strdup(upath); + return 0; + } else { + MAP_OR_FAIL(chdir); + int ret = UNIFYFS_REAL(chdir)(path); + + /* if the change dir was successful, + * update our current working direcotry */ + if (unifyfs_initialized && ret == 0) { + if (unifyfs_cwd != NULL) { + free(unifyfs_cwd); + } + + /* if we did a real chdir, let's use a real getcwd + * to get the current working directory */ + MAP_OR_FAIL(getcwd); + char* cwd = UNIFYFS_REAL(getcwd)(NULL, 0); + if (cwd != NULL) { + unifyfs_cwd = cwd; + + /* parts of the code may assume unifyfs_cwd is a max size */ + size_t len = strlen(cwd) + 1; + if (len > UNIFYFS_MAX_FILENAME) { + LOGERR("Current working dir longer (%lu bytes) " + "than UNIFYFS_MAX_FILENAME=%d", + (unsigned long) len, UNIFYFS_MAX_FILENAME); + } + } else { + /* ERROR */ + LOGERR("Failed to getcwd after chdir(%s) errno=%d %s", + path, errno, strerror(errno)); + } + } + + return ret; + } +} + +char* UNIFYFS_WRAP(__getcwd_chk)(char* path, size_t size, size_t buflen) +{ + /* if we're initialized, we're tracking the current working dir */ + if (unifyfs_initialized) { + /* check that we have a string, + * return unusual error in case we don't */ + if (unifyfs_cwd == NULL) { + errno = EACCES; + return NULL; + } + + /* If unifyfs_cwd is in unifyfs space, handle the cwd logic. + * Otherwise, call the real getcwd, and if actual cwd does + * not match what we expect, throw an error (the user somehow + * changed dir without us noticing, so there is a bug here) */ + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(unifyfs_cwd, upath)) { + /* man page if size=0 and path not NULL, return EINVAL */ + if (size == 0 && path != NULL) { + errno = EINVAL; + return NULL; + } + + /* get length of current working dir */ + size_t len = strlen(unifyfs_cwd) + 1; + + /* if user didn't provide a buffer, + * we attempt to allocate and return one for them */ + if (path == NULL) { + /* we'll allocate a buffer to return to the caller */ + char* buf = NULL; + + /* if path is NULL and size is positive, we must + * allocate a buffer of length size and copy into it */ + if (size > 0) { + /* check that size is big enough for the string */ + if (len <= size) { + /* path will fit, allocate buffer and copy */ + buf = (char*) malloc(size); + if (buf != NULL) { + strncpy(buf, unifyfs_cwd, size); + } else { + errno = ENOMEM; + } + return buf; + } else { + /* user's buffer limit is too small */ + errno = ERANGE; + return NULL; + } + } + + /* otherwise size == 0, so allocate a buffer + * that is big enough */ + buf = (char*) malloc(len); + if (buf != NULL) { + strncpy(buf, unifyfs_cwd, len); + } else { + errno = ENOMEM; + } + return buf; + } + + /* to get here, caller provided an actual buffer, + * check that path fits in the caller's buffer */ + if (len <= size) { + /* current working dir fits, copy and return */ + strncpy(path, unifyfs_cwd, size); + return path; + } else { + /* user's buffer is too small */ + errno = ERANGE; + return NULL; + } + } else { + /* current working dir is in real file system, + * fall through to real getcwd call */ + MAP_OR_FAIL(__getcwd_chk); + char* ret = UNIFYFS_REAL(__getcwd_chk)(path, size, buflen); + + /* check that current working dir is what we think + * it should be as a sanity check */ + if (ret != NULL && strcmp(unifyfs_cwd, ret) != 0) { + LOGERR("Expcted cwd=%s vs actual=%s", + unifyfs_cwd, ret); + } + + return ret; + } + } else { + /* not initialized, so fall through to real __getcwd_chk */ + MAP_OR_FAIL(__getcwd_chk); + char* ret = UNIFYFS_REAL(__getcwd_chk)(path, size, buflen); + return ret; + } +} + +char* UNIFYFS_WRAP(getcwd)(char* path, size_t size) +{ + /* if we're initialized, we're tracking the current working dir */ + if (unifyfs_initialized) { + + /* check that we have a string, + * return unusual error in case we don't */ + if (unifyfs_cwd == NULL) { + errno = EACCES; + return NULL; + } + + /* If unifyfs_cwd is in unifyfs space, handle the cwd logic. + * Otherwise, call the real getcwd, and if actual cwd does + * not match what we expect, throw an error (the user somehow + * changed dir without us noticing, so there is a bug here) */ + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(unifyfs_cwd, upath)) { + /* man page if size=0 and path not NULL, return EINVAL */ + if (size == 0 && path != NULL) { + errno = EINVAL; + return NULL; + } + + /* get length of current working dir */ + size_t len = strlen(unifyfs_cwd) + 1; + + /* if user didn't provide a buffer, + * we attempt to allocate and return one for them */ + if (path == NULL) { + /* we'll allocate a buffer to return to the caller */ + char* buf = NULL; + + /* if path is NULL and size is positive, we must + * allocate a buffer of length size and copy into it */ + if (size > 0) { + /* check that size is big enough for the string */ + if (len <= size) { + /* path will fit, allocate buffer and copy */ + buf = (char*) malloc(size); + if (buf != NULL) { + strncpy(buf, unifyfs_cwd, size); + } else { + errno = ENOMEM; + } + return buf; + } else { + /* user's buffer limit is too small */ + errno = ERANGE; + return NULL; + } + } + + /* otherwise size == 0, so allocate a buffer + * that is big enough */ + buf = (char*) malloc(len); + if (buf != NULL) { + strncpy(buf, unifyfs_cwd, len); + } else { + errno = ENOMEM; + } + return buf; + } + + /* to get here, caller provided an actual buffer, + * check that path fits in the caller's buffer */ + if (len <= size) { + /* current working dir fits, copy and return */ + strncpy(path, unifyfs_cwd, size); + return path; + } else { + /* user's buffer is too small */ + errno = ERANGE; + return NULL; + } + } else { + /* current working dir is in real file system, + * fall through to real getcwd call */ + MAP_OR_FAIL(getcwd); + char* ret = UNIFYFS_REAL(getcwd)(path, size); + + /* check that current working dir is what we think + * it should be as a sanity check */ + if (ret != NULL && strcmp(unifyfs_cwd, ret) != 0) { + LOGERR("Expcted cwd=%s vs actual=%s", + unifyfs_cwd, ret); + } + + return ret; + } + } else { + /* not initialized, so fall through to real getcwd */ + MAP_OR_FAIL(getcwd); + char* ret = UNIFYFS_REAL(getcwd)(path, size); + return ret; + } +} + +char* UNIFYFS_WRAP(getwd)(char* path) +{ + /* if we're initialized, we're tracking the current working dir */ + if (unifyfs_initialized) { + /* check that we have a string, + * return unusual error in case we don't */ + if (unifyfs_cwd == NULL) { + errno = EACCES; + return NULL; + } + + /* If unifyfs_cwd is in unifyfs space, handle the cwd logic. + * Otherwise, call the real getwd, and if actual cwd does + * not match what we expect, throw an error (the user somehow + * changed dir without us noticing, so there is a bug here) */ + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(unifyfs_cwd, upath)) { + /* check that we got a valid path */ + if (path == NULL) { + errno = EINVAL; + return NULL; + } + + /* finally get length of current working dir and check + * that it fits in the caller's buffer */ + size_t len = strlen(unifyfs_cwd) + 1; + if (len <= PATH_MAX) { + strncpy(path, unifyfs_cwd, PATH_MAX); + return path; + } else { + /* user's buffer is too small */ + errno = ENAMETOOLONG; + return NULL; + } + } else { + /* current working dir is in real file system, + * fall through to real getwd call */ + MAP_OR_FAIL(getwd); + char* ret = UNIFYFS_REAL(getwd)(path); + + /* check that current working dir is what we think + * it should be as a sanity check */ + if (ret != NULL && strcmp(unifyfs_cwd, ret) != 0) { + LOGERR("Expcted cwd=%s vs actual=%s", + unifyfs_cwd, ret); + } + + return ret; + } + } else { + /* not initialized, so fall through to real getwd */ + MAP_OR_FAIL(getwd); + char* ret = UNIFYFS_REAL(getwd)(path); + return ret; + } +} + +char* UNIFYFS_WRAP(get_current_dir_name)(void) +{ + /* if we're initialized, we're tracking the current working dir */ + if (unifyfs_initialized) { + /* check that we have a string, return unusual error + * in case we don't */ + if (unifyfs_cwd == NULL) { + errno = EACCES; + return NULL; + } + + /* If unifyfs_cwd is in unifyfs space, handle the cwd logic. + * Otherwise, call real get_current_dir_name, and if actual cwd does + * not match what we expect, throw an error (the user somehow + * changed dir without us noticing, so there is a bug here) */ + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(unifyfs_cwd, upath)) { + /* supposed to allocate a copy of the current working dir + * and return that to caller, to be freed by caller */ + char* ret = strdup(unifyfs_cwd); + if (ret == NULL) { + errno = ENOMEM; + } + return ret; + } else { + /* current working dir is in real file system, + * fall through to real get_current_dir_name call */ + MAP_OR_FAIL(get_current_dir_name); + char* ret = UNIFYFS_REAL(get_current_dir_name)(); + + /* check that current working dir is what we think + * it should be as a sanity check */ + if (ret != NULL && strcmp(unifyfs_cwd, ret) != 0) { + LOGERR("Expcted cwd=%s vs actual=%s", + unifyfs_cwd, ret); + } + + return ret; + } + } else { + /* not initialized, so fall through to real get_current_dir_name */ + MAP_OR_FAIL(get_current_dir_name); + char* ret = UNIFYFS_REAL(get_current_dir_name)(); + return ret; + } +} + int UNIFYFS_WRAP(rename)(const char* oldpath, const char* newpath) { /* TODO: allow oldpath / newpath to split across memfs and normal @@ -1440,6 +1787,65 @@ ssize_t UNIFYFS_WRAP(pwrite64)(int fd, const void* buf, size_t count, } } +int UNIFYFS_WRAP(fchdir)(int fd) +{ + /* determine whether we should intercept this path */ + if (unifyfs_intercept_fd(&fd)) { + /* lookup file id for file descriptor */ + int fid = unifyfs_get_fid_from_fd(fd); + if (fid < 0) { + errno = EBADF; + return -1; + } + + /* lookup path for fd */ + const char* path = unifyfs_path_from_fid(fid); + + /* TODO: test that path is not a file? */ + + /* we're happy to change into any directory in unifyfs + * should we check that we don't change into a file at least? */ + if (unifyfs_cwd != NULL) { + free(unifyfs_cwd); + } + unifyfs_cwd = strdup(path); + return 0; + } else { + MAP_OR_FAIL(fchdir); + int ret = UNIFYFS_REAL(fchdir)(fd); + + /* if the change dir was successful, + * update our current working direcotry */ + if (unifyfs_initialized && ret == 0) { + if (unifyfs_cwd != NULL) { + free(unifyfs_cwd); + } + + /* if we did a real chdir, let's use a real getcwd + * to get the current working directory */ + MAP_OR_FAIL(getcwd); + char* cwd = UNIFYFS_REAL(getcwd)(NULL, 0); + if (cwd != NULL) { + unifyfs_cwd = cwd; + + /* parts of the code may assume unifyfs_cwd is a max size */ + size_t len = strlen(cwd) + 1; + if (len > UNIFYFS_MAX_FILENAME) { + LOGERR("Current working dir longer (%lu bytes) " + "than UNIFYFS_MAX_FILENAME=%d", + (unsigned long) len, UNIFYFS_MAX_FILENAME); + } + } else { + /* ERROR */ + LOGERR("Failed to getcwd after fchdir(%d) errno=%d %s", + fd, errno, strerror(errno)); + } + } + + return ret; + } +} + int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) { /* check whether we should intercept this file descriptor */ diff --git a/client/src/unifyfs-sysio.h b/client/src/unifyfs-sysio.h index 45ba6f5cb..67b359988 100644 --- a/client/src/unifyfs-sysio.h +++ b/client/src/unifyfs-sysio.h @@ -55,6 +55,11 @@ UNIFYFS_DECL(access, int, (const char* pathname, int mode)); UNIFYFS_DECL(mkdir, int, (const char* path, mode_t mode)); UNIFYFS_DECL(rmdir, int, (const char* path)); +UNIFYFS_DECL(chdir, int, (const char* path)); +UNIFYFS_DECL(__getcwd_chk, char*, (char* path, size_t, size_t)); +UNIFYFS_DECL(getcwd, char*, (char* path, size_t)); +UNIFYFS_DECL(getwd, char*, (char* path)); +UNIFYFS_DECL(get_current_dir_name, char*, (void)); UNIFYFS_DECL(unlink, int, (const char* path)); UNIFYFS_DECL(remove, int, (const char* path)); UNIFYFS_DECL(rename, int, (const char* oldpath, const char* newpath)); @@ -88,6 +93,7 @@ UNIFYFS_DECL(pwrite64, ssize_t, (int fd, const void* buf, size_t count, UNIFYFS_DECL(posix_fadvise, int, (int fd, off_t offset, off_t len, int advice)); UNIFYFS_DECL(lseek, off_t, (int fd, off_t offset, int whence)); UNIFYFS_DECL(lseek64, off64_t, (int fd, off64_t offset, int whence)); +UNIFYFS_DECL(fchdir, int, (int fd)); UNIFYFS_DECL(ftruncate, int, (int fd, off_t length)); UNIFYFS_DECL(fstat, int, (int fd, struct stat* buf)); UNIFYFS_DECL(__fxstat, int, (int vers, int fd, struct stat* buf)); diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 668389337..bd9867b63 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -146,7 +146,7 @@ void* unifyfs_dirstream_stack; char* unifyfs_mount_prefix; size_t unifyfs_mount_prefixlen = 0; -/* to track current working directory within unifyfs namespace */ +/* to track current working directory */ char* unifyfs_cwd; /* mutex to lock stack operations */ @@ -2242,6 +2242,17 @@ static int unifyfs_init(void) free(unifyfs_cwd); unifyfs_cwd = NULL; } + } else { + /* user did not specify a CWD, so initialize with the actual + * current working dir */ + //MAP_OR_FAIL(getcwd); + //int cwd_rc = UNIFYFS_REAL(getcwd)(cwdpath, sizeof(cwdpath)); + char* cwd = getcwd(NULL, 0); + if (cwd != NULL) { + unifyfs_cwd = cwd; + } else { + LOGERR("Failed getcwd (%s)", strerror(errno)); + } } /* determine max number of files to store in file system */ diff --git a/configure.ac b/configure.ac index 84dc7b03c..aa09450ad 100755 --- a/configure.ac +++ b/configure.ac @@ -223,6 +223,11 @@ LIBS=$OLD_LIBS CP_WRAPPERS+=",-wrap,mkdir" CP_WRAPPERS+=",-wrap,rmdir" +CP_WRAPPERS+=",-wrap,chdir" +CP_WRAPPERS+=",-wrap,__getcwd_chk" +CP_WRAPPERS+=",-wrap,getcwd" +CP_WRAPPERS+=",-wrap,getwd" +CP_WRAPPERS+=",-wrap,get_current_dir_name" CP_WRAPPERS+=",-wrap,unlink" CP_WRAPPERS+=",-wrap,remove" CP_WRAPPERS+=",-wrap,rename" @@ -268,6 +273,7 @@ AC_CHECK_FUNCS(posix_fadvise, [ ],[]) CP_WRAPPERS+=",-wrap,lseek" CP_WRAPPERS+=",-wrap,lseek64" +CP_WRAPPERS+=",-wrap,fchdir" CP_WRAPPERS+=",-wrap,ftruncate" CP_WRAPPERS+=",-wrap,fsync" CP_WRAPPERS+=",-wrap,fdatasync" diff --git a/docs/configuration.rst b/docs/configuration.rst index 005fb5e0b..9e317fb23 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -77,11 +77,10 @@ a given section and key. The ``cwd`` setting is used to emulate the behavior one expects when changing into a working directory before starting a job and then using relative file names within the application. -If set, the value specified in ``cwd`` is prepended to any -relative path name when determining whether UnifyFS will intercept -a path. The value specified in ``cwd`` must be within the directory space +If set, the application changes its working directory to +the value specified in ``cwd`` when ``unifyfs_mount`` is called. +The value specified in ``cwd`` must be within the directory space of the UnifyFS mount point. -Setting ``cwd`` does not modify the job's actual current working directory. Enabling the ``local_extents`` optimization may significantly improve read performance. However, it should not be used by applications diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index 5161dae80..c5ab56f4f 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -83,12 +83,12 @@ test_ftn_flags = $(AM_FCFLAGS) $(MPI_FFLAGS) \ -I$(top_srcdir)/client/src -I$(top_srcdir)/common/src test_ftn_ldadd = $(top_builddir)/client/src/libunifyfsf.la -lrt -lm $(FCLIBS) test_ftn_ldflags = $(AM_LDFLAGS) $(MPI_FLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) $(SPATH_LDFLAGS) $(SPATH_LIBS) endif test_gotcha_ldadd = $(top_builddir)/client/src/libunifyfs_gotcha.la -lrt -lm test_gotcha_ldflags = $(AM_LDFLAGS) $(MPI_CLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) $(SPATH_LDFLAGS) $(SPATH_LIBS) test_posix_cppflags = $(AM_CPPFLAGS) $(MPI_CFLAGS) -DDISABLE_UNIFYFS test_posix_ldadd = -lrt -lm @@ -96,7 +96,7 @@ test_posix_ldflags = $(AM_LDFLAGS) $(MPI_CLDFLAGS) test_static_ldadd = $(top_builddir)/client/src/libunifyfs.la -lrt -lm test_static_ldflags = -static $(CP_WRAPPERS) $(AM_LDFLAGS) $(MPI_CLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) $(SPATH_LDFLAGS) $(SPATH_LIBS) # Per-target flags begin here diff --git a/m4/spath.m4 b/m4/spath.m4 index c7c2ce3d3..30a83c528 100644 --- a/m4/spath.m4 +++ b/m4/spath.m4 @@ -1,35 +1,13 @@ AC_DEFUN([UNIFYFS_AC_SPATH], [ - # preserve state of flags - SPATH_OLD_CFLAGS=$CFLAGS - SPATH_OLD_CXXFLAGS=$CXXFLAGS - SPATH_OLD_LDFLAGS=$LDFLAGS - - AC_ARG_WITH([spath], [AC_HELP_STRING([--with-spath=PATH], - [path to installed libspath [default=/usr/local]])], [ - SPATH_CFLAGS="-I${withval}/include" - SPATH_LDFLAGS="-L${withval}/lib64 -L${withval}/lib" - SPATH_LIBS="-lspath" - CFLAGS="$CFLAGS ${SPATH_CFLAGS}" - CXXFLAGS="$CXXFLAGS ${SPATH_CFLAGS}" - LDFLAGS="$LDFLAGS ${SPATH_LDFLAGS}" - ], []) - AC_CHECK_LIB([spath], [spath_strdup_reduce_str], [ - AC_SUBST(SPATH_CFLAGS) - AC_SUBST(SPATH_LDFLAGS) - AC_SUBST(SPATH_LIBS) + LIBS="$LIBS -lspath" AC_DEFINE([HAVE_SPATH], [1], [Defined if you have spath]) AM_CONDITIONAL([HAVE_AM_SPATH], [true]) ],[ - AC_MSG_WARN([couldn't find a suitable libspath, use --with-spath=PATH]) + AC_MSG_WARN([couldn't find a suitable libspath]) AM_CONDITIONAL([HAVE_AM_SPATH], [false]) ], [] ) - - # restore flags - CFLAGS=$SPATH_OLD_CFLAGS - CXXFLAGS=$SPATH_OLD_CXXFLAGS - LDFLAGS=$SPATH_OLD_LDFLAGS ]) diff --git a/t/Makefile.am b/t/Makefile.am index 5f6dd772d..d18b51867 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -128,7 +128,8 @@ sys_sysio_gotcha_t_SOURCES = sys/sysio_suite.h \ sys/write-read.c \ sys/write-read-hole.c \ sys/truncate.c \ - sys/unlink.c + sys/unlink.c \ + sys/chdir.c sys_sysio_gotcha_t_CPPFLAGS = $(test_cppflags) sys_sysio_gotcha_t_LDADD = $(test_ldadd) @@ -145,7 +146,8 @@ sys_sysio_static_t_SOURCES = sys/sysio_suite.h \ sys/write-read.c \ sys/write-read-hole.c \ sys/truncate.c \ - sys/unlink.c + sys/unlink.c \ + sys/chdir.c sys_sysio_static_t_CPPFLAGS = $(test_cppflags) sys_sysio_static_t_LDADD = $(test_static_ldadd) diff --git a/t/sys/chdir.c b/t/sys/chdir.c new file mode 100644 index 000000000..c1282e56f --- /dev/null +++ b/t/sys/chdir.c @@ -0,0 +1,361 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + + /* + * Test chdir/fchdir/getcwd/getwd/get_current_dir_name + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +int chdir_test(char* unifyfs_root) +{ + diag("Starting UNIFYFS_WRAP(chdir/fchdir/getcwd/getwd/" + "get_current_dir_name) tests"); + + char path[64]; + int rc; + char* str; + + errno = 0; + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* define a dir1 subdirectory within unifyfs space */ + char buf[64] = {0}; + snprintf(buf, sizeof(buf), "%s/dir1", unifyfs_root); + rc = mkdir(buf, 0700); + ok(rc == 0 || errno == EEXIST, "%s:%d mkdir(%s): %s", + __FILE__, __LINE__, buf, strerror(errno)); + errno = 0; + + /* define a dir2 subdirectory within unifyfs space */ + char buf2[64] = {0}; + snprintf(buf2, sizeof(buf2), "%s/dir2", unifyfs_root); + rc = mkdir(buf2, 0700); + ok(rc == 0 || errno == EEXIST, "%s:%d mkdir(%s): %s", + __FILE__, __LINE__, buf2, strerror(errno)); + errno = 0; + + /* change to root directory */ + rc = chdir("/"); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "/", strerror(errno)); + + /* check that we're in root directory */ + str = getcwd(path, sizeof(path)); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, "/") == 0, + "%s:%d getcwd returned %s expected %s", + __FILE__, __LINE__, str, "/"); + + /* change to unifyfs root directory */ + rc = chdir(unifyfs_root); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, unifyfs_root, strerror(errno)); + + /* check that we're in unifyfs root directory */ + str = getcwd(path, sizeof(path)); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, unifyfs_root) == 0, + "%s:%d getcwd returned %s expected %s", + __FILE__, __LINE__, str, unifyfs_root); + + /* get length of current directory */ + size_t len = strlen(str); + + /* try getcwd with a buffer short by one byte, + * should fail with ERANGE */ + str = getcwd(path, len); // pass + ok(str == NULL && errno == ERANGE, + "%s:%d getcwd(buf, %d): expected NULL got %s errno %s", + __FILE__, __LINE__, len, str, strerror(errno)); + errno = 0; + + /* try getcwd with a NULL buffer */ + str = getcwd(NULL, sizeof(path)); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + if (str != NULL) { + free(str); + } + + /* try getcwd with a NULL buffer but short size, + * should fail with ERANGE */ + str = getcwd(NULL, len); + ok(str == NULL && errno == ERANGE, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + if (str != NULL) { + free(str); + } + errno = 0; + + /* try getcwd with a buffer and 0 size, should fail with EINVAL */ + str = getcwd(path, 0); + ok(str == NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + if (str != NULL) { + free(str); + } + errno = 0; + + /* try getcwd with a NULL buffer and 0 size, + * getcwd should allocate buffer */ + str = getcwd(NULL, 0); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + if (str != NULL) { + free(str); + } + + /* change to unifyfs/dir1 */ + rc = chdir(buf); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, buf, strerror(errno)); + + /* check that we're in unifyfs/dir1 */ + str = getcwd(path, sizeof(path)); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, buf) == 0, + "%s:%d getcwd returned %s expected %s", + __FILE__, __LINE__, str, buf); + + /* change back to root unifyfs directory */ + rc = chdir(".."); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "..", strerror(errno)); + + /* check that we're in root unifyfs directory */ + str = getcwd(path, sizeof(path)); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, unifyfs_root) == 0, + "%s:%d getcwd returned %s expected %s", + __FILE__, __LINE__, str, unifyfs_root); + + /* change to unifyfs/dir1 directory using relative path */ + rc = chdir("dir1"); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "dir1", strerror(errno)); + + /* check that we're in unifyfs/dir1 directory */ + str = getcwd(path, sizeof(path)); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, buf) == 0, + "%s:%d getcwd returned %s expected %s", + __FILE__, __LINE__, str, buf); + + /* change to unifyfs/dir2 directory in strange way */ + rc = chdir("././.././dir2"); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "dir1", strerror(errno)); + + /* check that we're in unifyfs/dir2 directory */ + str = getcwd(path, sizeof(path)); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, buf2) == 0, + "%s:%d getcwd returned %s expected %s", + __FILE__, __LINE__, str, buf2); + + +#if 0 + /* change to root directory */ + rc = chdir("/"); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "/", strerror(errno)); + + /* check that we're in root directory */ + str = getwd(pathmax); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, "/") == 0, + "%s:%d getcwd returned %s expected %s", + __FILE__, __LINE__, str, "/"); + + /* change to unifyfs root directory */ + rc = chdir(unifyfs_root); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, unifyfs_root, strerror(errno)); + + /* check that we're in unifyfs root directory */ + str = getwd(pathmax); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, unifyfs_root) == 0, + "%s:%d getcwd returned %s expected %s", + __FILE__, __LINE__, str, unifyfs_root); + + /* change to directory within unifyfs */ + rc = chdir(buf); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, buf, strerror(errno)); + + /* check that we're in directory within unifyfs */ + str = getwd(pathmax); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, buf) == 0, + "%s:%d getcwd returned %s expected %s", + __FILE__, __LINE__, str, buf); + + /* change back to root unifyfs directory */ + rc = chdir(".."); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "..", strerror(errno)); + + /* check that we're in root unifyfs directory */ + str = getwd(pathmax); + ok(str != NULL, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, unifyfs_root) == 0, + "%s:%d getcwd returned %s expected %s", + __FILE__, __LINE__, str, unifyfs_root); +#endif + + /* change to root directory */ + rc = chdir("/"); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "/", strerror(errno)); + + /* check that we're in root directory */ + str = get_current_dir_name(); + ok(str != NULL, "%s:%d get_current_dir_name: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, "/") == 0, + "%s:%d get_current_dir_name returned %s expected %s", + __FILE__, __LINE__, str, "/"); + if (str != NULL) { + free(str); + } + + /* change to unifyfs root directory */ + rc = chdir(unifyfs_root); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, unifyfs_root, strerror(errno)); + + /* check that we're in unifyfs root directory */ + str = get_current_dir_name(); + ok(str != NULL, "%s:%d get_current_dir_name: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, unifyfs_root) == 0, + "%s:%d get_current_dir_name returned %s expected %s", + __FILE__, __LINE__, str, unifyfs_root); + if (str != NULL) { + free(str); + } + + /* change to unifyfs/dir1 */ + rc = chdir(buf); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, buf, strerror(errno)); + + /* check that we're in unifyfs/dir1 */ + str = get_current_dir_name(); + ok(str != NULL, "%s:%d get_current_dir_name: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, buf) == 0, + "%s:%d get_current_dir_name returned %s expected %s", + __FILE__, __LINE__, str, buf); + if (str != NULL) { + free(str); + } + + /* change back to root unifyfs directory */ + rc = chdir(".."); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "..", strerror(errno)); + + /* check that we're in root unifyfs directory */ + str = get_current_dir_name(); + ok(str != NULL, "%s:%d get_current_dir_name: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, unifyfs_root) == 0, + "%s:%d get_current_dir_name returned %s expected %s", + __FILE__, __LINE__, str, unifyfs_root); + if (str != NULL) { + free(str); + } + + /* change to unifyfs/dir2 directory */ + rc = chdir("dir2"); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "dir2", strerror(errno)); + + /* check that we're in unifyfs/dir2 */ + str = get_current_dir_name(); + ok(str != NULL, "%s:%d get_current_dir_name: %s", + __FILE__, __LINE__, strerror(errno)); + ok(str != NULL && strcmp(str, buf2) == 0, + "%s:%d get_current_dir_name returned %s expected %s", + __FILE__, __LINE__, str, buf2); + if (str != NULL) { + free(str); + } + + +#if 0 + /* change to root directory */ + rc = chdir("/"); + ok(rc == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "/", strerror(errno)); + + /* open a directory in unifyfs */ + DIR* dirp = opendir(buf); + ok(dirp != NULL, "%s:%d opendir(%s): %s", + __FILE__, __LINE__, buf, strerror(errno)); + + int fd = dirfd(dirp); + ok(fd >= 0, "%s:%d dirfd(%p): %s", + __FILE__, __LINE__, dirp, strerror(errno)); + + /* use fchdir to change into it */ + rc = fchdir(fd); + ok(rc == 0, "%s:%d fchdir(%d): %s", + __FILE__, __LINE__, fd, strerror(errno)); + + closedir(dirp); + + /* open root directory */ + dirp = opendir("/"); + ok(dirp != NULL, "%s:%d opendir(%s): %s", + __FILE__, __LINE__, "/", strerror(errno)); + + fd = dirfd(dirp); + ok(fd >= 0, "%s:%d dirfd(%p): %s", + __FILE__, __LINE__, dirp, strerror(errno)); + + /* use fchdir to change into it */ + rc = fchdir(fd); + ok(rc == 0, "%s:%d fchdir(%d): %s", + __FILE__, __LINE__, fd, strerror(errno)); + + closedir(dirp); +#endif + + return 0; +} diff --git a/t/sys/sysio_suite.c b/t/sys/sysio_suite.c index 235b9ccbb..dc00f65f3 100644 --- a/t/sys/sysio_suite.c +++ b/t/sys/sysio_suite.c @@ -100,6 +100,8 @@ int main(int argc, char* argv[]) unlink_test(unifyfs_root); + chdir_test(unifyfs_root); + MPI_Finalize(); done_testing(); diff --git a/t/sys/sysio_suite.h b/t/sys/sysio_suite.h index b69b82212..1556f222b 100644 --- a/t/sys/sysio_suite.h +++ b/t/sys/sysio_suite.h @@ -66,4 +66,6 @@ int truncate_trunc_before_sync(char* unifyfs_root); /* Test for UNIFYFS_WRAP(unlink) */ int unlink_test(char* unifyfs_root); +int chdir_test(char* unifyfs_root); + #endif /* SYSIO_SUITE_H */ From 9265efb6337542f399db728fe3be3d356bfbdab4 Mon Sep 17 00:00:00 2001 From: CamStan Date: Tue, 26 May 2020 13:06:06 -0700 Subject: [PATCH 124/168] Add OpenMPI to packages.yaml for spath dependency Spath needs mpi but is not set up to find mpi like UnifyFS. This adds the location of openmpi to the packages.yaml in travis so that spack doesn't try to build it can tell spath where to find it. --- .travis.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4f8a490ea..05fcc6470 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,6 @@ addons: - libtool-bin - m4 - openmpi-bin - - numactl before_install: # The default environment variable $CC is known to interfere with @@ -30,6 +29,8 @@ before_install: packages: all: target: [x86_64] + providers: + mpi: [openmpi] autoconf: buildable: False paths: @@ -37,7 +38,7 @@ before_install: automake: buildable: False paths: - automake@1.15: /usr + automake@1.15.1: /usr cmake: buildable: False paths: @@ -49,11 +50,11 @@ before_install: m4: buildable: False paths: - m4@4.17: /usr - numactl: + m4@1.4.18: /usr + openmpi: buildable: False paths: - numactl: /usr + openmpi@2.1.1: /usr EOF install: @@ -61,7 +62,7 @@ install: - spack install leveldb && spack load leveldb - spack install gotcha@1.0.3 && spack load gotcha@1.0.3 - spack install flatcc && spack load flatcc - - spack install margo^mercury+bmi~boostsys && spack load argobots && spack load mercury && spack load margo + - spack install margo^mercury+bmi~ofi~boostsys && spack load argobots && spack load mercury && spack load margo - spack install spath && spack load spath # prepare build environment - eval $(./scripts/git_log_test_env.sh) From dac083c678d95d2ae1a89eab8fdee2844896e724 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 2 Jun 2020 14:40:55 -0700 Subject: [PATCH 125/168] add libspath to LIBS, can drop SPATH-specific flags --- client/src/Makefile.am | 6 ------ examples/src/Makefile.am | 6 +++--- m4/spath.m4 | 2 -- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/client/src/Makefile.am b/client/src/Makefile.am index 06fd0b63e..0325df935 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -43,12 +43,6 @@ CLIENT_COMMON_LIBADD = \ $(FLATCC_LIBS) \ -lrt -lpthread -if HAVE_AM_SPATH -CLIENT_COMMON_CFLAGS += $(SPATH_CFLAGS) -CLIENT_COMMON_LDFLAGS += $(SPATH_LDFLAGS) -CLIENT_COMMON_LIBADD += $(SPATH_LIBS) -endif - CLIENT_COMMON_SOURCES = \ margo_client.c \ margo_client.h \ diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index c5ab56f4f..5161dae80 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -83,12 +83,12 @@ test_ftn_flags = $(AM_FCFLAGS) $(MPI_FFLAGS) \ -I$(top_srcdir)/client/src -I$(top_srcdir)/common/src test_ftn_ldadd = $(top_builddir)/client/src/libunifyfsf.la -lrt -lm $(FCLIBS) test_ftn_ldflags = $(AM_LDFLAGS) $(MPI_FLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) $(SPATH_LDFLAGS) $(SPATH_LIBS) + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) endif test_gotcha_ldadd = $(top_builddir)/client/src/libunifyfs_gotcha.la -lrt -lm test_gotcha_ldflags = $(AM_LDFLAGS) $(MPI_CLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) $(SPATH_LDFLAGS) $(SPATH_LIBS) + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) test_posix_cppflags = $(AM_CPPFLAGS) $(MPI_CFLAGS) -DDISABLE_UNIFYFS test_posix_ldadd = -lrt -lm @@ -96,7 +96,7 @@ test_posix_ldflags = $(AM_LDFLAGS) $(MPI_CLDFLAGS) test_static_ldadd = $(top_builddir)/client/src/libunifyfs.la -lrt -lm test_static_ldflags = -static $(CP_WRAPPERS) $(AM_LDFLAGS) $(MPI_CLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) $(SPATH_LDFLAGS) $(SPATH_LIBS) + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) # Per-target flags begin here diff --git a/m4/spath.m4 b/m4/spath.m4 index 30a83c528..183a4d54b 100644 --- a/m4/spath.m4 +++ b/m4/spath.m4 @@ -3,10 +3,8 @@ AC_DEFUN([UNIFYFS_AC_SPATH], [ [ LIBS="$LIBS -lspath" AC_DEFINE([HAVE_SPATH], [1], [Defined if you have spath]) - AM_CONDITIONAL([HAVE_AM_SPATH], [true]) ],[ AC_MSG_WARN([couldn't find a suitable libspath]) - AM_CONDITIONAL([HAVE_AM_SPATH], [false]) ], [] ) From 8f5de26efc277fbdebda5563fd469a6a45d4f415 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 2 Jun 2020 14:41:42 -0700 Subject: [PATCH 126/168] typos and suggestions from review --- client/src/unifyfs-sysio.c | 4 ++-- client/src/unifyfs.c | 2 -- t/sys/chdir.c | 11 ++++++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 917299c76..aa238c306 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -172,7 +172,7 @@ int UNIFYFS_WRAP(chdir)(const char* path) int ret = UNIFYFS_REAL(chdir)(path); /* if the change dir was successful, - * update our current working direcotry */ + * update our current working directory */ if (unifyfs_initialized && ret == 0) { if (unifyfs_cwd != NULL) { free(unifyfs_cwd); @@ -1815,7 +1815,7 @@ int UNIFYFS_WRAP(fchdir)(int fd) int ret = UNIFYFS_REAL(fchdir)(fd); /* if the change dir was successful, - * update our current working direcotry */ + * update our current working directory */ if (unifyfs_initialized && ret == 0) { if (unifyfs_cwd != NULL) { free(unifyfs_cwd); diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index bd9867b63..e9e52834d 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -2245,8 +2245,6 @@ static int unifyfs_init(void) } else { /* user did not specify a CWD, so initialize with the actual * current working dir */ - //MAP_OR_FAIL(getcwd); - //int cwd_rc = UNIFYFS_REAL(getcwd)(cwdpath, sizeof(cwdpath)); char* cwd = getcwd(NULL, 0); if (cwd != NULL) { unifyfs_cwd = cwd; diff --git a/t/sys/chdir.c b/t/sys/chdir.c index c1282e56f..dc73386fb 100644 --- a/t/sys/chdir.c +++ b/t/sys/chdir.c @@ -66,7 +66,7 @@ int chdir_test(char* unifyfs_root) str = getcwd(path, sizeof(path)); ok(str != NULL, "%s:%d getcwd: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, "/") == 0, + is(str, "/", "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, "/"); @@ -183,6 +183,9 @@ int chdir_test(char* unifyfs_root) __FILE__, __LINE__, str, buf2); +/* TODO: Some compilers throw a warning/error if one uses getwd(). + * For those compilers that allow it, it would be nice to execute + * these tests. For now, I'll leave this here as a reminder. */ #if 0 /* change to root directory */ rc = chdir("/"); @@ -193,7 +196,7 @@ int chdir_test(char* unifyfs_root) str = getwd(pathmax); ok(str != NULL, "%s:%d getcwd: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, "/") == 0, + is(str, "/", "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, "/"); @@ -246,7 +249,7 @@ int chdir_test(char* unifyfs_root) str = get_current_dir_name(); ok(str != NULL, "%s:%d get_current_dir_name: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, "/") == 0, + is(str, "/", "%s:%d get_current_dir_name returned %s expected %s", __FILE__, __LINE__, str, "/"); if (str != NULL) { @@ -318,6 +321,8 @@ int chdir_test(char* unifyfs_root) } +/* TODO: Our directory wrappers are not fully functioning yet, + * but when they do, we should check that fchdir works. */ #if 0 /* change to root directory */ rc = chdir("/"); From a23de5fa8f0a467d15bcbb8b7faa53b869a2af9c Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 2 Jun 2020 15:03:04 -0700 Subject: [PATCH 127/168] create common function for getcwd and __getcwd_chk --- client/src/unifyfs-sysio.c | 183 ++++++++++++++----------------------- 1 file changed, 71 insertions(+), 112 deletions(-) diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index aa238c306..b8ec89d98 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -203,6 +203,68 @@ int UNIFYFS_WRAP(chdir)(const char* path) } } +/* common logic for getcwd and __getcwd_chk */ +static char* _getcwd_impl(char* path, size_t size) +{ + /* man page if size=0 and path not NULL, return EINVAL */ + if (size == 0 && path != NULL) { + errno = EINVAL; + return NULL; + } + + /* get length of current working dir */ + size_t len = strlen(unifyfs_cwd) + 1; + + /* if user didn't provide a buffer, + * we attempt to allocate and return one for them */ + if (path == NULL) { + /* we'll allocate a buffer to return to the caller */ + char* buf = NULL; + + /* if path is NULL and size is positive, we must + * allocate a buffer of length size and copy into it */ + if (size > 0) { + /* check that size is big enough for the string */ + if (len <= size) { + /* path will fit, allocate buffer and copy */ + buf = (char*) malloc(size); + if (buf != NULL) { + strncpy(buf, unifyfs_cwd, size); + } else { + errno = ENOMEM; + } + return buf; + } else { + /* user's buffer limit is too small */ + errno = ERANGE; + return NULL; + } + } + + /* otherwise size == 0, so allocate a buffer + * that is big enough */ + buf = (char*) malloc(len); + if (buf != NULL) { + strncpy(buf, unifyfs_cwd, len); + } else { + errno = ENOMEM; + } + return buf; + } + + /* to get here, caller provided an actual buffer, + * check that path fits in the caller's buffer */ + if (len <= size) { + /* current working dir fits, copy and return */ + strncpy(path, unifyfs_cwd, size); + return path; + } else { + /* user's buffer is too small */ + errno = ERANGE; + return NULL; + } +} + char* UNIFYFS_WRAP(__getcwd_chk)(char* path, size_t size, size_t buflen) { /* if we're initialized, we're tracking the current working dir */ @@ -220,63 +282,15 @@ char* UNIFYFS_WRAP(__getcwd_chk)(char* path, size_t size, size_t buflen) * changed dir without us noticing, so there is a bug here) */ char upath[UNIFYFS_MAX_FILENAME]; if (unifyfs_intercept_path(unifyfs_cwd, upath)) { - /* man page if size=0 and path not NULL, return EINVAL */ - if (size == 0 && path != NULL) { - errno = EINVAL; - return NULL; - } - - /* get length of current working dir */ - size_t len = strlen(unifyfs_cwd) + 1; - - /* if user didn't provide a buffer, - * we attempt to allocate and return one for them */ - if (path == NULL) { - /* we'll allocate a buffer to return to the caller */ - char* buf = NULL; - - /* if path is NULL and size is positive, we must - * allocate a buffer of length size and copy into it */ - if (size > 0) { - /* check that size is big enough for the string */ - if (len <= size) { - /* path will fit, allocate buffer and copy */ - buf = (char*) malloc(size); - if (buf != NULL) { - strncpy(buf, unifyfs_cwd, size); - } else { - errno = ENOMEM; - } - return buf; - } else { - /* user's buffer limit is too small */ - errno = ERANGE; - return NULL; - } - } - - /* otherwise size == 0, so allocate a buffer - * that is big enough */ - buf = (char*) malloc(len); - if (buf != NULL) { - strncpy(buf, unifyfs_cwd, len); - } else { - errno = ENOMEM; - } - return buf; +#if 0 + /* TODO: what to do here? */ + if (size > buflen) { + __chk_fail(); } +#endif - /* to get here, caller provided an actual buffer, - * check that path fits in the caller's buffer */ - if (len <= size) { - /* current working dir fits, copy and return */ - strncpy(path, unifyfs_cwd, size); - return path; - } else { - /* user's buffer is too small */ - errno = ERANGE; - return NULL; - } + /* delegate the rest to our common getcwd function */ + return _getcwd_impl(path, size); } else { /* current working dir is in real file system, * fall through to real getcwd call */ @@ -318,63 +332,8 @@ char* UNIFYFS_WRAP(getcwd)(char* path, size_t size) * changed dir without us noticing, so there is a bug here) */ char upath[UNIFYFS_MAX_FILENAME]; if (unifyfs_intercept_path(unifyfs_cwd, upath)) { - /* man page if size=0 and path not NULL, return EINVAL */ - if (size == 0 && path != NULL) { - errno = EINVAL; - return NULL; - } - - /* get length of current working dir */ - size_t len = strlen(unifyfs_cwd) + 1; - - /* if user didn't provide a buffer, - * we attempt to allocate and return one for them */ - if (path == NULL) { - /* we'll allocate a buffer to return to the caller */ - char* buf = NULL; - - /* if path is NULL and size is positive, we must - * allocate a buffer of length size and copy into it */ - if (size > 0) { - /* check that size is big enough for the string */ - if (len <= size) { - /* path will fit, allocate buffer and copy */ - buf = (char*) malloc(size); - if (buf != NULL) { - strncpy(buf, unifyfs_cwd, size); - } else { - errno = ENOMEM; - } - return buf; - } else { - /* user's buffer limit is too small */ - errno = ERANGE; - return NULL; - } - } - - /* otherwise size == 0, so allocate a buffer - * that is big enough */ - buf = (char*) malloc(len); - if (buf != NULL) { - strncpy(buf, unifyfs_cwd, len); - } else { - errno = ENOMEM; - } - return buf; - } - - /* to get here, caller provided an actual buffer, - * check that path fits in the caller's buffer */ - if (len <= size) { - /* current working dir fits, copy and return */ - strncpy(path, unifyfs_cwd, size); - return path; - } else { - /* user's buffer is too small */ - errno = ERANGE; - return NULL; - } + /* delegate the rest to our common getcwd function */ + return _getcwd_impl(path, size); } else { /* current working dir is in real file system, * fall through to real getcwd call */ From f9a0aa9c76c96b7716eee62da7acdcf28ba836bd Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 2 Jun 2020 15:06:54 -0700 Subject: [PATCH 128/168] switch from ok to is --- t/sys/chdir.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/t/sys/chdir.c b/t/sys/chdir.c index dc73386fb..d7685e890 100644 --- a/t/sys/chdir.c +++ b/t/sys/chdir.c @@ -79,7 +79,7 @@ int chdir_test(char* unifyfs_root) str = getcwd(path, sizeof(path)); ok(str != NULL, "%s:%d getcwd: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, unifyfs_root) == 0, + is(str, unifyfs_root, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); @@ -139,7 +139,7 @@ int chdir_test(char* unifyfs_root) str = getcwd(path, sizeof(path)); ok(str != NULL, "%s:%d getcwd: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, buf) == 0, + is(str, buf, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, buf); @@ -152,7 +152,7 @@ int chdir_test(char* unifyfs_root) str = getcwd(path, sizeof(path)); ok(str != NULL, "%s:%d getcwd: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, unifyfs_root) == 0, + is(str, unifyfs_root, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); @@ -165,7 +165,7 @@ int chdir_test(char* unifyfs_root) str = getcwd(path, sizeof(path)); ok(str != NULL, "%s:%d getcwd: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, buf) == 0, + is(str, buf, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, buf); @@ -178,7 +178,7 @@ int chdir_test(char* unifyfs_root) str = getcwd(path, sizeof(path)); ok(str != NULL, "%s:%d getcwd: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, buf2) == 0, + is(str, buf2, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, buf2); @@ -209,7 +209,7 @@ int chdir_test(char* unifyfs_root) str = getwd(pathmax); ok(str != NULL, "%s:%d getcwd: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, unifyfs_root) == 0, + is(str, unifyfs_root, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); @@ -222,7 +222,7 @@ int chdir_test(char* unifyfs_root) str = getwd(pathmax); ok(str != NULL, "%s:%d getcwd: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, buf) == 0, + is(str, buf, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, buf); @@ -235,7 +235,7 @@ int chdir_test(char* unifyfs_root) str = getwd(pathmax); ok(str != NULL, "%s:%d getcwd: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, unifyfs_root) == 0, + is(str, unifyfs_root, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); #endif @@ -265,7 +265,7 @@ int chdir_test(char* unifyfs_root) str = get_current_dir_name(); ok(str != NULL, "%s:%d get_current_dir_name: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, unifyfs_root) == 0, + is(str, unifyfs_root, "%s:%d get_current_dir_name returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); if (str != NULL) { @@ -281,7 +281,7 @@ int chdir_test(char* unifyfs_root) str = get_current_dir_name(); ok(str != NULL, "%s:%d get_current_dir_name: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, buf) == 0, + is(str, buf, "%s:%d get_current_dir_name returned %s expected %s", __FILE__, __LINE__, str, buf); if (str != NULL) { @@ -297,7 +297,7 @@ int chdir_test(char* unifyfs_root) str = get_current_dir_name(); ok(str != NULL, "%s:%d get_current_dir_name: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, unifyfs_root) == 0, + is(str, unifyfs_root, "%s:%d get_current_dir_name returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); if (str != NULL) { @@ -313,7 +313,7 @@ int chdir_test(char* unifyfs_root) str = get_current_dir_name(); ok(str != NULL, "%s:%d get_current_dir_name: %s", __FILE__, __LINE__, strerror(errno)); - ok(str != NULL && strcmp(str, buf2) == 0, + is(str, buf2, "%s:%d get_current_dir_name returned %s expected %s", __FILE__, __LINE__, str, buf2); if (str != NULL) { From 0dc1e543861768aade9c2d01732f3d69d417f90a Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Thu, 4 Jun 2020 10:16:23 -0400 Subject: [PATCH 129/168] fix client assignment Occasionally, when running many clients per server, two (or more) clients could mount at the exact same time, leading to a duplicate client id assignment. This results in the duplicate clients both trying to use the same logio and shmem contexts, which leads to various problems, including read hangs. This PR fixes the problem by synchronizing updates to application state using an Argobots mutex. An early attempt to use a pthread mutex was not successful, because the Argobots user-level threads (ULTs) execute on the same system thread. Also included is a testutil fix to make sure unifyfs_mount() failures in the example programs are properly handled. --- client/src/margo_client.c | 1 + client/src/unifyfs-fixed.c | 7 +- common/src/seg_tree.c | 11 ++- examples/src/testutil.h | 1 + server/src/unifyfs_cmd_handler.c | 30 +++----- server/src/unifyfs_global.h | 10 +-- server/src/unifyfs_server.c | 122 ++++++++++++++++++++++--------- 7 files changed, 116 insertions(+), 66 deletions(-) diff --git a/client/src/margo_client.c b/client/src/margo_client.c index ec3d9f5ec..faefed5ee 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -290,6 +290,7 @@ int invoke_client_mount_rpc(void) LOGWARN("mismatch on app_id - using %d, server returned %d", unifyfs_app_id, srvr_app_id); } + LOGDBG("My client id is %d", unifyfs_client_id); /* free resources */ margo_free_output(handle, &out); diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 2a1eb0cbe..c17df0e4a 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -101,7 +101,7 @@ static int unifyfs_coalesce_index( /* determine number of bytes in next index that lie in current slice, * assume all bytes in the index will fit */ - long length = next_idx->length; + size_t length = next_idx->length; if (next_end > slice_end) { /* current index writes beyond end of slice, * so we can only coalesce bytes that fall within slice */ @@ -400,8 +400,9 @@ int unifyfs_fid_logio_write(int fid, LOGWARN("partial logio_write() @ offset=%zu (%zu of %zu bytes)", (size_t)log_off, *nwritten, count); } else { - LOGDBG("successful logio_write() @ log offset=%zu (%zu bytes)", - (size_t)log_off, count); + LOGDBG("fid=%d pos=%zu - successful logio_write() " + "@ log offset=%zu (%zu bytes)", + fid, (size_t)pos, (size_t)log_off, count); } /* update our write metadata for this write */ diff --git a/common/src/seg_tree.c b/common/src/seg_tree.c index 88d885542..2907c0a6c 100644 --- a/common/src/seg_tree.c +++ b/common/src/seg_tree.c @@ -27,12 +27,14 @@ * segments in the tree are non-overlapping. Added segments overwrite the old * segments in the tree. This is used to coalesce writes before an fsync. */ + #include #include #include #include #include #include + #include "seg_tree.h" #include "tree.h" @@ -277,7 +279,7 @@ int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, /* Check whether we can coalesce new extent with any preceding extent. */ prev = RB_PREV(inttree, &seg_tree->head, target); - if (prev != NULL && prev->end + 1 == target->start) { + if ((prev != NULL) && ((prev->end + 1) == target->start)) { /* * We found a extent that ends just before the new extent starts. * Check whether they are also contiguous in the log. @@ -306,7 +308,7 @@ int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, /* Check whether we can coalesce new extent with any trailing extent. */ next = RB_NEXT(inttree, &seg_tree->head, target); - if (next != NULL && target->end + 1 == next->start) { + if ((next != NULL) && ((target->end + 1) == next->start)) { /* * We found a extent that starts just after the new extent ends. * Check whether they are also contiguous in the log. @@ -415,6 +417,7 @@ struct seg_tree_node* seg_tree_iter(struct seg_tree* seg_tree, struct seg_tree_node* start) { struct seg_tree_node* next = NULL; + struct seg_tree_node* tmp = NULL; if (start == NULL) { /* Initial case, no starting node */ next = RB_MIN(inttree, &seg_tree->head); @@ -425,8 +428,8 @@ seg_tree_iter(struct seg_tree* seg_tree, struct seg_tree_node* start) * We were given a valid start node. Look it up to start our traversal * from there. */ - next = RB_FIND(inttree, &seg_tree->head, start); - if (!next) { + tmp = RB_FIND(inttree, &seg_tree->head, start); + if (!tmp) { /* Some kind of error */ return NULL; } diff --git a/examples/src/testutil.h b/examples/src/testutil.h index 80677cc18..48875f337 100644 --- a/examples/src/testutil.h +++ b/examples/src/testutil.h @@ -1157,6 +1157,7 @@ int test_init(int argc, char** argv, if (rc) { test_print(cfg, "ERROR: unifyfs_mount() failed (rc=%d)", rc); test_abort(cfg, rc); + return -1; } #endif test_barrier(cfg); diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index 22636d3cc..cd08893dd 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -58,7 +58,6 @@ * client */ static void unifyfs_mount_rpc(hg_handle_t handle) { - int rc; int ret = (int)UNIFYFS_SUCCESS; /* get input params */ @@ -73,34 +72,28 @@ static void unifyfs_mount_rpc(hg_handle_t handle) /* lookup app_config for given app_id */ app_config* app_cfg = get_application(app_id); if (app_cfg == NULL) { - /* don't have an app_config for this app_id, - * so allocate and fill a new one */ - app_cfg = (app_config*) calloc(1, sizeof(app_config)); - app_cfg->app_id = app_id; - /* insert new app_config into our app_configs array */ - LOGDBG("creating new application"); - rc = new_application(app_cfg); - if (rc != UNIFYFS_SUCCESS) { - ret = rc; - free(app_cfg); - app_cfg = NULL; + LOGDBG("creating new application for app_id=%d", app_id); + app_cfg = new_application(app_id); + if (NULL == app_cfg) { + ret = UNIFYFS_FAILURE; } } else { LOGDBG("using existing app_config for app_id=%d", app_id); } if (NULL != app_cfg) { - LOGDBG("creating new client"); - app_client* client = create_app_client(app_cfg, - in.client_addr_str, - in.dbg_rank); + LOGDBG("creating new app client for %s", in.client_addr_str); + app_client* client = new_app_client(app_cfg, + in.client_addr_str, + in.dbg_rank); if (NULL == client) { - LOGERR("create_app_client() failed for app_id=%d dbg_rank=%d", + LOGERR("failed to create new client for app_id=%d dbg_rank=%d", app_id, (int)in.dbg_rank); ret = (int)UNIFYFS_FAILURE; } else { client_id = client->client_id; + LOGDBG("created new application client %d:%d", app_id, client_id); } } @@ -139,8 +132,7 @@ static void unifyfs_attach_rpc(hg_handle_t handle) /* lookup client structure and attach it */ app_client* client = get_app_client(app_id, client_id); if (NULL != client) { - LOGDBG("attaching client (app_id=%d, client_id=%d)", - app_id, client_id); + LOGDBG("attaching client %d:%d", app_id, client_id); ret = attach_app_client(client, in.logio_spill_dir, in.logio_spill_size, diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index db5157336..897b3e279 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -138,7 +138,7 @@ struct reqmgr_thrd; * logio and shared memory contexts, margo rpc address, etc. */ typedef struct app_client { - int app_id; /* index of associated app in app_configs */ + int app_id; /* index of app in server app_configs array */ int client_id; /* this client's index in app's clients array */ int dbg_rank; /* client debug rank - NOT CURRENTLY USED */ int connected; /* is client currently connected? */ @@ -172,16 +172,16 @@ typedef struct app_config { app_client* clients[MAX_APP_CLIENTS]; } app_config; -extern app_config* app_configs[MAX_NUM_APPS]; /* list of apps */ - app_config* get_application(int app_id); -unifyfs_rc new_application(app_config* new_app); +app_config* new_application(int app_id); + +unifyfs_rc cleanup_application(app_config* app); app_client* get_app_client(int app_id, int client_id); -app_client* create_app_client(app_config* app, +app_client* new_app_client(app_config* app, const char* margo_addr_str, const int dbg_rank); diff --git a/server/src/unifyfs_server.c b/server/src/unifyfs_server.c index 86813c073..51776972c 100644 --- a/server/src/unifyfs_server.c +++ b/server/src/unifyfs_server.c @@ -58,7 +58,8 @@ server_info_t* glb_servers; // array of server_info_t unifyfs_cfg_t server_cfg; -app_config* app_configs[MAX_NUM_APPS]; /* list of apps */ +static ABT_mutex app_configs_abt_sync; +static app_config* app_configs[MAX_NUM_APPS]; /* list of apps */ /** * @brief create a ready status file to notify that all servers are ready for @@ -351,6 +352,8 @@ int main(int argc, char* argv[]) } LOGDBG("initializing rpc service"); + ABT_init(argc, argv); + ABT_mutex_create(&app_configs_abt_sync); rc = configurator_bool_val(server_cfg.margo_tcp, &margo_use_tcp); rc = margo_server_rpc_init(); if (rc != UNIFYFS_SUCCESS) { @@ -571,26 +574,19 @@ static int unifyfs_exit(void) int ret = UNIFYFS_SUCCESS; /* iterate over each active application and free resources */ + ABT_mutex_lock(app_configs_abt_sync); for (int i = 0; i < MAX_NUM_APPS; i++) { /* get pointer to app config for this app_id */ app_config* app = app_configs[i]; - if (NULL == app) { - /* skip to next app_id if this slot is empty */ - continue; - } - - /* free resources allocated for each client */ - int app_id = app->app_id; - for (int j = 1; j <= MAX_APP_CLIENTS; j++) { - app_client* client = get_app_client(app_id, j); - if (NULL != client) { - int rc = cleanup_app_client(client); - if (rc != UNIFYFS_SUCCESS) { - ret = rc; - } + if (NULL != app) { + app_configs[i] = NULL; + unifyfs_rc rc = cleanup_application(app); + if (rc != UNIFYFS_SUCCESS) { + ret = rc; } } } + ABT_mutex_unlock(app_configs_abt_sync); /* TODO: notify the service threads to exit */ @@ -621,41 +617,84 @@ static int unifyfs_exit(void) /* get pointer to app config for this app_id */ app_config* get_application(int app_id) { + ABT_mutex_lock(app_configs_abt_sync); for (int i = 0; i < MAX_NUM_APPS; i++) { app_config* app_cfg = app_configs[i]; if ((NULL != app_cfg) && (app_cfg->app_id == app_id)) { + ABT_mutex_unlock(app_configs_abt_sync); return app_cfg; } } + ABT_mutex_unlock(app_configs_abt_sync); return NULL; } /* insert a new app config in app_configs[] */ -unifyfs_rc new_application(app_config* new_app) +app_config* new_application(int app_id) { + ABT_mutex_lock(app_configs_abt_sync); + + /* don't have an app_config for this app_id, + * so allocate and fill a new one */ + app_config* new_app = (app_config*) calloc(1, sizeof(app_config)); if (NULL == new_app) { - LOGERR("NULL app_config pointer"); - return EINVAL; + LOGERR("failed to allocate application structure") + ABT_mutex_unlock(app_configs_abt_sync); + return NULL; } - /* check for existing app structure with given app_id */ - int app_id = new_app->app_id; - app_config* existing = get_application(app_id); - if (NULL != existing) { - LOGERR("application with app_id %d already exists", app_id); - return EINVAL; - } + new_app->app_id = app_id; /* insert the given app_config in an empty slot */ for (int i = 0; i < MAX_NUM_APPS; i++) { - existing = app_configs[i]; + app_config* existing = app_configs[i]; if (NULL == existing) { app_configs[i] = new_app; - return UNIFYFS_SUCCESS; + ABT_mutex_unlock(app_configs_abt_sync); + return new_app; + } else if (existing->app_id == app_id) { + /* someone beat us to it, use existing */ + LOGDBG("found existing application for id=%d", app_id); + ABT_mutex_unlock(app_configs_abt_sync); + free(new_app); + return existing; } } + + ABT_mutex_unlock(app_configs_abt_sync); + + /* no empty slots found */ LOGERR("insert into app_configs[] failed"); - return UNIFYFS_FAILURE; + free(new_app); + return NULL; +} + +/* free application state */ +unifyfs_rc cleanup_application(app_config* app) +{ + unifyfs_rc ret = UNIFYFS_SUCCESS; + + if (NULL == app) { + return EINVAL; + } + + int app_id = app->app_id; + LOGDBG("cleaning application %d", app_id); + + /* free resources allocated for each client */ + for (int j = 1; j <= MAX_APP_CLIENTS; j++) { + app_client* client = get_app_client(app_id, j); + if (NULL != client) { + unifyfs_rc rc = cleanup_app_client(client); + if (rc != UNIFYFS_SUCCESS) { + ret = rc; + } + } + } + + free(app); + + return ret; } app_client* get_app_client(int app_id, @@ -728,9 +767,9 @@ static unifyfs_rc attach_to_client_shmem(app_client* client, * Sets up logio and shmem region contexts, request manager thread, * margo rpc address, etc. */ -app_client* create_app_client(app_config* app, - const char* margo_addr_str, - const int debug_rank) +app_client* new_app_client(app_config* app, + const char* margo_addr_str, + const int debug_rank) { if ((NULL == app) || (NULL == margo_addr_str)) { return NULL; @@ -741,6 +780,8 @@ app_client* create_app_client(app_config* app, return NULL; } + ABT_mutex_lock(app_configs_abt_sync); + int app_id = app->app_id; int client_id = app->num_clients + 1; /* next client id */ int client_ndx = client_id - 1; /* clients array index is (id - 1) */ @@ -768,14 +809,20 @@ app_client* create_app_client(app_config* app, if (failure) { LOGERR("failed to initialize application client"); + ABT_mutex_unlock(app_configs_abt_sync); cleanup_app_client(client); - client = NULL; - } else { - app->num_clients++; - app->clients[client_ndx] = client; + return NULL; } + + /* update app state */ + app->num_clients++; + app->clients[client_ndx] = client; + } else { + LOGERR("failed to allocate client structure"); } + ABT_mutex_unlock(app_configs_abt_sync); + return client; } @@ -885,6 +932,9 @@ unifyfs_rc cleanup_app_client(app_client* client) return EINVAL; } + LOGDBG("cleaning application client %d:%d", + client->app_id, client->client_id); + disconnect_app_client(client); /* close client logio context */ @@ -897,9 +947,11 @@ unifyfs_rc cleanup_app_client(app_client* client) app_config* app = get_application(client->app_id); if (NULL != app) { int client_ndx = client->client_id - 1; /* client ids start at 1 */ + ABT_mutex_lock(app_configs_abt_sync); if (client == app->clients[client_ndx]) { app->clients[client_ndx] = NULL; } + ABT_mutex_unlock(app_configs_abt_sync); } /* free client structure */ From acc2002e40e9c1e65e625b5e51c15974b00813fe Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Wed, 10 Jun 2020 17:46:07 -0400 Subject: [PATCH 130/168] Enhancing the documentation: assumptions section. Specifically: - discussing the unifyfs data staging functionalities - fix the code example of lamination: remove unnecessary fsync() - clarify some sentences --- docs/assumptions.rst | 137 +++++++++++++++++++++++++------------------ docs/start-stop.rst | 1 + 2 files changed, 80 insertions(+), 58 deletions(-) diff --git a/docs/assumptions.rst b/docs/assumptions.rst index 0c099c2dc..c2af68b97 100644 --- a/docs/assumptions.rst +++ b/docs/assumptions.rst @@ -5,63 +5,84 @@ Assumptions In this section, we provide assumptions we make about the behavior of applications that use UnifyFS, and about how UnifyFS currently functions. +--------------------------- +System Requirements +--------------------------- + +UnifyFS uses node-local storage devices, e.g., RAM and SSD, for storing the +application data. Therefore, a system should support the following requirements +to run UnifyFS. + + - A compute node is equipped with a local storage device that UnifyFS can + use for storing file data, e.g., SSD or RAM. + + - An ability for UnifyFS to launch user-level daemon processes on compute + nodes, which run concurrently with user application processes + --------------------------- Application Behavior --------------------------- - - Workload supported is globally synchronous checkpointing. - - I/O occurs in write and read phases. Files are not read and written at - the same time. There is some (good) amount of time between the two phases. - For example, files are written during checkpoint phases and only read - during recovery or restart. +UnifyFS is specifically designed to support globally synchronous checkpointing +workloads. In such a workload, the expected application behavior is as follows. - - Processes on any node can read any byte in the file (not just local - data), but the common case will be processes read only their local bytes. + - I/O operations occur in separate write and read phases, and thus files are + not read and written simultaneously. For instance, files are only written + during the checkpointing (a write phase) and only read during the + recovery/restart (a read phase). - - Assume general parallel I/O concurrency semantics where processes can - write to the same offset concurrently. We assume the outcome of concurrent - writes to the same offset or other conflicting concurrent accesses is - undefined. For example, if a command in the job renames a file while the - parallel application is writing to it, the outcome is undefined. It could - be a failure or not, depending on timing. + - During the read phase, a process can read any byte in a file including + remote data that has been written by processes in remote compute nodes. + However, reading the local data (which has been written by processes in + the same compute node) will be faster than reading the remote data. + + - During the write phase, the result of concurrently writing to the same + file offset by multiple processes is undefined. Similarly, multiple + processes writing to an overlapped region also leads to an undefined + result. For example, if a command in the job renames a file while the + parallel application is writing to it, the outcome is undefined, i.e., it + could be a success or failure depending on timing. --------------------------- Consistency Model --------------------------- + One key aspect of UnifyFS is the idea of "laminating" a file. After a file is -laminated, it becomes "set in stone" and its data is accessible across all the -nodes. Laminated files are permanently read-only, and cannot be modified in -any way (but can be deleted). If the application process group fails before a -file has been laminated, UnifyFS may delete the file. +laminated, it becomes "set in stone," and its data is accessible across all the +nodes. Laminated files are permanently read-only and cannot be further modified, +except for being renamed or deleted. If the application process group fails +before a file has been laminated, UnifyFS may delete the file. -The typical use case is to laminate your checkpoint files after they've been -written. To laminate a file, first call fsync() to sync all your writes to the -server, then call chmod() to remove all the write bits. Removing the write -bits does the actual lamination. A typical checkpoint will look like this: +A typical use case is to laminate application checkpoint files after they have +been successfully written. To laminate a file, an application can simply call +chmod() to remove all the write bits, after its write phase is completed. When +write bits of a file are all canceled, UnifyFS will internally laminate the +file. A typical checkpoint will look like: .. code-block:: C - fp = fopen("checkpoint1.chk") - write(fp, ) - fsync(fp) - fclose(fp) + fd = open("checkpoint1.chk", O_WRONLY) + write(fd, , ) + close(fd) chmod("checkpoint1.chk", 0444) Future versions of UnifyFS may support different laminate semantics, such as -laminate on close(), or laminate via an explicit API call. +laminate on close() or laminate via an explicit API call. + +We define the laminated consistency model to enable certain optimizations while +supporting the perceived requirements of application checkpoints. Since remote +processes are not permitted to read arbitrary bytes of a file until its +lamination, UnifyFS can buffer all data and metadata of the file locally +(instead of exchanging indexing information between compute nodes) before the +lamination occurs. Also, since file contents cannot change after lamination, +aggressive caching may be used during the read phase with minimal locking. +Further, since a file may be lost on application failure unless laminated, data +redundancy schemes can be delayed until lamination. -We define the laminated consistency model to enable certain optimizations -while supporting the perceived requirements of application checkpoints. -Since remote processes are not permitted to read arbitrary bytes within the -file until lamination, -global exchange of file data and/or data index information can be buffered -locally on each node until the point of lamination. -Since file contents cannot change after lamination, -aggressive caching may be used during the read-only phase with minimal locking. -Since a file may be lost on application failure unless laminated, -data redundancy schemes can be delayed until lamination. +The following lists summarize available application I/O operations according to +our consistency model. -Behavior before lamination: +Behavior before lamination (write phase): - open/close: A process may open/close a file multiple times. @@ -77,7 +98,7 @@ Behavior before lamination: - unlink: A process may delete a file. -Behavior after lamination: +Behavior after lamination (read phase): - open/close: A process may open/close a file multiple times. @@ -95,32 +116,32 @@ Behavior after lamination: File System Behavior --------------------------- - - The file system exists on node local storage only and is not persisted to - stable storage like a parallel file system (PFS). Can be coupled with +The additional behavior of UnifyFS can be summarized as follows. - - SymphonyFS or high level I/O or checkpoint library (VeloC) to move data to - PFS periodically, or data can be moved manually + - UnifyFS exists on node local storage only and is not automatically + persisted to stable storage like a parallel file system (PFS). When the + data needs to be persisted to an external file system, users can use + :ref:`unifyfs utility ` with its data staging + options. - - Can be used with checkpointing libraries (VeloC) or I/O libraries to - support shared files on burst buffers + - UnifyFS also can be coupled with SymphonyFS_, high level I/O libraries, or + a checkpoint library (VeloC_) to move data to PFS periodically. - - File system starts empty at job start. User job must populate the file - system. + - UnifyFS can be used with checkpointing libraries (VeloC_) or other I/O + libraries to support shared files on burst buffers. - - Shared file system namespace across all compute nodes in a job, even if - an application process is not running on all compute nodes + - UnifyFS starts empty at job start. User job must populate the file system + manually or by using + :ref:`unifyfs utility `. - - Survives application termination and/or relaunch within a job + - UnifyFS creates a shared file system namespace across all compute nodes in + a job, even if an application process is not running on all compute nodes. - - Will transparently intercept system level I/O calls of applications and - I/O libraries + - UnifyFS survives across multiple application runs within a job. ---------------------------- -System Characteristics ---------------------------- + - UnifyFS will transparently intercept system level I/O calls of + applications and I/O libraries. - - There is some storage available for storing file data on a compute node, - e.g. SSD or RAM disk +.. _SymphonyFS: https://code.ornl.gov/techint/SymphonyFS +.. _VeloC: https://github.com/ECP-VeloC/VELOC - - We can run user-level daemon processes on compute nodes concurrently with - a user application diff --git a/docs/start-stop.rst b/docs/start-stop.rst index 8792d54b2..98cb9c32b 100644 --- a/docs/start-stop.rst +++ b/docs/start-stop.rst @@ -50,6 +50,7 @@ for further details on customizing the UnifyFS runtime configuration. unifyfs start --share-dir=/path/to/shared/file/system +.. _unifyfs_utility_label: ``unifyfs`` provides command-line options to choose the client mountpoint, adjust the consistency model, and control stage-in and stage-out of files. From c81207f647ad716fee8d6a7a55555bddf8a2868e Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Wed, 17 Jun 2020 15:40:49 -0400 Subject: [PATCH 131/168] fix server application cleanup deadlock --- server/src/unifyfs_global.h | 6 +++--- server/src/unifyfs_request_manager.c | 3 +-- server/src/unifyfs_server.c | 32 +++++++++++++++------------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index 897b3e279..28e911917 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -182,8 +182,8 @@ app_client* get_app_client(int app_id, int client_id); app_client* new_app_client(app_config* app, - const char* margo_addr_str, - const int dbg_rank); + const char* margo_addr_str, + const int dbg_rank); unifyfs_rc attach_app_client(app_client* client, const char* logio_spill_dir, @@ -196,6 +196,6 @@ unifyfs_rc attach_app_client(app_client* client, unifyfs_rc disconnect_app_client(app_client* clnt); -unifyfs_rc cleanup_app_client(app_client* clnt); +unifyfs_rc cleanup_app_client(app_config* app, app_client* clnt); #endif // UNIFYFS_GLOBAL_H diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 7315bd70f..f5aed3f11 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -1907,9 +1907,8 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, (NULL != del_reads->resp)); /* look up client shared memory region */ - app_config* app_cfg = get_application(rdreq->app_id); app_client* clnt = get_app_client(rdreq->app_id, rdreq->client_id); - if ((NULL == app_cfg) || (NULL == clnt)) { + if (NULL == clnt) { return (int)UNIFYFS_FAILURE; } client_shm = clnt->shmem_data; diff --git a/server/src/unifyfs_server.c b/server/src/unifyfs_server.c index 51776972c..166e8f630 100644 --- a/server/src/unifyfs_server.c +++ b/server/src/unifyfs_server.c @@ -669,7 +669,11 @@ app_config* new_application(int app_id) return NULL; } -/* free application state */ +/* free application state + * + * NOTE: the application state mutex (app_configs_abt_sync) should be locked + * before calling this function + */ unifyfs_rc cleanup_application(app_config* app) { unifyfs_rc ret = UNIFYFS_SUCCESS; @@ -682,10 +686,10 @@ unifyfs_rc cleanup_application(app_config* app) LOGDBG("cleaning application %d", app_id); /* free resources allocated for each client */ - for (int j = 1; j <= MAX_APP_CLIENTS; j++) { - app_client* client = get_app_client(app_id, j); + for (int j = 0; j < MAX_APP_CLIENTS; j++) { + app_client* client = app->clients[j]; if (NULL != client) { - unifyfs_rc rc = cleanup_app_client(client); + unifyfs_rc rc = cleanup_app_client(app, client); if (rc != UNIFYFS_SUCCESS) { ret = rc; } @@ -809,8 +813,8 @@ app_client* new_app_client(app_config* app, if (failure) { LOGERR("failed to initialize application client"); + cleanup_app_client(app, client); ABT_mutex_unlock(app_configs_abt_sync); - cleanup_app_client(client); return NULL; } @@ -925,10 +929,13 @@ unifyfs_rc disconnect_app_client(app_client* client) * * This function may be called due to a failed initialization, so we can't * assume any particular state is valid, other than app_id and client_id. + * + * NOTE: the application state mutex (app_configs_abt_sync) should be locked + * before calling this function */ -unifyfs_rc cleanup_app_client(app_client* client) +unifyfs_rc cleanup_app_client(app_config* app, app_client* client) { - if (NULL == client) { + if ((NULL == app) || (NULL == client)) { return EINVAL; } @@ -944,14 +951,9 @@ unifyfs_rc cleanup_app_client(app_client* client) } /* reset app->clients array index if set */ - app_config* app = get_application(client->app_id); - if (NULL != app) { - int client_ndx = client->client_id - 1; /* client ids start at 1 */ - ABT_mutex_lock(app_configs_abt_sync); - if (client == app->clients[client_ndx]) { - app->clients[client_ndx] = NULL; - } - ABT_mutex_unlock(app_configs_abt_sync); + int client_ndx = client->client_id - 1; /* client ids start at 1 */ + if (client == app->clients[client_ndx]) { + app->clients[client_ndx] = NULL; } /* free client structure */ From 1516f3f1acca150dab96a0ed03bd71c468bdc958 Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Wed, 17 Jun 2020 13:49:20 -0400 Subject: [PATCH 132/168] logio fixes for empty shmem or spill --- client/src/margo_client.c | 8 +++++--- client/src/unifyfs.c | 16 +++++++++++++--- server/src/unifyfs_server.c | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/client/src/margo_client.c b/client/src/margo_client.c index faefed5ee..c47615c01 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -229,9 +229,6 @@ int invoke_client_attach_rpc(void) hg_return_t hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); - /* free memory on input struct */ - free((void*)in.logio_spill_dir); - /* decode response */ unifyfs_attach_out_t out; hret = margo_get_output(handle, &out); @@ -239,6 +236,11 @@ int invoke_client_attach_rpc(void) int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); + /* free memory on input struct */ + if (NULL != in.logio_spill_dir) { + free((void*)in.logio_spill_dir); + } + /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index e9e52834d..2b1258941 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -2456,9 +2456,19 @@ void fill_client_attach_info(unifyfs_attach_in_t* in) in->shmem_super_size = shm_super_ctx->size; in->meta_offset = meta_offset; in->meta_size = meta_size; - in->logio_mem_size = logio_ctx->shmem->size; - in->logio_spill_size = logio_ctx->spill_sz; - in->logio_spill_dir = strdup(client_cfg.logio_spill_dir); + + if (NULL != logio_ctx->shmem) { + in->logio_mem_size = logio_ctx->shmem->size; + } else { + in->logio_mem_size = 0; + } + + in->logio_spill_size = logio_ctx->spill_sz; + if (logio_ctx->spill_sz) { + in->logio_spill_dir = strdup(client_cfg.logio_spill_dir); + } else { + in->logio_spill_dir = NULL; + } } /** diff --git a/server/src/unifyfs_server.c b/server/src/unifyfs_server.c index 166e8f630..d0297644f 100644 --- a/server/src/unifyfs_server.c +++ b/server/src/unifyfs_server.c @@ -842,7 +842,7 @@ unifyfs_rc attach_app_client(app_client* client, const size_t super_meta_offset, const size_t super_meta_size) { - if ((NULL == client) || (NULL == logio_spill_dir)) { + if (NULL == client) { return EINVAL; } From 8b9fe0643ea3c506c5d87f1cec6e639bec2c0e7f Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Wed, 20 May 2020 15:55:36 -0400 Subject: [PATCH 133/168] create unifyfs-stage and unify.c calls it Creates unifyfs-stage infrastructure and binary. That's used to stage files in and out at the startup and termination of unifyfs, respectively. The unifyfs (sourced in unifyfs.c) calls that binary through command line flags --stage-in and --stage-out. Because unifyfs-stage has ssl-based tools for verification that files were transferred correctly, now have a dependency on an SSL package. --- client/src/unifyfs.c | 48 +- common/src/unifyfs_const.h | 1 + configure.ac | 10 +- m4/openssl.m4 | 20 + util/Makefile.am | 2 +- util/unifyfs-stage/Makefile.am | 1 + util/unifyfs-stage/src/Makefile.am | 21 + .../src/unifyfs-stage-transfer.c | 403 ++++++++++++++++ util/unifyfs-stage/src/unifyfs-stage.c | 300 ++++++++++++ util/unifyfs-stage/src/unifyfs-stage.h | 61 +++ util/unifyfs/src/Makefile.am | 3 +- util/unifyfs/src/unifyfs-rm.c | 456 +++++++++++++++++- util/unifyfs/src/unifyfs.c | 82 +++- util/unifyfs/src/unifyfs.h | 1 + 14 files changed, 1355 insertions(+), 54 deletions(-) create mode 100644 m4/openssl.m4 create mode 100644 util/unifyfs-stage/Makefile.am create mode 100644 util/unifyfs-stage/src/Makefile.am create mode 100644 util/unifyfs-stage/src/unifyfs-stage-transfer.c create mode 100644 util/unifyfs-stage/src/unifyfs-stage.c create mode 100644 util/unifyfs-stage/src/unifyfs-stage.h diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 2b1258941..8f8040a99 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -2682,7 +2682,7 @@ int unifyfs_unmount(void) return ret; } -#define UNIFYFS_TX_BUFSIZE (64*(1<<10)) +#define UNIFYFS_TX_BUFSIZE (1<<20) enum { UNIFYFS_TX_STAGE_OUT = 0, @@ -2705,14 +2705,14 @@ ssize_t do_transfer_data(int fd_src, int fd_dst, off_t offset, size_t count) pos = lseek(fd_src, offset, SEEK_SET); if (pos == (off_t) -1) { LOGERR("lseek failed (%d: %s)\n", errno, strerror(errno)); - ret = -1; + ret = errno; goto out; } pos = lseek(fd_dst, offset, SEEK_SET); if (pos == (off_t) -1) { LOGERR("lseek failed (%d: %s)\n", errno, strerror(errno)); - ret = -1; + ret = errno; goto out; } @@ -2772,6 +2772,9 @@ static int do_transfer_file_serial(const char* src, const char* dst, goto out_close_src; } + LOGDBG("serial transfer (%d/%d): offset=0, length=%lu", + client_rank, global_rank_cnt, (unsigned long) sb_src->st_size); + ret = do_transfer_data(fd_src, fd_dst, 0, sb_src->st_size); if (ret < 0) { LOGERR("do_transfer_data failed!"); @@ -2802,15 +2805,10 @@ static int do_transfer_file_parallel(const char* src, const char* dst, fd_src = open(src, O_RDONLY); if (fd_src < 0) { + LOGERR("failed to open file %s", src); return errno; } - fd_dst = open(dst, O_CREAT | O_WRONLY | O_TRUNC, 0644); - if (fd_dst < 0) { - ret = errno; - goto out_close_src; - } - /* * if the file is smaller than (rankcount*buffersize), just do with the * serial mode. @@ -2848,18 +2846,35 @@ static int do_transfer_file_parallel(const char* src, const char* dst, if (client_rank == (global_rank_cnt - 1)) { len = (n_chunks - 1) * UNIFYFS_TX_BUFSIZE; - len += size % UNIFYFS_TX_BUFSIZE; + remainder = size % UNIFYFS_TX_BUFSIZE; + len += (remainder > 0 ? remainder : UNIFYFS_TX_BUFSIZE); } else { len = n_chunks * UNIFYFS_TX_BUFSIZE; } - LOGDBG("parallel transfer (%d/%d): offset=%lu, length=%lu", - client_rank, global_rank_cnt, - (unsigned long) offset, (unsigned long) len); + if (len > 0) { + LOGDBG("parallel transfer (%d/%d): " + "nchunks=%lu, offset=%lu, length=%lu", + client_rank, global_rank_cnt, + n_chunks, (unsigned long) offset, (unsigned long) len); - ret = do_transfer_data(fd_src, fd_dst, offset, len); + fd_dst = open(dst, O_WRONLY); + if (fd_dst < 0) { + LOGERR("failed to open file %s", dst); + ret = errno; + goto out_close_src; + } + + ret = do_transfer_data(fd_src, fd_dst, offset, len); + if (ret) { + LOGERR("failed to transfer data (ret=%d, %s)", ret, strerror(ret)); + } else { + fsync(fd_dst); + } + + close(fd_dst); + } - close(fd_dst); out_close_src: close(fd_src); @@ -2914,7 +2929,8 @@ int unifyfs_transfer_file(const char* src, const char* dst, int parallel) } if (unify_src + unify_dst != 1) { - return -EINVAL; + // we may fail the operation with EINVAL, but useful for testing + LOGDBG("WARNING: none of pathnames points to unifyfs volume"); } if (parallel) { diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h index d3890ba82..ea09ce091 100644 --- a/common/src/unifyfs_const.h +++ b/common/src/unifyfs_const.h @@ -68,6 +68,7 @@ /* timeout (in seconds) of waiting for initialization of all servers */ #define UNIFYFS_DEFAULT_INIT_TIMEOUT 120 #define UNIFYFSD_PID_FILENAME "unifyfsd.pids" +#define UNIFYFS_STAGE_STATUS_FILENAME "unifyfs-stage.status" // Client #define UNIFYFS_MAX_FILES 128 diff --git a/configure.ac b/configure.ac index aa09450ad..af9106edd 100755 --- a/configure.ac +++ b/configure.ac @@ -66,9 +66,6 @@ AC_CHECK_HEADERS([wchar.h wctype.h]) AC_CHECK_HEADERS([sys/mount.h sys/socket.h sys/statfs.h sys/time.h]) AC_CHECK_HEADERS([arpa/inet.h netdb.h netinet/in.h]) -AC_CHECK_HEADER([openssl/md5.h], [], - [AC_MSG_FAILURE([*** openssl/md5.h missing, openssl-devel package required])]) - # Checks for library functions. AC_FUNC_MALLOC AC_FUNC_MMAP @@ -168,6 +165,9 @@ UNIFYFS_AC_SPATH UNIFYFS_AC_MARGO UNIFYFS_AC_FLATCC +# openssl for md5 checksum +UNIFYFS_AC_OPENSSL + # checks to see how we can print 64 bit values on this architecture gt_INTTYPES_PRI @@ -389,7 +389,9 @@ AC_CONFIG_FILES([Makefile util/scripts/Makefile util/scripts/lsfcsm/Makefile util/unifyfs/Makefile - util/unifyfs/src/Makefile]) + util/unifyfs/src/Makefile + util/unifyfs-stage/Makefile + util/unifyfs-stage/src/Makefile]) AC_CONFIG_FILES([client/unifyfs-config], [chmod +x client/unifyfs-config]) AC_CONFIG_FILES([util/scripts/lsfcsm/unifyfs_lsfcsm_prolog], [chmod +x util/scripts/lsfcsm/unifyfs_lsfcsm_prolog]) diff --git a/m4/openssl.m4 b/m4/openssl.m4 new file mode 100644 index 000000000..c40ac5cd2 --- /dev/null +++ b/m4/openssl.m4 @@ -0,0 +1,20 @@ +AC_DEFUN([UNIFYFS_AC_OPENSSL], [ + # preserve state of flags + OPENSSL_OLD_CFLAGS=$CFLAGS + OPENSSL_OLD_CXXFLAGS=$CXXFLAGS + OPENSSL_OLD_LDFLAGS=$LDFLAGS + + PKG_CHECK_MODULES([OPENSSL],[openssl], + [ + AC_SUBST(OPENSSL_CFLAGS) + AC_SUBST(OPENSSL_LIBS) + ], + [AC_MSG_ERROR(m4_normalize([ + couldn't find a suitable openssl-devel + ]))]) + + # restore flags + CFLAGS=$OPENSSL_OLD_CFLAGS + CXXFLAGS=$OPENSSL_OLD_CXXFLAGS + LDFLAGS=$OPENSSL_OLD_LDFLAGS +]) diff --git a/util/Makefile.am b/util/Makefile.am index ca451f459..69e40c9c1 100644 --- a/util/Makefile.am +++ b/util/Makefile.am @@ -1 +1 @@ -SUBDIRS = scripts unifyfs +SUBDIRS = scripts unifyfs unifyfs-stage diff --git a/util/unifyfs-stage/Makefile.am b/util/unifyfs-stage/Makefile.am new file mode 100644 index 000000000..af437a64d --- /dev/null +++ b/util/unifyfs-stage/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = src diff --git a/util/unifyfs-stage/src/Makefile.am b/util/unifyfs-stage/src/Makefile.am new file mode 100644 index 000000000..704ad8639 --- /dev/null +++ b/util/unifyfs-stage/src/Makefile.am @@ -0,0 +1,21 @@ +libexec_PROGRAMS = unifyfs-stage + +unifyfs_stage_SOURCES = unifyfs-stage.c \ + unifyfs-stage-transfer.c + +noinst_HEADERS = unifyfs-stage.h + +unifyfs_stage_CPPFLAGS = $(AM_CPPFLAGS) $(MPI_CFLAGS) \ + $(OPENSSL_CFLAGS) \ + -I$(top_srcdir)/client/src \ + -I$(top_srcdir)/common/src + +unifyfs_stage_LDADD = $(top_builddir)/client/src/libunifyfs.la -lrt -lm + +unifyfs_stage_LDFLAGS = -static $(CP_WRAPPERS) $(AM_LDFLAGS) \ + $(MPI_CLDFLAGS) $(FLATCC_LDFLAGS) $(FLATCC_LIBS) \ + $(OPENSSL_LIBS) + +AM_CFLAGS = -Wall -Werror + +CLEANFILES = $(libexec_PROGRAMS) diff --git a/util/unifyfs-stage/src/unifyfs-stage-transfer.c b/util/unifyfs-stage/src/unifyfs-stage-transfer.c new file mode 100644 index 000000000..4b601aec6 --- /dev/null +++ b/util/unifyfs-stage/src/unifyfs-stage-transfer.c @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unifyfs-stage.h" + +/** + * @brief Run md5 checksum on specified file, send back + * digest. + * + * @param path path to the target file + * @param digest hash of the file + * + * @return 0 on success, errno otherwise + */ +static int md5_checksum(const char* path, unsigned char* digest) +{ + int ret = 0; + size_t len = 0; + int fd = -1; + unsigned char data[UNIFYFS_STAGE_MD5_BLOCKSIZE] = { 0, }; + MD5_CTX md5; + + fd = open(path, O_RDONLY); + if (fd < 0) { + perror("open"); + return errno; + } + + ret = MD5_Init(&md5); + if (!ret) { + fprintf(stderr, "failed to create md5 context\n"); + goto out; + } + + while ((len = read(fd, (void*) data, UNIFYFS_STAGE_MD5_BLOCKSIZE)) != 0) { + ret = MD5_Update(&md5, data, len); + if (!ret) { + fprintf(stderr, "failed to update checksum\n"); + goto out; + } + } + + ret = MD5_Final(digest, &md5); + if (!ret) { + fprintf(stderr, "failed to finalize md5\n"); + } + +out: + /* MD5_xx returns 1 for success */ + ret = ret == 1 ? 0 : EIO; + close(fd); + + return ret; +} + +/** + * @brief prints md5 checksum into string + * + * @param buf buffer to print into + * @param digest hash of the file + * + * @return buffer that has been printed to + */ +static char* checksum_str(char* buf, unsigned char* digest) +{ + int i = 0; + char* pos = buf; + + for (i = 0; i < MD5_DIGEST_LENGTH; i++) { + pos += sprintf(pos, "%02x", digest[i]); + } + + pos[0] = '\0'; + + return buf; +} + +/** + * @brief takes check sums of two files and compares + * + * @param src path to one file + * @param dst path to the other file + * + * @return 0 if files are identical, non-zero if not, or other error + */ +static int verify_checksum(const char* src, const char* dst) +{ + int ret = 0; + int i = 0; + char md5src[2 * MD5_DIGEST_LENGTH + 1] = { 0, }; + char md5dst[2 * MD5_DIGEST_LENGTH + 1] = { 0, }; + unsigned char src_digest[MD5_DIGEST_LENGTH + 1] = { 0, }; + unsigned char dst_digest[MD5_DIGEST_LENGTH + 1] = { 0, }; + + src_digest[MD5_DIGEST_LENGTH] = '\0'; + dst_digest[MD5_DIGEST_LENGTH] = '\0'; + + ret = md5_checksum(src, src_digest); + if (ret) { + fprintf(stderr, "failed to calculate checksum for %s (%s)\n", + src, strerror(ret)); + return ret; + } + + ret = md5_checksum(dst, dst_digest); + if (ret) { + fprintf(stderr, "failed to calculate checksum for %s (%s)\n", + dst, strerror(ret)); + return ret; + } + + if (verbose) { + printf("[%d] src: %s, dst: %s\n", rank, + checksum_str(md5src, src_digest), + checksum_str(md5dst, dst_digest)); + } + + for (i = 0; i < MD5_DIGEST_LENGTH; i++) { + if (src_digest[i] != dst_digest[i]) { + fprintf(stderr, "[%d] checksum verification failed: " + "(src=%s, dst=%s)\n", rank, + checksum_str(md5src, src_digest), + checksum_str(md5dst, dst_digest)); + ret = EIO; + } + } + + return ret; +} + +/* + * Parse a line from the manifest in the form of: + * + * + * + * If the paths have spaces, they must be quoted. + * + * On success, return 0 along with allocated src and dest strings. These + * must be freed when you're finished with them. On failure return non-zero, + * and set src and dest to NULL. + * + * Note, leading and tailing whitespace are ok. They just get ignored. + * Lines with only whitespace are ignored. A line of all whitespace will + * return 0, with src and dest being NULL, so users should not check for + * 'if (*src == NULL)' to see if the function failed. They should be looking + * at the return code. + */ +/** + * @brief parses manifest file line, passes back src and dst strings + * + * @param line input manifest file line + * @param src return val of src filename + * @param dst return val of dst filename + * + * @return 0 if all was well, or there was nothing; non-zero on error + */ +int +unifyfs_parse_manifest_line(char* line, char** src, char** dest) +{ + char* new_src = NULL; + char* new_dest = NULL; + char* copy; + char* tmp; + unsigned long copy_len; + int i; + unsigned int tmp_count; + int in_quotes = 0; + int rc = 0; + + copy = strdup(line); + copy_len = strlen(copy) + 1;/* +1 for '\0' */ + + /* Replace quotes and separator with '\0' */ + for (i = 0; i < copy_len; i++) { + if (copy[i] == '"') { + in_quotes ^= 1;/* toggle */ + copy[i] = '\0'; + } else if (isspace(copy[i]) && !in_quotes) { + /* + * Allow any whitespace for our separator + */ + copy[i] = '\0'; + } + } + + /* + * copy[] now contains a series of strings, one after the other + * (possibly containing some NULL strings, which we ignore) + */ + tmp = copy; + while (tmp < copy + copy_len) { + tmp_count = strlen(tmp); + if (tmp_count > 0) { + /* We have a real string */ + if (!new_src) { + new_src = strdup(tmp); + } else { + if (!new_dest) { + new_dest = strdup(tmp); + } else { + /* Error: a third file name */ + rc = 1; + break; + } + } + } + tmp += tmp_count + 1; + } + + /* Some kind of error parsing a line */ + if (rc != 0 || (new_src && !new_dest)) { + fprintf(stderr, "manifest file line >>%s<< is invalid!\n", + line); + free(new_src); + free(new_dest); + new_src = NULL; + new_dest = NULL; + if (rc == 0) { + rc = 1; + } + } + + *src = new_src; + *dest = new_dest; + + free(copy); + return rc; +} + +/** + * @brief controls the action of the stage-in or stage-out. Opens up + * the manifest file, sends each line to be parsed, and fires + * each source/destination to be staged. + * + * @param ctx stage context and instructions + * + * @return 0 indicates success, non-zero is error + */ +int unifyfs_stage_transfer(unifyfs_stage_t* ctx) +{ + int ret = 0; + int count = 0; + FILE* fp = NULL; + char* src = NULL; + char* dst = NULL; + char linebuf[LINE_MAX] = { 0, }; + struct stat sb = { 0, }; + + if (!ctx) { + return EINVAL; + } + + fp = fopen(ctx->manifest_file, "r"); + if (!fp) { + fprintf(stderr, "failed to open file %s: %s\n", + ctx->manifest_file, strerror(errno)); + ret = errno; + goto out; + } + + while (NULL != fgets(linebuf, LINE_MAX-1, fp)) { + if (strlen(linebuf) < 5) { + // the manifest file perhaps ends with a couple of characters + // and/or a newline not meant to be a transfer spec. + if (linebuf[0] == '\n') { + goto out; + } else{ + fprintf(stderr, "Short (bad) manifest file line: >%s<\n", + linebuf); + ret = -EINVAL; + goto out; + } + } + ret = unifyfs_parse_manifest_line(linebuf, &src, &dst); + if (ret < 0) { + fprintf(stderr, "failed to parse %s (%s)\n", + linebuf, strerror(ret)); + goto out; + } + if (ctx->mode == UNIFYFS_STAGE_SERIAL) { + if (count % total_ranks == rank) { + if (verbose) { + fprintf(stdout, "[%d] serial transfer: src=%s, dst=%s\n", + rank, src, dst); + } + + ret = unifyfs_transfer_file_serial(src, dst); + if (ret) { + goto out; + } + + if (ret < 0) { + fprintf(stderr, "stat on %s failed (err=%d, %s)\n", + dst, errno, strerror(errno)); + ret = errno; + goto out; + } + + if (ctx->checksum) { + ret = verify_checksum(src, dst); + if (ret) { + fprintf(stderr, "checksums for >%s< and >%s< differ!\n", + src, dst); + goto out; + } + } + } + } else { + if (0 == rank) { + int fd = -1; + + if (verbose) { + fprintf(stdout, "[%d] parallel transfer: src=%s, dst=%s\n", + rank, src, dst); + } + + /* FIXME: Since we cannot use the mpi barrier inside the + * unifyfs_transfer_file_parallel(), we need to create the file + * here before others start to access. It would be better if we + * can skip this step. + */ + fd = open(dst, O_WRONLY|O_CREAT|O_TRUNC, 0600); + if (fd < 0) { + fprintf(stderr, "[%d] failed to create the file %s\n", + rank, dst); + goto out; + } + + close(fd); + } + + MPI_Barrier(MPI_COMM_WORLD); + + ret = unifyfs_transfer_file_parallel(src, dst); + if (ret) { + goto out; + } + + MPI_Barrier(MPI_COMM_WORLD); + + ret = stat(dst, &sb); + if (ret < 0) { + fprintf(stderr, "stat on %s failed (err=%d, %s)\n", + dst, errno, strerror(errno)); + ret = errno; + goto out; + } + + if (ctx->checksum && 0 == rank) { + ret = verify_checksum(src, dst); + if (ret) { + goto out; + } + } + } + + count++; + } +out: + if (ret) { + fprintf(stderr, "failed to transfer file (src=%s, dst=%s): %s\n", + src, dst, strerror(ret)); + } + + if (fp) { + fclose(fp); + fp = NULL; + } + + return ret; +} + diff --git a/util/unifyfs-stage/src/unifyfs-stage.c b/util/unifyfs-stage/src/unifyfs-stage.c new file mode 100644 index 000000000..590732a3b --- /dev/null +++ b/util/unifyfs-stage/src/unifyfs-stage.c @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ +/* unifyfs-stage: this application is supposed to excuted by the unifyfs + * command line utility for: + * - stage in: moving files in pfs to unifyfs volume before user starts + * application, + * e.g., unifyfs start --stage-in= + * - stage out: moving files in the unifyfs volume to parallel file system + * after user application completes, + * e.g., unifyfs terminate --stage-out= + * + * Currently, we request users to pass the to specify target + * files to be transferred. The should list all target files + * and their destinations, line by line. + * + * This supports two transfer modes (although both are technically parallel): + * + * - serial: Each process will transfer a file. Data of a single file will + * reside in a single compute node. + * - parallel (-p, --parallel): Each file will be split and transferred by all + * processes. Data of a single file will be spread evenly across all + * available compute nodes. + * + * TODO: + * Maybe later on, it would be better to have a size threshold. Based on the + * threshold, we can determine whether a file needs to transferred serially (if + * smaller than threshold), or parallelly. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unifyfs_const.h" +#include "unifyfs-stage.h" + +int rank; +int total_ranks; +int verbose; + +static int debug; +static int checksum; +static int mode; +static char* manifest_file; +static char* mountpoint = "/unifyfs"; +static char* share_dir; + +static unifyfs_stage_t _ctx; + +/** + * @brief create a status (lock) file to notify the unifyfs executable + * when the staging is finished + * + * @param status 0 indicates success + * + * @return 0 on success, errno otherwise + */ +static int create_status_file(int status) +{ + char filename[PATH_MAX]; + FILE* fp = NULL; + const char* msg = status ? "fail" : "success"; + int return_val_from_scnprintf; + + return_val_from_scnprintf = + scnprintf(filename, PATH_MAX, + "%s/%s", share_dir, UNIFYFS_STAGE_STATUS_FILENAME); + if (return_val_from_scnprintf > (PATH_MAX-1)) { + fprintf(stderr, "Stage status file is too long!\n"); + return -ENOMEM; + } + + fp = fopen(filename, "w"); + if (!fp) { + fprintf(stderr, "failed to create %s (%s)\n", + filename, strerror(errno)); + return errno; + } + + fprintf(fp, "%s\n", msg); + + fclose(fp); + + return 0; +} + +static struct option long_opts[] = { + { "checksum", 0, 0, 'c' }, + { "debug", 0, 0, 'd' }, + { "help", 0, 0, 'h' }, + { "mountpoint", 1, 0, 'm' }, + { "parallel", 0, 0, 'p' }, + { "share-dir", 1, 0, 's' }, + { "verbose", 0, 0, 'v' }, + { 0, 0, 0, 0 }, +}; + +static char* short_opts = "cdhm:ps:v"; + +static const char* usage_str = + "\n" + "Usage: %s [OPTION]... \n" + "\n" + "Transfer files between unifyfs volume and external file system.\n" + "The should contain list of files to be transferred,\n" + "and each line should be formatted as\n" + "\n" + " /source/file/path,/destination/file/path\n" + "\n" + "Specifying directories is not supported.\n" + "\n" + "Available options:\n" + "\n" + " -c, --checksum verify md5 checksum for each transfer\n" + " -h, --help print this usage\n" + " -m, --mountpoint= use as unifyfs mountpoint\n" + " (default: /unifyfs)\n" + " -p, --parallel transfer each file in parallel\n" + " (experimental)\n" + " -s, --share-dir= directory path for creating status file\n" + " -v, --verbose print noisy outputs\n" + "\n" + "Without the '-p, --parallel' option, a file is transferred by a single\n" + "process. If the '-p, --parallel' option is specified, each file will be\n" + "divided by multiple processes and transferred in parallel.\n" + "\n"; + +static char* program; + +static void print_usage(void) +{ + if (0 == rank) { + fprintf(stdout, usage_str, program); + } +} + +static inline +void debug_pause(int rank, const char* fmt, ...) +{ + if (rank == 0) { + va_list args; + + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + + fprintf(stderr, " ENTER to continue ... "); + + (void) getchar(); + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* internal accept() call from mpi may set errno */ + errno = 0; +} + +static int parse_option(int argc, char** argv) +{ + int ch = 0; + int optidx = 0; + char* filepath = NULL; + + if (argc < 2) { + return EINVAL; + } + + while ((ch = getopt_long(argc, argv, + short_opts, long_opts, &optidx)) >= 0) { + switch (ch) { + case 'c': + checksum = 1; + break; + + case 'd': + debug = 1; + break; + + case 'm': + mountpoint = strdup(optarg); + break; + + case 'p': + mode = UNIFYFS_STAGE_PARALLEL; + break; + + case 's': + share_dir = strdup(optarg); + break; + + case 'v': + verbose = 1; + break; + + case 'h': + default: + break; + } + } + + if (argc - optind != 1) { + return EINVAL; + } + + filepath = argv[optind]; + + manifest_file = realpath(filepath, NULL); + if (!manifest_file) { + fprintf(stderr, "problem with accessing file %s: %s\n", + filepath, strerror(errno)); + return errno; + } + + return 0; +} + +int main(int argc, char** argv) +{ + int ret = 0; + unifyfs_stage_t* ctx = &_ctx; + + program = basename(strdup(argv[0])); + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &total_ranks); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + ret = parse_option(argc, argv); + if (ret) { + if (EINVAL == ret) { + print_usage(); + } + goto out; + } + + ctx->rank = rank; + ctx->total_ranks = total_ranks; + ctx->checksum = checksum; + ctx->mode = mode; + ctx->mountpoint = mountpoint; + ctx->manifest_file = manifest_file; + + if (verbose) { + unifyfs_stage_print(ctx); + } + + if (debug) { + debug_pause(rank, "About to mount unifyfs.. "); + } + + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); + if (ret) { + fprintf(stderr, "failed to mount unifyfs at %s (%s)", + ctx->mountpoint, strerror(ret)); + goto out; + } + + MPI_Barrier(MPI_COMM_WORLD); + + ret = unifyfs_stage_transfer(ctx); + if (ret) { + fprintf(stderr, "data transfer failed (%s)\n", strerror(errno)); + } + + if (share_dir) { + ret = create_status_file(ret); + if (ret) { + fprintf(stderr, "failed to create the status file (%s)\n", + strerror(errno)); + } + } + + ret = unifyfs_unmount(); + if (ret) { + fprintf(stderr, "unmounting unifyfs failed (ret=%d)\n", ret); + } +out: + MPI_Finalize(); + + return ret; +} + diff --git a/util/unifyfs-stage/src/unifyfs-stage.h b/util/unifyfs-stage/src/unifyfs-stage.h new file mode 100644 index 000000000..7e4c0eaa8 --- /dev/null +++ b/util/unifyfs-stage/src/unifyfs-stage.h @@ -0,0 +1,61 @@ +#ifndef __UNIFYFS_STAGE_H +#define __UNIFYFS_STAGE_H + +#include +#include +#include + +#define UNIFYFS_STAGE_MD5_BLOCKSIZE (1048576) + +/* + * serial: each file is tranferred by a process. + * parallel: a file is transferred by all processes. + */ +enum { + UNIFYFS_STAGE_SERIAL = 0, + UNIFYFS_STAGE_PARALLEL = 1, +}; + +struct _unifyfs_stage { + int rank; /* my rank */ + int total_ranks; /* mpi world size */ + + int checksum; /* perform checksum? 0:no, 1:yes */ + int mode; /* transfer mode? 0:serial, 1:parallel */ + char* mountpoint; /* unifyfs mountpoint */ + char* manifest_file; /* manifest file containing the transfer list */ +}; + +typedef struct _unifyfs_stage unifyfs_stage_t; + +static inline void unifyfs_stage_print(unifyfs_stage_t* ctx) +{ + printf("== unifyfs stage context ==\n" + "rank = %d\n" + "total ranks = %d\n" + "checksum = %d\n" + "mode = %d\n" + "mountpoint = %s\n" + "manifest file = %s\n", + ctx->rank, + ctx->total_ranks, + ctx->checksum, + ctx->mode, + ctx->mountpoint, + ctx->manifest_file); +} + +/** + * @brief transfer files specified in @ctx + * + * @param ctx unifyfs_stage_t data transfer context + * + * @return 0 on success, errno otherwise + */ +int unifyfs_stage_transfer(unifyfs_stage_t* ctx); + +extern int verbose; +extern int rank; +extern int total_ranks; + +#endif /* __UNIFYFS_STAGE_H */ diff --git a/util/unifyfs/src/Makefile.am b/util/unifyfs/src/Makefile.am index 770e66be0..c82fc8b8f 100644 --- a/util/unifyfs/src/Makefile.am +++ b/util/unifyfs/src/Makefile.am @@ -9,7 +9,8 @@ unifyfs_LDADD = $(top_builddir)/common/src/libunifyfs_common.la AM_CPPFLAGS = -I$(top_srcdir)/common/src \ -DBINDIR=\"$(bindir)\" \ - -DSBINDIR=\"$(sbindir)\" + -DSBINDIR=\"$(sbindir)\" \ + -DLIBEXECDIR=\"$(libexecdir)\" AM_CFLAGS = -Wall -Werror diff --git a/util/unifyfs/src/unifyfs-rm.c b/util/unifyfs/src/unifyfs-rm.c index c59790c76..bada93e82 100644 --- a/util/unifyfs/src/unifyfs-rm.c +++ b/util/unifyfs/src/unifyfs-rm.c @@ -46,6 +46,7 @@ #include #include #include +#include #include "unifyfs.h" @@ -57,12 +58,16 @@ typedef int (*unifyfs_rm_launch_t)(unifyfs_resource_t* resource, typedef int (*unifyfs_rm_terminate_t)(unifyfs_resource_t* resource, unifyfs_args_t* args); +typedef int (*unifyfs_rm_stage_t)(unifyfs_resource_t* resource, + unifyfs_args_t* args); + struct _ucr_resource_manager { const char* type; unifyfs_rm_read_resource_t read_resource; unifyfs_rm_launch_t launch; unifyfs_rm_terminate_t terminate; + unifyfs_rm_stage_t stage; }; typedef struct _ucr_resource_manager _ucr_resource_manager_t; @@ -102,7 +107,7 @@ static int parse_hostfile(unifyfs_resource_t* resource, int i = 0; FILE* fp = NULL; char** nodes = NULL; - char buf[1024] = { 0, }; + char buf[1024]; if (hostfile == NULL) { return -EINVAL; @@ -201,10 +206,17 @@ static int wait_server_initialization(unifyfs_resource_t* resource, unsigned int interval = 3; unsigned int wait_time = 0; FILE* fp = NULL; - char linebuf[32] = { 0, }; - char filename[PATH_MAX] = { 0, }; - - sprintf(filename, "%s/%s", args->share_dir, UNIFYFSD_PID_FILENAME); + char linebuf[32]; + char filename[PATH_MAX]; + int return_val_from_scnprintf; + + return_val_from_scnprintf = + scnprintf(filename, PATH_MAX, + "%s/%s", args->share_dir, UNIFYFSD_PID_FILENAME); + if (return_val_from_scnprintf > (PATH_MAX - 2)) { + fprintf(stderr, "Unifyfs status filename is too long!\n"); + return -ENOMEM; + } while (1) { fp = fopen(filename, "r"); @@ -227,7 +239,7 @@ static int wait_server_initialization(unifyfs_resource_t* resource, if (errno != ENOENT) { fprintf(stderr, "failed to open file %s (%s)\n", - filename, strerror(errno)); + filename, strerror(errno)); ret = -errno; break; } @@ -244,6 +256,111 @@ static int wait_server_initialization(unifyfs_resource_t* resource, return ret; } +enum { + UNIFYFS_STAGE_IN = 0, + UNIFYFS_STAGE_OUT = 1, +}; + +static inline unsigned int estimate_timeout(const char* manifest_file, int op) +{ + /* FIXME: we just wait for 20 mins. + * In fact, we can roughly estimate time for stage in by looking at the + * total file size. However, such a method is not possible for stage out + * because we have no idea on what the total transfer size is. + */ + return 20 * 60; +} + + +/** + * @brief wait until data stage operation finishes + * + * @param resource + * @param args + * + * @return + */ +static +int wait_stage(unifyfs_resource_t* resource, unifyfs_args_t* args, int op) +{ + int ret = UNIFYFS_SUCCESS; + unsigned int interval = 5; + unsigned int wait_time = 0; + unsigned int timeout = 0; + FILE* fp = NULL; + const char* manifest_file = NULL; + char filename[PATH_MAX]; + char linebuf[16]; + int return_val_from_scnprintf; + + return_val_from_scnprintf = + scnprintf(filename, PATH_MAX, + "%s/%s", args->share_dir, UNIFYFS_STAGE_STATUS_FILENAME); + if (return_val_from_scnprintf > (PATH_MAX - 2)) { + fprintf(stderr, "Unifyfs status filename is too long!\n"); + return -ENOMEM; + } + + + if (op == UNIFYFS_STAGE_IN) { + manifest_file = args->stage_in; + } else { + manifest_file = args->stage_out; + } + + if (args->stage_timeout > 0) { + timeout = args->stage_timeout; + } else { + timeout = estimate_timeout(manifest_file, op); + } + + while (1) { + fp = fopen(filename, "r"); + if (fp) { + char* line = fgets(linebuf, 15, fp); + if (0 == strncmp("success", line, strlen("success"))) { + ret = 0; + break; // transfer completed + } else if (0 == strncmp("fail", line, strlen("fail"))) { + ret = -EIO; + break; // transfer failed + } else { + fclose(fp); // try again + } + } + + + if (errno != ENOENT) { + fprintf(stderr, "failed to open file %s (%s)\n", + UNIFYFS_STAGE_STATUS_FILENAME, strerror(errno)); + ret = -errno; + break; + } + + wait_time += interval; + sleep(interval); + + if (wait_time > timeout) { + ret = UNIFYFS_FAILURE; + break; + } + } + + return ret; +} + +static inline int wait_stage_in(unifyfs_resource_t* resource, + unifyfs_args_t* args) +{ + return wait_stage(resource, args, UNIFYFS_STAGE_IN); +} + +static inline int wait_stage_out(unifyfs_resource_t* resource, + unifyfs_args_t* args) +{ + return wait_stage(resource, args, UNIFYFS_STAGE_OUT); +} + /** * @brief remove server pid file if exists (possibly from previous run). * returns 0 (success) if the pid file does not exist. @@ -253,9 +370,16 @@ static int wait_server_initialization(unifyfs_resource_t* resource, static int remove_server_pid_file(unifyfs_args_t* args) { int ret = 0; - char filename[PATH_MAX] = { 0, }; - - sprintf(filename, "%s/%s", args->share_dir, UNIFYFSD_PID_FILENAME); + char filename[PATH_MAX]; + int return_val_from_scnprintf; + + return_val_from_scnprintf = + scnprintf(filename, PATH_MAX, + "%s/%s", args->share_dir, UNIFYFSD_PID_FILENAME); + if (return_val_from_scnprintf > (PATH_MAX - 2)) { + fprintf(stderr, "Unifyfs status filename is too long!\n"); + return -ENOMEM; + } ret = unlink(filename); if (ret) { @@ -263,7 +387,41 @@ static int remove_server_pid_file(unifyfs_args_t* args) ret = 0; } else { fprintf(stderr, "failed to unlink existing pid file %s (%s)\n", - filename, strerror(errno)); + filename, strerror(errno)); + ret = -errno; + } + } + + return ret; +} + +/** + * @brief remove stagein/out status file if exists (possibly from previous run). + * returns 0 (success) if the pid file does not exist. + * + * @return 0 on success, negative errno otherwise + */ +static int remove_stage_status_file(unifyfs_args_t* args) +{ + int ret = 0; + char filename[PATH_MAX]; + int return_val_from_scnprintf; + + return_val_from_scnprintf = + scnprintf(filename, PATH_MAX, + "%s/%s", args->share_dir, UNIFYFS_STAGE_STATUS_FILENAME); + if (return_val_from_scnprintf > (PATH_MAX - 2)) { + fprintf(stderr, "Unifyfs stage status filename is too long!\n"); + return -ENOMEM; + } + + ret = unlink(filename); + if (ret) { + if (ENOENT == errno) { + ret = 0; + } else { + fprintf(stderr, "failed to unlink existing stage status file " + "%s (%s)\n", filename, strerror(errno)); ret = -errno; } } @@ -472,8 +630,16 @@ static int slurm_read_resource(unifyfs_resource_t* resource) return ret; } +// construct_server_argv(): +// This function is called in two ways. +// Call it once with server_argv==NULL and it +// will count up the number of arguments you'll have, but +// doesn't construct the list itself. Call it again with +// the same args but with a buffer in server_argv, and it will +// construct the argument list there. /** - * @brief Default server launch routine + * @brief Constructs argument chain to mpi-start (or terminate) + * unifyfs server processes. * * @param args The command-line options * @param server_args Server argument vector to be filled @@ -539,6 +705,51 @@ static size_t construct_server_argv(unifyfs_args_t* args, return argc; } +// construct_stage_argv: +// this is currently set up to create one rank per compute node, +// mirroring the configuration of the servers. However, in the +// future, this may be reconfigured to have more, to support +// more files being staged in or out more quickly. +/** + * @brief Constructs argument chain to mpi-start (or terminate) + * unifyfs-stage stagein/out process. + * + * @param args The command-line options + * @param stage_args unifyfs-stage argument vector to be filled + * + * @return number of server arguments + */ +static size_t construct_stage_argv(unifyfs_args_t* args, + char** stage_argv) +{ + size_t argc = 0; + + if (stage_argv != NULL) { + stage_argv[0] = strdup(LIBEXECDIR "/unifyfs-stage"); + } + argc = 1; + + if (args->mountpoint != NULL) { + if (stage_argv != NULL) { + stage_argv[argc] = strdup("-m"); + stage_argv[argc + 1] = strdup(args->mountpoint); + } + argc += 2; + } + + if (stage_argv != NULL) { + char* manifest_file = args->stage_in ? args->stage_in + : args->stage_out; + + stage_argv[argc] = strdup("-s"); + stage_argv[argc + 1] = strdup(args->share_dir); + stage_argv[argc + 2] = strdup(manifest_file); + } + argc += 3; + + return argc; +} + /** * @brief Default server launch routine * @@ -567,6 +778,20 @@ static int invalid_terminate(unifyfs_resource_t* resource, return -ENOSYS; } +/** + * @brief Default data stage routine + * + * @param resource Not used + * @param args Not used + * + * @return -ENOSYS + */ +static int invalid_stage(unifyfs_resource_t* resource, + unifyfs_args_t* args) +{ + return -ENOSYS; +} + /** * @brief Launch servers using IBM jsrun * @@ -651,6 +876,50 @@ static int jsrun_terminate(unifyfs_resource_t* resource, return -errno; } +/** + * @brief Launch data stage using IBM jsrun + * + * @param resource The job resource record + * @param args The command-line options + * + * @return + */ +static int jsrun_stage(unifyfs_resource_t* resource, + unifyfs_args_t* args) +{ + size_t argc, jsrun_argc, stage_argc; + char** argv = NULL; + char n_nodes[16]; + + // full command: jsrun + jsrun_argc = 13; + snprintf(n_nodes, sizeof(n_nodes), "%zu", resource->n_nodes); + + stage_argc = construct_stage_argv(args, NULL); + + // setup full command argv + argc = 1 + jsrun_argc + stage_argc; + argv = calloc(argc, sizeof(char*)); + argv[0] = strdup("jsrun"); + argv[1] = strdup("--immediate"); + argv[2] = strdup("-e"); + argv[3] = strdup("individual"); + argv[4] = strdup("--stdio_stderr"); + argv[5] = strdup("unifyfs-stage.err.%h.%p"); + argv[6] = strdup("--stdio_stdout"); + argv[7] = strdup("unifyfs-stage.out.%h.%p"); + argv[8] = strdup("--nrs"); + argv[9] = strdup(n_nodes); + argv[10] = strdup("-r1"); + argv[11] = strdup("-c1"); + argv[12] = strdup("-a1"); + construct_stage_argv(args, argv + jsrun_argc); + + execvp(argv[0], argv); + perror("failed to execvp() mpirun to handle data stage"); + return -errno; +} + /** * @brief Launch servers using mpirun (OpenMPI) * @@ -724,6 +993,42 @@ static int mpirun_terminate(unifyfs_resource_t* resource, return -errno; } +/** + * @brief Launch unifyfs-stage using mpirun (OpenMPI) + * + * @param resource The job resource record + * @param args The command-line options + * + * @return + */ +static int mpirun_stage(unifyfs_resource_t* resource, + unifyfs_args_t* args) +{ + size_t argc, mpirun_argc, stage_argc; + char** argv = NULL; + char n_nodes[16]; + + // full command: mpirun + + mpirun_argc = 5; + snprintf(n_nodes, sizeof(n_nodes), "%zu", resource->n_nodes); + + stage_argc = construct_stage_argv(args, NULL); + + // setup full command argv + argc = 1 + mpirun_argc + stage_argc; + argv = calloc(argc, sizeof(char*)); + argv[0] = strdup("mpirun"); + argv[1] = strdup("-np"); + argv[2] = strdup(n_nodes); + argv[3] = strdup("--map-by"); + argv[4] = strdup("ppr:1:node"); + construct_stage_argv(args, argv + mpirun_argc); + + execvp(argv[0], argv); + perror("failed to execvp() mpirun to handle data stage"); + return -errno; +} /** * @brief Launch servers using SLURM srun @@ -798,6 +1103,43 @@ static int srun_terminate(unifyfs_resource_t* resource, return -errno; } +/** + * @brief Launch unifyfs-stage using SLURM srun + * + * @param resource The job resource record + * @param args The command-line options + * + * @return + */ +static int srun_stage(unifyfs_resource_t* resource, + unifyfs_args_t* args) +{ + size_t argc, srun_argc, stage_argc; + char** argv = NULL; + char n_nodes[16]; + + // full command: srun + + srun_argc = 5; + snprintf(n_nodes, sizeof(n_nodes), "%zu", resource->n_nodes); + + stage_argc = construct_stage_argv(args, NULL); + + // setup full command argv + argc = 1 + srun_argc + stage_argc; + argv = calloc(argc, sizeof(char*)); + argv[0] = strdup("srun"); + argv[1] = strdup("-N"); + argv[2] = strdup(n_nodes); + argv[3] = strdup("--ntasks-per-node"); + argv[4] = strdup("1"); + construct_stage_argv(args, argv + srun_argc); + + execvp(argv[0], argv); + perror("failed to execvp() srun to launch unifyfsd"); + return -errno; +} + /** * @brief Launch servers using custom script * @@ -869,11 +1211,41 @@ static int script_terminate(unifyfs_resource_t* resource, * match the definition in common/src/rm_enumerator.h */ static _ucr_resource_manager_t resource_managers[] = { - { "none", &invalid_read_resource, &invalid_launch, &invalid_terminate }, - { "pbs", &pbs_read_resource, &mpirun_launch, &mpirun_terminate }, - { "slurm", &slurm_read_resource, &srun_launch, &srun_terminate }, - { "lsf", &lsf_read_resource, &mpirun_launch, &mpirun_terminate }, - { "lsfcsm", &lsf_read_resource, &jsrun_launch, &jsrun_terminate }, + { + .type = "none", + .read_resource = &invalid_read_resource, + .launch = &invalid_launch, + .terminate = &invalid_terminate, + .stage = &invalid_stage, + }, + { + .type = "pbs", + .read_resource = &pbs_read_resource, + .launch = &mpirun_launch, + .terminate = &mpirun_terminate, + .stage = &mpirun_stage, + }, + { + .type = "slurm", + .read_resource = &slurm_read_resource, + .launch = &srun_launch, + .terminate = &srun_terminate, + .stage = &srun_stage, + }, + { + .type = "lsf", + .read_resource = &lsf_read_resource, + .launch = &mpirun_launch, + .terminate = &mpirun_terminate, + .stage = &mpirun_stage, + }, + { + .type = "lsfcsm", + .read_resource = &lsf_read_resource, + .launch = &jsrun_launch, + .terminate = &jsrun_terminate, + .stage = &jsrun_stage, + }, }; int unifyfs_detect_resources(unifyfs_resource_t* resource) @@ -919,8 +1291,8 @@ int unifyfs_start_servers(unifyfs_resource_t* resource, pid = fork(); if (pid < 0) { - fprintf(stderr, "failed to create server launch server process (%s)\n", - strerror(errno)); + fprintf(stderr, "failed to create server launch process (%s)\n", + strerror(errno)); return -errno; } else if (pid == 0) { if (args->script != NULL) { @@ -935,17 +1307,65 @@ int unifyfs_start_servers(unifyfs_resource_t* resource, fprintf(stderr, "ERROR: failed to wait for server initialization\n"); } + if (args->stage_in) { + rc = remove_stage_status_file(args); + if (rc) { + fprintf(stderr, "ERROR: failed to remove stage status file\n"); + return rc; + } + + pid = fork(); + if (pid < 0) { + fprintf(stderr, "failed to create stage-in launch process (%s)\n", + strerror(errno)); + return -errno; + } else if (pid == 0) { + return resource_managers[resource->rm].stage(resource, args); + } + + rc = wait_stage_in(resource, args); + if (rc) { + fprintf(stderr, "failed to detect the stage in status (rc=%d)\n", + rc); + } + } + return rc; } int unifyfs_stop_servers(unifyfs_resource_t* resource, unifyfs_args_t* args) { + int rc; + pid_t pid; if ((resource == NULL) || (args == NULL)) { return -EINVAL; } + if (args->stage_out) { + rc = remove_stage_status_file(args); + if (rc) { + fprintf(stderr, "ERROR: failed to remove stage status file\n"); + return rc; + } + + pid = fork(); + if (pid < 0) { + fprintf(stderr, "failed to create stage-out launch process (%s)\n", + strerror(errno)); + return -errno; + } else if (pid == 0) { + return resource_managers[resource->rm].stage(resource, args); + } + + rc = wait_stage_out(resource, args); + if (rc) { + fprintf(stderr, "failed to detect the data out status (rc=%d)\n", + rc); + } + } + if (args->script != NULL) { return script_terminate(resource, args); } else { diff --git a/util/unifyfs/src/unifyfs.c b/util/unifyfs/src/unifyfs.c index be54a52f7..a518dfd8b 100644 --- a/util/unifyfs/src/unifyfs.c +++ b/util/unifyfs/src/unifyfs.c @@ -44,6 +44,7 @@ #include #include #include +#include #include "unifyfs.h" @@ -80,7 +81,7 @@ static struct option const long_opts[] = { }; static char* program; -static char* short_opts = ":cC:de:hi:m:o:s:S:t:"; +static char* short_opts = ":cC:de:hi:m:o:s:S:t:T:"; static char* usage_str = "\n" "Usage: %s [options...]\n" @@ -94,18 +95,21 @@ static char* usage_str = " -h, --help print usage\n" "\n" "Command options for \"start\":\n" - " -C, --consistency= [OPTIONAL] consistency model (NONE | LAMINATED | POSIX)\n" - " -e, --exe= [OPTIONAL] where unifyfsd is installed\n" - " -m, --mount= [OPTIONAL] mount UnifyFS at \n" - " -s, --script= [OPTIONAL] to custom launch script\n" - " -t, --timeout= [OPTIONAL] wait until all servers become ready\n" - " -S, --share-dir= [REQUIRED] shared file system for use by servers\n" - " -c, --cleanup [OPTIONAL] clean up the UnifyFS storage upon server exit\n" - " -i, --stage-in= [OPTIONAL, NOT YET SUPPORTED] stage in file(s) at \n" - " -o, --stage-out= [OPTIONAL, NOT YET SUPPORTED] stage out file(s) to on termination\n" + " -C, --consistency= [OPTIONAL] consistency model (NONE | LAMINATED | POSIX)\n" + " -e, --exe= [OPTIONAL] where unifyfsd is installed\n" + " -m, --mount= [OPTIONAL] mount UnifyFS at \n" + " -s, --script= [OPTIONAL] to custom launch script\n" + " -t, --timeout= [OPTIONAL] wait until all servers become ready\n" + " -S, --share-dir= [REQUIRED] shared file system for use by servers\n" + " -c, --cleanup [OPTIONAL] clean up the UnifyFS storage upon server exit\n" + " -i, --stage-in= [OPTIONAL] stage in file(s) listed in file\n" + " -T, --stage-timeout= [OPTIONAL] timeout for stage-in operation\n" "\n" "Command options for \"terminate\":\n" - " -s, --script= to custom termination script\n" + " -o, --stage-out= [OPTIONAL] stage out file(s) listed in on termination\n" + " -T, --stage-timeout= [OPTIONAL] timeout for stage-out operation\n" + " -s, --script= [OPTIONAL] to custom termination script\n" + " -S, --share-dir= [REQUIRED for --stage-out] shared file system for use by servers\n" "\n"; static int debug; @@ -122,6 +126,7 @@ static void parse_cmd_arguments(int argc, char** argv) int optidx = 2; int cleanup = 0; int timeout = UNIFYFS_DEFAULT_INIT_TIMEOUT; + int stage_timeout = -1; unifyfs_cm_e consistency = UNIFYFS_CM_LAMINATED; char* mountpoint = NULL; char* script = NULL; @@ -130,6 +135,8 @@ static void parse_cmd_arguments(int argc, char** argv) char* stage_in = NULL; char* stage_out = NULL; + int argument_count = 1; + while ((ch = getopt_long(argc, argv, short_opts, long_opts, &optidx)) >= 0) { switch (ch) { @@ -169,21 +176,28 @@ static void parse_cmd_arguments(int argc, char** argv) timeout = atoi(optarg); break; + case 'T': + stage_timeout = atoi(optarg); + break; + case 'i': - printf("WARNING: stage-in not yet supported!\n"); stage_in = strdup(optarg); break; case 'o': - printf("WARNING: stage-out not yet supported!\n"); stage_out = strdup(optarg); break; case 'h': - default: usage(0); break; + + default: + printf("\n\nArgument %d is invalid!\n", argument_count); + usage(-EINVAL); + break; } + argument_count++; } cli_args.debug = debug; @@ -195,6 +209,7 @@ static void parse_cmd_arguments(int argc, char** argv) cli_args.share_dir = share_dir; cli_args.stage_in = stage_in; cli_args.stage_out = stage_out; + cli_args.stage_timeout = stage_timeout; cli_args.timeout = timeout; } @@ -237,6 +252,7 @@ int main(int argc, char** argv) printf("server:\t%s\n", cli_args.server_path); printf("stage_in:\t%s\n", cli_args.stage_in); printf("stage_out:\t%s\n", cli_args.stage_out); + printf("stage_timeout:\t%d\n", cli_args.stage_timeout); } ret = unifyfs_detect_resources(&resource); @@ -258,9 +274,47 @@ int main(int argc, char** argv) if (NULL == cli_args.share_dir) { printf("USAGE ERROR: shared directory (-S) is required!\n"); usage(1); + return -EINVAL; + } + if (cli_args.stage_in != NULL) { + if (cli_args.script) { + fprintf(stderr, + "WARNING! You are using a script with --stage-in.\n"); + fprintf(stderr, "This is won't work.\n"); + return -EINVAL; + } + if (access(cli_args.stage_in, R_OK)) { + fprintf(stderr, + "Cannot read stagein manifest file:%s\n", + cli_args.stage_in); + return -ENOENT; + } } return unifyfs_start_servers(&resource, &cli_args); } else if (action == ACT_TERMINATE) { + if (cli_args.stage_out != NULL) { + // status directory isn't required just to terminate the servers + // but it IS required if we're calling for stage-out, so that + // the stage-out can be started, then the server terminate waits + // until the stage-out is done. + if (NULL == cli_args.share_dir) { + printf("USAGE ERROR: shared directory (-S) is required!\n"); + usage(1); + return -EINVAL; + } + if (cli_args.script) { + fprintf(stderr, + "WARNING! You are using a script with --stage-out.\n"); + fprintf(stderr, "This won't work.\n"); + return -EINVAL; + } + if (access(cli_args.stage_out, R_OK)) { + fprintf(stderr, + "Cannot read stageout manifest file:%s\n", + cli_args.stage_out); + return -ENOENT; + } + } return unifyfs_stop_servers(&resource, &cli_args); } else { fprintf(stderr, "INTERNAL ERROR: unhandled action %d\n", (int)action); diff --git a/util/unifyfs/src/unifyfs.h b/util/unifyfs/src/unifyfs.h index 28f46001c..a14e99592 100644 --- a/util/unifyfs/src/unifyfs.h +++ b/util/unifyfs/src/unifyfs.h @@ -65,6 +65,7 @@ struct _unifyfs_args { char* share_hostfile; /* full path to shared server hostfile */ char* stage_in; /* data path to stage-in */ char* stage_out; /* data path to stage-out (drain) */ + int stage_timeout; /* timeout of (in or out) file staging*/ char* script; /* path to custom launch/terminate script */ }; typedef struct _unifyfs_args unifyfs_args_t; From 4e8092f5d89f4456780ea3952184788170162621 Mon Sep 17 00:00:00 2001 From: Craig P Steffen Date: Wed, 10 Jun 2020 17:19:26 -0400 Subject: [PATCH 134/168] tests 0700 and 9300 for unifyfs-stage Added sharness tests for unifyfs-stage. These tests were created and tested on summitdev. Test 0700 stages a file into the unifyfs file space, and then to a different directory coming back out. This requires unifyfsd to be running. Test 9300 exersizes the unifyfs-stage machinery to move a file from one directory in the temp test area to a different directory also in the test area; thus it does not require unifyfsd to be running. However, it still requires mpirun (or equivalent) infrastructure, so typically needs to be run within a job environment. --- t/0700-unifyfs-stage-full.t | 64 ++++++++++++++++++ t/9300-unifyfs-stage-isolated.t | 62 +++++++++++++++++ t/Makefile.am | 4 ++ .../src/unifyfs-stage-transfer.c | 67 ++++++++----------- util/unifyfs-stage/src/unifyfs-stage.c | 61 +++++++++++------ util/unifyfs-stage/src/unifyfs-stage.h | 3 + util/unifyfs/src/unifyfs-rm.c | 51 +++++--------- util/unifyfs/src/unifyfs.c | 8 +-- 8 files changed, 219 insertions(+), 101 deletions(-) create mode 100755 t/0700-unifyfs-stage-full.t create mode 100755 t/9300-unifyfs-stage-isolated.t diff --git a/t/0700-unifyfs-stage-full.t b/t/0700-unifyfs-stage-full.t new file mode 100755 index 000000000..5448b7d94 --- /dev/null +++ b/t/0700-unifyfs-stage-full.t @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Test unifyfs-stage executable for basic functionality +# + +test_description="Test basic functionality of unifyfs-stage executable" + +. $(dirname $0)/sharness.sh + +test_expect_success "unifyfs-stage exists" ' + test_path_is_file ${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage +' +test_expect_success "testing temp dir exists" ' + test_path_is_dir ${UNIFYFS_TEST_TMPDIR} +' + +mkdir -p ${UNIFYFS_TEST_TMPDIR}/config_0700 +mkdir -p ${UNIFYFS_TEST_TMPDIR}/stage_source +mkdir -p ${UNIFYFS_TEST_TMPDIR}/stage_destination_0700 + +test_expect_success "stage testing dirs exist" ' + test_path_is_dir ${UNIFYFS_TEST_TMPDIR}/config_0700 + test_path_is_dir ${UNIFYFS_TEST_TMPDIR}/stage_source + test_path_is_dir ${UNIFYFS_TEST_TMPDIR}/stage_destination_0700 +' + +dd if=/dev/urandom bs=4M count=1 of=${UNIFYFS_TEST_TMPDIR}/stage_source/source_0700.file + +test_expect_success "source.file exists" ' + test_path_is_file ${UNIFYFS_TEST_TMPDIR}/stage_source/source_0700.file +' + +rm -f ${UNIFYFS_TEST_TMPDIR}/config_0700/* +rm -f ${UNIFYFS_TEST_TMPDIR}/stage_destination_0700/* + +test_expect_success "config_0700 directory is empty" ' + test_dir_is_empty ${UNIFYFS_TEST_TMPDIR}/config_0700 +' + +echo "\"${UNIFYFS_TEST_TMPDIR}/stage_source/source_0700.file\" \"${UNIFYFS_TEST_MOUNT}/intermediate.file\"" > ${UNIFYFS_TEST_TMPDIR}/config_0700/test_IN.manifest +echo "\"${UNIFYFS_TEST_MOUNT}/intermediate.file\" \"${UNIFYFS_TEST_TMPDIR}/stage_destination_0700/destination_0700.file\"" > ${UNIFYFS_TEST_TMPDIR}/config_0700/test_OUT.manifest + +test_expect_success "config_0700 directory now has manifest files" ' + test_path_is_file ${UNIFYFS_TEST_TMPDIR}/config_0700/test_IN.manifest + test_path_is_file ${UNIFYFS_TEST_TMPDIR}/config_0700/test_OUT.manifest +' + +test_expect_success "target directory is empty" ' + test_dir_is_empty ${UNIFYFS_TEST_TMPDIR}/stage_destination_0700 +' + +${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage -m ${UNIFYFS_TEST_MOUNT} ${UNIFYFS_TEST_TMPDIR}/config_0700/test_IN.manifest > ${UNIFYFS_TEST_TMPDIR}/config_0700/stage_IN_output.OUT 2>&1 + +${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage -m ${UNIFYFS_TEST_MOUNT} ${UNIFYFS_TEST_TMPDIR}/config_0700/test_OUT.manifest > ${UNIFYFS_TEST_TMPDIR}/config_0700/stage_OUT_output.OUT 2>&1 + +test_expect_success "input file has been staged to output" ' + test_path_is_file ${UNIFYFS_TEST_TMPDIR}/stage_destination_0700/destination_0700.file +' + +test_expect_success "final output is identical to initial input" ' + test_might_fail test_cmp ${UNIFYFS_TEST_TMPDIR}/stage_source/source_0700.file ${UNIFYFS_TEST_TMPDIR}/stage_destination_0700/destination_0700.file +' + +test_done diff --git a/t/9300-unifyfs-stage-isolated.t b/t/9300-unifyfs-stage-isolated.t new file mode 100755 index 000000000..c8a389f5e --- /dev/null +++ b/t/9300-unifyfs-stage-isolated.t @@ -0,0 +1,62 @@ +#!/bin/bash +# +# Test unifyfs-stage executable for basic functionality +# + +test_description="Test basic functionality of unifyfs-stage executable" + +. $(dirname $0)/sharness.sh + +test_expect_success "unifyfs-stage exists" ' + test_path_is_file ${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage +' +test_expect_success "testing temp dir exists" ' + test_path_is_dir ${UNIFYFS_TEST_TMPDIR} +' + +mkdir -p ${UNIFYFS_TEST_TMPDIR}/config_9300 +mkdir -p ${UNIFYFS_TEST_TMPDIR}/stage_source +mkdir -p ${UNIFYFS_TEST_TMPDIR}/stage_destination_9300 + +test_expect_success "stage testing dirs exist" ' + test_path_is_dir ${UNIFYFS_TEST_TMPDIR}/config_9300 + test_path_is_dir ${UNIFYFS_TEST_TMPDIR}/stage_source + test_path_is_dir ${UNIFYFS_TEST_TMPDIR}/stage_destination_9300 +' + +# NOTE: we're using the unifyfs-stage binary as its own transfer data target +# because we know it's there and it's filled with non-zero data. +cp ${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage ${UNIFYFS_TEST_TMPDIR}/stage_source/source_9300.file + +test_expect_success "source.file exists" ' + test_path_is_file ${UNIFYFS_TEST_TMPDIR}/stage_source/source_9300.file +' + +rm - f $ {UNIFYFS_TEST_TMPDIR} / config_9300/* +rm -f ${UNIFYFS_TEST_TMPDIR}/stage_destination_9300/* + +test_expect_success "config_9300 directory is empty" ' + test_dir_is_empty ${UNIFYFS_TEST_TMPDIR}/config_9300 +' + +echo "\"${UNIFYFS_TEST_TMPDIR}/stage_source/source_9300.file\" \"${UNIFYFS_TEST_TMPDIR}/stage_destination_9300/destination_9300.file\"" > ${UNIFYFS_TEST_TMPDIR}/config_9300/test_INOUT.manifest + +test_expect_success "config_9300 directory now has manifest files" ' + test_path_is_file ${UNIFYFS_TEST_TMPDIR}/config_9300/test_INOUT.manifest +' + +test_expect_success "target directory is empty" ' + test_dir_is_empty ${UNIFYFS_TEST_TMPDIR}/stage_destination_9300 +' + +${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage -N ${UNIFYFS_TEST_TMPDIR}/config_9300/test_INOUT.manifest > ${UNIFYFS_TEST_TMPDIR}/config_9300/stage_INOUT_output.OUT 2>&1 + +test_expect_success "input file has been staged to output" ' + test_path_is_file ${UNIFYFS_TEST_TMPDIR}/stage_destination_9300/destination_9300.file +' + +test_expect_success "final output is identical to initial input" ' + test_cmp ${UNIFYFS_TEST_TMPDIR}/stage_source/source_9300.file ${UNIFYFS_TEST_TMPDIR}/stage_destination_9300/destination_9300.file +' + +test_done diff --git a/t/Makefile.am b/t/Makefile.am index d18b51867..45f3c758d 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -9,12 +9,14 @@ TESTS = \ 0200-stdio-gotcha.t \ 0500-sysio-static.t \ 0600-stdio-static.t \ + 0700-unifyfs-stage-full.t \ 9005-unifyfs-unmount.t \ 9010-stop-unifyfsd.t \ 9020-mountpoint-empty.t \ 9100-metadata-api.t \ 9200-seg-tree-test.t \ 9201-slotmap-test.t \ + 9300-unifyfs-stage-isolated.t \ 9999-cleanup.t check_SCRIPTS = \ @@ -23,12 +25,14 @@ check_SCRIPTS = \ 0200-stdio-gotcha.t \ 0500-sysio-static.t \ 0600-stdio-static.t \ + 0700-unifyfs-stage-full.t \ 9005-unifyfs-unmount.t \ 9010-stop-unifyfsd.t \ 9020-mountpoint-empty.t \ 9100-metadata-api.t \ 9200-seg-tree-test.t \ 9201-slotmap-test.t \ + 9300-unifyfs-stage-isolated.t \ 9999-cleanup.t EXTRA_DIST = \ diff --git a/util/unifyfs-stage/src/unifyfs-stage-transfer.c b/util/unifyfs-stage/src/unifyfs-stage-transfer.c index 4b601aec6..2c5d5b69d 100644 --- a/util/unifyfs-stage/src/unifyfs-stage-transfer.c +++ b/util/unifyfs-stage/src/unifyfs-stage-transfer.c @@ -129,14 +129,14 @@ static int verify_checksum(const char* src, const char* dst) ret = md5_checksum(src, src_digest); if (ret) { fprintf(stderr, "failed to calculate checksum for %s (%s)\n", - src, strerror(ret)); + src, strerror(ret)); return ret; } ret = md5_checksum(dst, dst_digest); if (ret) { fprintf(stderr, "failed to calculate checksum for %s (%s)\n", - dst, strerror(ret)); + dst, strerror(ret)); return ret; } @@ -149,9 +149,9 @@ static int verify_checksum(const char* src, const char* dst) for (i = 0; i < MD5_DIGEST_LENGTH; i++) { if (src_digest[i] != dst_digest[i]) { fprintf(stderr, "[%d] checksum verification failed: " - "(src=%s, dst=%s)\n", rank, - checksum_str(md5src, src_digest), - checksum_str(md5dst, dst_digest)); + "(src=%s, dst=%s)\n", rank, + checksum_str(md5src, src_digest), + checksum_str(md5dst, dst_digest)); ret = EIO; } } @@ -275,7 +275,6 @@ int unifyfs_stage_transfer(unifyfs_stage_t* ctx) char* src = NULL; char* dst = NULL; char linebuf[LINE_MAX] = { 0, }; - struct stat sb = { 0, }; if (!ctx) { return EINVAL; @@ -284,35 +283,34 @@ int unifyfs_stage_transfer(unifyfs_stage_t* ctx) fp = fopen(ctx->manifest_file, "r"); if (!fp) { fprintf(stderr, "failed to open file %s: %s\n", - ctx->manifest_file, strerror(errno)); + ctx->manifest_file, strerror(errno)); ret = errno; goto out; } - while (NULL != fgets(linebuf, LINE_MAX-1, fp)) { + while (NULL != fgets(linebuf, LINE_MAX - 1, fp)) { if (strlen(linebuf) < 5) { - // the manifest file perhaps ends with a couple of characters - // and/or a newline not meant to be a transfer spec. if (linebuf[0] == '\n') { - goto out; - } else{ - fprintf(stderr, "Short (bad) manifest file line: >%s<\n", - linebuf); - ret = -EINVAL; + // manifest file ends in a blank line + goto out; + } else { + fprintf(stderr, "Short (bad) manifest file line: >%s<\n", + linebuf); + ret = -EINVAL; goto out; } - } - ret = unifyfs_parse_manifest_line(linebuf, &src, &dst); - if (ret < 0) { - fprintf(stderr, "failed to parse %s (%s)\n", - linebuf, strerror(ret)); + } + ret = unifyfs_parse_manifest_line(linebuf, &src, &dst); + if (ret < 0) { + fprintf(stderr, "failed to parse %s (%s)\n", + linebuf, strerror(ret)); goto out; - } + } if (ctx->mode == UNIFYFS_STAGE_SERIAL) { if (count % total_ranks == rank) { if (verbose) { fprintf(stdout, "[%d] serial transfer: src=%s, dst=%s\n", - rank, src, dst); + rank, src, dst); } ret = unifyfs_transfer_file_serial(src, dst); @@ -322,7 +320,7 @@ int unifyfs_stage_transfer(unifyfs_stage_t* ctx) if (ret < 0) { fprintf(stderr, "stat on %s failed (err=%d, %s)\n", - dst, errno, strerror(errno)); + dst, errno, strerror(errno)); ret = errno; goto out; } @@ -342,21 +340,15 @@ int unifyfs_stage_transfer(unifyfs_stage_t* ctx) if (verbose) { fprintf(stdout, "[%d] parallel transfer: src=%s, dst=%s\n", - rank, src, dst); + rank, src, dst); } - /* FIXME: Since we cannot use the mpi barrier inside the - * unifyfs_transfer_file_parallel(), we need to create the file - * here before others start to access. It would be better if we - * can skip this step. - */ - fd = open(dst, O_WRONLY|O_CREAT|O_TRUNC, 0600); + fd = open(dst, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (fd < 0) { fprintf(stderr, "[%d] failed to create the file %s\n", - rank, dst); + rank, dst); goto out; } - close(fd); } @@ -369,13 +361,8 @@ int unifyfs_stage_transfer(unifyfs_stage_t* ctx) MPI_Barrier(MPI_COMM_WORLD); - ret = stat(dst, &sb); - if (ret < 0) { - fprintf(stderr, "stat on %s failed (err=%d, %s)\n", - dst, errno, strerror(errno)); - ret = errno; - goto out; - } + // possible lamination check or force lamination + // may need to go here if (ctx->checksum && 0 == rank) { ret = verify_checksum(src, dst); @@ -390,7 +377,7 @@ int unifyfs_stage_transfer(unifyfs_stage_t* ctx) out: if (ret) { fprintf(stderr, "failed to transfer file (src=%s, dst=%s): %s\n", - src, dst, strerror(ret)); + src, dst, strerror(ret)); } if (fp) { diff --git a/util/unifyfs-stage/src/unifyfs-stage.c b/util/unifyfs-stage/src/unifyfs-stage.c index 590732a3b..c65b86bec 100644 --- a/util/unifyfs-stage/src/unifyfs-stage.c +++ b/util/unifyfs-stage/src/unifyfs-stage.c @@ -59,6 +59,7 @@ int verbose; static int debug; static int checksum; static int mode; +static int should_we_mount_unifyfs = 1; static char* manifest_file; static char* mountpoint = "/unifyfs"; static char* share_dir; @@ -82,8 +83,8 @@ static int create_status_file(int status) return_val_from_scnprintf = scnprintf(filename, PATH_MAX, - "%s/%s", share_dir, UNIFYFS_STAGE_STATUS_FILENAME); - if (return_val_from_scnprintf > (PATH_MAX-1)) { + "%s/%s", share_dir, UNIFYFS_STAGE_STATUS_FILENAME); + if (return_val_from_scnprintf > (PATH_MAX - 1)) { fprintf(stderr, "Stage status file is too long!\n"); return -ENOMEM; } @@ -91,7 +92,7 @@ static int create_status_file(int status) fp = fopen(filename, "w"); if (!fp) { fprintf(stderr, "failed to create %s (%s)\n", - filename, strerror(errno)); + filename, strerror(errno)); return errno; } @@ -110,10 +111,11 @@ static struct option long_opts[] = { { "parallel", 0, 0, 'p' }, { "share-dir", 1, 0, 's' }, { "verbose", 0, 0, 'v' }, + { "no-mount-unifyfs", 0, 0, 'N' }, { 0, 0, 0, 0 }, }; -static char* short_opts = "cdhm:ps:v"; +static char* short_opts = "cdhm:ps:vN"; static const char* usage_str = "\n" @@ -123,9 +125,13 @@ static const char* usage_str = "The should contain list of files to be transferred,\n" "and each line should be formatted as\n" "\n" - " /source/file/path,/destination/file/path\n" + " /source/file/path /destination/file/path\n" "\n" - "Specifying directories is not supported.\n" + "OR in the case of filenames with spaces or special characters:\n" + "\n" + " \"/source/file/path\" \"/destination/file/path\"\n" + "\n" + "One file per line; Specifying directories is not supported.\n" "\n" "Available options:\n" "\n" @@ -137,6 +143,7 @@ static const char* usage_str = " (experimental)\n" " -s, --share-dir= directory path for creating status file\n" " -v, --verbose print noisy outputs\n" + " -N, --no-mount-unifyfs don't mount unifyfs file system (for testing)\n" "\n" "Without the '-p, --parallel' option, a file is transferred by a single\n" "process. If the '-p, --parallel' option is specified, each file will be\n" @@ -152,7 +159,7 @@ static void print_usage(void) } } -static inline +static void debug_pause(int rank, const char* fmt, ...) { if (rank == 0) { @@ -210,6 +217,11 @@ static int parse_option(int argc, char** argv) verbose = 1; break; + case 'N': + fprintf(stderr, "WARNING: not mounting unifyfs file system!\n"); + should_we_mount_unifyfs = 0; + break; + case 'h': default: break; @@ -225,7 +237,7 @@ static int parse_option(int argc, char** argv) manifest_file = realpath(filepath, NULL); if (!manifest_file) { fprintf(stderr, "problem with accessing file %s: %s\n", - filepath, strerror(errno)); + filepath, strerror(errno)); return errno; } @@ -239,18 +251,18 @@ int main(int argc, char** argv) program = basename(strdup(argv[0])); - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &total_ranks); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - ret = parse_option(argc, argv); if (ret) { if (EINVAL == ret) { print_usage(); } - goto out; + goto preMPIout; } + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &total_ranks); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + ctx->rank = rank; ctx->total_ranks = total_ranks; ctx->checksum = checksum; @@ -266,11 +278,13 @@ int main(int argc, char** argv) debug_pause(rank, "About to mount unifyfs.. "); } - ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); - if (ret) { - fprintf(stderr, "failed to mount unifyfs at %s (%s)", - ctx->mountpoint, strerror(ret)); - goto out; + if (should_we_mount_unifyfs) { + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); + if (ret) { + fprintf(stderr, "failed to mount unifyfs at %s (%s)", + ctx->mountpoint, strerror(ret)); + goto out; + } } MPI_Barrier(MPI_COMM_WORLD); @@ -284,16 +298,19 @@ int main(int argc, char** argv) ret = create_status_file(ret); if (ret) { fprintf(stderr, "failed to create the status file (%s)\n", - strerror(errno)); + strerror(errno)); } } - ret = unifyfs_unmount(); - if (ret) { - fprintf(stderr, "unmounting unifyfs failed (ret=%d)\n", ret); + if (should_we_mount_unifyfs) { + ret = unifyfs_unmount(); + if (ret) { + fprintf(stderr, "unmounting unifyfs failed (ret=%d)\n", ret); + } } out: MPI_Finalize(); +preMPIout: return ret; } diff --git a/util/unifyfs-stage/src/unifyfs-stage.h b/util/unifyfs-stage/src/unifyfs-stage.h index 7e4c0eaa8..d9a12697c 100644 --- a/util/unifyfs-stage/src/unifyfs-stage.h +++ b/util/unifyfs-stage/src/unifyfs-stage.h @@ -22,6 +22,7 @@ struct _unifyfs_stage { int checksum; /* perform checksum? 0:no, 1:yes */ int mode; /* transfer mode? 0:serial, 1:parallel */ + int should_we_mount_unifyfs; /* mount? 0:no (for testing), 1: yes */ char* mountpoint; /* unifyfs mountpoint */ char* manifest_file; /* manifest file containing the transfer list */ }; @@ -35,12 +36,14 @@ static inline void unifyfs_stage_print(unifyfs_stage_t* ctx) "total ranks = %d\n" "checksum = %d\n" "mode = %d\n" + "should_we_mount_unifyfs = %d\n" "mountpoint = %s\n" "manifest file = %s\n", ctx->rank, ctx->total_ranks, ctx->checksum, ctx->mode, + ctx->should_we_mount_unifyfs, ctx->mountpoint, ctx->manifest_file); } diff --git a/util/unifyfs/src/unifyfs-rm.c b/util/unifyfs/src/unifyfs-rm.c index bada93e82..8518c14c9 100644 --- a/util/unifyfs/src/unifyfs-rm.c +++ b/util/unifyfs/src/unifyfs-rm.c @@ -261,13 +261,9 @@ enum { UNIFYFS_STAGE_OUT = 1, }; -static inline unsigned int estimate_timeout(const char* manifest_file, int op) +static inline unsigned int estimate_timeout(const char* manifest_file) { - /* FIXME: we just wait for 20 mins. - * In fact, we can roughly estimate time for stage in by looking at the - * total file size. However, such a method is not possible for stage out - * because we have no idea on what the total transfer size is. - */ + /* crude guess: 20 minutes */ return 20 * 60; } @@ -281,7 +277,7 @@ static inline unsigned int estimate_timeout(const char* manifest_file, int op) * @return */ static -int wait_stage(unifyfs_resource_t* resource, unifyfs_args_t* args, int op) +int wait_stage(unifyfs_resource_t* resource, unifyfs_args_t* args) { int ret = UNIFYFS_SUCCESS; unsigned int interval = 5; @@ -301,17 +297,10 @@ int wait_stage(unifyfs_resource_t* resource, unifyfs_args_t* args, int op) return -ENOMEM; } - - if (op == UNIFYFS_STAGE_IN) { - manifest_file = args->stage_in; - } else { - manifest_file = args->stage_out; - } - if (args->stage_timeout > 0) { timeout = args->stage_timeout; } else { - timeout = estimate_timeout(manifest_file, op); + timeout = estimate_timeout(manifest_file); } while (1) { @@ -319,9 +308,13 @@ int wait_stage(unifyfs_resource_t* resource, unifyfs_args_t* args, int op) if (fp) { char* line = fgets(linebuf, 15, fp); if (0 == strncmp("success", line, strlen("success"))) { + fclose(fp); + fp = NULL; ret = 0; break; // transfer completed } else if (0 == strncmp("fail", line, strlen("fail"))) { + fclose(fp); + fp = NULL; ret = -EIO; break; // transfer failed } else { @@ -349,18 +342,6 @@ int wait_stage(unifyfs_resource_t* resource, unifyfs_args_t* args, int op) return ret; } -static inline int wait_stage_in(unifyfs_resource_t* resource, - unifyfs_args_t* args) -{ - return wait_stage(resource, args, UNIFYFS_STAGE_IN); -} - -static inline int wait_stage_out(unifyfs_resource_t* resource, - unifyfs_args_t* args) -{ - return wait_stage(resource, args, UNIFYFS_STAGE_OUT); -} - /** * @brief remove server pid file if exists (possibly from previous run). * returns 0 (success) if the pid file does not exist. @@ -1279,19 +1260,19 @@ int unifyfs_start_servers(unifyfs_resource_t* resource, rc = write_hostfile(resource, args); if (rc) { - fprintf(stderr, "ERROR: failed to write shared server hostfile\n"); + fprintf(stderr, "Failed to write shared server hostfile!\n"); return rc; } rc = remove_server_pid_file(args); if (rc) { - fprintf(stderr, "ERROR: failed to remove server pid file\n"); + fprintf(stderr, "Failed to remove server pid file!\n"); return rc; } pid = fork(); if (pid < 0) { - fprintf(stderr, "failed to create server launch process (%s)\n", + fprintf(stderr, "Failed to create server launch process (%s)\n", strerror(errno)); return -errno; } else if (pid == 0) { @@ -1304,13 +1285,13 @@ int unifyfs_start_servers(unifyfs_resource_t* resource, rc = wait_server_initialization(resource, args); if (rc) { - fprintf(stderr, "ERROR: failed to wait for server initialization\n"); + fprintf(stderr, "Failed to wait for server initialization\n"); } if (args->stage_in) { rc = remove_stage_status_file(args); if (rc) { - fprintf(stderr, "ERROR: failed to remove stage status file\n"); + fprintf(stderr, "Failed to remove stage status file\n"); return rc; } @@ -1323,7 +1304,7 @@ int unifyfs_start_servers(unifyfs_resource_t* resource, return resource_managers[resource->rm].stage(resource, args); } - rc = wait_stage_in(resource, args); + rc = wait_stage(resource, args); if (rc) { fprintf(stderr, "failed to detect the stage in status (rc=%d)\n", rc); @@ -1346,7 +1327,7 @@ int unifyfs_stop_servers(unifyfs_resource_t* resource, if (args->stage_out) { rc = remove_stage_status_file(args); if (rc) { - fprintf(stderr, "ERROR: failed to remove stage status file\n"); + fprintf(stderr, "Failed to remove stage status file\n"); return rc; } @@ -1359,7 +1340,7 @@ int unifyfs_stop_servers(unifyfs_resource_t* resource, return resource_managers[resource->rm].stage(resource, args); } - rc = wait_stage_out(resource, args); + rc = wait_stage(resource, args); if (rc) { fprintf(stderr, "failed to detect the data out status (rc=%d)\n", rc); diff --git a/util/unifyfs/src/unifyfs.c b/util/unifyfs/src/unifyfs.c index a518dfd8b..db5f596de 100644 --- a/util/unifyfs/src/unifyfs.c +++ b/util/unifyfs/src/unifyfs.c @@ -279,8 +279,8 @@ int main(int argc, char** argv) if (cli_args.stage_in != NULL) { if (cli_args.script) { fprintf(stderr, - "WARNING! You are using a script with --stage-in.\n"); - fprintf(stderr, "This is won't work.\n"); + "WARNING! You are using a script with --stage-in.\n" + "This will not work.\n"); return -EINVAL; } if (access(cli_args.stage_in, R_OK)) { @@ -304,8 +304,8 @@ int main(int argc, char** argv) } if (cli_args.script) { fprintf(stderr, - "WARNING! You are using a script with --stage-out.\n"); - fprintf(stderr, "This won't work.\n"); + "WARNING! You are using a script with --stage-out.\n" + "This will not work.\n"); return -EINVAL; } if (access(cli_args.stage_out, R_OK)) { From 9ed0f1480882fd8a74e7277eaa17446c06c9049e Mon Sep 17 00:00:00 2001 From: CamStan Date: Tue, 23 Jun 2020 16:06:49 -0700 Subject: [PATCH 135/168] Update bootstrap.sh to build on power9 systems This changes bootstrap.sh to checkout a stable version of mercury in order to build on power9 systems. Also required adding the `git submodule update --init` command since using checksum, per the mercury build instructions. --- bootstrap.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 73913c61e..3f8ce7b12 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -70,15 +70,18 @@ cd .. echo "### building mercury ###" cd mercury +git checkout v1.0.1 +git submodule update --init mkdir -p build && cd build cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ + -DMERCURY_USE_SELF_FORWARD=ON \ -DMERCURY_USE_BOOST_PP=ON \ -DMERCURY_USE_CHECKSUMS=ON \ -DMERCURY_USE_EAGER_BULK=ON \ -DMERCURY_USE_SYSTEM_MCHECKSUM=OFF \ -DNA_USE_BMI=ON \ -DMERCURY_USE_XDR=OFF \ - -DBUILD_SHARED_LIBS=on .. + -DBUILD_SHARED_LIBS=ON .. make -j $(nproc) && make install cd .. cd .. @@ -88,7 +91,7 @@ cd margo git checkout v0.4.3 export PKG_CONFIG_PATH="$INSTALL_DIR/lib/pkgconfig" ./prepare.sh -./configure --prefix="$INSTALL_DIR" +./configure --prefix="$INSTALL_DIR" --enable-shared make -j $(nproc) && make install cd .. From 52fa3e17fbbccae4c28be09c7244efd8f455a698 Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Thu, 18 Jun 2020 13:20:55 -0400 Subject: [PATCH 136/168] support per-file sync This updates the unifyfs_sync() method in the client to support sync of either all outstanding writes or writes to a specific file. The server rpc handler did not require any modifications to accommodate this change. The biggest change to how the client operates is that we no longer put writes in the shmem index as they happen. This avoids any interleaving of writes to different files. Instead, each sync rpc only handles the writes for a single file, and we always generate the contents of the shmem write index from each file's extents_sync segment tree at the time of the sync. This behavior matches what we were doing before when flatten_writes was enabled (which was the default mode of operation). Also included were fixes for several places where the wrong in/out structure types were used in the rpc invokers and handlers, removing a couple client global variables that are no longer used, and related documentation updates. --- client/src/margo_client.c | 96 +++---- client/src/unifyfs-dirops.c | 6 +- client/src/unifyfs-fixed.c | 363 +++++++++------------------ client/src/unifyfs-fixed.h | 12 +- client/src/unifyfs-internal.h | 10 +- client/src/unifyfs-sysio.c | 10 +- client/src/unifyfs.c | 132 +++++----- common/src/unifyfs_client_rpcs.h | 25 +- common/src/unifyfs_configurator.h | 1 - common/src/unifyfs_logio.c | 28 +++ common/src/unifyfs_logio.h | 12 + docs/add-rpcs.rst | 9 - docs/configuration.rst | 1 - server/src/unifyfs_cmd_handler.c | 22 +- server/src/unifyfs_request_manager.c | 6 +- 15 files changed, 306 insertions(+), 427 deletions(-) diff --git a/client/src/margo_client.c b/client/src/margo_client.c index c47615c01..afbb4f0a2 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -15,65 +15,31 @@ static void register_client_rpcs(client_rpc_context_t* ctx) /* shorter name for our margo instance id */ margo_instance_id mid = ctx->mid; - ctx->rpcs.attach_id = MARGO_REGISTER(mid, "unifyfs_attach_rpc", - unifyfs_attach_in_t, - unifyfs_attach_out_t, - NULL); - - ctx->rpcs.mount_id = MARGO_REGISTER(mid, "unifyfs_mount_rpc", - unifyfs_mount_in_t, - unifyfs_mount_out_t, - NULL); - - ctx->rpcs.unmount_id = MARGO_REGISTER(mid, "unifyfs_unmount_rpc", - unifyfs_unmount_in_t, - unifyfs_unmount_out_t, - NULL); - - ctx->rpcs.metaset_id = MARGO_REGISTER(mid, "unifyfs_metaset_rpc", - unifyfs_metaset_in_t, - unifyfs_metaset_out_t, - NULL); - - ctx->rpcs.metaget_id = MARGO_REGISTER(mid, "unifyfs_metaget_rpc", - unifyfs_metaget_in_t, - unifyfs_metaget_out_t, - NULL); - - ctx->rpcs.filesize_id = MARGO_REGISTER(mid, "unifyfs_filesize_rpc", - unifyfs_filesize_in_t, - unifyfs_filesize_out_t, - NULL); - - ctx->rpcs.truncate_id = MARGO_REGISTER(mid, "unifyfs_truncate_rpc", - unifyfs_truncate_in_t, - unifyfs_truncate_out_t, - NULL); - - ctx->rpcs.unlink_id = MARGO_REGISTER(mid, "unifyfs_unlink_rpc", - unifyfs_unlink_in_t, - unifyfs_unlink_out_t, - NULL); - - ctx->rpcs.laminate_id = MARGO_REGISTER(mid, "unifyfs_laminate_rpc", - unifyfs_laminate_in_t, - unifyfs_laminate_out_t, - NULL); - - ctx->rpcs.sync_id = MARGO_REGISTER(mid, "unifyfs_sync_rpc", - unifyfs_sync_in_t, - unifyfs_sync_out_t, - NULL); - - ctx->rpcs.read_id = MARGO_REGISTER(mid, "unifyfs_read_rpc", - unifyfs_read_in_t, - unifyfs_read_out_t, - NULL); - - ctx->rpcs.mread_id = MARGO_REGISTER(mid, "unifyfs_mread_rpc", - unifyfs_mread_in_t, - unifyfs_mread_out_t, - NULL); + hg_id_t hgid; + +#define CLIENT_REGISTER_RPC(name) \ + do { \ + hgid = MARGO_REGISTER(mid, "unifyfs_" #name "_rpc", \ + unifyfs_##name##_in_t, \ + unifyfs_##name##_out_t, \ + NULL); \ + ctx->rpcs.name##_id = hgid; \ + } while (0) + + CLIENT_REGISTER_RPC(attach); + CLIENT_REGISTER_RPC(mount); + CLIENT_REGISTER_RPC(unmount); + CLIENT_REGISTER_RPC(metaset); + CLIENT_REGISTER_RPC(metaget); + CLIENT_REGISTER_RPC(filesize); + CLIENT_REGISTER_RPC(truncate); + CLIENT_REGISTER_RPC(unlink); + CLIENT_REGISTER_RPC(laminate); + CLIENT_REGISTER_RPC(sync); + CLIENT_REGISTER_RPC(read); + CLIENT_REGISTER_RPC(mread); + +#undef CLIENT_REGISTER_RPC } /* initialize margo client-server rpc */ @@ -281,10 +247,6 @@ int invoke_client_mount_rpc(void) int32_t ret = out.ret; LOGDBG("Got response ret=%" PRIi32, ret); - /* get slice size for write index key/value store */ - unifyfs_key_slice_range = out.meta_slice_sz; - LOGDBG("set unifyfs_key_slice_range=%zu", unifyfs_key_slice_range); - /* get assigned client id, and verify app_id */ unifyfs_client_id = (int) out.client_id; int srvr_app_id = (int) out.app_id; @@ -497,7 +459,7 @@ int invoke_client_truncate_rpc(int gfid, size_t filesize) assert(hret == HG_SUCCESS); /* decode response */ - unifyfs_filesize_out_t out; + unifyfs_truncate_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); int32_t ret = out.ret; @@ -532,7 +494,7 @@ int invoke_client_unlink_rpc(int gfid) assert(hret == HG_SUCCESS); /* decode response */ - unifyfs_filesize_out_t out; + unifyfs_unlink_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); int32_t ret = out.ret; @@ -556,7 +518,7 @@ int invoke_client_laminate_rpc(int gfid) hg_handle_t handle = create_handle(client_rpc_context->rpcs.laminate_id); /* fill in input struct */ - unifyfs_unlink_in_t in; + unifyfs_laminate_in_t in; in.app_id = (int32_t) unifyfs_app_id; in.client_id = (int32_t) unifyfs_client_id; in.gfid = (int32_t) gfid; @@ -567,7 +529,7 @@ int invoke_client_laminate_rpc(int gfid) assert(hret == HG_SUCCESS); /* decode response */ - unifyfs_filesize_out_t out; + unifyfs_laminate_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); int32_t ret = out.ret; diff --git a/client/src/unifyfs-dirops.c b/client/src/unifyfs-dirops.c index dcb7ca710..2353c7eb4 100644 --- a/client/src/unifyfs-dirops.c +++ b/client/src/unifyfs-dirops.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -121,6 +121,7 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) unifyfs_filemeta_t* meta = NULL; if (fid >= 0) { meta = unifyfs_get_meta_from_fid(fid); + assert(meta != NULL); /* * FIXME: We found an inconsistent status between local cache and @@ -139,6 +140,7 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) } meta = unifyfs_get_meta_from_fid(fid); + assert(meta != NULL); meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; /* set as directory */ } diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index c17df0e4a..d5520b2d7 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -50,76 +50,6 @@ * Operations on client write index * --------------------------------------- */ -/* Merges write at next_idx into prev_idx if possible. - * Updates both prev_idx and next_idx leaving any - * portion that could not be merged in next_idx */ -static int unifyfs_coalesce_index( - unifyfs_index_t* prev_idx, /* existing index entry to coalesce into */ - unifyfs_index_t* next_idx, /* new index entry we'd like to add */ - long slice_size) /* byte size of slice of key-value store */ -{ - /* check whether last index and next index refer to same file */ - if (prev_idx->gfid != next_idx->gfid) { - /* two index values are for different files, can't combine */ - return UNIFYFS_SUCCESS; - } - - /* got same file, - * check whether last index and next index refer to - * contiguous bytes */ - off_t prev_offset = prev_idx->file_pos + prev_idx->length; - off_t next_offset = next_idx->file_pos; - if (prev_offset != next_offset) { - /* index values are not contiguous, can't combine */ - return UNIFYFS_SUCCESS; - } - - /* got contiguous bytes in the same file, - * check whether both index values fall in the same slice */ - off_t prev_slice = prev_idx->file_pos / slice_size; - off_t next_slice = next_idx->file_pos / slice_size; - if (prev_slice != next_slice) { - /* index values refer to different slices, can't combine */ - return UNIFYFS_SUCCESS; - } - - /* check whether last index and next index refer to - * contiguous bytes in the log */ - off_t prev_log = prev_idx->log_pos + prev_idx->length; - off_t next_log = next_idx->log_pos; - if (prev_log != next_log) { - /* index values are not contiguous in log, can't combine */ - return UNIFYFS_SUCCESS; - } - - /* if we get here, we can coalesce next index value into previous */ - - /* get ending offset of write in next index, - * and ending offset of current slice */ - off_t next_end = next_idx->file_pos + next_idx->length; - off_t slice_end = (next_slice * slice_size) + slice_size; - - /* determine number of bytes in next index that lie in current slice, - * assume all bytes in the index will fit */ - size_t length = next_idx->length; - if (next_end > slice_end) { - /* current index writes beyond end of slice, - * so we can only coalesce bytes that fall within slice */ - length = slice_end - next_offset; - } - - /* extend length of last index to include beginning portion - * of current index */ - prev_idx->length += length; - - /* adjust current index to subtract off those bytes */ - next_idx->file_pos += length; - next_idx->log_pos += length; - next_idx->length -= length; - - return UNIFYFS_SUCCESS; -} - /* * Clear all entries in the log index. This only clears the metadata, * not the data itself. @@ -129,137 +59,58 @@ static void clear_index(void) *unifyfs_indices.ptr_num_entries = 0; } -static void add_index_entry_to_seg_tree(unifyfs_filemeta_t* meta, - unifyfs_index_t* index) +/* Add the metadata for a single write to the index */ +static int add_write_meta_to_index(unifyfs_filemeta_t* meta, + off_t file_pos, + off_t log_pos, + size_t length) { - /* add index to our local log */ + /* add write extent to our segment trees */ if (unifyfs_local_extents) { - seg_tree_add(&meta->extents, index->file_pos, - index->file_pos + index->length - 1, - index->log_pos); + /* record write extent in our local cache */ + seg_tree_add(&meta->extents, + file_pos, + file_pos + length - 1, + log_pos); } - if (!unifyfs_flatten_writes) { - /* We're not flattening writes. Nothing to do */ - return; - } - - /* to update the global running segment count, we need to capture - * the count in this tree before adding and the count after to - * add the difference */ - unsigned long count_before = seg_tree_count(&meta->extents_sync); - - /* - * Store the write in our segment tree. We will later use this for - * flattening writes. - */ - seg_tree_add(&meta->extents_sync, index->file_pos, - index->file_pos + index->length - 1, - index->log_pos); - /* - * We want to make sure the next write following this wont overflow the - * max number of index entries (if it were synced). A write can at most - * create two new nodes in the seg_tree. If we're close to potentially + * We want to make sure this write will not overflow the maximum + * number of index entries we can sync with server. A write can at most + * create two new nodes in the seg_tree. If we're close to potentially * filling up the index, sync it out. */ - if (unifyfs_segment_count >= (unifyfs_max_index_entries - 2)) { + unsigned long count_before = seg_tree_count(&meta->extents_sync); + if (count_before >= (unifyfs_max_index_entries - 2)) { /* this will flush our segments, sync them, and set the running * segment count back to 0 */ - unifyfs_sync(); - } else { - /* increase the running global segment count by the number of - * new entries we added to this tree */ - unsigned long count_after = seg_tree_count(&meta->extents_sync); - unifyfs_segment_count += (count_after - count_before); - } -} - -/* Add the metadata for a single write to the index */ -static int add_write_meta_to_index(unifyfs_filemeta_t* meta, - off_t file_pos, - off_t log_pos, - size_t length) -{ - /* global file id for this entry */ - int gfid = meta->gfid; - - /* define an new index entry for this write operation */ - unifyfs_index_t cur_idx; - cur_idx.gfid = gfid; - cur_idx.file_pos = file_pos; - cur_idx.log_pos = log_pos; - cur_idx.length = length; - - /* lookup number of existing index entries */ - off_t num_entries = *(unifyfs_indices.ptr_num_entries); - - /* get pointer to index array */ - unifyfs_index_t* idxs = unifyfs_indices.index_entry; - - /* attempt to coalesce contiguous index entries if we - * have an existing index in the buffer */ - if (num_entries > 0) { - /* get pointer to last element in index array */ - unifyfs_index_t* prev_idx = &idxs[num_entries - 1]; - - /* attempt to coalesce current index with last index, - * updates fields in last index and current index - * accordingly */ - unifyfs_coalesce_index(prev_idx, &cur_idx, - unifyfs_key_slice_range); - if (cur_idx.length == 0) { - /* We were able to coalesce this write into prev_idx */ - add_index_entry_to_seg_tree(meta, prev_idx); - } + unifyfs_sync(meta->fid); } - /* add new index entry if needed */ - if (cur_idx.length > 0) { - /* remaining entries we can fit in the shared memory region */ - off_t remaining_entries = unifyfs_max_index_entries - num_entries; - - /* if we have filled the key/value buffer, flush it to server */ - if (0 == remaining_entries) { - /* index buffer is full, flush it */ - int ret = unifyfs_sync(); - if (ret != UNIFYFS_SUCCESS) { - /* something went wrong when trying to flush key/values */ - LOGERR("failed to flush key/value index to server"); - return EIO; - } - } - - /* copy entry into index buffer */ - idxs[num_entries] = cur_idx; - - /* Add index entry to our seg_tree */ - add_index_entry_to_seg_tree(meta, &idxs[num_entries]); - - /* account for entries we just added */ - num_entries += 1; - - /* update number of entries in index array */ - (*unifyfs_indices.ptr_num_entries) = num_entries; - } + /* store the write in our segment tree used for syncing with server. */ + seg_tree_add(&meta->extents_sync, + file_pos, + file_pos + length - 1, + log_pos); return UNIFYFS_SUCCESS; } /* * Remove all entries in the current index and re-write it using the write - * metadata stored in all the file's seg_trees. This only re-writes the - * metadata in the index. All the actual data is still kept in the log and - * will be referenced correctly by the new metadata. + * metadata stored in the target file's extents_sync segment tree. This only + * re-writes the metadata in the index. All the actual data is still kept + * in the write log and will be referenced correctly by the new metadata. * - * After this function is done 'unifyfs_indices' will have been totally - * re-written. The writes in the index will be flattened, non-overlapping, - * and sequential, for each file, one after another. All seg_trees will be - * cleared. + * After this function is done, 'unifyfs_indices' will have been totally + * re-written. The writes in the index will be flattened, non-overlapping, + * and sequential. The extents_sync segment tree will be cleared. * - * This function is called when we sync our extents. + * This function is called when we sync our extents with the server. + * + * Returns maximum write log offset for synced extents. */ -void unifyfs_rewrite_index_from_seg_tree(void) +off_t unifyfs_rewrite_index_from_seg_tree(unifyfs_filemeta_t* meta) { /* get pointer to index buffer */ unifyfs_index_t* indexes = unifyfs_indices.index_entry; @@ -270,84 +121,114 @@ void unifyfs_rewrite_index_from_seg_tree(void) /* count up number of entries we wrote to buffer */ unsigned long idx = 0; - /* For each fid .. */ - for (int i = 0; i < UNIFYFS_MAX_FILEDESCS; i++) { - /* get file id for each file descriptor */ - int fid = unifyfs_fds[i].fid; - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (!meta) { - continue; - } - - int gfid = unifyfs_gfid_from_fid(fid); + /* record maximum write log offset */ + off_t max_log_offset = 0; - seg_tree_rdlock(&meta->extents_sync); + int gfid = meta->gfid; - /* For each write in this file's seg_tree ... */ - struct seg_tree_node* node = NULL; - while ((node = seg_tree_iter(&meta->extents_sync, node))) { - indexes[idx].file_pos = node->start; - indexes[idx].log_pos = node->ptr; - indexes[idx].length = node->end - node->start + 1; - indexes[idx].gfid = gfid; - idx++; + seg_tree_rdlock(&meta->extents_sync); + /* For each write in this file's seg_tree ... */ + struct seg_tree_node* node = NULL; + while ((node = seg_tree_iter(&meta->extents_sync, node))) { + indexes[idx].file_pos = node->start; + indexes[idx].log_pos = node->ptr; + indexes[idx].length = node->end - node->start + 1; + indexes[idx].gfid = gfid; + idx++; + if ((off_t)(node->end) > max_log_offset) { + max_log_offset = (off_t) node->end; } - - seg_tree_unlock(&meta->extents_sync); - - /* All done processing this files writes. Clear its seg_tree */ - seg_tree_clear(&meta->extents_sync); } - - /* reset our segment count since we just dumped them all */ - unifyfs_segment_count = 0; + seg_tree_unlock(&meta->extents_sync); + /* All done processing this files writes. Clear its seg_tree */ + seg_tree_clear(&meta->extents_sync); /* record total number of entries in index buffer */ *unifyfs_indices.ptr_num_entries = idx; + + return max_log_offset; } /* - * Sync all the extents to the server. Clears the metadata index afterwards. + * Sync all the write extents for the target file(s) to the server. + * The target_fid identifies a specific file, or all files (-1). + * Clears the metadata index afterwards. * * Returns 0 on success, nonzero otherwise. */ -int unifyfs_sync(void) +int unifyfs_sync(int target_fid) { - /* NOTE: we currently ignore gfid and sync extents for all files in - * the index. If we ever switch to storing extents in a per-file index, - * we can support per-file sync. */ - - /* write contents from segment tree to index buffer - * if we're using that optimization */ - if (unifyfs_flatten_writes) { - unifyfs_rewrite_index_from_seg_tree(); - } + int ret = UNIFYFS_SUCCESS; + off_t max_log_offset = 0; - /* if there are no index entries, we've got nothing to sync */ - if (*unifyfs_indices.ptr_num_entries == 0) { - return UNIFYFS_SUCCESS; - } + /* For each open file descriptor .. */ + for (int i = 0; i < UNIFYFS_MAX_FILEDESCS; i++) { + /* get file id for each file descriptor */ + int fid = unifyfs_fds[i].fid; + if (-1 == fid) { + /* file descriptor is not currently in use */ + continue; + } - /* ensure any data written to the spill over file is flushed */ - int ret = unifyfs_logio_sync(logio_ctx); - if (ret != UNIFYFS_SUCCESS) { - LOGERR("failed to sync logio data"); - return EIO; - } + /* is this the target file? */ + if ((target_fid != -1) && (fid != target_fid)) { + continue; + } + + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if ((NULL == meta) || (meta->fid != fid)) { + LOGERR("missing filemeta for fid=%d", fid); + if (fid == target_fid) { + return UNIFYFS_FAILURE; + } + continue; + } + + if (meta->needs_sync) { + /* write contents from segment tree to index buffer */ + off_t max_log_off = unifyfs_rewrite_index_from_seg_tree(meta); + if (max_log_off > max_log_offset) { + max_log_offset = max_log_off; + } - /* tell the server to grab our new extents */ - ret = invoke_client_sync_rpc(); - if (ret != UNIFYFS_SUCCESS) { - /* something went wrong when trying to flush key/values */ - LOGERR("failed to flush key/value index to server"); - return EIO; + /* if there are no index entries, we've got nothing to sync */ + if (*unifyfs_indices.ptr_num_entries == 0) { + if (fid == target_fid) { + return UNIFYFS_SUCCESS; + } + } + + /* tell the server to grab our new extents */ + ret = invoke_client_sync_rpc(); + if (ret != UNIFYFS_SUCCESS) { + /* something went wrong when trying to flush extents */ + LOGERR("failed to flush write index to server for gfid=%d", + meta->gfid); + } + meta->needs_sync = 0; + + /* flushed, clear buffer and refresh number of entries + * and number remaining */ + clear_index(); + } + + /* break out of loop when targeting a specific file */ + if (fid == target_fid) { + break; + } } - /* flushed, clear buffer and refresh number of entries - * and number remaining */ - clear_index(); + /* ensure any data written to the spillover file is flushed */ + off_t logio_shmem_size; + unifyfs_logio_get_sizes(logio_ctx, &logio_shmem_size, NULL); + if (max_log_offset >= logio_shmem_size) { + ret = unifyfs_logio_sync(logio_ctx); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("failed to sync logio data"); + } + } - return UNIFYFS_SUCCESS; + return ret; } /* --------------------------------------- diff --git a/client/src/unifyfs-fixed.h b/client/src/unifyfs-fixed.h index 3898cfcd3..8086973dc 100644 --- a/client/src/unifyfs-fixed.h +++ b/client/src/unifyfs-fixed.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -45,11 +45,11 @@ #include "unifyfs-internal.h" -/* rewrite client's index of all file write extents */ -void unifyfs_rewrite_index_from_seg_tree(void); +/* rewrite client's shared memory index of file write extents */ +off_t unifyfs_rewrite_index_from_seg_tree(unifyfs_filemeta_t* meta); -/* sync all writes from client's index with local server */ -int unifyfs_sync(void); +/* sync all writes for target file(s) with the server */ +int unifyfs_sync(int target_fid); /* write data to file using log-based I/O */ int unifyfs_fid_logio_write( diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index d6dca5260..ce89a168e 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -276,6 +276,7 @@ typedef struct { int storage; /* FILE_STORAGE type */ + int fid; /* local file index in filemetas array */ int gfid; /* global file id for this file */ int needs_sync; /* have unsynced writes */ @@ -327,9 +328,6 @@ typedef struct { extern unifyfs_index_buf_t unifyfs_indices; extern unsigned long unifyfs_max_index_entries; -/* tracks total number of unsync'd segments for all files */ -extern unsigned long unifyfs_segment_count; - /* shmem context for read-request replies data region */ extern shm_context* shm_recv_ctx; @@ -338,7 +336,6 @@ extern logio_context* logio_ctx; extern int unifyfs_app_id; extern int unifyfs_client_id; -extern size_t unifyfs_key_slice_range; /* ------------------------------- * Global varaible declarations @@ -394,7 +391,6 @@ extern void* unifyfs_dirstream_stack; extern pthread_mutex_t unifyfs_stack_mutex; extern int unifyfs_max_files; /* maximum number of files to store */ -extern bool unifyfs_flatten_writes; /* enable write flattening */ extern bool unifyfs_local_extents; /* enable tracking of local extents */ /* ------------------------------- diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index b8ec89d98..669a88e8a 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -2113,15 +2113,15 @@ static int __chmod(int fid, mode_t mode) /* lookup metadata for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (!meta) { - LOGDBG("chmod: %s no metadata info", path); + if (NULL == meta) { + LOGDBG("no metadata info for %s", path); errno = ENOENT; return -1; } /* Once a file is laminated, you can't modify it in any way */ if (meta->is_laminated) { - LOGDBG("chmod: %s is already laminated", path); + LOGDBG("%s is already laminated", path); errno = EROFS; return -1; } diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 8f8040a99..1e024527a 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -68,9 +68,6 @@ unifyfs_index_buf_t unifyfs_indices; static size_t unifyfs_index_buf_size; /* size of metadata log */ unsigned long unifyfs_max_index_entries; /* max metadata log entries */ -/* tracks total number of unsync'd segments for all files */ -unsigned long unifyfs_segment_count; - int global_rank_cnt; /* count of world ranks */ int client_rank; /* client-provided rank (for debugging) */ @@ -80,7 +77,6 @@ shm_context* shm_recv_ctx; // = NULL int unifyfs_app_id; int unifyfs_client_id; -size_t unifyfs_key_slice_range; static int unifyfs_use_single_shm = 0; static int unifyfs_page_size = 0; @@ -92,7 +88,6 @@ static off_t unifyfs_min_long; /* TODO: moved these to fixed file */ int unifyfs_max_files; /* maximum number of files to store */ -bool unifyfs_flatten_writes; /* flatten our writes (true = enabled) */ bool unifyfs_local_extents; /* track data extents in client to read local */ /* log-based I/O context */ @@ -505,7 +500,10 @@ unifyfs_filemeta_t* unifyfs_get_meta_from_fid(int fid) int unifyfs_fid_is_laminated(int fid) { unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - return meta->is_laminated; + if ((meta != NULL) && (meta->fid == fid)) { + return meta->is_laminated; + } + return 0; } int unifyfs_fd_is_laminated(int fd) @@ -523,11 +521,28 @@ static int fid_store_alloc(int fid) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if ((meta != NULL) && (meta->fid == fid)) { + /* indicate that we're using LOGIO to store data for this file */ + meta->storage = FILE_STORAGE_LOGIO; - /* indicate that we're using LOGIO to store data for this file */ - meta->storage = FILE_STORAGE_LOGIO; + /* Initialize our segment tree that will record our writes */ + int rc = seg_tree_init(&meta->extents_sync); + if (rc != 0) { + return rc; + } - return UNIFYFS_SUCCESS; + /* Initialize our segment tree to track extents for all writes + * by this process, can be used to read back local data */ + if (unifyfs_local_extents) { + rc = seg_tree_init(&meta->extents); + if (rc != 0) { + return rc; + } + } + + return UNIFYFS_SUCCESS; + } + return UNIFYFS_FAILURE; } /* free data management resource for file */ @@ -535,21 +550,21 @@ static int fid_store_free(int fid) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if ((meta != NULL) && (meta->fid == fid)) { + /* set storage type back to NULL */ + meta->storage = FILE_STORAGE_NULL; - /* set storage type back to NULL */ - meta->storage = FILE_STORAGE_NULL; - - /* Free our write seg_tree */ - if (unifyfs_flatten_writes) { + /* Free our write seg_tree */ seg_tree_destroy(&meta->extents_sync); - } - /* Free our extent seg_tree */ - if (unifyfs_local_extents) { - seg_tree_destroy(&meta->extents); - } + /* Free our extent seg_tree */ + if (unifyfs_local_extents) { + seg_tree_destroy(&meta->extents); + } - return UNIFYFS_SUCCESS; + return UNIFYFS_SUCCESS; + } + return UNIFYFS_FAILURE; } /* ======================================= @@ -803,6 +818,7 @@ static void service_local_reqs( /* get pointer to extents for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + assert(meta != NULL); struct seg_tree* extents = &meta->extents; /* lock the extent tree for reading */ @@ -1208,7 +1224,7 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) int unifyfs_fid_is_dir(int fid) { unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (meta && meta->mode & S_IFDIR) { + if ((meta != NULL) && (meta->mode & S_IFDIR)) { return 1; } else { /* if it doesn't exist, then it's not a directory? */ @@ -1225,7 +1241,11 @@ int unifyfs_gfid_from_fid(const int fid) /* return global file id, cached in file meta struct */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - return meta->gfid; + if (meta != NULL) { + return meta->gfid; + } else { + return -1; + } } /* scan list of files and return fid corresponding to target gfid, @@ -1291,7 +1311,7 @@ off_t unifyfs_fid_global_size(int fid) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (NULL != meta) { + if (meta != NULL) { return meta->global_size; } return (off_t)-1; @@ -1361,7 +1381,7 @@ int unifyfs_fid_update_file_meta(int fid, unifyfs_file_attr_t* gfattr) /* lookup local metadata for file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (NULL != meta) { + if (meta != NULL) { /* update lamination state */ meta->is_laminated = gfattr->is_laminated; if (meta->is_laminated) { @@ -1443,6 +1463,7 @@ int unifyfs_set_global_file_meta_from_fid(int fid, int create) /* lookup local metadata for file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + assert(meta != NULL); /* copy our file name */ const char* path = unifyfs_path_from_fid(fid); @@ -1525,38 +1546,21 @@ int unifyfs_fid_create_file(const char* path) /* copy file name into slot */ strcpy((void*)&unifyfs_filelist[fid].filename, path); - LOGDBG("Filename %s got unifyfs fd %d", + LOGDBG("Filename %s got unifyfs fid %d", unifyfs_filelist[fid].filename, fid); /* initialize meta data */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + assert(meta != NULL); meta->global_size = 0; meta->flock_status = UNLOCKED; meta->storage = FILE_STORAGE_NULL; + meta->fid = fid; meta->gfid = unifyfs_generate_gfid(path); meta->needs_sync = 0; meta->is_laminated = 0; meta->mode = UNIFYFS_STAT_DEFAULT_FILE_MODE; - if (unifyfs_flatten_writes) { - /* Initialize our segment tree that will record our writes */ - rc = seg_tree_init(&meta->extents_sync); - if (rc != 0) { - errno = rc; - fid = -1; - } - } - - /* Initialize our segment tree to track extents for all writes - * by this process, can be used to read back local data */ - if (unifyfs_local_extents) { - rc = seg_tree_init(&meta->extents); - if (rc != 0) { - errno = rc; - fid = -1; - } - } - /* PTHREAD_PROCESS_SHARED allows Process-Shared Synchronization */ pthread_spin_init(&meta->fspinlock, PTHREAD_PROCESS_SHARED); @@ -1619,6 +1623,7 @@ int unifyfs_fid_create_directory(const char* path) /* Set as directory */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + assert(meta != NULL); meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; /* insert global meta data for directory */ @@ -1655,6 +1660,7 @@ int unifyfs_fid_write( /* get meta for this file id */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + assert(meta != NULL); /* determine storage type to write file data */ if (meta->storage == FILE_STORAGE_LOGIO) { @@ -1680,6 +1686,7 @@ int unifyfs_fid_truncate(int fid, off_t length) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + assert(meta != NULL); if (meta->is_laminated) { /* Can't truncate a laminated file */ return EINVAL; @@ -1713,16 +1720,10 @@ int unifyfs_fid_sync(int fid) /* sync any writes to disk */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + assert(meta != NULL); if (meta->needs_sync) { - /* TODO: no way to just sync data for this file, - * so we sync every file for now */ /* sync data with server */ - ret = unifyfs_sync(); - - /* just synced writes for this file */ - if (ret == UNIFYFS_SUCCESS) { - meta->needs_sync = 0; - } + ret = unifyfs_sync(fid); } return ret; @@ -2124,11 +2125,10 @@ static int init_superblock_shm(size_t super_sz) if (unifyfs_filelist[i].in_use) { /* got a live file, get pointer to its metadata */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(i); + assert(meta != NULL); /* Reset our segment tree that will record our writes */ - if (unifyfs_flatten_writes) { - seg_tree_init(&meta->extents_sync); - } + seg_tree_init(&meta->extents_sync); /* Reset our segment tree to track extents for all writes * by this process, can be used to read back local data */ @@ -2263,16 +2263,6 @@ static int unifyfs_init(void) } } - /* Determine if we should flatten writes or not */ - unifyfs_flatten_writes = 1; - cfgval = client_cfg.client_flatten_writes; - if (cfgval != NULL) { - rc = configurator_bool_val(cfgval, &b); - if (rc == 0) { - unifyfs_flatten_writes = (bool)b; - } - } - /* Determine if we should track all write extents and use them * to service read requests if all data is local */ unifyfs_local_extents = 0; @@ -2627,6 +2617,14 @@ int unifyfs_unmount(void) return UNIFYFS_SUCCESS; } + /* sync any outstanding writes */ + LOGDBG("syncing data"); + rc = unifyfs_sync(-1); + if (rc) { + LOGERR("client sync failed"); + ret = UNIFYFS_FAILURE; + } + /************************ * tear down connection to server ************************/ diff --git a/common/src/unifyfs_client_rpcs.h b/common/src/unifyfs_client_rpcs.h index 16db3d167..900ec10b3 100644 --- a/common/src/unifyfs_client_rpcs.h +++ b/common/src/unifyfs_client_rpcs.h @@ -1,5 +1,19 @@ -#ifndef __UNIFYFS_CLIENT_RPCS_H -#define __UNIFYFS_CLIENT_RPCS_H +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef UNIFYFS_CLIENT_RPCS_H +#define UNIFYFS_CLIENT_RPCS_H /* * Declarations for client-server margo RPCs (shared-memory) @@ -40,7 +54,6 @@ MERCURY_GEN_PROC(unifyfs_mount_in_t, ((hg_const_string_t)(mount_prefix)) ((hg_const_string_t)(client_addr_str))) MERCURY_GEN_PROC(unifyfs_mount_out_t, - ((hg_size_t)(meta_slice_sz)) ((int32_t)(app_id)) ((int32_t)(client_id)) ((int32_t)(ret))) @@ -102,9 +115,9 @@ DECLARE_MARGO_RPC_HANDLER(unifyfs_metaget_rpc) /* unifyfs_sync_rpc (client => server) * - * given app_id and client_id as input, read all write extents - * from client index in shared memory and insert corresponding - * key/value pairs into our global metadata */ + * given a client identified by (app_id, client_id) as input, read the write + * extents for one or more of the client's files from the shared memory index + * and update the global metadata for the file(s) */ MERCURY_GEN_PROC(unifyfs_sync_in_t, ((int32_t)(app_id)) ((int32_t)(client_id))) diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index 27787706b..1f6f7bf5e 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -70,7 +70,6 @@ UNIFYFS_CFG_CLI(unifyfs, daemonize, BOOL, on, "enable server daemonization", NULL, 'D', "on|off") \ UNIFYFS_CFG_CLI(unifyfs, mountpoint, STRING, /unifyfs, "mountpoint directory", NULL, 'm', "specify full path to desired mountpoint") \ UNIFYFS_CFG(client, max_files, INT, UNIFYFS_MAX_FILES, "client max file count", NULL) \ - UNIFYFS_CFG(client, flatten_writes, BOOL, on, "flatten writes", NULL) \ UNIFYFS_CFG(client, local_extents, BOOL, off, "track extents to service reads of local data", NULL) \ UNIFYFS_CFG(client, recv_data_size, INT, UNIFYFS_DATA_RECV_SIZE, "shared memory segment size in bytes for receiving data from server", NULL) \ UNIFYFS_CFG(client, write_index_size, INT, UNIFYFS_INDEX_BUF_SIZE, "write metadata index buffer size", NULL) \ diff --git a/common/src/unifyfs_logio.c b/common/src/unifyfs_logio.c index b6d38482d..6bdeb2a73 100644 --- a/common/src/unifyfs_logio.c +++ b/common/src/unifyfs_logio.c @@ -805,3 +805,31 @@ int unifyfs_logio_sync(logio_context* ctx) } return UNIFYFS_SUCCESS; } + +/* Get the shmem and spill data sizes */ +int unifyfs_logio_get_sizes(logio_context* ctx, + off_t* shmem_sz, + off_t* spill_sz) +{ + if (NULL == ctx) { + return EINVAL; + } + + if (NULL != shmem_sz) { + *shmem_sz = 0; + if (NULL != ctx->shmem) { + log_header* shmem_hdr = (log_header*) ctx->shmem->addr; + *shmem_sz = (off_t) shmem_hdr->data_sz; + } + } + + if (NULL != spill_sz) { + *spill_sz = 0; + if (NULL != ctx->spill_hdr) { + log_header* spill_hdr = (log_header*) ctx->spill_hdr; + *spill_sz = (off_t) spill_hdr->data_sz; + } + } + + return UNIFYFS_SUCCESS; +} diff --git a/common/src/unifyfs_logio.h b/common/src/unifyfs_logio.h index 10a5a3135..2cdd1e2a2 100644 --- a/common/src/unifyfs_logio.h +++ b/common/src/unifyfs_logio.h @@ -136,6 +136,18 @@ int unifyfs_logio_write(logio_context* ctx, */ int unifyfs_logio_sync(logio_context* ctx); +/** + * Get the shmem and spill data sizes. + * + * @param ctx pointer to logio context + * @param[out] shmem_sz if non-NULL, set to size of shmem data storage + * @param[out] spill_sz if non-NULL, set to size of spillover data storage + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_logio_get_sizes(logio_context* ctx, + off_t* shmem_sz, + off_t* spill_sz); + #ifdef __cplusplus } // extern "C" #endif diff --git a/docs/add-rpcs.rst b/docs/add-rpcs.rst index 3fa370e70..4c95951ae 100644 --- a/docs/add-rpcs.rst +++ b/docs/add-rpcs.rst @@ -30,19 +30,10 @@ Common .. code-block:: C MERCURY_GEN_PROC(unifyfs_mount_in_t, - ((int32_t)(client_id)) ((int32_t)(dbg_rank)) - ((hg_size_t)(shmem_data_size)) - ((hg_size_t)(shmem_super_size)) - ((hg_size_t)(meta_offset)) - ((hg_size_t)(meta_size)) - ((hg_size_t)(logio_mem_size)) - ((hg_size_t)(logio_spill_size)) - ((hg_const_string_t)(logio_spill_dir)) ((hg_const_string_t)(mount_prefix)) ((hg_const_string_t)(client_addr_str))) MERCURY_GEN_PROC(unifyfs_mount_out_t, - ((hg_size_t)(meta_slice_sz)) ((int32_t)(app_id)) ((int32_t)(client_id)) ((int32_t)(ret))) diff --git a/docs/configuration.rst b/docs/configuration.rst index 9e317fb23..42832d291 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -68,7 +68,6 @@ a given section and key. ================ ====== ================================================================= cwd STRING effective starting current working directory max_files INT maximum number of open files per client process (default: 128) - flatten_writes BOOL enable flattening writes (optimization for overwrite-heavy codes) local_extents BOOL service reads from local data if possible (default: off) recv_data_size INT maximum size (B) of memory buffer for receiving data from server write_index_size INT maximum size (B) of memory buffer for storing write log metadata diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index cd08893dd..72e22124f 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -99,7 +99,6 @@ static void unifyfs_mount_rpc(hg_handle_t handle) /* build output structure to return to caller */ unifyfs_mount_out_t out; - out.meta_slice_sz = meta_slice_sz; out.app_id = (int32_t) app_id; out.client_id = (int32_t) client_id; out.ret = ret; @@ -279,9 +278,9 @@ static void unifyfs_metaset_rpc(hg_handle_t handle) } DEFINE_MARGO_RPC_HANDLER(unifyfs_metaset_rpc) -/* given app_id and client_id as input, read all extents from client - * write index in shared memory and insert corresponding key/value pairs - * into the global metadata */ +/* given a client identified by (app_id, client_id) as input, read the write + * extents for one or more of the client's files from the shared memory index + * and update the global metadata for the file(s) */ static void unifyfs_sync_rpc(hg_handle_t handle) { /* get input params */ @@ -289,12 +288,12 @@ static void unifyfs_sync_rpc(hg_handle_t handle) hg_return_t hret = margo_get_input(handle, &in); assert(hret == HG_SUCCESS); - /* given global file id, read index metadata from client and - * insert into global index key/value store */ + /* read the write indices for all client files from shmem and + * propagate their extents to our global metadata */ int ret = rm_cmd_sync(in.app_id, in.client_id); /* build our output values */ - unifyfs_metaset_out_t out; + unifyfs_sync_out_t out; out.ret = ret; /* return to caller */ @@ -307,7 +306,6 @@ static void unifyfs_sync_rpc(hg_handle_t handle) } DEFINE_MARGO_RPC_HANDLER(unifyfs_sync_rpc) - /* given an app_id, client_id, global file id, * return current file size */ static void unifyfs_filesize_rpc(hg_handle_t handle) @@ -370,7 +368,7 @@ DEFINE_MARGO_RPC_HANDLER(unifyfs_truncate_rpc) static void unifyfs_unlink_rpc(hg_handle_t handle) { /* get input params */ - unifyfs_truncate_in_t in; + unifyfs_unlink_in_t in; hg_return_t hret = margo_get_input(handle, &in); assert(hret == HG_SUCCESS); @@ -378,7 +376,7 @@ static void unifyfs_unlink_rpc(hg_handle_t handle) int ret = rm_cmd_unlink(in.app_id, in.client_id, in.gfid); /* build our output values */ - unifyfs_truncate_out_t out; + unifyfs_unlink_out_t out; out.ret = (int32_t) ret; /* return to caller */ @@ -396,7 +394,7 @@ DEFINE_MARGO_RPC_HANDLER(unifyfs_unlink_rpc) static void unifyfs_laminate_rpc(hg_handle_t handle) { /* get input params */ - unifyfs_truncate_in_t in; + unifyfs_laminate_in_t in; hg_return_t hret = margo_get_input(handle, &in); assert(hret == HG_SUCCESS); @@ -404,7 +402,7 @@ static void unifyfs_laminate_rpc(hg_handle_t handle) int ret = rm_cmd_laminate(in.app_id, in.client_id, in.gfid); /* build our output values */ - unifyfs_truncate_out_t out; + unifyfs_laminate_out_t out; out.ret = (int32_t) ret; /* return to caller */ diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index f5aed3f11..65333b26b 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -1550,7 +1550,7 @@ int rm_cmd_sync(int app_id, int client_id) (NULL == val_lens)) { LOGERR("failed to allocate memory for file extents"); ret = ENOMEM; - goto rm_cmd_fsync_exit; + goto rm_cmd_sync_exit; } /* create file extent key/values for insertion into MDHIM */ @@ -1579,10 +1579,10 @@ int rm_cmd_sync(int app_id, int client_id) if (ret != UNIFYFS_SUCCESS) { /* TODO: need proper error handling */ LOGERR("unifyfs_set_file_extents() failed"); - goto rm_cmd_fsync_exit; + goto rm_cmd_sync_exit; } -rm_cmd_fsync_exit: +rm_cmd_sync_exit: /* clean up memory */ if (NULL != keys) { From 4998aabf1450f925bffc2b4f901645a67322bbf4 Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Thu, 25 Jun 2020 12:19:59 -0400 Subject: [PATCH 137/168] Fixes the problem of request manager becoming unresponsive, when handling a read request that needs to fetch data from local and remote servers. - Fix the request manager (removing the PARTIAL read status) - An example to examine the read operations --- examples/src/Makefile.am | 17 +- examples/src/read-data.c | 269 +++++++++++++++++++++++++++ server/src/unifyfs_global.h | 3 +- server/src/unifyfs_request_manager.c | 6 +- 4 files changed, 287 insertions(+), 8 deletions(-) create mode 100644 examples/src/read-data.c diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index 5161dae80..790834399 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -25,7 +25,8 @@ if HAVE_LD_WRAP size-static \ simul-static \ chmod-static \ - multi-write-static + multi-write-static \ + read-data-static endif if HAVE_GOTCHA @@ -46,9 +47,10 @@ if HAVE_GOTCHA app-tileio-gotcha \ transfer-gotcha \ size-gotcha \ - simul-gotcha \ + simul-gotcha \ chmod-gotcha \ - multi-write-gotcha + multi-write-gotcha \ + read-data-gotcha endif if HAVE_FORTRAN @@ -338,3 +340,12 @@ multi_write_static_CPPFLAGS = $(test_cppflags) multi_write_static_LDADD = $(test_static_ldadd) multi_write_static_LDFLAGS = $(test_static_ldflags) +read_data_gotcha_SOURCES = read-data.c +read_data_gotcha_CPPFLAGS = $(test_cppflags) +read_data_gotcha_LDADD = $(test_gotcha_ldadd) +read_data_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +read_data_static_SOURCES = read-data.c +read_data_static_CPPFLAGS = $(test_cppflags) +read_data_static_LDADD = $(test_static_ldadd) +read_data_static_LDFLAGS = $(test_static_ldflags) diff --git a/examples/src/read-data.c b/examples/src/read-data.c new file mode 100644 index 000000000..49f72c666 --- /dev/null +++ b/examples/src/read-data.c @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ +/* read-data: This program aims to test reading data from a file with arbitrary + * offset and length. This program can run either interactively (only + * specifying filename) or non-interactively (specifying filename with offset + * and length). + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "testutil.h" + +#define DEFAULT_MOUNTPOINT "/unifyfs" + +static char filename[PATH_MAX]; +static char mountpoint[PATH_MAX]; + +static char* buf; +static unsigned int bufsize; + +static int parse_line(char* line, unsigned long* offset, unsigned long* length) +{ + char* pos = NULL; + + line[strlen(line)-1] = '\0'; + + if (strncmp("quit", line, strlen("quit")) == 0) { + return 1; + } + + pos = strchr(line, ','); + if (!pos) { + return -1; + } + + *pos = '\0'; + pos++; + + *offset = strtoul(line, NULL, 0); + *length = strtoul(pos, NULL, 0); + + return 0; +} + +static void alloc_buf(unsigned long length) +{ + if (!buf) { + bufsize = length; + buf = malloc(bufsize); + } else { + if (bufsize < length) { + buf = realloc(buf, length); + } + } + + if (!buf) { + perror("failed to allocate buffer"); + exit(1); + } +} + +static void do_pread(int fd, size_t length, off_t offset) +{ + int ret = 0; + + alloc_buf(length); + + errno = 0; + + ret = pread(fd, buf, length, offset); + + printf(" -> pread(off=%lu, len=%lu) = %d", offset, length, ret); + if (errno) { + printf(" (err=%d, %s)\n", errno, strerror(errno)); + } else { + printf("\n"); + } +} + +static void run_interactive(int fd) +{ + int ret = 0; + unsigned long offset = 0; + unsigned long length = 0; + char* line = NULL; + char linebuf[LINE_MAX]; + + while (1) { + printf("\nread (offset,length)> "); + fflush(stdout); + line = fgets(linebuf, LINE_MAX-1, stdin); + if (!line) { + continue; + } + + ret = parse_line(line, &offset, &length); + if (ret < 0) { + continue; + } else if (1 == ret) { + printf("terminating..\n"); + break; + } + + do_pread(fd, length, offset); + } +} + +static struct option long_opts[] = { + { "help", 0, 0, 'h' }, + { "mount", 1, 0, 'm' }, + { "offset", 1, 0, 'o' }, + { "length", 1, 0, 'l' }, + { 0, 0, 0, 0}, +}; + +static char* short_opts = "hm:o:l:"; + +static const char* usage_str = +"\n" +"Usage: %s [options...] \n" +"\n" +"Test reading data from a file . should be a full\n" +"pathname. If running without --offset and --length options, this will run\n" +"in an interactive mode where the offset and length should be specified\n" +"with a separating comma between them, e.g., ','.\n" +"'quit' will terminate the program.\n" +"\n" +"Available options:\n" +" -h, --help help message\n" +" -m, --mount= use for unifyfs (default: /unifyfs)\n" +" -o, --offset= read from \n" +" -l, --length= read bytes\n" +"\n"; + +static char* program; + +static void print_usage(void) +{ + printf(usage_str, program); + exit(0); +} + +int main(int argc, char** argv) +{ + int ret = 0; + int ch = 0; + int optidx = 0; + int unifyfs = 0; + int fd = -1; + unsigned long offset = 0; + unsigned long length = 0; + struct stat sb; + char* tmp_program = NULL; + + tmp_program = strdup(argv[0]); + if (!tmp_program) { + perror("failed to allocate memory"); + return -1; + } + + program = basename(tmp_program); + + while ((ch = getopt_long(argc, argv, + short_opts, long_opts, &optidx)) >= 0) { + switch (ch) { + case 'm': + sprintf(mountpoint, "%s", optarg); + break; + + case 'o': + offset = strtoul(optarg, NULL, 0); + break; + + case 'l': + length = strtoul(optarg, NULL, 0); + break; + + case 'h': + default: + print_usage(); + break; + } + } + + if (argc - optind != 1) { + print_usage(); + return -1; + } + + sprintf(filename, "%s", argv[optind]); + + if (mountpoint[0] == '\0') { + sprintf(mountpoint, "%s", DEFAULT_MOUNTPOINT); + } + + if (strncmp(filename, mountpoint, strlen(mountpoint)) == 0) { + printf("mounting unifyfs at %s ..\n", mountpoint); + + ret = unifyfs_mount(mountpoint, 0, 1, 0); + if (ret) { + fprintf(stderr, "unifyfs_mount failed (return = %d)\n", ret); + return -1; + } + + unifyfs = 1; + } + + fd = open(filename, O_RDONLY); + if (fd < 0) { + perror("open failed"); + return -1; + } + + ret = stat(filename, &sb); + if (ret < 0) { + perror("stat failed"); + goto out; + } + + printf("%s (size = %lu)\n", filename, sb.st_size); + + if (offset == 0 && length == 0) { + run_interactive(fd); + } else { + do_pread(fd, length, offset); + } + + ret = 0; +out: + close(fd); + + if (buf) { + free(buf); + } + + if (unifyfs) { + unifyfs_unmount(); + } + + if (tmp_program) { + free(tmp_program); + } + + return ret; +} diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index 28e911917..653bf00a4 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -91,8 +91,7 @@ typedef enum { READREQ_NULL = 0, /* request not initialized */ READREQ_READY, /* request ready to be issued */ READREQ_STARTED, /* chunk requests issued */ - READREQ_PARTIAL_COMPLETE, /* some reads completed */ - READREQ_COMPLETE /* all reads completed */ + READREQ_COMPLETE, /* all reads completed */ } readreq_status_e; typedef struct { diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 65333b26b..36d263ecd 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -1961,15 +1961,15 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, } data_buf += data_sz; } + /* cleanup */ free((void*)responses); del_reads->resp = NULL; /* update request status */ del_reads->status = READREQ_COMPLETE; - if (rdreq->status == READREQ_STARTED) { - rdreq->status = READREQ_PARTIAL_COMPLETE; - } + + /* if all remote reads are complete, mark the request as complete */ int completed_remote_reads = 0; for (i = 0; i < rdreq->num_remote_reads; i++) { if (rdreq->remote_reads[i].status != READREQ_COMPLETE) { From 4ac866bd8befb8fecd357bddf7817836be76efda Mon Sep 17 00:00:00 2001 From: CamStan Date: Wed, 24 Jun 2020 13:27:57 -0700 Subject: [PATCH 138/168] Update Gitlab CI for recent changes This better organizes and updates our Gitlab CI files and test suite for recent changes to how Gitlab CI works and how UnifyFS works. Jobs specific to individual systems are now organized in their own .gitlab/.yml files and templates are better organized. For dependencies, this uses Spack to load the appropriate dependencies in the environment before building/testing. However, this wont install any new dependencies when needed to avoid interfering with the user's home environment. So to test with new versions or compilers, the dependencies will need to be installed in the home environment that Gitlab uses first. This can be adjusted once Gitlab has the service user feature working. --- .gitlab-ci.yml | 177 +++++++++++++++++------------------- .gitlab/catalyst.yml | 45 +++++++++ .gitlab/lassen.yml | 45 +++++++++ docs/testing.rst | 9 -- t/ci/001-setup.sh | 12 +-- t/ci/002-start-server.sh | 4 +- t/ci/100-writeread-tests.sh | 16 ++-- t/ci/110-write-tests.sh | 16 ++-- t/ci/120-read-tests.sh | 16 ++-- 9 files changed, 206 insertions(+), 134 deletions(-) create mode 100644 .gitlab/catalyst.yml create mode 100644 .gitlab/lassen.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6b4632af2..a26a75440 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,46 +1,69 @@ +# Both testing stages depend on the build stage. By using the "needs" +# keyword, we prevent the testing stages from blocking, in favor of a +# DAG. stages: - build - test-unit - test-integ -cache: - paths: - - spack_ci/ - -##### Templates ##### - -.catalyst-template: &catalyst_template - tags: - - catalyst - - shell - variables: - LLNL_SERVICE_USER: "unifysrv" +##### System Templates ##### + +# Generic system templates used to contruct the final jobs on specific +# systems within their respective .yml file. Currently +# these are LLNL specific, but can be adjusted or added to as new +# systems become available. +# +# The NNODES, WALL_TIME, and STORAGE_SIZE variables can be altered in +# Gitlab interface if/when the defaults need to be changed. + +.base-template: + rules: + - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "dev"' + - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "dev" && $CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "dev" && $CI_PIPELINE_SOURCE == "external_pull_request_event"' retry: max: 1 when: - unknown_failure - stuck_or_timeout_failure -.butte-template: &butte_template - tags: - - butte - - shell +.slurm-single-node-template: + extends: .base-template variables: - LLNL_SERVICE_USER: "unifysrv" - retry: - max: 1 - when: - - unknown_failure - - stuck_or_timeout_failure + LLNL_SLURM_SCHEDULER_PARAMETERS: "-N 1 -p pbatch -t $UNIT_WALL_TIME -J unifyfs-unit-tests" + +.slurm-multi-node-template: + extends: .base-template + variables: + LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -p pbatch -t $INTEG_WALL_TIME -J unifyfs-integ-tests" + +.lsf-single-node-template: + extends: .base-template + variables: + LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -q pbatch -W $UNIT_WALL_TIME -J unifyfs-unit-tests" + +.lsf-multi-node-template: + extends: .base-template + variables: + LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes $NNODES -stage storage=${STORAGE_SIZE} -q pbatch -W $INTEG_WALL_TIME -J unifyfs-integ-tests" -.build-template: &build_template +##### Job Templates ##### + +# Build script used by each system. The CC and FC variables are set in +# the specific job scripts and evaluated in the before_script in order +# to customize which compiler will be used for each job. +# An artifact is created to pass on to the testing stages. The +# test-unit stage requires the unifyfs-build/ files and the test-integ +# stage requires the unifyfs-install/ files. +.build-template: stage: build script: - ./autogen.sh - mkdir -p unifyfs-build unifyfs-install && cd unifyfs-build - - ../configure --prefix=$CI_PROJECT_DIR/unifyfs-install --enable-fortran --disable-silent-rules + - ../configure CC=$CC_PATH FC=$FC_PATH --prefix=$CI_PROJECT_DIR/unifyfs-install --enable-fortran --disable-silent-rules - make V=1 - make V=1 install + needs: [] artifacts: name: "${CI_JOB_NAME}-${CI_PIPELINE_ID}" untracked: true @@ -49,84 +72,54 @@ cache: - unifyfs-build/ - unifyfs-install/ -.unit-test-template: &unit_test_template +.unit-test-template: stage: test-unit script: - cd unifyfs-build/t && make check after_script: - - rm -rf /tmp/unify* /tmp/tmp.* /tmp/mdhim* /tmp/na_sm + - rm -rf /tmp/unify* /tmp/tmp.* /tmp/mdhim* /tmp/na_sm | true -.catalyst-batch-variables: - variables: &catalyst_batch_variables - LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -p pbatch -t $WALL_TIME" - LLNL_SERVICE_USER: "unifysrv" - CI_PROJDIR: "$CI_PROJECT_DIR" +# Variables here are used for the integration test suite and can be +# adjusted in the Gitlab interface. See our testing documentation for +# full details. +.integ-test-template: + stage: test-integ + variables: UNIFYFS_INSTALL: "$CI_PROJECT_DIR/unifyfs-install" - CI_NPROCS: "$NPROCS" - -.butte-batch-variables: - variables: &butte_batch_variables - LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes $NNODES -q pbatch -W $WALL_TIME" - LLNL_SERVICE_USER: "unifysrv" CI_PROJDIR: "$CI_PROJECT_DIR" - UNIFYFS_INSTALL: "$CI_PROJECT_DIR/unifyfs-install" CI_NPROCS: "$NPROCS" + script: + - cd t/ci && prove -v RUN_CI_TESTS.sh ##### Jobs ##### +# Since Gitlab currently runs in the user's home environment, the +# before_script is currently only set up to load the proper Spack +# modules, if they are available, to prevent changing the user's +# environment. Install any needed modules in the user's environment +# prior to running when new compilers or architectures need to be +# tested. +# +# The COMPILER, CC_PATH, and FC_PATH variables are evaluated here. Set +# them in their specific job scripts. +# SPACK_COMPILER and SPACK_ARCH are then set to load the matching +# dependencies for the desired compiler. before_script: - # HERE BE DRAGONS!: Since on HPC and running as user, Spack might already - # exist and can get complicated if we install it again. - # - # check for sourced spack || check for unsourced spack in $HOME/spack and - # source it || check for cached spack, clone if none, and source it - - which spack || ((cd $HOME/spack && git describe) && . $HOME/spack/share/spack/setup-env.sh) || (((cd spack_ci && git describe) || git clone https://github.com/CamStan/spack spack_ci) && . spack_ci/share/spack/setup-env.sh) + - which spack || ((cd $HOME/spack && git describe) && . $HOME/spack/share/spack/setup-env.sh) + - module load $COMPILER + - CC_PATH=$($CC_COMMAND) + - FC_PATH=$($FC_COMMAND) + - SPACK_COMPILER=${COMPILER//\//@} - SPACK_ARCH="$(spack arch -p)-$(spack arch -o)-$(uname -m)" - - spack install leveldb && spack load leveldb arch=$SPACK_ARCH - - spack install gotcha@0.0.2 && spack load gotcha@0.0.2 arch=$SPACK_ARCH - - spack install flatcc && spack load flatcc arch=$SPACK_ARCH - - spack install margo^mercury+bmi~boostsys^argobots~debug && spack load argobots arch=$SPACK_ARCH && spack load mercury arch=$SPACK_ARCH && spack load margo arch=$SPACK_ARCH - -build-catalyst: - <<: *catalyst_template - <<: *build_template - -build-butte: - <<: *butte_template - <<: *build_template - -unit-test-catalyst: - <<: *catalyst_template - <<: *unit_test_template - dependencies: - - build-catalyst - -unit-test-butte: - <<: *butte_template - <<: *unit_test_template - dependencies: - - build-butte - -integ-test-catalyst: - <<: *catalyst_template - stage: test-integ - tags: - - catalyst - - batch - variables: *catalyst_batch_variables - script: - - cd t/ci && prove -v RUN_CI_TESTS.sh - dependencies: - - build-catalyst - -integ-test-butte: - <<: *butte_template - stage: test-integ - tags: - - butte - - batch - variables: *butte_batch_variables - script: - - cd t/ci && prove -v RUN_CI_TESTS.sh - dependencies: - - build-butte + - spack load flatcc %$SPACK_COMPILER arch=$SPACK_ARCH + - spack load gotcha %$SPACK_COMPILER arch=$SPACK_ARCH + - spack load leveldb %$SPACK_COMPILER arch=$SPACK_ARCH + - spack load argobots %$SPACK_COMPILER arch=$SPACK_ARCH + - spack load mercury %$SPACK_COMPILER arch=$SPACK_ARCH + - spack load margo %$SPACK_COMPILER arch=$SPACK_ARCH + - spack load spath %$SPACK_COMPILER arch=$SPACK_ARCH + +# System specific jobs +include: + - local: .gitlab/catalyst.yml + - local: .gitlab/lassen.yml diff --git a/.gitlab/catalyst.yml b/.gitlab/catalyst.yml new file mode 100644 index 000000000..4eb7d19ee --- /dev/null +++ b/.gitlab/catalyst.yml @@ -0,0 +1,45 @@ +# Catalyst Templates + +# The RUN_CATALYST variable can be toggled in the Gitlab interface to +# toggle whether jobs should be run on this system. +.catalyst-template: + rules: + - if: '$RUN_CATALYST != "ON"' + when: never + +.catalyst-shell-template: + extends: .catalyst-template + tags: + - catalyst + - shell + +.catalyst-batch-template: + extends: .catalyst-template + tags: + - catalyst + - batch + +##### All Catalyst Jobs ##### + +catalyst-gcc-4_9_3-build: + variables: + COMPILER: gcc/4.9.3 + CC_COMMAND: "which gcc" + FC_COMMAND: "which gfortran" + extends: [.catalyst-shell-template, .build-template] + +catalyst-gcc-4_9_3-unit-test: + variables: + COMPILER: gcc/4.9.3 + CC_COMMAND: "which gcc" + FC_COMMAND: "which gfortran" + extends: [.slurm-single-node-template, .catalyst-batch-template, .unit-test-template] + needs: ["catalyst-gcc-4_9_3-build"] + +catalyst-gcc-4_9_3-integ-test: + variables: + COMPILER: gcc/4.9.3 + CC_COMMAND: "which gcc" + FC_COMMAND: "which gfortran" + extends: [.slurm-multi-node-template, .catalyst-batch-template, .integ-test-template] + needs: ["catalyst-gcc-4_9_3-build"] diff --git a/.gitlab/lassen.yml b/.gitlab/lassen.yml new file mode 100644 index 000000000..7caf6e0fa --- /dev/null +++ b/.gitlab/lassen.yml @@ -0,0 +1,45 @@ +##### Lassen Templates ##### + +# The RUN_LASSEN variable can be toggled in the Gitlab interface to +# toggle whether jobs should be run on this system. +.lassen-template: + rules: + - if: '$RUN_LASSEN != "ON"' + when: never + +.lassen-shell-template: + extends: .lassen-template + tags: + - lassen + - shell + +.lassen-batch-template: + extends: .lassen-template + tags: + - lassen + - batch + +##### All Lassen Jobs ##### + +lassen-gcc-4_9_3-build: + variables: + COMPILER: gcc/4.9.3 + CC_COMMAND: "which gcc" + FC_COMMAND: "which gfortran" + extends: [.lassen-shell-template, .build-template] + +lassen-gcc-4_9_3-unit-test: + variables: + COMPILER: gcc/4.9.3 + CC_COMMAND: "which gcc" + FC_COMMAND: "which gfortran" + extends: [.lsf-single-node-template, .lassen-batch-template, .unit-test-template] + needs: ["lassen-gcc-4_9_3-build"] + +lassen-gcc-4_9_3-integ-test: + variables: + COMPILER: gcc/4.9.3 + CC_COMMAND: "which gcc" + FC_COMMAND: "which gfortran" + extends: [.lsf-multi-node-template, .lassen-batch-template, .integ-test-template] + needs: ["lassen-gcc-4_9_3-build"] diff --git a/docs/testing.rst b/docs/testing.rst index 1816c199e..d235b7cc8 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -576,15 +576,6 @@ Can be used as a shortcut to set ``UNIFYFS_RUNSTATE_DIR`` and ``UNIFYFS_META_DB_PATH`` to the same path. This envar defaults to ``CI_TEMP_DIR=${TMPDIR}/unifyfs.${USER}.${JOB_ID}``. -``CI_STORAGE_DIR`` -""""""""""""""""""" - -USAGE: ``CI_STORAGE_DIR=/path/for/storage/files/`` - -Can be used as a shortcut to set ``UNIFYFS_SPILLOVER_DATA_DIR`` and -``UNIFYFS_SPILLOVER_META_DIR`` to the same path. This envar defaults to -``CI_STORAGE_DIR=${TMPDIR}/unifyfs.${USER}.${JOB_ID}``. - ``CI_TEST_POSIX`` """"""""""""""""" diff --git a/t/ci/001-setup.sh b/t/ci/001-setup.sh index 54b9a8689..9e60aaa30 100755 --- a/t/ci/001-setup.sh +++ b/t/ci/001-setup.sh @@ -204,14 +204,10 @@ echo "$infomsg Set CI_TEMP_DIR to change both of these to same path" # storage nls=$nlt -export CI_STORAGE_DIR=${CI_STORAGE_DIR:-$nls} -export UNIFYFS_SPILLOVER_SIZE=${UNIFYFS_SPILLOVER_SIZE:-$((5 * GB))} -export UNIFYFS_SPILLOVER_ENABLED=${UNIFYFS_SPILLOVER_ENABLED:-yes} -export UNIFYFS_SPILLOVER_DATA_DIR=${UNIFYFS_SPILLOVER_DATA_DIR:-$CI_STORAGE_DIR} -export UNIFYFS_SPILLOVER_META_DIR=${UNIFYFS_SPILLOVER_META_DIR:-$CI_STORAGE_DIR} -echo "$infomsg UNIFYFS_SPILLOVER_DATA_DIR set as $UNIFYFS_SPILLOVER_DATA_DIR" -echo "$infomsg UNIFYFS_SPILLOVER_META_DIR set as $UNIFYFS_SPILLOVER_META_DIR" -echo "$infomsg Set CI_STORAGE_DIR to change both of these to same path" +export UNIFYFS_LOGIO_SPILL_SIZE=${UNIFYFS_LOGIO_SPILL_SIZE:-$((5 * GB))} +export UNIFYFS_LOGIO_SPILL_DIR=${UNIFYFS_LOGIO_SPILL_DIR:-$nls} +echo "$infomsg UNIFYFS_LOGIO_SPILL_SIZE set as $UNIFYFS_LOGIO_SPILL_SIZE" +echo "$infomsg UNIFYFS_LOGIO_SPILL_DIR set as $UNIFYFS_LOGIO_SPILL_DIR" ########## Set up mountpoints and sharness testing prereqs ########## diff --git a/t/ci/002-start-server.sh b/t/ci/002-start-server.sh index 8d9d0a4a5..9d962fd19 100755 --- a/t/ci/002-start-server.sh +++ b/t/ci/002-start-server.sh @@ -55,7 +55,7 @@ test_expect_success "unifyfsd hasn't started yet" ' ' $UNIFYFS_BIN/unifyfs start -c -d -S $UNIFYFS_SHAREDFS_DIR \ - -e $UNIFYFS_BIN/unifyfsd &> ${UNIFYFS_LOG_DIR}/unifyfs.start.out & + -e $UNIFYFS_BIN/unifyfsd &> ${UNIFYFS_LOG_DIR}/unifyfs.start.out test_expect_success "unifyfsd started" ' process_is_running unifyfsd 10 || @@ -95,3 +95,5 @@ clean_fail() { cleanup_hosts } trap 'clean_fail $BASH_SOURCE' EXIT + +#TODO: can't call cleanup_hosts if PDSH prereq isn't set... diff --git a/t/ci/100-writeread-tests.sh b/t/ci/100-writeread-tests.sh index 4d5ad4cf4..7bc8bd18c 100755 --- a/t/ci/100-writeread-tests.sh +++ b/t/ci/100-writeread-tests.sh @@ -215,18 +215,18 @@ unify_test_writeread $runmode "$app_args" runmode=static unify_test_writeread $runmode "$app_args" -# Increase sizes: -n 32 -c 1MB -b 16MB +# Increase sizes: -n 32 -c 4MB -b 16MB -# writeread-static -p n1 -n 32 -c 1MB -b 16MB -io_sizes="-n 32 -c $MB -b $((16 * $MB))" +# writeread-static -p n1 -n 32 -c 4MB -b 16MB +io_sizes="-n 32 -c $((4 * $MB)) -b $((16 * $MB))" app_args="$io_pattern $io_sizes" unify_test_writeread $runmode "$app_args" -# writeread-gotcha -p n1 -n 32 -c 1MB -b 16MB +# writeread-gotcha -p n1 -n 32 -c 4MB -b 16MB runmode=gotcha unify_test_writeread $runmode "$app_args" -# writeread-posix -p n1 -n 32 -c 1MB -b 16MB +# writeread-posix -p n1 -n 32 -c 4MB -b 16MB runmode=posix unify_test_writeread_posix "$app_args" @@ -234,13 +234,13 @@ unify_test_writeread_posix "$app_args" io_pattern="-p nn" app_args="$io_pattern $io_sizes" -# writeread-posix -p nn -n 32 -c 1MB -b 16MB +# writeread-posix -p nn -n 32 -c 4MB -b 16MB unify_test_writeread_posix "$app_args" -# writeread-gotcha -p nn -n 32 -c 1MB -b 16MB +# writeread-gotcha -p nn -n 32 -c 4MB -b 16MB runmode=gotcha unify_test_writeread $runmode "$app_args" -# writeread-static -p nn -n 32 -c 1MB -b 16MB +# writeread-static -p nn -n 32 -c 4MB -b 16MB runmode=static unify_test_writeread $runmode "$app_args" diff --git a/t/ci/110-write-tests.sh b/t/ci/110-write-tests.sh index 27474c857..066296b27 100755 --- a/t/ci/110-write-tests.sh +++ b/t/ci/110-write-tests.sh @@ -215,18 +215,18 @@ unify_test_write $runmode "$app_args" runmode=static unify_test_write $runmode "$app_args" -# Increase sizes: -n 32 -c 1MB -b 16MB +# Increase sizes: -n 32 -c 4MB -b 16MB -# write-static -p n1 -n 32 -c 1MB -b 16MB -io_sizes="-n 32 -c $MB -b $((16 * $MB))" +# write-static -p n1 -n 32 -c 4MB -b 16MB +io_sizes="-n 32 -c $((4 * $MB)) -b $((16 * $MB))" app_args="$io_pattern $io_sizes" unify_test_write $runmode "$app_args" -# write-gotcha -p n1 -n 32 -c 1MB -b 16MB +# write-gotcha -p n1 -n 32 -c 4MB -b 16MB runmode=gotcha unify_test_write $runmode "$app_args" -# write-posix -p n1 -n 32 -c 1MB -b 16MB +# write-posix -p n1 -n 32 -c 4MB -b 16MB runmode=posix unify_test_write_posix "$app_args" @@ -234,13 +234,13 @@ unify_test_write_posix "$app_args" io_pattern="-p nn" app_args="$io_pattern $io_sizes" -# write-posix -p nn -n 32 -c 1MB -b 16MB +# write-posix -p nn -n 32 -c 4MB -b 16MB unify_test_write_posix "$app_args" -# write-gotcha -p nn -n 32 -c 1MB -b 16MB +# write-gotcha -p nn -n 32 -c 4MB -b 16MB runmode=gotcha unify_test_write $runmode "$app_args" -# write-static -p nn -n 32 -c 1MB -b 16MB +# write-static -p nn -n 32 -c 4MB -b 16MB runmode=static unify_test_write $runmode "$app_args" diff --git a/t/ci/120-read-tests.sh b/t/ci/120-read-tests.sh index c61549321..6acc0d45a 100755 --- a/t/ci/120-read-tests.sh +++ b/t/ci/120-read-tests.sh @@ -209,18 +209,18 @@ unify_test_read $runmode "$app_args" runmode=static unify_test_read $runmode "$app_args" -# Increase sizes: -n 32 -c 1MB -b 16MB +# Increase sizes: -n 32 -c 4MB -b 16MB -# read-static -p n1 -n 32 -c 1MB -b 16MB -io_sizes="-n 32 -c $MB -b $((16 * $MB))" +# read-static -p n1 -n 32 -c 4MB -b 16MB +io_sizes="-n 32 -c $((4 * $MB)) -b $((16 * $MB))" app_args="$io_pattern $io_sizes" unify_test_read $runmode "$app_args" -# read-gotcha -p n1 -n 32 -c 1MB -b 16MB +# read-gotcha -p n1 -n 32 -c 4MB -b 16MB runmode=gotcha unify_test_read $runmode "$app_args" -# read-posix -p n1 -n 32 -c 1MB -b 16MB +# read-posix -p n1 -n 32 -c 4MB -b 16MB runmode=posix unify_test_read_posix "$app_args" @@ -228,13 +228,13 @@ unify_test_read_posix "$app_args" io_pattern="-p nn" app_args="$io_pattern $io_sizes" -# read-posix -p n1 -n 32 -c 1MB -b 16MB +# read-posix -p n1 -n 32 -c 4MB -b 16MB unify_test_read_posix "$app_args" -# read-gotcha -p n1 -n 32 -c 1MB -b 16MB +# read-gotcha -p n1 -n 32 -c 4MB -b 16MB runmode=gotcha unify_test_read $runmode "$app_args" -# read-static -p n1 -n 32 -c 1MB -b 16MB +# read-static -p n1 -n 32 -c 4MB -b 16MB runmode=static unify_test_read $runmode "$app_args" From 63692d449cce8225e199d1606c945a977c051258 Mon Sep 17 00:00:00 2001 From: CamStan Date: Thu, 2 Jul 2020 20:23:56 -0700 Subject: [PATCH 139/168] Fix Gitlab CI rules for pipeline Current rules were not defined properly and end up any pipeline creation, even manually from the web interface. This moves the rules to the workflow key to define when pipelines should be created, but not interfere with when jobs should run. --- .gitlab-ci.yml | 18 ++++++++++-------- .gitlab/catalyst.yml | 2 ++ .gitlab/lassen.yml | 2 ++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a26a75440..762df64f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,16 @@ stages: - test-unit - test-integ +# Rules to determine when a pipeline will be generated. If none of these rules +# match, no pipeline will be generated. +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_PIPELINE_SOURCE == "web"' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "dev"' + - if: '$CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "dev"' + - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "dev"' + ##### System Templates ##### # Generic system templates used to contruct the final jobs on specific @@ -17,10 +27,6 @@ stages: # Gitlab interface if/when the defaults need to be changed. .base-template: - rules: - - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "dev"' - - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "dev" && $CI_PIPELINE_SOURCE == "merge_request_event"' - - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "dev" && $CI_PIPELINE_SOURCE == "external_pull_request_event"' retry: max: 1 when: @@ -28,22 +34,18 @@ stages: - stuck_or_timeout_failure .slurm-single-node-template: - extends: .base-template variables: LLNL_SLURM_SCHEDULER_PARAMETERS: "-N 1 -p pbatch -t $UNIT_WALL_TIME -J unifyfs-unit-tests" .slurm-multi-node-template: - extends: .base-template variables: LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -p pbatch -t $INTEG_WALL_TIME -J unifyfs-integ-tests" .lsf-single-node-template: - extends: .base-template variables: LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -q pbatch -W $UNIT_WALL_TIME -J unifyfs-unit-tests" .lsf-multi-node-template: - extends: .base-template variables: LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes $NNODES -stage storage=${STORAGE_SIZE} -q pbatch -W $INTEG_WALL_TIME -J unifyfs-integ-tests" diff --git a/.gitlab/catalyst.yml b/.gitlab/catalyst.yml index 4eb7d19ee..f3533548c 100644 --- a/.gitlab/catalyst.yml +++ b/.gitlab/catalyst.yml @@ -3,9 +3,11 @@ # The RUN_CATALYST variable can be toggled in the Gitlab interface to # toggle whether jobs should be run on this system. .catalyst-template: + extends: .base-template rules: - if: '$RUN_CATALYST != "ON"' when: never + - when: on_success .catalyst-shell-template: extends: .catalyst-template diff --git a/.gitlab/lassen.yml b/.gitlab/lassen.yml index 7caf6e0fa..a34a6f587 100644 --- a/.gitlab/lassen.yml +++ b/.gitlab/lassen.yml @@ -3,9 +3,11 @@ # The RUN_LASSEN variable can be toggled in the Gitlab interface to # toggle whether jobs should be run on this system. .lassen-template: + extends: .base-template rules: - if: '$RUN_LASSEN != "ON"' when: never + - when: on_success .lassen-shell-template: extends: .lassen-template From 6d86c59d05adf8f228033498bd88a948fc4a34b3 Mon Sep 17 00:00:00 2001 From: CamStan Date: Wed, 20 May 2020 17:22:35 -0700 Subject: [PATCH 140/168] Add unit test to reproduce issue #488 This adds a unit test to reproduce issue #488. In particular, this is the self-contained version of the test that doesn't bring in any outside files. The failing test is wrapped in a `todo()` to mark it as an XFAIL to allow the test suite to pass. Once the issue is fixed and the test is passing, the `todo()` and `end_todo` can be removed. --- t/sys/sysio_suite.c | 1 + t/sys/sysio_suite.h | 2 ++ t/sys/write-read.c | 56 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/t/sys/sysio_suite.c b/t/sys/sysio_suite.c index dc00f65f3..63fa2acc2 100644 --- a/t/sys/sysio_suite.c +++ b/t/sys/sysio_suite.c @@ -84,6 +84,7 @@ int main(int argc, char* argv[]) lseek_test(unifyfs_root); write_read_test(unifyfs_root); + write_pre_existing_file_test(unifyfs_root); write_read_hole_test(unifyfs_root); diff --git a/t/sys/sysio_suite.h b/t/sys/sysio_suite.h index 1556f222b..a24d7a9b5 100644 --- a/t/sys/sysio_suite.h +++ b/t/sys/sysio_suite.h @@ -50,6 +50,8 @@ int lseek_test(char* unifyfs_root); int write_read_test(char* unifyfs_root); +int write_pre_existing_file_test(char* unifyfs_root); + /* test reading from file with holes */ int write_read_hole_test(char* unifyfs_root); diff --git a/t/sys/write-read.c b/t/sys/write-read.c index d98d54c8a..768b3d2d3 100644 --- a/t/sys/write-read.c +++ b/t/sys/write-read.c @@ -175,3 +175,59 @@ int write_read_test(char* unifyfs_root) return 0; } + +/* Test to reproduce issue 488 */ +int write_pre_existing_file_test(char* unifyfs_root) +{ + diag("Starting write-to-pre-existing-file tests"); + + char path[64]; + char buf[300] = {0}; + int fd = -1; + size_t global; + + errno = 0; + + testutil_rand_path(path, sizeof(path), unifyfs_root); + + fd = open(path, O_RDWR | O_CREAT, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* Write 300 bytes to a file */ + ok(write(fd, "a", 300) == 300, + "%s:%d write() a 300 byte file: %s", + __FILE__, __LINE__, strerror(errno)); + + ok(close(fd) == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(errno)); + + /* Check global size is 300 */ + testutil_get_size(path, &global); + ok(global == 300, "%s:%d global size of 300 byte file is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + + /* Reopen the same file */ + fd = open(path, O_RDWR, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* Overwrite the first 100 bytes of same file */ + ok(write(fd, buf, 100) == 100, + "%s:%d overwrite first 100 bytes of same file: %s", + __FILE__, __LINE__, strerror(errno)); + + ok(close(fd) == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(errno)); + + /* Check global size is 300 */ + testutil_get_size(path, &global); + todo("File is now 100 bytes instead of 300. See issue #488 for details"); + ok(global == 300, "%s:%d global size of 300 byte file is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + end_todo; + + diag("Finished write-to-pre-existing-file tests"); + + return 0; +} From cd0c9b45ea843b5af3e403526057488953bd092b Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Tue, 7 Jul 2020 07:56:06 -0400 Subject: [PATCH 141/168] remove runstate file but keep runstate dir TEST_CHECKPATCH_SKIP_FILES="common/src/unifyfs_configurator.h" --- client/src/unifyfs.c | 10 +--- common/src/Makefile.am | 2 - common/src/unifyfs_configurator.h | 4 +- common/src/unifyfs_keyval.c | 1 - common/src/unifyfs_keyval.h | 1 - common/src/unifyfs_runstate.c | 97 ------------------------------- common/src/unifyfs_runstate.h | 21 ------- docs/configuration.rst | 8 +-- server/src/unifyfs_server.c | 9 --- t/0001-setup.t | 10 ---- 10 files changed, 7 insertions(+), 156 deletions(-) delete mode 100644 common/src/unifyfs_runstate.c delete mode 100644 common/src/unifyfs_runstate.h diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 1e024527a..710bb92d3 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -42,7 +42,6 @@ #include "unifyfs-internal.h" #include "unifyfs-fixed.h" -#include "unifyfs_runstate.h" #include @@ -2523,19 +2522,12 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, l_app_id, unifyfs_app_id); } - // update configuration from runstate file - rc = unifyfs_read_runstate(&client_cfg, NULL); - if (rc) { - LOGERR("failed to update configuration from runstate."); - return UNIFYFS_FAILURE; - } - // initialize k-v store access kv_rank = client_rank; kv_nranks = size; rc = unifyfs_keyval_init(&client_cfg, &kv_rank, &kv_nranks); if (rc) { - LOGERR("failed to update configuration from runstate."); + LOGERR("failed to initialize kvstore"); return UNIFYFS_FAILURE; } if ((client_rank != kv_rank) || (size != kv_nranks)) { diff --git a/common/src/Makefile.am b/common/src/Makefile.am index fa0c02df9..c7a25e210 100644 --- a/common/src/Makefile.am +++ b/common/src/Makefile.am @@ -41,8 +41,6 @@ BASE_SRCS = \ unifyfs_server_rpcs.h \ unifyfs_rc.h \ unifyfs_rc.c \ - unifyfs_runstate.h \ - unifyfs_runstate.c \ unifyfs_shm.h \ unifyfs_shm.c diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index 1f6f7bf5e..1c4f41f6f 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -86,10 +86,10 @@ UNIFYFS_CFG(meta, db_path, STRING, RUNDIR, "metadata database path", configurator_directory_check) \ UNIFYFS_CFG(meta, server_ratio, INT, META_DEFAULT_SERVER_RATIO, "metadata server ratio", NULL) \ UNIFYFS_CFG(meta, range_size, INT, META_DEFAULT_RANGE_SZ, "metadata range size", NULL) \ - UNIFYFS_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server runstate file") \ + UNIFYFS_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server-local state") \ UNIFYFS_CFG_CLI(server, hostfile, STRING, NULLSTRING, "server hostfile name", NULL, 'H', "specify full path to server hostfile") \ - UNIFYFS_CFG_CLI(sharedfs, dir, STRING, NULLSTRING, "shared file system directory", configurator_directory_check, 'S', "specify full path to directory to contain server shared files") \ UNIFYFS_CFG_CLI(server, init_timeout, INT, UNIFYFS_DEFAULT_INIT_TIMEOUT, "timeout of waiting for server initialization", NULL, 't', "timeout in seconds to wait for servers to be ready for clients") \ + UNIFYFS_CFG_CLI(sharedfs, dir, STRING, NULLSTRING, "shared file system directory", configurator_directory_check, 'S', "specify full path to directory to contain files shared across servers") \ #ifdef __cplusplus diff --git a/common/src/unifyfs_keyval.c b/common/src/unifyfs_keyval.c index 85ef01e0f..30f69b3d8 100644 --- a/common/src/unifyfs_keyval.c +++ b/common/src/unifyfs_keyval.c @@ -34,7 +34,6 @@ #include // UnifyFS keys -const char* const key_runstate = "unifyfs.runstate"; const char* const key_unifyfsd_socket = "unifyfsd.socket"; const char* const key_unifyfsd_margo_shm = "unifyfsd.margo-shm"; const char* const key_unifyfsd_margo_svr = "unifyfsd.margo-svr"; diff --git a/common/src/unifyfs_keyval.h b/common/src/unifyfs_keyval.h index b3ff18883..33f88b194 100644 --- a/common/src/unifyfs_keyval.h +++ b/common/src/unifyfs_keyval.h @@ -22,7 +22,6 @@ extern "C" { #endif // keys we use -extern const char* const key_runstate; // path to runstate file extern const char* const key_unifyfsd_socket; // server domain socket path extern const char* const key_unifyfsd_margo_shm; // client-server margo address extern const char* const key_unifyfsd_margo_svr; // server-server margo address diff --git a/common/src/unifyfs_runstate.c b/common/src/unifyfs_runstate.c deleted file mode 100644 index db9a75b41..000000000 --- a/common/src/unifyfs_runstate.c +++ /dev/null @@ -1,97 +0,0 @@ -#include -#include -#include -#include -#include - -#include "unifyfs_keyval.h" -#include "unifyfs_log.h" -#include "unifyfs_runstate.h" - -const char* runstate_file = "unifyfs-runstate.conf"; - -int unifyfs_read_runstate(unifyfs_cfg_t* cfg, - const char* runstate_path) -{ - int rc = (int)UNIFYFS_SUCCESS; - int uid = (int)getuid(); - char runstate_fname[UNIFYFS_MAX_FILENAME] = {0}; - - if (cfg == NULL) { - LOGERR("NULL config"); - return EINVAL; - } - - if (runstate_path == NULL) { - if (cfg->runstate_dir == NULL) { - LOGERR("bad runstate dir config setting"); - return (int)UNIFYFS_ERROR_BADCONFIG; - } - snprintf(runstate_fname, sizeof(runstate_fname), - "%s/%s.%d", cfg->runstate_dir, runstate_file, uid); - } else { - snprintf(runstate_fname, sizeof(runstate_fname), - "%s", runstate_path); - } - - if (unifyfs_config_process_ini_file(cfg, runstate_fname) != 0) { - LOGERR("failed to process runstate file %s", runstate_fname); - rc = (int)UNIFYFS_ERROR_BADCONFIG; - } - - return rc; -} - -int unifyfs_write_runstate(unifyfs_cfg_t* cfg) -{ - int rc = (int)UNIFYFS_SUCCESS; - int uid = (int)getuid(); - FILE* runstate_fp = NULL; - char runstate_fname[UNIFYFS_MAX_FILENAME] = {0}; - - if (cfg == NULL) { - LOGERR("NULL config"); - return EINVAL; - } - - snprintf(runstate_fname, sizeof(runstate_fname), - "%s/%s.%d", cfg->runstate_dir, runstate_file, uid); - - runstate_fp = fopen(runstate_fname, "w"); - if (runstate_fp == NULL) { - rc = errno; - LOGERR("failed to create file %s", runstate_fname); - } else { - if ((unifyfs_log_stream != NULL) && - (unifyfs_log_level >= LOG_INFO)) { - unifyfs_config_print(cfg, unifyfs_log_stream); - } - unifyfs_config_print_ini(cfg, runstate_fp); - fclose(runstate_fp); - } - - return rc; -} - -int unifyfs_clean_runstate(unifyfs_cfg_t* cfg) -{ - int rc = (int)UNIFYFS_SUCCESS; - int uid = (int)getuid(); - char runstate_fname[UNIFYFS_MAX_FILENAME] = {0}; - - if (cfg == NULL) { - LOGERR("invalid config arg"); - return EINVAL; - } - - snprintf(runstate_fname, sizeof(runstate_fname), - "%s/%s.%d", cfg->runstate_dir, runstate_file, uid); - - rc = unlink(runstate_fname); - if (rc != 0) { - rc = errno; - LOGERR("failed to remove file %s", runstate_fname); - } - - return rc; -} diff --git a/common/src/unifyfs_runstate.h b/common/src/unifyfs_runstate.h deleted file mode 100644 index e32c67372..000000000 --- a/common/src/unifyfs_runstate.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef _UNIFYFS_RUNSTATE_H_ -#define _UNIFYFS_RUNSTATE_H_ - -#include "unifyfs_configurator.h" - -#ifdef __cplusplus -extern "C" { -#endif - -int unifyfs_read_runstate(unifyfs_cfg_t* cfg, - const char* runstate_path); - -int unifyfs_write_runstate(unifyfs_cfg_t* cfg); - -int unifyfs_clean_runstate(unifyfs_cfg_t* cfg); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif // UNIFYFS_RUNSTATE_H diff --git a/docs/configuration.rst b/docs/configuration.rst index 42832d291..7468f7ce6 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -125,11 +125,11 @@ files. .. table:: ``[runstate]`` section - server runstate settings :widths: auto - ======== ====== ================================================== + ======== ====== =============================================== Key Type Description - ======== ====== ================================================== - dir STRING path to directory to contain server runstate file - ======== ====== ================================================== + ======== ====== =============================================== + dir STRING path to directory to contain server-local state + ======== ====== =============================================== .. table:: ``[server]`` section - server settings :widths: auto diff --git a/server/src/unifyfs_server.c b/server/src/unifyfs_server.c index d0297644f..39afa8a46 100644 --- a/server/src/unifyfs_server.c +++ b/server/src/unifyfs_server.c @@ -34,7 +34,6 @@ // common headers #include "unifyfs_configurator.h" #include "unifyfs_keyval.h" -#include "unifyfs_runstate.h" // server components #include "unifyfs_global.h" @@ -346,11 +345,6 @@ int main(int argc, char* argv[]) rc = allocate_servers((size_t)kv_nranks); } - rc = unifyfs_write_runstate(&server_cfg); - if (rc != (int)UNIFYFS_SUCCESS) { - exit(1); - } - LOGDBG("initializing rpc service"); ABT_init(argc, argv); ABT_mutex_create(&app_configs_abt_sync); @@ -403,9 +397,6 @@ int main(int argc, char* argv[]) LOGDBG("stopping service manager thread"); rc = svcmgr_fini(); - LOGDBG("cleaning run state"); - rc = unifyfs_clean_runstate(&server_cfg); - return unifyfs_exit(); } diff --git a/t/0001-setup.t b/t/0001-setup.t index 18cfbc5fe..4cf3042a3 100755 --- a/t/0001-setup.t +++ b/t/0001-setup.t @@ -66,15 +66,5 @@ if process_is_not_running unifyfsd 5; then exit 1 fi -# -# Make sure unifyfsd successfully generated client runstate file -# -uid=$(id -u) -if ! test -f $UNIFYFS_RUNSTATE_DIR/unifyfs-runstate.conf.$uid ; then - cat $UNIFYFS_LOG_DIR/${UNIFYFS_LOG_FILE}* >&3 - echo not ok 1 - unifyfsd runstate - exit 1 -fi - echo ok 1 - unifyfsd running exit 0 From b4ea3eeceac1c9bf4dffc95ac7479ae3003bd3e3 Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Thu, 9 Jul 2020 13:53:40 -0400 Subject: [PATCH 142/168] Handling large transfer between servers. This fixes the large data transfer failures (in the mercury tranport layer) between servers. - common/unifyfs_const.h: defined MAX_BULK_TX_SIZE (8MB) for data transfer size. - src/server/unifyfs_request_manager.c: fixed request manager to initiate multiple transfers for large bulk data. - some relevant debugging messages. - type fix in examples/src/read-data.c --- client/src/unifyfs.c | 5 ++- common/src/unifyfs_const.h | 1 + examples/src/read-data.c | 4 +- server/src/unifyfs_request_manager.c | 60 +++++++++++++++++++++------- 4 files changed, 53 insertions(+), 17 deletions(-) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 710bb92d3..c374e8a6a 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -1107,7 +1107,7 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) /* * ToDo: Exception handling when some of the requests * are missed - * */ + */ /* spin waiting for read data to come back from the server, * we process it in batches as it comes in, eventually the @@ -1121,12 +1121,15 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) } else { tmp_rc = process_read_data(read_reqs, count, &done); if (tmp_rc != UNIFYFS_SUCCESS) { + LOGERR("failed to process data from server"); rc = UNIFYFS_FAILURE; } delegator_signal(); } } + LOGDBG("fetched all data from server for %d requests", count); + /* got all of the data we'll get from the server, * check for short reads and whether those short * reads are from errors, holes, or the end of the file */ diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h index ea09ce091..f6fa1670f 100644 --- a/common/src/unifyfs_const.h +++ b/common/src/unifyfs_const.h @@ -54,6 +54,7 @@ #define REQ_BUF_LEN (MAX_META_PER_SEND * 64) /* chunk read reqs buffer size */ #define SHM_WAIT_INTERVAL 1000 /* unit: ns */ #define RM_MAX_ACTIVE_REQUESTS 64 /* number of concurrent read requests */ +#define MAX_BULK_TX_SIZE (8 * MIB) /* bulk transfer size */ // Server - Service Manager #define LARGE_BURSTY_DATA (512 * MIB) diff --git a/examples/src/read-data.c b/examples/src/read-data.c index 49f72c666..83c71e8e0 100644 --- a/examples/src/read-data.c +++ b/examples/src/read-data.c @@ -85,7 +85,7 @@ static void alloc_buf(unsigned long length) static void do_pread(int fd, size_t length, off_t offset) { - int ret = 0; + ssize_t ret = 0; alloc_buf(length); @@ -93,7 +93,7 @@ static void do_pread(int fd, size_t length, off_t offset) ret = pread(fd, buf, length, offset); - printf(" -> pread(off=%lu, len=%lu) = %d", offset, length, ret); + printf(" -> pread(off=%lu, len=%lu) = %zd", offset, length, ret); if (errno) { printf(" (err=%d, %s)\n", errno, strerror(errno)); } else { diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 36d263ecd..9b1384836 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -2255,6 +2255,7 @@ static void chunk_read_response_rpc(hg_handle_t handle) { int32_t ret; hg_return_t hret; + chunk_read_response_out_t out; /* get input params */ chunk_read_response_in_t in; @@ -2269,6 +2270,9 @@ static void chunk_read_response_rpc(hg_handle_t handle) int num_chks = (int)in.num_chks; size_t bulk_sz = (size_t)in.bulk_size; + LOGDBG("received chunk read response from server %d (%d chunks)", + src_rank, num_chks); + /* The input parameters specify the info for a bulk transfer * buffer on the sending process. We use that info to pull data * from the sender into a local buffer. This buffer contains @@ -2306,21 +2310,49 @@ static void chunk_read_response_rpc(hg_handle_t handle) hg_bulk_t bulk_handle; hret = margo_bulk_create(mid, 1, (void**)&resp_buf, &in.bulk_size, HG_BULK_WRITE_ONLY, &bulk_handle); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("failed to prepare bulk transfer"); + ret = UNIFYFS_ERROR_MARGO; + goto out_respond; + } /* execute the transfer to pull data from remote side - * into our local bulk transfer buffer */ - hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, - in.bulk_handle, 0, bulk_handle, 0, in.bulk_size); - assert(hret == HG_SUCCESS); - - /* process read replies (headers and data) we just - * received */ - rc = rm_post_chunk_read_responses(app_id, client_id, - src_rank, req_id, num_chks, bulk_sz, resp_buf); - if (rc != (int)UNIFYFS_SUCCESS) { - LOGERR("failed to handle chunk read responses") - ret = rc; + * into our local bulk transfer buffer. + * NOTE: mercury/margo bulk transfer does not check the maximum + * transfer size that the underlying transport supports, and a + * large bulk transfer may result in failure. */ + int i = 0; + hg_size_t remain = in.bulk_size; + + do { + hg_size_t offset = i * MAX_BULK_TX_SIZE; + hg_size_t len = remain < MAX_BULK_TX_SIZE + ? remain : MAX_BULK_TX_SIZE; + + hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, + in.bulk_handle, offset, + bulk_handle, offset, len); + if (hret != HG_SUCCESS) { + break; + } + + remain -= len; + i++; + } while (remain > 0); + + if (hret == HG_SUCCESS) { + LOGDBG("transferred bulk data (%lu bytes)", in.bulk_size); + + /* process read replies (headers and data) we just received */ + rc = rm_post_chunk_read_responses(app_id, client_id, + src_rank, req_id, num_chks, bulk_sz, resp_buf); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to handle chunk read responses") + ret = rc; + } + } else { + LOGERR("failed to perform bulk transfer"); + ret = UNIFYFS_ERROR_MARGO; } /* deregister our bulk transfer buffer */ @@ -2328,8 +2360,8 @@ static void chunk_read_response_rpc(hg_handle_t handle) } } +out_respond: /* fill output structure */ - chunk_read_response_out_t out; out.ret = ret; /* return to caller */ From 9aad7579d1b2a3fd4d0a2e2c651bbdff7bcd3ba8 Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Wed, 22 Jul 2020 17:44:37 -0400 Subject: [PATCH 143/168] Fixing bugs in the unifyfs-transfer: - adding a missing barrier in the unifyfs-stage utility. - adding a missing long option (--stage-timeout) in the unifyfs utility. - chaning the transfer buffer size to 8MB (from 1MB). - remove error (space) in the testscript --- client/src/unifyfs.c | 15 +++++++++++++-- t/9300-unifyfs-stage-isolated.t | 2 +- util/unifyfs-stage/src/unifyfs-stage.c | 5 ++++- util/unifyfs/src/unifyfs.c | 1 + 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index c374e8a6a..96073de5e 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -2675,7 +2675,7 @@ int unifyfs_unmount(void) return ret; } -#define UNIFYFS_TX_BUFSIZE (1<<20) +#define UNIFYFS_TX_BUFSIZE (8*(1<<20)) enum { UNIFYFS_TX_STAGE_OUT = 0, @@ -2693,7 +2693,13 @@ ssize_t do_transfer_data(int fd_src, int fd_dst, off_t offset, size_t count) ssize_t n_left = 0; ssize_t n_processed = 0; size_t len = UNIFYFS_TX_BUFSIZE; - char buf[UNIFYFS_TX_BUFSIZE] = { 0, }; + char* buf = NULL; + + buf = malloc(UNIFYFS_TX_BUFSIZE); + if (!buf) { + LOGERR("failed to allocate transfer buffer"); + return ENOMEM; + } pos = lseek(fd_src, offset, SEEK_SET); if (pos == (off_t) -1) { @@ -2740,6 +2746,11 @@ ssize_t do_transfer_data(int fd_src, int fd_dst, off_t offset, size_t count) } out: + if (buf) { + free(buf); + buf = NULL; + } + return ret; } diff --git a/t/9300-unifyfs-stage-isolated.t b/t/9300-unifyfs-stage-isolated.t index c8a389f5e..f6696ae05 100755 --- a/t/9300-unifyfs-stage-isolated.t +++ b/t/9300-unifyfs-stage-isolated.t @@ -32,7 +32,7 @@ test_expect_success "source.file exists" ' test_path_is_file ${UNIFYFS_TEST_TMPDIR}/stage_source/source_9300.file ' -rm - f $ {UNIFYFS_TEST_TMPDIR} / config_9300/* +rm -f ${UNIFYFS_TEST_TMPDIR}/config_9300/* rm -f ${UNIFYFS_TEST_TMPDIR}/stage_destination_9300/* test_expect_success "config_9300 directory is empty" ' diff --git a/util/unifyfs-stage/src/unifyfs-stage.c b/util/unifyfs-stage/src/unifyfs-stage.c index c65b86bec..169c6a325 100644 --- a/util/unifyfs-stage/src/unifyfs-stage.c +++ b/util/unifyfs-stage/src/unifyfs-stage.c @@ -294,7 +294,10 @@ int main(int argc, char** argv) fprintf(stderr, "data transfer failed (%s)\n", strerror(errno)); } - if (share_dir) { + /* wait until all processes are done */ + MPI_Barrier(MPI_COMM_WORLD); + + if (share_dir && rank == 0) { ret = create_status_file(ret); if (ret) { fprintf(stderr, "failed to create the status file (%s)\n", diff --git a/util/unifyfs/src/unifyfs.c b/util/unifyfs/src/unifyfs.c index db5f596de..23438b71d 100644 --- a/util/unifyfs/src/unifyfs.c +++ b/util/unifyfs/src/unifyfs.c @@ -77,6 +77,7 @@ static struct option const long_opts[] = { { "stage-in", required_argument, NULL, 'i' }, { "stage-out", required_argument, NULL, 'o' }, { "timeout", required_argument, NULL, 't' }, + { "stage-timeout", required_argument, NULL, 'T' }, { 0, 0, 0, 0 }, }; From 91173d17f10e7ba1edff5416f73b39ff0fedb90c Mon Sep 17 00:00:00 2001 From: CamStan Date: Thu, 23 Jul 2020 17:51:40 -0700 Subject: [PATCH 144/168] Implement Gitlab CI on Ascent This adds an `ascent.yml` file and sets up the scheduler parameters needed to get started using Gitlab CI on Ascent at ORNL. --- .gitlab-ci.yml | 2 ++ .gitlab/ascent.yml | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 .gitlab/ascent.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 762df64f8..ae0e095b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,10 +44,12 @@ workflow: .lsf-single-node-template: variables: LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -q pbatch -W $UNIT_WALL_TIME -J unifyfs-unit-tests" + SCHEDULER_PARAMETERS: "-nnodes 1 -P $PROJECT_ID -W $UNIT_WALL_TIME -J unifyfs-unit-tests" .lsf-multi-node-template: variables: LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes $NNODES -stage storage=${STORAGE_SIZE} -q pbatch -W $INTEG_WALL_TIME -J unifyfs-integ-tests" + SCHEDULER_PARAMETERS: "-nnodes $NNODES -P $PROJECT_ID -W $INTEG_WALL_TIME -J unifyfs-integ-tests" ##### Job Templates ##### diff --git a/.gitlab/ascent.yml b/.gitlab/ascent.yml new file mode 100644 index 000000000..b5a051901 --- /dev/null +++ b/.gitlab/ascent.yml @@ -0,0 +1,43 @@ +##### Ascent Templates ##### + +# The RUN_ASCENT variable can be toggled in the Gitlab interface to +# toggle whether jobs should be run on this system. +.ascent-template: + extends: .base-template + rules: + - if: '$RUN_ASCENT != "ON"' + when: never + - when: on_success + +.ascent-shell-template: + extends: .ascent-template + tags: [nobatch] + +.ascent-batch-template: + extends: .ascent-template + tags: [batch] + +##### All Ascent Jobs ##### + +ascent-gcc-4_8_5-build: + variables: + COMPILER: gcc/4.8.5 + CC_COMMAND: "which gcc" + FC_COMMAND: "which gfortran" + extends: [.ascent-shell-template, .build-template] + +ascent-gcc-4_8_5-unit-test: + variables: + COMPILER: gcc/4.8.5 + CC_COMMAND: "which gcc" + FC_COMMAND: "which gfortran" + extends: [.lsf-single-node-template, .ascent-batch-template, .unit-test-template] + needs: ["ascent-gcc-4_8_5-build"] + +ascent-gcc-4_8_5-integ-test: + variables: + COMPILER: gcc/4.8.5 + CC_COMMAND: "which gcc" + FC_COMMAND: "which gfortran" + extends: [.lsf-multi-node-template, .ascent-batch-template, .integ-test-template] + needs: ["ascent-gcc-4_8_5-build"] From c2f38d60b329217426aec9a1644e2f4298a9b877 Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Mon, 27 Jul 2020 17:16:46 -0400 Subject: [PATCH 145/168] make clients per application a runtime config TEST_CHECKPATCH_SKIP_FILES="common/src/unifyfs_configurator.h" --- common/src/unifyfs_configurator.h | 5 +++-- common/src/unifyfs_const.h | 7 +++---- server/src/unifyfs_global.h | 3 ++- server/src/unifyfs_server.c | 29 +++++++++++++++++++++++++---- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index 1c4f41f6f..839aa2db5 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -86,10 +86,11 @@ UNIFYFS_CFG(meta, db_path, STRING, RUNDIR, "metadata database path", configurator_directory_check) \ UNIFYFS_CFG(meta, server_ratio, INT, META_DEFAULT_SERVER_RATIO, "metadata server ratio", NULL) \ UNIFYFS_CFG(meta, range_size, INT, META_DEFAULT_RANGE_SZ, "metadata range size", NULL) \ - UNIFYFS_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server-local state") \ + UNIFYFS_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server runstate file") \ UNIFYFS_CFG_CLI(server, hostfile, STRING, NULLSTRING, "server hostfile name", NULL, 'H', "specify full path to server hostfile") \ UNIFYFS_CFG_CLI(server, init_timeout, INT, UNIFYFS_DEFAULT_INIT_TIMEOUT, "timeout of waiting for server initialization", NULL, 't', "timeout in seconds to wait for servers to be ready for clients") \ - UNIFYFS_CFG_CLI(sharedfs, dir, STRING, NULLSTRING, "shared file system directory", configurator_directory_check, 'S', "specify full path to directory to contain files shared across servers") \ + UNIFYFS_CFG(server, max_app_clients, INT, MAX_APP_CLIENTS, "maximum number of clients per application", NULL) \ + UNIFYFS_CFG_CLI(sharedfs, dir, STRING, NULLSTRING, "shared file system directory", configurator_directory_check, 'S', "specify full path to directory to contain server shared files") \ #ifdef __cplusplus diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h index f6fa1670f..72c7ae1cd 100644 --- a/common/src/unifyfs_const.h +++ b/common/src/unifyfs_const.h @@ -64,10 +64,9 @@ #define SLEEP_SLICE_PER_UNIT 50 /* unit: us */ // Server - General -#define MAX_NUM_APPS 64 /* max # apps supported by a single server */ -#define MAX_APP_CLIENTS 64 /* app processes per server */ -/* timeout (in seconds) of waiting for initialization of all servers */ -#define UNIFYFS_DEFAULT_INIT_TIMEOUT 120 +#define MAX_NUM_APPS 64 /* max # apps/mountpoints supported */ +#define MAX_APP_CLIENTS 256 /* max # clients per application */ +#define UNIFYFS_DEFAULT_INIT_TIMEOUT 120 /* server init timeout (seconds) */ #define UNIFYFSD_PID_FILENAME "unifyfsd.pids" #define UNIFYFS_STAGE_STATUS_FILENAME "unifyfs-stage.status" diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index 653bf00a4..a47e633e3 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -168,7 +168,8 @@ typedef struct app_config { /* array of clients associated with this app */ size_t num_clients; - app_client* clients[MAX_APP_CLIENTS]; + size_t clients_sz; + app_client** clients; } app_config; app_config* get_application(int app_id); diff --git a/server/src/unifyfs_server.c b/server/src/unifyfs_server.c index 39afa8a46..cdd5b2d29 100644 --- a/server/src/unifyfs_server.c +++ b/server/src/unifyfs_server.c @@ -59,6 +59,7 @@ unifyfs_cfg_t server_cfg; static ABT_mutex app_configs_abt_sync; static app_config* app_configs[MAX_NUM_APPS]; /* list of apps */ +static size_t clients_per_app = MAX_APP_CLIENTS; /** * @brief create a ready status file to notify that all servers are ready for @@ -266,6 +267,7 @@ int main(int argc, char* argv[]) } server_cfg.ptype = UNIFYFS_SERVER; + // to daemon or not to daemon, that is the question rc = configurator_bool_val(server_cfg.unifyfs_daemonize, &daemon); if (rc != 0) { exit(1); @@ -293,6 +295,15 @@ int main(int argc, char* argv[]) rc = sigaction(SIGQUIT, &sa, NULL); rc = sigaction(SIGTERM, &sa, NULL); + // update clients_per_app based on configuration + if (server_cfg.server_max_app_clients != NULL) { + long l; + rc = configurator_int_val(server_cfg.server_max_app_clients, &l); + if (0 == rc) { + clients_per_app = l; + } + } + // initialize empty app_configs[] memset(app_configs, 0, sizeof(app_configs)); @@ -640,6 +651,14 @@ app_config* new_application(int app_id) for (int i = 0; i < MAX_NUM_APPS; i++) { app_config* existing = app_configs[i]; if (NULL == existing) { + new_app->clients = (app_client**) calloc(clients_per_app, + sizeof(app_client*)); + if (NULL == new_app->clients) { + LOGERR("failed to allocate application clients arrays") + ABT_mutex_unlock(app_configs_abt_sync); + return NULL; + } + new_app->clients_sz = clients_per_app; app_configs[i] = new_app; ABT_mutex_unlock(app_configs_abt_sync); return new_app; @@ -677,7 +696,7 @@ unifyfs_rc cleanup_application(app_config* app) LOGDBG("cleaning application %d", app_id); /* free resources allocated for each client */ - for (int j = 0; j < MAX_APP_CLIENTS; j++) { + for (int j = 0; j < app->clients_sz; j++) { app_client* client = app->clients[j]; if (NULL != client) { unifyfs_rc rc = cleanup_app_client(app, client); @@ -686,7 +705,9 @@ unifyfs_rc cleanup_application(app_config* app) } } } - + if (NULL != app->clients) { + free(app->clients); + } free(app); return ret; @@ -699,7 +720,7 @@ app_client* get_app_client(int app_id, app_config* app_cfg = get_application(app_id); if ((NULL == app_cfg) || (client_id <= 0) || - (client_id > MAX_APP_CLIENTS)) { + (client_id > (int)app_cfg->clients_sz)) { return NULL; } @@ -770,7 +791,7 @@ app_client* new_app_client(app_config* app, return NULL; } - if (app->num_clients == MAX_APP_CLIENTS) { + if (app->num_clients == app->clients_sz) { LOGERR("reached maximum number of application clients"); return NULL; } From 5a99d59a2787f22d49065e517b1e43c81ef778ad Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Tue, 28 Jul 2020 13:59:34 -0400 Subject: [PATCH 146/168] fixup - merge conflict --- common/src/unifyfs_configurator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index 839aa2db5..86c21000a 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -86,7 +86,7 @@ UNIFYFS_CFG(meta, db_path, STRING, RUNDIR, "metadata database path", configurator_directory_check) \ UNIFYFS_CFG(meta, server_ratio, INT, META_DEFAULT_SERVER_RATIO, "metadata server ratio", NULL) \ UNIFYFS_CFG(meta, range_size, INT, META_DEFAULT_RANGE_SZ, "metadata range size", NULL) \ - UNIFYFS_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server runstate file") \ + UNIFYFS_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server-local state") \ UNIFYFS_CFG_CLI(server, hostfile, STRING, NULLSTRING, "server hostfile name", NULL, 'H', "specify full path to server hostfile") \ UNIFYFS_CFG_CLI(server, init_timeout, INT, UNIFYFS_DEFAULT_INIT_TIMEOUT, "timeout of waiting for server initialization", NULL, 't', "timeout in seconds to wait for servers to be ready for clients") \ UNIFYFS_CFG(server, max_app_clients, INT, MAX_APP_CLIENTS, "maximum number of clients per application", NULL) \ From 470a0a92e40954214bbbbef5d416910884fc0275 Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Wed, 22 Jul 2020 17:44:37 -0400 Subject: [PATCH 147/168] Fixing the race condition between server and client during mread. When multiple file descriptors are involved in a single lio_listio call, UnifyFS hangs due to incorrect coordination for exchanging data between server and client. This patch fixes it. - client issues mread rpc with margo_iforward. - modifying the server data structures to handle multiple file requests correctly. - also removing some extra global lock statements in the request manager (unifyfs_request_manager.c). - some extra debugging messages --- client/src/margo_client.c | 73 ++++++++--- client/src/margo_client.h | 28 ++++- client/src/unifyfs-internal.h | 1 + client/src/unifyfs-sysio.c | 7 +- client/src/unifyfs.c | 41 ++++++- server/src/unifyfs_global.h | 2 + server/src/unifyfs_request_manager.c | 175 ++++++++++++--------------- server/src/unifyfs_request_manager.h | 2 +- server/src/unifyfs_service_manager.c | 1 + 9 files changed, 212 insertions(+), 118 deletions(-) diff --git a/client/src/margo_client.c b/client/src/margo_client.c index afbb4f0a2..2a0188f6b 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -609,19 +609,62 @@ int invoke_client_read_rpc(int gfid, size_t offset, size_t length) /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); - return (int)ret; + + return ret; +} + +int unifyfs_mread_rpc_status_check(unifyfs_mread_rpc_ctx_t* ctx) +{ + int ret = 0; + int flag = 0; + + if (!ctx) { + return -EINVAL; + } + + ret = margo_test(ctx->req, &flag); + if (ret) { + return -EINVAL; /* assume that the given ctx is invalid */ + } + + /* flag becomes 1 when rpc is complete (otherwise 0) */ + if (flag) { + unifyfs_mread_out_t out; + + hg_return_t hret = margo_get_output(ctx->handle, &out); + if (hret == HG_SUCCESS) { + ctx->rpc_ret = out.ret; + margo_free_output(ctx->handle, &out); + } else { + /* we failed to get the correct response from the server and + * assume that the rpc failed. */ + ctx->rpc_ret = UNIFYFS_ERROR_MARGO; + } + + margo_destroy(ctx->handle); + } + + return flag; } /* invokes the client mread rpc function */ -int invoke_client_mread_rpc(int read_count, size_t size, void* buffer) +int invoke_client_mread_rpc(int read_count, size_t size, void* buffer, + unifyfs_mread_rpc_ctx_t* ctx) { + int ret = UNIFYFS_SUCCESS; + /* check that we have initialized margo */ if (NULL == client_rpc_context) { return UNIFYFS_FAILURE; } + if (NULL == ctx) { + return UNIFYFS_FAILURE; + } + /* get handle to rpc function */ hg_handle_t handle = create_handle(client_rpc_context->rpcs.mread_id); + margo_request req; unifyfs_mread_in_t in; hg_return_t hret = margo_bulk_create( @@ -636,20 +679,18 @@ int invoke_client_mread_rpc(int read_count, size_t size, void* buffer) in.bulk_size = (hg_size_t) size; /* call rpc function */ - LOGDBG("invoking the read rpc function in client"); - hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); - - /* decode response */ - unifyfs_mread_out_t out; - hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); + LOGDBG("invoking the mread rpc function in client"); + hret = margo_iforward(handle, &in, &req); + if (HG_SUCCESS == hret) { + ctx->handle = handle; + ctx->req = req; + } else { + ret = UNIFYFS_FAILURE; + } - /* free resources */ + /* margo_iforward serializes all data before returning, and it's safe to + * free the rpc params */ margo_bulk_free(in.bulk_handle); - margo_free_output(handle, &out); - margo_destroy(handle); - return (int)ret; + + return ret; } diff --git a/client/src/margo_client.h b/client/src/margo_client.h index 0d2139603..7ea974b8b 100644 --- a/client/src/margo_client.h +++ b/client/src/margo_client.h @@ -61,6 +61,32 @@ int invoke_client_sync_rpc(void); int invoke_client_read_rpc(int gfid, size_t offset, size_t length); -int invoke_client_mread_rpc(int read_count, size_t size, void* buffer); +/* + * mread rpc function is non-blocking (using margo_iforward), and the response + * from the server should be checked by the caller manually using + * the unifyfs_mread_rpc_status_check function. + */ +struct unifyfs_mread_rpc_ctx { + margo_request req; /* margo request for track iforward result */ + hg_handle_t handle; /* rpc handle */ + int rpc_ret; /* rpc response from the server */ +}; + +typedef struct unifyfs_mread_rpc_ctx unifyfs_mread_rpc_ctx_t; + +/** + * @brief track the progress of the submitted rpc. if the rpc is done, this + * funtcion returns 1 with the server response being stored in @ctx->rpc_ret. + * + * @param ctx pointer to the rpc ctx + * + * @return 1 if rpc is done (received response from the server), 0 if still in + * progress. -EINVAL if the @ctx is invalid. + */ +int unifyfs_mread_rpc_status_check(unifyfs_mread_rpc_ctx_t* ctx); + + +int invoke_client_mread_rpc(int read_count, size_t size, void* buffer, + unifyfs_mread_rpc_ctx_t* ctx); #endif // MARGO_CLIENT_H diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index ce89a168e..3e68becdf 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -318,6 +318,7 @@ typedef struct { size_t length; /* number of bytes to read */ size_t nread; /* number of bytes actually read */ char* buf; /* pointer to user buffer to place data */ + struct aiocb* aiocbp; /* the original request from application */ } read_req_t; typedef struct { diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 669a88e8a..47b9f2b85 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -1581,6 +1581,7 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], reqs[reqcnt].nread = 0; reqs[reqcnt].errcode = EINPROGRESS; reqs[reqcnt].buf = (char*)(cbp->aio_buf); + reqs[reqcnt].aiocbp = cbp; reqcnt++; } } else { @@ -1609,12 +1610,10 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], } /* update aiocb fields to record error status and return value */ - ndx = 0; for (i = 0; i < reqcnt; i++) { - char* buf = reqs[i].buf; - for (; ndx < nitems; ndx++) { + for (ndx = 0; ndx < nitems; ndx++) { cbp = aiocb_list[ndx]; - if ((char*)(cbp->aio_buf) == buf) { + if (cbp == reqs[i].aiocbp) { AIOCB_ERROR_CODE(cbp) = reqs[i].errcode; if (0 == reqs[i].errcode) { AIOCB_RETURN_VAL(cbp) = reqs[i].length; diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 96073de5e..300b6c59b 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -665,6 +665,10 @@ static int process_read_data(read_req_t* read_reqs, int count, int* done) char* rep_buf = shmptr; shmptr += rep->length; + LOGDBG("processing data response from server: " + "[%zu] (gfid=%d, offset=%lu, length=%lu, errcode=%d)", + i, rep->gfid, rep->offset, rep->length, rep->errcode); + /* get start and end offset of reply */ size_t rep_start = rep->offset; size_t rep_end = rep->offset + rep->length; @@ -747,6 +751,8 @@ static int process_read_data(read_req_t* read_reqs, int count, int* done) char* rep_ptr = rep_buf + rep_offset; memcpy(req_ptr, rep_ptr, length); + LOGDBG("copied data to application buffer (%lu bytes)", length); + /* update max number of bytes we have written to in the * request buffer */ size_t nread = end - req_start; @@ -1047,6 +1053,9 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) /* prepare our shared memory buffer for delegator */ delegator_signal(); + /* for mread, we need to manually track the rpc progress */ + unifyfs_mread_rpc_ctx_t mread_ctx = { 0, }; + /* we select different rpcs depending on the number of * read requests */ if (count > 1) { @@ -1078,7 +1087,7 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) count, buffer, size); /* invoke multi-read rpc */ - read_rc = invoke_client_mread_rpc(count, size, buffer); + read_rc = invoke_client_mread_rpc(count, size, buffer, &mread_ctx); /* free flat buffer resources */ flatcc_builder_clear(&builder); @@ -1113,6 +1122,7 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) * we process it in batches as it comes in, eventually the * server will tell us it's sent us everything it can */ int done = 0; + int rpc_done = 0; while (!done) { int tmp_rc = delegator_wait(); if (tmp_rc != UNIFYFS_SUCCESS) { @@ -1126,6 +1136,30 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) } delegator_signal(); } + + /* if this was mread, track the progress */ + if (count > 1 && !rpc_done) { + tmp_rc = unifyfs_mread_rpc_status_check(&mread_ctx); + if (tmp_rc < 0) { + LOGERR("failed to check the rpc progress"); + continue; + } + + /* if we received a response from the server, check any errors. + * for any errors, we do not have to wait for the data anymore. */ + if (tmp_rc) { + LOGDBG("received rpc response from the server (ret=%d)", + mread_ctx.rpc_ret); + + if (mread_ctx.rpc_ret != UNIFYFS_SUCCESS) { + LOGERR("mread rpc failed on server (ret=%d)", + mread_ctx.rpc_ret); + return UNIFYFS_FAILURE; + } + + rpc_done = 1; + } + } } LOGDBG("fetched all data from server for %d requests", count); @@ -1137,6 +1171,11 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) /* get pointer to next read request */ read_req_t* req = &read_reqs[i]; + /* no error message was received from server, set it success */ + if (req->errcode == EINPROGRESS) { + req->errcode = UNIFYFS_SUCCESS; + } + /* if we hit an error on our read, nothing else to do */ if (req->errcode != UNIFYFS_SUCCESS) { continue; diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index a47e633e3..9205d4f08 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -95,6 +95,7 @@ typedef enum { } readreq_status_e; typedef struct { + int gfid; /* gfid */ size_t nbytes; /* size of data chunk */ size_t offset; /* file offset */ size_t log_offset; /* remote log offset */ @@ -103,6 +104,7 @@ typedef struct { } chunk_read_req_t; typedef struct { + int gfid; /* gfid */ size_t offset; /* file offset */ size_t nbytes; /* requested read size */ ssize_t read_rc; /* bytes read (or negative error code) */ diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 9b1384836..10f1d6ee3 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -228,9 +228,8 @@ void free_value_array(unifyfs_val_t** array) static void debug_print_read_req(server_read_req_t* req) { if (NULL != req) { - LOGDBG("server_read_req[%d] status=%d, gfid=%d, num_remote=%d", - req->req_ndx, req->status, req->extent.gfid, - req->num_remote_reads); + LOGDBG("server_read_req[%d] status=%d, num_remote=%d", + req->req_ndx, req->status, req->num_remote_reads); } } @@ -241,18 +240,19 @@ static server_read_req_t* reserve_read_req(reqmgr_thrd_t* thrd_ctrl) if (thrd_ctrl->num_read_reqs < RM_MAX_ACTIVE_REQUESTS) { if (thrd_ctrl->next_rdreq_ndx < (RM_MAX_ACTIVE_REQUESTS - 1)) { rdreq = thrd_ctrl->read_reqs + thrd_ctrl->next_rdreq_ndx; - assert((rdreq->req_ndx == 0) && (rdreq->extent.gfid == 0)); + assert((rdreq->req_ndx == 0) && (rdreq->in_use == 0)); rdreq->req_ndx = thrd_ctrl->next_rdreq_ndx++; } else { // search for unused slot for (int i = 0; i < RM_MAX_ACTIVE_REQUESTS; i++) { rdreq = thrd_ctrl->read_reqs + i; - if ((rdreq->req_ndx == 0) && (rdreq->extent.gfid == 0)) { + if ((rdreq->req_ndx == 0) && (rdreq->in_use == 0)) { rdreq->req_ndx = i; break; } } } thrd_ctrl->num_read_reqs++; + rdreq->in_use = 1; LOGDBG("reserved read req %d (active=%d, next=%d)", rdreq->req_ndx, thrd_ctrl->num_read_reqs, thrd_ctrl->next_rdreq_ndx); debug_print_read_req(rdreq); @@ -263,11 +263,13 @@ static server_read_req_t* reserve_read_req(reqmgr_thrd_t* thrd_ctrl) return rdreq; } -static int release_read_req(reqmgr_thrd_t* thrd_ctrl, - server_read_req_t* rdreq) +static int __release_read_req(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq) { + // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked + int rc = (int)UNIFYFS_SUCCESS; - RM_LOCK(thrd_ctrl); + if (rdreq != NULL) { LOGDBG("releasing read req %d", rdreq->req_ndx); if (rdreq->req_ndx == (thrd_ctrl->next_rdreq_ndx - 1)) { @@ -288,7 +290,18 @@ static int release_read_req(reqmgr_thrd_t* thrd_ctrl, rc = EINVAL; LOGERR("NULL read_req"); } + + return rc; +} + +static int release_read_req(reqmgr_thrd_t* thrd_ctrl, server_read_req_t* rdreq) +{ + int rc = (int)UNIFYFS_SUCCESS; + + RM_LOCK(thrd_ctrl); + rc = __release_read_req(thrd_ctrl, rdreq); RM_UNLOCK(thrd_ctrl); + return rc; } @@ -374,6 +387,7 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, chunk_read_req_t* chk = all_chunk_reads + i; /* fill in chunk read request */ + chk->gfid = keyvals[i].key.gfid; chk->nbytes = keyvals[i].val.len; chk->offset = keyvals[i].key.offset; chk->log_offset = keyvals[i].val.addr; @@ -1033,63 +1047,68 @@ int rm_cmd_laminate( return rc; } -int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, - int gfid, int app_id, int client_id, - int num_keys, unifyfs_key_t** keys, int* keylens) +static int submit_read_request(reqmgr_thrd_t* thrd_ctrl, int num_keys, + unifyfs_key_t** keys, int* keylens) { - /* lookup all key/value pairs for given range */ + int ret = UNIFYFS_SUCCESS; + int app_id = -1; + int client_id = -1; int num_vals = 0; unifyfs_keyval_t* keyvals = NULL; - int rc = unifyfs_get_file_extents(num_keys, keys, keylens, - &num_vals, &keyvals); + + if (!thrd_ctrl || num_keys < 0 || !keys || !keylens) { + return EINVAL; + } + + app_id = thrd_ctrl->app_id; + client_id = thrd_ctrl->client_id; + + /* lookup all key/value pairs for given range */ + ret = unifyfs_get_file_extents(num_keys, keys, keylens, + &num_vals, &keyvals); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("failed to get file extents (ret=%d)", ret); + return UNIFYFS_ERROR_MDHIM; + } /* this is to maintain limits imposed in previous code * that would throw fatal errors */ if (num_vals >= UNIFYFS_MAX_SPLIT_CNT || num_vals >= MAX_META_PER_SEND) { LOGERR("too many key/values returned in range lookup"); - if (NULL != keyvals) { - free(keyvals); - keyvals = NULL; - } - return ENOMEM; + ret = ENOMEM; + goto out_free; } - if (UNIFYFS_SUCCESS != rc) { - /* failed to find any key / value pairs */ - rc = UNIFYFS_FAILURE; - } else { - /* if we get more than one write index entry - * sort them by file id and then by delegator rank */ - if (num_vals > 1) { - qsort(keyvals, (size_t)num_vals, sizeof(unifyfs_keyval_t), - compare_kv_gfid_rank); - } + /* if we get more than one write index entry + * sort them by file id and then by delegator rank */ + if (num_vals > 1) { + qsort(keyvals, (size_t)num_vals, sizeof(unifyfs_keyval_t), + compare_kv_gfid_rank); + } - server_read_req_t* rdreq = reserve_read_req(thrd_ctrl); - if (NULL == rdreq) { - rc = UNIFYFS_FAILURE; - } else { - rdreq->app_id = app_id; - rdreq->client_id = client_id; - rdreq->extent.gfid = gfid; - rdreq->extent.errcode = EINPROGRESS; + server_read_req_t* rdreq = reserve_read_req(thrd_ctrl); + if (NULL == rdreq) { + LOGERR("failed to allocate server_read_req_t"); + ret = UNIFYFS_FAILURE; + } else { + rdreq->app_id = app_id; + rdreq->client_id = client_id; - rc = create_chunk_requests(thrd_ctrl, rdreq, - num_vals, keyvals); - if (rc != (int)UNIFYFS_SUCCESS) { - release_read_req(thrd_ctrl, rdreq); - } + ret = create_chunk_requests(thrd_ctrl, rdreq, num_vals, keyvals); + if (ret != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to submit read requests"); + release_read_req(thrd_ctrl, rdreq); } } - /* free off key/value buffer returned from get_file_extents */ +out_free: if (NULL != keyvals) { free(keyvals); keyvals = NULL; } - return rc; + return ret; } /* return number of slice ranges needed to cover range */ @@ -1293,8 +1312,7 @@ int rm_cmd_read( split_request(keys, key_lens, gfid, offset, length); /* queue up the read operations */ - int rc = create_gfid_chunk_reads(thrd_ctrl, gfid, - app_id, client_id, key_cnt, keys, key_lens); + int rc = submit_read_request(thrd_ctrl, key_cnt, keys, key_lens); /* free memory allocated for key storage */ free_key_array(keys); @@ -1362,30 +1380,12 @@ int rm_cmd_mread( return ENOMEM; } - /* get chunks corresponding to requested client read extents */ - int ret; + /* we need to create a single server_read_req_t structure even with + * multiple gfids. */ int num_keys = 0; - int last_gfid = -1; for (j = 0; j < req_num; j++) { /* get the file id for this request */ int gfid = unifyfs_Extent_fid(unifyfs_Extent_vec_at(extents, j)); - - /* if we have switched to a different file, create chunk reads - * for the previous file */ - if (j && (gfid != last_gfid)) { - /* create requests for all extents of last_gfid */ - ret = create_gfid_chunk_reads(thrd_ctrl, last_gfid, - app_id, client_id, num_keys, keys, key_lens); - if (ret != UNIFYFS_SUCCESS) { - LOGERR("Error creating chunk reads for gfid=%d", last_gfid); - rc = ret; - } - - /* reset key counter for the current gfid */ - num_keys = 0; - } - - /* get offset and length of current read request */ size_t off = unifyfs_Extent_offset(unifyfs_Extent_vec_at(extents, j)); size_t len = unifyfs_Extent_length(unifyfs_Extent_vec_at(extents, j)); LOGDBG("gfid:%d, offset:%zu, length:%zu", gfid, off, len); @@ -1402,21 +1402,12 @@ int rm_cmd_mread( /* split range of read request at boundaries used for * MDHIM range query */ - int used = split_request(&keys[num_keys], &key_lens[num_keys], - gfid, off, len); - num_keys += used; - - /* keep track of the last gfid value that we processed */ - last_gfid = gfid; + num_keys += split_request(&keys[num_keys], &key_lens[num_keys], + gfid, off, len); } - /* create requests for all extents of final gfid */ - ret = create_gfid_chunk_reads(thrd_ctrl, last_gfid, - app_id, client_id, num_keys, keys, key_lens); - if (ret != UNIFYFS_SUCCESS) { - LOGERR("Error creating chunk reads for gfid=%d", last_gfid); - rc = ret; - } + /* queue the read operations */ + rc = submit_read_request(thrd_ctrl, num_keys, keys, key_lens); /* free memory allocated for key storage */ free_key_array(keys); @@ -1750,8 +1741,6 @@ static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) } } else if ((req->num_remote_reads == 0) && (req->status == READREQ_STARTED)) { - RM_LOCK(thrd_ctrl); - /* look up client shared memory region */ int app_id = req->app_id; int client_id = req->client_id; @@ -1771,13 +1760,11 @@ static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) client_wait(shm_hdr); } - rc = release_read_req(thrd_ctrl, req); + rc = __release_read_req(thrd_ctrl, req); if (rc != (int)UNIFYFS_SUCCESS) { LOGERR("failed to release server_read_req_t"); ret = rc; } - - RM_UNLOCK(thrd_ctrl); } } @@ -1891,6 +1878,8 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, server_read_req_t* rdreq, remote_chunk_reads_t* del_reads) { + // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked + int errcode, gfid, i, num_chks, rc, thrd_id; int ret = (int)UNIFYFS_SUCCESS; chunk_read_resp_t* responses = NULL; @@ -1914,26 +1903,23 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, client_shm = clnt->shmem_data; shm_hdr = (shm_data_header*) client_shm->addr; - RM_LOCK(thrd_ctrl); - num_chks = del_reads->num_chunks; - gfid = rdreq->extent.gfid; if (del_reads->status != READREQ_STARTED) { LOGERR("chunk read response for non-started req @ index=%d", rdreq->req_ndx); ret = (int32_t)EINVAL; } else if (0 == del_reads->total_sz) { - LOGERR("empty chunk read response for gfid=%d", gfid); + LOGERR("empty chunk read response from delegator %d", del_reads->rank); ret = (int32_t)EINVAL; } else { LOGDBG("handling chunk read responses from server %d: " - "gfid=%d num_chunks=%d buf_size=%zu", - del_reads->rank, gfid, num_chks, - del_reads->total_sz); + "num_chunks=%d buf_size=%zu", + del_reads->rank, num_chks, del_reads->total_sz); responses = del_reads->resp; data_buf = (char*)(responses + num_chks); for (i = 0; i < num_chks; i++) { chunk_read_resp_t* resp = responses + i; + gfid = resp->gfid; if (resp->read_rc < 0) { errcode = (int)-(resp->read_rc); data_sz = 0; @@ -1942,7 +1928,8 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, data_sz = resp->nbytes; } offset = resp->offset; - LOGDBG("chunk response for offset=%zu: sz=%zu", offset, data_sz); + LOGDBG("chunk response for gfid=%d (offset=%zu, sz=%zu)", + gfid, offset, data_sz); /* allocate and register local target buffer for bulk access */ meta = reserve_shmem_meta(client_shm, shm_hdr, data_sz); @@ -1986,15 +1973,13 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, /* wait for client to read data */ client_wait(shm_hdr); - rc = release_read_req(thrd_ctrl, rdreq); + rc = __release_read_req(thrd_ctrl, rdreq); if (rc != (int)UNIFYFS_SUCCESS) { LOGERR("failed to release server_read_req_t"); } } } - RM_UNLOCK(thrd_ctrl); - return ret; } diff --git a/server/src/unifyfs_request_manager.h b/server/src/unifyfs_request_manager.h index 4c4310e89..bb0924338 100644 --- a/server/src/unifyfs_request_manager.h +++ b/server/src/unifyfs_request_manager.h @@ -38,7 +38,7 @@ typedef struct { int app_id; /* app id of requesting client process */ int client_id; /* client id of requesting client process */ int num_remote_reads; /* size of remote_reads array */ - client_read_req_t extent; /* client read extent, includes gfid */ + int in_use; /* occupied by a thread */ chunk_read_req_t* chunks; /* array of chunk-reads */ remote_chunk_reads_t* remote_reads; /* per-delegator remote reads array */ } server_read_req_t; diff --git a/server/src/unifyfs_service_manager.c b/server/src/unifyfs_service_manager.c index 3821708e2..3368c66bc 100644 --- a/server/src/unifyfs_service_manager.c +++ b/server/src/unifyfs_service_manager.c @@ -172,6 +172,7 @@ int sm_issue_chunk_reads(int src_rank, size_t log_offset = rreq->log_offset; /* record request metadata in response */ + rresp->gfid = rreq->gfid; rresp->read_rc = 0; rresp->nbytes = nbytes; rresp->offset = rreq->offset; From 6b422c3954cb988007293cf863f2b974d601cd5c Mon Sep 17 00:00:00 2001 From: CamStan Date: Tue, 4 Aug 2020 17:52:06 -0700 Subject: [PATCH 148/168] Add ascent.yml to gitlab-ci.yml Add missing include for ascent.yml to .gitlab-ci.yml. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae0e095b3..b4fd79e8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -125,5 +125,6 @@ before_script: # System specific jobs include: + - local: .gitlab/ascent.yml - local: .gitlab/catalyst.yml - local: .gitlab/lassen.yml From 08be75659f5e41550999f97c4231176f891c290e Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Wed, 29 Jul 2020 14:10:42 -0400 Subject: [PATCH 149/168] Modifying the read-data example to verify the data content. - modifying lipsum_check function (testutil.h) to check arbitrary byte offset - modifying read-data example to verify data content - modifying read-data example to generate ramdom offset/length --- examples/src/read-data.c | 155 ++++++++++++++++++++++++++++++++++----- examples/src/testutil.h | 15 ++++ 2 files changed, 150 insertions(+), 20 deletions(-) diff --git a/examples/src/read-data.c b/examples/src/read-data.c index 83c71e8e0..fa3c962c9 100644 --- a/examples/src/read-data.c +++ b/examples/src/read-data.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "testutil.h" @@ -40,9 +41,10 @@ static char filename[PATH_MAX]; static char mountpoint[PATH_MAX]; static char* buf; -static unsigned int bufsize; +static uint64_t bufsize; +static int check; -static int parse_line(char* line, unsigned long* offset, unsigned long* length) +static int parse_line(char* line, uint64_t* offset, uint64_t* length) { char* pos = NULL; @@ -60,13 +62,13 @@ static int parse_line(char* line, unsigned long* offset, unsigned long* length) *pos = '\0'; pos++; - *offset = strtoul(line, NULL, 0); - *length = strtoul(pos, NULL, 0); + *offset = strtoull(line, NULL, 0); + *length = strtoull(pos, NULL, 0); return 0; } -static void alloc_buf(unsigned long length) +static void alloc_buf(uint64_t length) { if (!buf) { bufsize = length; @@ -83,29 +85,80 @@ static void alloc_buf(unsigned long length) } } +static void aligned_offlen(uint64_t filesize, uint64_t blocksize, + uint64_t* off, uint64_t* len) +{ + uint64_t block_count = filesize / blocksize; + + *off = (random() % (block_count - 1)) * blocksize; + *len = blocksize; +} + + +static void random_offlen(uint64_t filesize, uint64_t maxoff, uint64_t maxlen, + uint64_t* off, uint64_t* len) +{ + uint64_t _off; + uint64_t _len; + + _off = random() % maxoff; + _len = random() % maxlen; + + while (_off + _len > filesize) { + _len = _len / 2 + 1; + } + + *len = _len; + *off = _off; +} + static void do_pread(int fd, size_t length, off_t offset) { ssize_t ret = 0; + struct timespec ts1, ts2; + double ts1nsec, ts2nsec; + double elapsed_sec, mbps; alloc_buf(length); errno = 0; + clock_gettime(CLOCK_REALTIME, &ts1); + ret = pread(fd, buf, length, offset); + clock_gettime(CLOCK_REALTIME, &ts2); + + ts1nsec = 1e9 * 1.0 * ts1.tv_sec + 1.0 * ts1.tv_nsec; + ts2nsec = 1e9 * 1.0 * ts2.tv_sec + 1.0 * ts2.tv_nsec; + elapsed_sec = (ts2nsec - ts1nsec) / (1e9); + + mbps = (1.0 * length / (1<<20)) / elapsed_sec; + printf(" -> pread(off=%lu, len=%lu) = %zd", offset, length, ret); if (errno) { printf(" (err=%d, %s)\n", errno, strerror(errno)); } else { - printf("\n"); + printf(" (%.3f sec, %.3lf MB/s)\n", elapsed_sec, mbps); + + if (check) { + uint64_t error_offset; + ret = lipsum_check(buf, length, offset, &error_offset); + if (ret < 0) { + printf(" * data verification failed at offset %" PRIu64 "\n", + error_offset); + } else { + printf(" * data verification success\n"); + } + } } } static void run_interactive(int fd) { int ret = 0; - unsigned long offset = 0; - unsigned long length = 0; + uint64_t offset = 0; + uint64_t length = 0; char* line = NULL; char linebuf[LINE_MAX]; @@ -131,13 +184,18 @@ static void run_interactive(int fd) static struct option long_opts[] = { { "help", 0, 0, 'h' }, + { "check", 0, 0, 'c' }, { "mount", 1, 0, 'm' }, { "offset", 1, 0, 'o' }, { "length", 1, 0, 'l' }, + { "random", 1, 0, 'r' }, + { "max-offset", 1, 0, 'O' }, + { "max-length", 1, 0, 'L' }, + { "aligned", 1, 0, 'a' }, { 0, 0, 0, 0}, }; -static char* short_opts = "hm:o:l:"; +static char* short_opts = "hcm:o:l:r:O:L:f:"; static const char* usage_str = "\n" @@ -150,10 +208,17 @@ static const char* usage_str = "'quit' will terminate the program.\n" "\n" "Available options:\n" -" -h, --help help message\n" -" -m, --mount= use for unifyfs (default: /unifyfs)\n" -" -o, --offset= read from \n" -" -l, --length= read bytes\n" +" -h, --help help message\n" +" -c, --check verify data content. data should be written using\n" +" the write example with --check option\n" +" -m, --mount= use for unifyfs (default: /unifyfs)\n" +" -o, --offset= read from \n" +" -l, --length= read bytes\n" +" -r, --random= generate random offset and length times,\n" +" only workin in the non-interactive mode\n" +" -O, --max-offset= generate a random offset not exceeding \n" +" -L, --max-length= generate a random length not exceeding \n" +" -f, --aligned= generate requests aligned with a blocksize \n" "\n"; static char* program; @@ -171,8 +236,13 @@ int main(int argc, char** argv) int optidx = 0; int unifyfs = 0; int fd = -1; - unsigned long offset = 0; - unsigned long length = 0; + int random = 0; + uint64_t offset = 0; + uint64_t length = 0; + uint64_t maxoff = 0; + uint64_t maxlen = 0; + uint64_t aligned = 0; + uint64_t filesize = 0; struct stat sb; char* tmp_program = NULL; @@ -187,16 +257,36 @@ int main(int argc, char** argv) while ((ch = getopt_long(argc, argv, short_opts, long_opts, &optidx)) >= 0) { switch (ch) { + case 'c': + check = 1; + break; + case 'm': sprintf(mountpoint, "%s", optarg); break; case 'o': - offset = strtoul(optarg, NULL, 0); + offset = strtoull(optarg, NULL, 0); break; case 'l': - length = strtoul(optarg, NULL, 0); + length = strtoull(optarg, NULL, 0); + break; + + case 'r': + random = atoi(optarg); + break; + + case 'O': + maxoff = strtoull(optarg, NULL, 0); + break; + + case 'L': + maxlen = strtoull(optarg, NULL, 0); + break; + + case 'a': + aligned = strtoull(optarg, NULL, 0); break; case 'h': @@ -241,12 +331,37 @@ int main(int argc, char** argv) goto out; } - printf("%s (size = %lu)\n", filename, sb.st_size); + filesize = sb.st_size; + printf("%s (size = %lu)\n", filename, filesize); - if (offset == 0 && length == 0) { + if (offset == 0 && length == 0 && random == 0) { run_interactive(fd); } else { - do_pread(fd, length, offset); + if (random) { + struct timespec ts; + + clock_gettime(CLOCK_REALTIME, &ts); + srandom(ts.tv_nsec % ts.tv_sec); + + if (0 == maxoff) { + maxoff = filesize / 2; + } + + if (0 == maxlen) { + maxlen = filesize / 2; + } + + for (int i = 0; i < random; i++) { + if (aligned) { + aligned_offlen(filesize, aligned, &offset, &length); + } else { + random_offlen(filesize, maxoff, maxlen, &offset, &length); + } + do_pread(fd, length, offset); + } + } else { + do_pread(fd, length, offset); + } } ret = 0; diff --git a/examples/src/testutil.h b/examples/src/testutil.h index 48875f337..b646c4d50 100644 --- a/examples/src/testutil.h +++ b/examples/src/testutil.h @@ -759,8 +759,23 @@ int lipsum_check(const char* buf, uint64_t len, uint64_t offset, uint64_t i, val; uint64_t start = offset / sizeof(uint64_t); uint64_t count = len / sizeof(uint64_t); + uint64_t skip = 0; + uint64_t remain = 0; const uint64_t* ibuf = (uint64_t*) buf; + /* check if we have any extra bytes at the front and end */ + if (offset % sizeof(uint64_t)) { + skip = sizeof(uint64_t) - (offset % sizeof(uint64_t)); + remain = (len - skip) % sizeof(uint64_t); + + ibuf = (uint64_t*) &buf[skip]; + start++; + + if (skip + remain >= sizeof(uint64_t)) { + count--; + } + } + for (i = 0; i < count; i++) { val = start + i; if (ibuf[i] != val) { From 626a74796eb081925c5ea919e7a299cef1472a3b Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Mon, 3 Aug 2020 18:02:00 -0400 Subject: [PATCH 150/168] remove flatcc dependency --- .gitlab-ci.yml | 1 - .travis.yml | 1 - bootstrap.sh | 13 +- client/src/Makefile.am | 7 +- client/src/unifyfs.c | 55 +- common/src/Makefile.am | 7 +- common/src/flatbuffers_common_builder.h | 658 ------------------------ common/src/flatbuffers_common_reader.h | 512 ------------------ common/src/ucr_read_builder.h | 74 --- common/src/ucr_read_reader.h | 74 --- common/src/unifyfs_meta.h | 8 + configure.ac | 1 - docs/build-intercept.rst | 6 +- docs/dependencies.rst | 2 - examples/src/Makefile.am | 9 +- m4/flatcc.m4 | 29 -- m4/leveldb.m4 | 2 +- server/src/Makefile.am | 7 +- server/src/unifyfs_metadata.c | 2 - server/src/unifyfs_request_manager.c | 125 +++-- t/Makefile.am | 7 +- t/ci/001-setup.sh | 2 +- util/unifyfs-stage/src/Makefile.am | 3 +- 23 files changed, 105 insertions(+), 1500 deletions(-) delete mode 100644 common/src/flatbuffers_common_builder.h delete mode 100644 common/src/flatbuffers_common_reader.h delete mode 100644 common/src/ucr_read_builder.h delete mode 100644 common/src/ucr_read_reader.h delete mode 100644 m4/flatcc.m4 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4fd79e8e..370f9ec95 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,7 +115,6 @@ before_script: - FC_PATH=$($FC_COMMAND) - SPACK_COMPILER=${COMPILER//\//@} - SPACK_ARCH="$(spack arch -p)-$(spack arch -o)-$(uname -m)" - - spack load flatcc %$SPACK_COMPILER arch=$SPACK_ARCH - spack load gotcha %$SPACK_COMPILER arch=$SPACK_ARCH - spack load leveldb %$SPACK_COMPILER arch=$SPACK_ARCH - spack load argobots %$SPACK_COMPILER arch=$SPACK_ARCH diff --git a/.travis.yml b/.travis.yml index 05fcc6470..3a4113bea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,7 +61,6 @@ install: - . $HOME/spack/share/spack/setup-env.sh - spack install leveldb && spack load leveldb - spack install gotcha@1.0.3 && spack load gotcha@1.0.3 - - spack install flatcc && spack load flatcc - spack install margo^mercury+bmi~ofi~boostsys && spack load argobots && spack load mercury && spack load margo - spack install spath && spack load spath # prepare build environment diff --git a/bootstrap.sh b/bootstrap.sh index 3f8ce7b12..e29f69ae4 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -17,7 +17,6 @@ repos=( https://xgitlab.cels.anl.gov/sds/bmi.git https://github.com/pmodels/argobots.git https://github.com/mercury-hpc/mercury.git https://xgitlab.cels.anl.gov/sds/margo.git - https://github.com/dvidelabs/flatcc.git ) for i in "${repos[@]}" ; do @@ -95,23 +94,13 @@ export PKG_CONFIG_PATH="$INSTALL_DIR/lib/pkgconfig" make -j $(nproc) && make install cd .. -echo "### building flatcc ###" -cd flatcc -# need -DBUILD_SHARED_LIBS=ye -mkdir -p build && cd build -cmake -DBUILD_SHARED_LIBS=on -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -DFLATCC_INSTALL=on .. -make -j $(nproc) && make install -cd .. -cd .. - cd "$ROOT" echo "*************************************************************************" echo "Dependencies are all built. You can now build UnifyFS with:" echo "" echo -n " export PKG_CONFIG_PATH=$INSTALL_DIR/lib/pkgconfig && " -echo "export LEVELDB_ROOT=$INSTALL_DIR && export FLATCC_ROOT=$INSTALL_DIR" +echo "export LEVELDB_ROOT=$INSTALL_DIR" echo -n " ./autogen.sh && ./configure --with-gotcha=$INSTALL_DIR" echo " --prefix=$INSTALL_DIR" echo " make" diff --git a/client/src/Makefile.am b/client/src/Makefile.am index 0325df935..9df435752 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -28,19 +28,16 @@ CLIENT_COMMON_CFLAGS = \ $(MPI_CFLAGS) \ $(MERCURY_CFLAGS) \ $(ARGOBOTS_CFLAGS) \ - $(MARGO_CFLAGS) \ - $(FLATCC_CFLAGS) + $(MARGO_CFLAGS) CLIENT_COMMON_LDFLAGS = \ -version-info $(LIBUNIFYFS_LT_VERSION) \ $(MPI_CLDFLAGS) \ - $(MARGO_LDFLAGS) \ - $(FLATCC_LDFLAGS) + $(MARGO_LDFLAGS) CLIENT_COMMON_LIBADD = \ $(top_builddir)/common/src/libunifyfs_common.la \ $(MARGO_LIBS) \ - $(FLATCC_LIBS) \ -lrt -lpthread CLIENT_COMMON_SOURCES = \ diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 300b6c59b..5521fce73 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -50,7 +50,6 @@ #include "unifyfs_rpc_util.h" #include "margo_client.h" #include "seg_tree.h" -#include "ucr_read_builder.h" #ifdef HAVE_SPATH #include "spath.h" @@ -593,20 +592,20 @@ static int compare_read_req(const void* a, const void* b) } } -/* notify our delegator that the shared memory buffer +/* notify our reqmgr that the shared memory buffer * is now clear and ready to hold more read data */ static void delegator_signal(void) { LOGDBG("receive buffer now empty"); - /* set shm flag to signal delegator we're done */ + /* set shm flag to signal reqmgr we're done */ shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); hdr->state = SHMEM_REGION_EMPTY; /* TODO: MEM_FLUSH */ } -/* wait for delegator to inform us that shared memory buffer +/* wait for reqmgr to inform us that shared memory buffer * is filled with read data */ static int delegator_wait(void) { @@ -640,7 +639,7 @@ static int delegator_wait(void) } /* copy read data from shared memory buffer to user buffers from read - * calls, sets done=1 on return when delegator informs us it has no + * calls, sets done=1 on return when reqmgr informs us it has no * more data */ static int process_read_data(read_req_t* read_reqs, int count, int* done) { @@ -967,7 +966,7 @@ static void service_local_reqs( /* * get data for a list of read requests from the - * delegator + * reqmgr * * @param read_reqs: a list of read requests * @param count: number of read requests @@ -1050,7 +1049,7 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) /* order read request by increasing file id, then increasing offset */ qsort(read_reqs, count, sizeof(read_req_t), compare_read_req); - /* prepare our shared memory buffer for delegator */ + /* prepare our shared memory buffer for reqmgr */ delegator_signal(); /* for mread, we need to manually track the rpc progress */ @@ -1059,38 +1058,28 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) /* we select different rpcs depending on the number of * read requests */ if (count > 1) { - /* got multiple read requests, - * build up a flat buffer to include them all */ - flatcc_builder_t builder; - flatcc_builder_init(&builder); - - /* create request vector */ - unifyfs_Extent_vec_start(&builder); - - /* fill in values for each request entry */ + /* got multiple read requests */ + size_t size = (size_t)count * sizeof(unifyfs_extent_t); + void* buffer = malloc(size); + if (NULL == buffer) { + return ENOMEM; + } + unifyfs_extent_t* extents = (unifyfs_extent_t*)buffer; + unifyfs_extent_t* ext; + read_req_t* req; for (i = 0; i < count; i++) { - unifyfs_Extent_vec_push_create(&builder, - read_reqs[i].gfid, read_reqs[i].offset, read_reqs[i].length); + ext = extents + i; + req = read_reqs + i; + ext->gfid = req->gfid; + ext->offset = req->offset; + ext->length = req->length; } - /* complete the array */ - unifyfs_Extent_vec_ref_t extents = unifyfs_Extent_vec_end(&builder); - unifyfs_ReadRequest_create_as_root(&builder, extents); - //unifyfs_ReadRequest_end_as_root(&builder); - - /* allocate our buffer to be sent */ - size_t size = 0; - void* buffer = flatcc_builder_finalize_buffer(&builder, &size); - assert(buffer); - - LOGDBG("mread: n_reqs:%d, flatcc buffer (%p) sz:%zu", + LOGDBG("mread: n_reqs:%d, reqs(%p) sz:%zu", count, buffer, size); /* invoke multi-read rpc */ read_rc = invoke_client_mread_rpc(count, size, buffer, &mread_ctx); - - /* free flat buffer resources */ - flatcc_builder_clear(&builder); free(buffer); } else { /* got a single read request */ @@ -1566,8 +1555,6 @@ int unifyfs_fid_free(int fid) * returns the new fid, or negative value on error */ int unifyfs_fid_create_file(const char* path) { - int rc; - /* check that pathname is within bounds */ size_t pathlen = strlen(path) + 1; if (pathlen > UNIFYFS_MAX_FILENAME) { diff --git a/common/src/Makefile.am b/common/src/Makefile.am index c7a25e210..11f09771d 100644 --- a/common/src/Makefile.am +++ b/common/src/Makefile.am @@ -11,8 +11,6 @@ BASE_SRCS = \ cm_enumerator.c \ rm_enumerator.h \ rm_enumerator.c \ - flatbuffers_common_builder.h \ - flatbuffers_common_reader.h \ seg_tree.h \ seg_tree.c \ slotmap.h \ @@ -20,8 +18,6 @@ BASE_SRCS = \ tinyexpr.h \ tinyexpr.c \ tree.h \ - ucr_read_builder.h \ - ucr_read_reader.h \ unifyfs_const.h \ unifyfs_configurator.h \ unifyfs_configurator.c \ @@ -64,8 +60,7 @@ libunifyfs_common_la_CPPFLAGS = \ $(OPT_FLAGS) \ $(MERCURY_CFLAGS) \ $(ARGOBOTS_CFLAGS) \ - $(MARGO_CFLAGS) \ - $(FLATCC_CFLAGS) + $(MARGO_CFLAGS) libunifyfs_common_la_CPPFLAGS += -DSYSCONFDIR="$(sysconfdir)" diff --git a/common/src/flatbuffers_common_builder.h b/common/src/flatbuffers_common_builder.h deleted file mode 100644 index 07d840f01..000000000 --- a/common/src/flatbuffers_common_builder.h +++ /dev/null @@ -1,658 +0,0 @@ -#ifndef FLATBUFFERS_COMMON_BUILDER_H -#define FLATBUFFERS_COMMON_BUILDER_H - -/* Generated by flatcc 0.5.3-pre FlatBuffers schema compiler for C by dvide.com */ - -/* Common FlatBuffers build functionality for C. */ - -#include "flatcc/flatcc_prologue.h" -#ifndef FLATBUILDER_H -#include "flatcc/flatcc_builder.h" -#endif -typedef flatcc_builder_t flatbuffers_builder_t; -typedef flatcc_builder_ref_t flatbuffers_ref_t; -typedef flatcc_builder_ref_t flatbuffers_vec_ref_t; -typedef flatcc_builder_union_ref_t flatbuffers_union_ref_t; -typedef flatcc_builder_union_vec_ref_t flatbuffers_union_vec_ref_t; -/* integer return code (ref and ptr always fail on 0) */ -#define flatbuffers_failed(x) ((x) < 0) -typedef flatbuffers_ref_t flatbuffers_root_t; -#define flatbuffers_root(ref) ((flatbuffers_root_t)(ref)) - -#define __flatbuffers_memoize_begin(B, src)\ -do { flatcc_builder_ref_t _ref; if ((_ref = flatcc_builder_refmap_find((B), (src)))) return _ref; } while (0) -#define __flatbuffers_memoize_end(B, src, op) do { return flatcc_builder_refmap_insert((B), (src), (op)); } while (0) -#define __flatbuffers_memoize(B, src, op) do { __flatbuffers_memoize_begin(B, src); __flatbuffers_memoize_end(B, src, op); } while (0) - -#define __flatbuffers_build_buffer(NS)\ -typedef NS ## ref_t NS ## buffer_ref_t;\ -static inline int NS ## buffer_start(NS ## builder_t *B, const NS ##fid_t fid)\ -{ return flatcc_builder_start_buffer(B, fid, 0, 0); }\ -static inline int NS ## buffer_start_with_size(NS ## builder_t *B, const NS ##fid_t fid)\ -{ return flatcc_builder_start_buffer(B, fid, 0, flatcc_builder_with_size); }\ -static inline int NS ## buffer_start_aligned(NS ## builder_t *B, NS ##fid_t fid, uint16_t block_align)\ -{ return flatcc_builder_start_buffer(B, fid, block_align, 0); }\ -static inline int NS ## buffer_start_aligned_with_size(NS ## builder_t *B, NS ##fid_t fid, uint16_t block_align)\ -{ return flatcc_builder_start_buffer(B, fid, block_align, flatcc_builder_with_size); }\ -static inline NS ## buffer_ref_t NS ## buffer_end(NS ## builder_t *B, NS ## ref_t root)\ -{ return flatcc_builder_end_buffer(B, root); } - -#define __flatbuffers_build_table_root(NS, N, FID, TFID)\ -static inline int N ## _start_as_root(NS ## builder_t *B)\ -{ return NS ## buffer_start(B, FID) ? -1 : N ## _start(B); }\ -static inline int N ## _start_as_root_with_size(NS ## builder_t *B)\ -{ return NS ## buffer_start_with_size(B, FID) ? -1 : N ## _start(B); }\ -static inline int N ## _start_as_typed_root(NS ## builder_t *B)\ -{ return NS ## buffer_start(B, TFID) ? -1 : N ## _start(B); }\ -static inline int N ## _start_as_typed_root_with_size(NS ## builder_t *B)\ -{ return NS ## buffer_start_with_size(B, TFID) ? -1 : N ## _start(B); }\ -static inline NS ## buffer_ref_t N ## _end_as_root(NS ## builder_t *B)\ -{ return NS ## buffer_end(B, N ## _end(B)); }\ -static inline NS ## buffer_ref_t N ## _end_as_typed_root(NS ## builder_t *B)\ -{ return NS ## buffer_end(B, N ## _end(B)); }\ -static inline NS ## buffer_ref_t N ## _create_as_root(NS ## builder_t *B __ ## N ## _formal_args)\ -{ if (NS ## buffer_start(B, FID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ -static inline NS ## buffer_ref_t N ## _create_as_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ -{ if (NS ## buffer_start_with_size(B, FID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ -static inline NS ## buffer_ref_t N ## _create_as_typed_root(NS ## builder_t *B __ ## N ## _formal_args)\ -{ if (NS ## buffer_start(B, TFID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ -static inline NS ## buffer_ref_t N ## _create_as_typed_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ -{ if (NS ## buffer_start_with_size(B, TFID)) return 0; return NS ## buffer_end(B, N ## _create(B __ ## N ## _call_args)); }\ -static inline NS ## buffer_ref_t N ## _clone_as_root(NS ## builder_t *B, N ## _table_t t)\ -{ if (NS ## buffer_start(B, FID)) return 0; return NS ## buffer_end(B, N ## _clone(B, t)); }\ -static inline NS ## buffer_ref_t N ## _clone_as_root_with_size(NS ## builder_t *B, N ## _table_t t)\ -{ if (NS ## buffer_start_with_size(B, FID)) return 0; return NS ## buffer_end(B, N ## _clone(B, t)); }\ -static inline NS ## buffer_ref_t N ## _clone_as_typed_root(NS ## builder_t *B, N ## _table_t t)\ -{ if (NS ## buffer_start(B, TFID)) return 0;return NS ## buffer_end(B, N ## _clone(B, t)); }\ -static inline NS ## buffer_ref_t N ## _clone_as_typed_root_with_size(NS ## builder_t *B, N ## _table_t t)\ -{ if (NS ## buffer_start_with_size(B, TFID)) return 0; return NS ## buffer_end(B, N ## _clone(B, t)); } - -#define __flatbuffers_build_table_prolog(NS, N, FID, TFID)\ -__flatbuffers_build_table_vector_ops(NS, N ## _vec, N)\ -__flatbuffers_build_table_root(NS, N, FID, TFID) - -#define __flatbuffers_build_struct_root(NS, N, A, FID, TFID)\ -static inline N ## _t *N ## _start_as_root(NS ## builder_t *B)\ -{ return NS ## buffer_start(B, FID) ? 0 : N ## _start(B); }\ -static inline N ## _t *N ## _start_as_root_with_size(NS ## builder_t *B)\ -{ return NS ## buffer_start_with_size(B, FID) ? 0 : N ## _start(B); }\ -static inline N ## _t *N ## _start_as_typed_root(NS ## builder_t *B)\ -{ return NS ## buffer_start(B, TFID) ? 0 : N ## _start(B); }\ -static inline N ## _t *N ## _start_as_typed_root_with_size(NS ## builder_t *B)\ -{ return NS ## buffer_start_with_size(B, TFID) ? 0 : N ## _start(B); }\ -static inline NS ## buffer_ref_t N ## _end_as_root(NS ## builder_t *B)\ -{ return NS ## buffer_end(B, N ## _end(B)); }\ -static inline NS ## buffer_ref_t N ## _end_as_typed_root(NS ## builder_t *B)\ -{ return NS ## buffer_end(B, N ## _end(B)); }\ -static inline NS ## buffer_ref_t N ## _end_pe_as_root(NS ## builder_t *B)\ -{ return NS ## buffer_end(B, N ## _end_pe(B)); }\ -static inline NS ## buffer_ref_t N ## _end_pe_as_typed_root(NS ## builder_t *B)\ -{ return NS ## buffer_end(B, N ## _end_pe(B)); }\ -static inline NS ## buffer_ref_t N ## _create_as_root(NS ## builder_t *B __ ## N ## _formal_args)\ -{ return flatcc_builder_create_buffer(B, FID, 0,\ - N ## _create(B __ ## N ## _call_args), A, 0); }\ -static inline NS ## buffer_ref_t N ## _create_as_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ -{ return flatcc_builder_create_buffer(B, FID, 0,\ - N ## _create(B __ ## N ## _call_args), A, flatcc_builder_with_size); }\ -static inline NS ## buffer_ref_t N ## _create_as_typed_root(NS ## builder_t *B __ ## N ## _formal_args)\ -{ return flatcc_builder_create_buffer(B, TFID, 0,\ - N ## _create(B __ ## N ## _call_args), A, 0); }\ -static inline NS ## buffer_ref_t N ## _create_as_typed_root_with_size(NS ## builder_t *B __ ## N ## _formal_args)\ -{ return flatcc_builder_create_buffer(B, TFID, 0,\ - N ## _create(B __ ## N ## _call_args), A, flatcc_builder_with_size); }\ -static inline NS ## buffer_ref_t N ## _clone_as_root(NS ## builder_t *B, N ## _struct_t p)\ -{ return flatcc_builder_create_buffer(B, FID, 0, N ## _clone(B, p), A, 0); }\ -static inline NS ## buffer_ref_t N ## _clone_as_root_with_size(NS ## builder_t *B, N ## _struct_t p)\ -{ return flatcc_builder_create_buffer(B, FID, 0, N ## _clone(B, p), A, flatcc_builder_with_size); }\ -static inline NS ## buffer_ref_t N ## _clone_as_typed_root(NS ## builder_t *B, N ## _struct_t p)\ -{ return flatcc_builder_create_buffer(B, TFID, 0, N ## _clone(B, p), A, 0); }\ -static inline NS ## buffer_ref_t N ## _clone_as_typed_root_with_size(NS ## builder_t *B, N ## _struct_t p)\ -{ return flatcc_builder_create_buffer(B, TFID, 0, N ## _clone(B, p), A, flatcc_builder_with_size); } - -#define __flatbuffers_build_nested_table_root(NS, N, TN, FID, TFID)\ -static inline int N ## _start_as_root(NS ## builder_t *B)\ -{ return NS ## buffer_start(B, FID) ? -1 : TN ## _start(B); }\ -static inline int N ## _start_as_typed_root(NS ## builder_t *B)\ -{ return NS ## buffer_start(B, TFID) ? -1 : TN ## _start(B); }\ -static inline int N ## _end_as_root(NS ## builder_t *B)\ -{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ -static inline int N ## _end_as_typed_root(NS ## builder_t *B)\ -{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ -static inline int N ## _nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ -{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ - align ? align : 8, FLATBUFFERS_COUNT_MAX(1))); }\ -static inline int N ## _typed_nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ -{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ - align ? align : 8, FLATBUFFERS_COUNT_MAX(1))); }\ -static inline int N ## _clone_as_root(NS ## builder_t *B, TN ## _table_t t)\ -{ return N ## _add(B, TN ## _clone_as_root(B, t)); }\ -static inline int N ## _clone_as_typed_root(NS ## builder_t *B, TN ## _table_t t)\ -{ return N ## _add(B, TN ## _clone_as_typed_root(B, t)); } - -#define __flatbuffers_build_nested_struct_root(NS, N, TN, A, FID, TFID)\ -static inline TN ## _t *N ## _start_as_root(NS ## builder_t *B)\ -{ return NS ## buffer_start(B, FID) ? 0 : TN ## _start(B); }\ -static inline TN ## _t *N ## _start_as_typed_root(NS ## builder_t *B)\ -{ return NS ## buffer_start(B, FID) ? 0 : TN ## _start(B); }\ -static inline int N ## _end_as_root(NS ## builder_t *B)\ -{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ -static inline int N ## _end_as_typed_root(NS ## builder_t *B)\ -{ return N ## _add(B, NS ## buffer_end(B, TN ## _end(B))); }\ -static inline int N ## _end_pe_as_root(NS ## builder_t *B)\ -{ return N ## _add(B, NS ## buffer_end(B, TN ## _end_pe(B))); }\ -static inline int N ## _create_as_root(NS ## builder_t *B __ ## TN ## _formal_args)\ -{ return N ## _add(B, flatcc_builder_create_buffer(B, FID, 0,\ - TN ## _create(B __ ## TN ## _call_args), A, flatcc_builder_is_nested)); }\ -static inline int N ## _create_as_typed_root(NS ## builder_t *B __ ## TN ## _formal_args)\ -{ return N ## _add(B, flatcc_builder_create_buffer(B, TFID, 0,\ - TN ## _create(B __ ## TN ## _call_args), A, flatcc_builder_is_nested)); }\ -static inline int N ## _nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ -{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ - align < A ? A : align, FLATBUFFERS_COUNT_MAX(1))); }\ -static inline int N ## _typed_nest(NS ## builder_t *B, void *data, size_t size, uint16_t align)\ -{ return N ## _add(B, flatcc_builder_create_vector(B, data, size, 1,\ - align < A ? A : align, FLATBUFFERS_COUNT_MAX(1))); }\ -static inline int N ## _clone_as_root(NS ## builder_t *B, TN ## _struct_t p)\ -{ return N ## _add(B, TN ## _clone_as_root(B, p)); }\ -static inline int N ## _clone_as_typed_root(NS ## builder_t *B, TN ## _struct_t p)\ -{ return N ## _add(B, TN ## _clone_as_typed_root(B, p)); } - -#define __flatbuffers_build_vector_ops(NS, V, N, TN, T)\ -static inline T *V ## _extend(NS ## builder_t *B, size_t len)\ -{ return (T *)flatcc_builder_extend_vector(B, len); }\ -static inline T *V ## _append(NS ## builder_t *B, const T *data, size_t len)\ -{ return (T *)flatcc_builder_append_vector(B, data, len); }\ -static inline int V ## _truncate(NS ## builder_t *B, size_t len)\ -{ return flatcc_builder_truncate_vector(B, len); }\ -static inline T *V ## _edit(NS ## builder_t *B)\ -{ return (T *)flatcc_builder_vector_edit(B); }\ -static inline size_t V ## _reserved_len(NS ## builder_t *B)\ -{ return flatcc_builder_vector_count(B); }\ -static inline T *V ## _push(NS ## builder_t *B, const T *p)\ -{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? (memcpy(_p, p, TN ## __size()), _p) : 0; }\ -static inline T *V ## _push_copy(NS ## builder_t *B, const T *p)\ -{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? TN ## _copy(_p, p) : 0; }\ -static inline T *V ## _push_clone(NS ## builder_t *B, const T *p)\ -{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? TN ## _copy(_p, p) : 0; }\ -static inline T *V ## _push_create(NS ## builder_t *B __ ## TN ## _formal_args)\ -{ T *_p; return (_p = (T *)flatcc_builder_extend_vector(B, 1)) ? TN ## _assign(_p __ ## TN ## _call_args) : 0; } - -#define __flatbuffers_build_vector(NS, N, T, S, A)\ -typedef NS ## ref_t N ## _vec_ref_t;\ -static inline int N ## _vec_start(NS ## builder_t *B)\ -{ return flatcc_builder_start_vector(B, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ -static inline N ## _vec_ref_t N ## _vec_end_pe(NS ## builder_t *B)\ -{ return flatcc_builder_end_vector(B); }\ -static inline N ## _vec_ref_t N ## _vec_end(NS ## builder_t *B)\ -{ if (!NS ## is_native_pe()) { size_t i, n; T *p = (T *)flatcc_builder_vector_edit(B);\ - for (i = 0, n = flatcc_builder_vector_count(B); i < n; ++i)\ - { N ## _to_pe(N ## __ptr_add(p, i)); }} return flatcc_builder_end_vector(B); }\ -static inline N ## _vec_ref_t N ## _vec_create_pe(NS ## builder_t *B, const T *data, size_t len)\ -{ return flatcc_builder_create_vector(B, data, len, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ -static inline N ## _vec_ref_t N ## _vec_create(NS ## builder_t *B, const T *data, size_t len)\ -{ if (!NS ## is_native_pe()) { size_t i; T *p; int ret = flatcc_builder_start_vector(B, S, A, FLATBUFFERS_COUNT_MAX(S)); if (ret) { return ret; }\ - p = (T *)flatcc_builder_extend_vector(B, len); if (!p) return 0;\ - for (i = 0; i < len; ++i) { N ## _copy_to_pe(N ## __ptr_add(p, i), N ## __const_ptr_add(data, i)); }\ - return flatcc_builder_end_vector(B); } else return flatcc_builder_create_vector(B, data, len, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ -static inline N ## _vec_ref_t N ## _vec_clone(NS ## builder_t *B, N ##_vec_t vec)\ -{ __flatbuffers_memoize(B, vec, flatcc_builder_create_vector(B, vec, N ## _vec_len(vec), S, A, FLATBUFFERS_COUNT_MAX(S))); }\ -static inline N ## _vec_ref_t N ## _vec_slice(NS ## builder_t *B, N ##_vec_t vec, size_t index, size_t len)\ -{ size_t n = N ## _vec_len(vec); if (index >= n) index = n; n -= index; if (len > n) len = n;\ - return flatcc_builder_create_vector(B, N ## __const_ptr_add(vec, index), len, S, A, FLATBUFFERS_COUNT_MAX(S)); }\ -__flatbuffers_build_vector_ops(NS, N ## _vec, N, N, T) - -#define __flatbuffers_build_union_vector_ops(NS, V, N, TN)\ -static inline TN ## _union_ref_t *V ## _extend(NS ## builder_t *B, size_t len)\ -{ return flatcc_builder_extend_union_vector(B, len); }\ -static inline TN ## _union_ref_t *V ## _append(NS ## builder_t *B, const TN ## _union_ref_t *data, size_t len)\ -{ return flatcc_builder_append_union_vector(B, data, len); }\ -static inline int V ## _truncate(NS ## builder_t *B, size_t len)\ -{ return flatcc_builder_truncate_union_vector(B, len); }\ -static inline TN ## _union_ref_t *V ## _edit(NS ## builder_t *B)\ -{ return (TN ## _union_ref_t *) flatcc_builder_union_vector_edit(B); }\ -static inline size_t V ## _reserved_len(NS ## builder_t *B)\ -{ return flatcc_builder_union_vector_count(B); }\ -static inline TN ## _union_ref_t *V ## _push(NS ## builder_t *B, const TN ## _union_ref_t ref)\ -{ return flatcc_builder_union_vector_push(B, ref); }\ -static inline TN ## _union_ref_t *V ## _push_clone(NS ## builder_t *B, TN ## _union_t u)\ -{ return TN ## _vec_push(B, TN ## _clone(B, u)); } - -#define __flatbuffers_build_union_vector(NS, N)\ -static inline int N ## _vec_start(NS ## builder_t *B)\ -{ return flatcc_builder_start_union_vector(B); }\ -static inline N ## _union_vec_ref_t N ## _vec_end(NS ## builder_t *B)\ -{ return flatcc_builder_end_union_vector(B); }\ -static inline N ## _union_vec_ref_t N ## _vec_create(NS ## builder_t *B, const N ## _union_ref_t *data, size_t len)\ -{ return flatcc_builder_create_union_vector(B, data, len); }\ -__flatbuffers_build_union_vector_ops(NS, N ## _vec, N, N)\ -/* Preserves DAG structure separately for type and value vector, so a type vector could be shared for many value vectors. */\ -static inline N ## _union_vec_ref_t N ## _vec_clone(NS ## builder_t *B, N ##_union_vec_t vec)\ -{ N ## _union_vec_ref_t _uvref, _ret = { 0, 0 }; NS ## union_ref_t _uref; size_t _i, _len; flatcc_builder_ref_t *_p;\ - if (vec.type == 0) return _ret;\ - _uvref.type = flatcc_builder_refmap_find(B, vec.type); _uvref.value = flatcc_builder_refmap_find(B, vec.value);\ - _len = N ## _union_vec_len(vec); if (_uvref.type == 0) {\ - _uvref.type = flatcc_builder_refmap_insert(B, vec.type, (flatcc_builder_create_type_vector(B, vec.type, _len))); }\ - if (_uvref.type == 0) return _ret; if (_uvref.value == 0) {\ - if (flatcc_builder_start_offset_vector(B)) return _ret;\ - _p = flatcc_builder_extend_offset_vector(B, _len); if (!_p) return _ret;\ - for (_i = 0; _i < _len; ++_i) { _uref = N ## _clone(B, N ## _union_vec_at(vec, _i)); _p[_i] = _uref.value; }\ - _uvref.value = flatcc_builder_refmap_insert(B, vec.value, flatcc_builder_end_offset_vector(B));\ - if (_uvref.value == 0) return _ret; } return _uvref; } - -#define __flatbuffers_build_string_vector_ops(NS, N)\ -static inline int N ## _push_start(NS ## builder_t *B)\ -{ return NS ## string_start(B); }\ -static inline NS ## string_ref_t *N ## _push_end(NS ## builder_t *B)\ -{ return NS ## string_vec_push(B, NS ## string_end(B)); }\ -static inline NS ## string_ref_t *N ## _push_create(NS ## builder_t *B, const char *s, size_t len)\ -{ return NS ## string_vec_push(B, NS ## string_create(B, s, len)); }\ -static inline NS ## string_ref_t *N ## _push_create_str(NS ## builder_t *B, const char *s)\ -{ return NS ## string_vec_push(B, NS ## string_create_str(B, s)); }\ -static inline NS ## string_ref_t *N ## _push_create_strn(NS ## builder_t *B, const char *s, size_t max_len)\ -{ return NS ## string_vec_push(B, NS ## string_create_strn(B, s, max_len)); }\ -static inline NS ## string_ref_t *N ## _push_clone(NS ## builder_t *B, NS ## string_t string)\ -{ return NS ## string_vec_push(B, NS ## string_clone(B, string)); }\ -static inline NS ## string_ref_t *N ## _push_slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ -{ return NS ## string_vec_push(B, NS ## string_slice(B, string, index, len)); } - -#define __flatbuffers_build_table_vector_ops(NS, N, TN)\ -static inline int N ## _push_start(NS ## builder_t *B)\ -{ return TN ## _start(B); }\ -static inline TN ## _ref_t *N ## _push_end(NS ## builder_t *B)\ -{ return N ## _push(B, TN ## _end(B)); }\ -static inline TN ## _ref_t *N ## _push_create(NS ## builder_t *B __ ## TN ##_formal_args)\ -{ return N ## _push(B, TN ## _create(B __ ## TN ## _call_args)); } - -#define __flatbuffers_build_offset_vector_ops(NS, V, N, TN)\ -static inline TN ## _ref_t *V ## _extend(NS ## builder_t *B, size_t len)\ -{ return flatcc_builder_extend_offset_vector(B, len); }\ -static inline TN ## _ref_t *V ## _append(NS ## builder_t *B, const TN ## _ref_t *data, size_t len)\ -{ return flatcc_builder_append_offset_vector(B, data, len); }\ -static inline int V ## _truncate(NS ## builder_t *B, size_t len)\ -{ return flatcc_builder_truncate_offset_vector(B, len); }\ -static inline TN ## _ref_t *V ## _edit(NS ## builder_t *B)\ -{ return (TN ## _ref_t *)flatcc_builder_offset_vector_edit(B); }\ -static inline size_t V ## _reserved_len(NS ## builder_t *B)\ -{ return flatcc_builder_offset_vector_count(B); }\ -static inline TN ## _ref_t *V ## _push(NS ## builder_t *B, const TN ## _ref_t ref)\ -{ return ref ? flatcc_builder_offset_vector_push(B, ref) : 0; } - -#define __flatbuffers_build_offset_vector(NS, N)\ -typedef NS ## ref_t N ## _vec_ref_t;\ -static inline int N ## _vec_start(NS ## builder_t *B)\ -{ return flatcc_builder_start_offset_vector(B); }\ -static inline N ## _vec_ref_t N ## _vec_end(NS ## builder_t *B)\ -{ return flatcc_builder_end_offset_vector(B); }\ -static inline N ## _vec_ref_t N ## _vec_create(NS ## builder_t *B, const N ## _ref_t *data, size_t len)\ -{ return flatcc_builder_create_offset_vector(B, data, len); }\ -__flatbuffers_build_offset_vector_ops(NS, N ## _vec, N, N)\ -static inline N ## _vec_ref_t N ## _vec_clone(NS ## builder_t *B, N ##_vec_t vec)\ -{ int _ret; N ## _ref_t *_p; size_t _i, _len; __flatbuffers_memoize_begin(B, vec);\ - _len = N ## _vec_len(vec); if (flatcc_builder_start_offset_vector(B)) return 0;\ - _p = flatcc_builder_extend_offset_vector(B, _len); if (!_p) return 0;\ - for (_i = 0; _i < _len; ++_i) { if (!(_p[_i] = N ## _clone(B, N ## _vec_at(vec, _i)))) return 0; }\ - __flatbuffers_memoize_end(B, vec, flatcc_builder_end_offset_vector(B)); }\ - -#define __flatbuffers_build_string_ops(NS, N)\ -static inline char *N ## _append(NS ## builder_t *B, const char *s, size_t len)\ -{ return flatcc_builder_append_string(B, s, len); }\ -static inline char *N ## _append_str(NS ## builder_t *B, const char *s)\ -{ return flatcc_builder_append_string_str(B, s); }\ -static inline char *N ## _append_strn(NS ## builder_t *B, const char *s, size_t len)\ -{ return flatcc_builder_append_string_strn(B, s, len); }\ -static inline size_t N ## _reserved_len(NS ## builder_t *B)\ -{ return flatcc_builder_string_len(B); }\ -static inline char *N ## _extend(NS ## builder_t *B, size_t len)\ -{ return flatcc_builder_extend_string(B, len); }\ -static inline char *N ## _edit(NS ## builder_t *B)\ -{ return flatcc_builder_string_edit(B); }\ -static inline int N ## _truncate(NS ## builder_t *B, size_t len)\ -{ return flatcc_builder_truncate_string(B, len); } - -#define __flatbuffers_build_string(NS)\ -typedef NS ## ref_t NS ## string_ref_t;\ -static inline int NS ## string_start(NS ## builder_t *B)\ -{ return flatcc_builder_start_string(B); }\ -static inline NS ## string_ref_t NS ## string_end(NS ## builder_t *B)\ -{ return flatcc_builder_end_string(B); }\ -static inline NS ## ref_t NS ## string_create(NS ## builder_t *B, const char *s, size_t len)\ -{ return flatcc_builder_create_string(B, s, len); }\ -static inline NS ## ref_t NS ## string_create_str(NS ## builder_t *B, const char *s)\ -{ return flatcc_builder_create_string_str(B, s); }\ -static inline NS ## ref_t NS ## string_create_strn(NS ## builder_t *B, const char *s, size_t len)\ -{ return flatcc_builder_create_string_strn(B, s, len); }\ -static inline NS ## string_ref_t NS ## string_clone(NS ## builder_t *B, NS ## string_t string)\ -{ __flatbuffers_memoize(B, string, flatcc_builder_create_string(B, string, NS ## string_len(string))); }\ -static inline NS ## string_ref_t NS ## string_slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ -{ size_t n = NS ## string_len(string); if (index >= n) index = n; n -= index; if (len > n) len = n;\ - return flatcc_builder_create_string(B, string + index, len); }\ -__flatbuffers_build_string_ops(NS, NS ## string)\ -__flatbuffers_build_offset_vector(NS, NS ## string) - -#define __flatbuffers_copy_from_pe(P, P2, N) (*(P) = N ## _cast_from_pe(*P2), (P)) -#define __flatbuffers_from_pe(P, N) (*(P) = N ## _cast_from_pe(*P), (P)) -#define __flatbuffers_copy_to_pe(P, P2, N) (*(P) = N ## _cast_to_pe(*P2), (P)) -#define __flatbuffers_to_pe(P, N) (*(P) = N ## _cast_to_pe(*P), (P)) -#define __flatbuffers_define_scalar_primitives(NS, N, T)\ -static inline T *N ## _from_pe(T *p) { return __ ## NS ## from_pe(p, N); }\ -static inline T *N ## _to_pe(T *p) { return __ ## NS ## to_pe(p, N); }\ -static inline T *N ## _copy(T *p, const T *p2) { *p = *p2; return p; }\ -static inline T *N ## _copy_from_pe(T *p, const T *p2)\ -{ return __ ## NS ## copy_from_pe(p, p2, N); }\ -static inline T *N ## _copy_to_pe(T *p, const T *p2) \ -{ return __ ## NS ## copy_to_pe(p, p2, N); }\ -static inline T *N ## _assign(T *p, const T v0) { *p = v0; return p; }\ -static inline T *N ## _assign_from_pe(T *p, T v0)\ -{ *p = N ## _cast_from_pe(v0); return p; }\ -static inline T *N ## _assign_to_pe(T *p, T v0)\ -{ *p = N ## _cast_to_pe(v0); return p; } -#define __flatbuffers_build_scalar(NS, N, T)\ -__ ## NS ## define_scalar_primitives(NS, N, T)\ -__ ## NS ## build_vector(NS, N, T, sizeof(T), sizeof(T)) -/* Depends on generated copy_to/from_pe functions, and the type. */ -#define __flatbuffers_define_struct_primitives(NS, N)\ -static inline N ## _t *N ##_to_pe(N ## _t *p)\ -{ if (!NS ## is_native_pe()) { N ## _copy_to_pe(p, p); }; return p; }\ -static inline N ## _t *N ##_from_pe(N ## _t *p)\ -{ if (!NS ## is_native_pe()) { N ## _copy_from_pe(p, p); }; return p; }\ -static inline N ## _t *N ## _clear(N ## _t *p) { return (N ## _t *)memset(p, 0, N ## __size()); } - -/* Depends on generated copy/assign_to/from_pe functions, and the type. */ -#define __flatbuffers_build_struct(NS, N, S, A, FID, TFID)\ -__ ## NS ## define_struct_primitives(NS, N)\ -typedef NS ## ref_t N ## _ref_t;\ -static inline N ## _t *N ## _start(NS ## builder_t *B)\ -{ return (N ## _t *)flatcc_builder_start_struct(B, S, A); }\ -static inline N ## _ref_t N ## _end(NS ## builder_t *B)\ -{ if (!NS ## is_native_pe()) { N ## _to_pe((N ## _t *)flatcc_builder_struct_edit(B)); }\ - return flatcc_builder_end_struct(B); }\ -static inline N ## _ref_t N ## _end_pe(NS ## builder_t *B)\ -{ return flatcc_builder_end_struct(B); }\ -static inline N ## _ref_t N ## _create(NS ## builder_t *B __ ## N ## _formal_args)\ -{ N ## _t *_p = N ## _start(B); if (!_p) return 0; N ##_assign_to_pe(_p __ ## N ## _call_args);\ - return N ## _end_pe(B); }\ -static inline N ## _ref_t N ## _clone(NS ## builder_t *B, N ## _struct_t p)\ -{ N ## _t *_p; __flatbuffers_memoize_begin(B, p); _p = N ## _start(B); if (!_p) return 0;\ - N ## _copy(_p, p); __flatbuffers_memoize_end(B, p, N ##_end_pe(B)); }\ -__flatbuffers_build_vector(NS, N, N ## _t, S, A)\ -__flatbuffers_build_struct_root(NS, N, A, FID, TFID) - -#define __flatbuffers_build_table(NS, N, K)\ -static inline int N ## _start(NS ## builder_t *B)\ -{ return flatcc_builder_start_table(B, K); }\ -static inline N ## _ref_t N ## _end(NS ## builder_t *B)\ -{ assert(flatcc_builder_check_required(B, __ ## N ## _required,\ - sizeof(__ ## N ## _required) / sizeof(__ ## N ## _required[0]) - 1));\ - return flatcc_builder_end_table(B); }\ -__flatbuffers_build_offset_vector(NS, N) - -#define __flatbuffers_build_table_field(ID, NS, N, TN, TT)\ -static inline int N ## _add(NS ## builder_t *B, TN ## _ref_t ref)\ -{ TN ## _ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ?\ - ((*_p = ref), 0) : -1; }\ -static inline int N ## _start(NS ## builder_t *B)\ -{ return TN ## _start(B); }\ -static inline int N ## _end(NS ## builder_t *B)\ -{ return N ## _add(B, TN ## _end(B)); }\ -static inline TN ## _ref_t N ## _create(NS ## builder_t *B __ ## TN ##_formal_args)\ -{ return N ## _add(B, TN ## _create(B __ ## TN ## _call_args)); }\ -static inline int N ## _clone(NS ## builder_t *B, TN ## _table_t p)\ -{ return N ## _add(B, TN ## _clone(B, p)); }\ -static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ -{ TN ## _table_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } - -#define __flatbuffers_build_union_field(ID, NS, N, TN, TT)\ -static inline int N ## _add(NS ## builder_t *B, TN ## _union_ref_t uref)\ -{ NS ## ref_t *_p; TN ## _union_type_t *_pt; if (uref.type == TN ## _NONE) return 0; if (uref.value == 0) return -1;\ - if (!(_pt = (TN ## _union_type_t *)flatcc_builder_table_add(B, ID - 1, sizeof(*_pt), sizeof(*_pt))) ||\ - !(_p = flatcc_builder_table_add_offset(B, ID))) return -1; *_pt = uref.type; *_p = uref.value; return 0; }\ -static inline int N ## _add_type(NS ## builder_t *B, TN ## _union_type_t type)\ -{ TN ## _union_type_t *_pt; if (type == TN ## _NONE) return 0; return (_pt = (TN ## _union_type_t *)flatcc_builder_table_add(B, ID - 1,\ - sizeof(*_pt), sizeof(*_pt))) ? ((*_pt = type), 0) : -1; }\ -static inline int N ## _add_value(NS ## builder_t *B, TN ## _union_ref_t uref)\ -{ NS ## ref_t *p; if (uref.type == TN ## _NONE) return 0; return (p = flatcc_builder_table_add_offset(B, ID)) ?\ - ((*p = uref.value), 0) : -1; }\ -static inline int N ## _clone(NS ## builder_t *B, TN ## _union_t p)\ -{ return N ## _add(B, TN ## _clone(B, p)); }\ -static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ -{ TN ## _union_t _p = N ## _union(t); return _p.type ? N ## _clone(B, _p) : 0; } - -/* M is the union value name and T is its type, i.e. the qualified name. */ -#define __flatbuffers_build_union_table_value_field(NS, N, NU, M, T)\ -static inline int N ## _ ## M ## _add(NS ## builder_t *B, T ## _ref_t ref)\ -{ return N ## _add(B, NU ## _as_ ## M (ref)); }\ -static inline int N ## _ ## M ## _start(NS ## builder_t *B)\ -{ return T ## _start(B); }\ -static inline int N ## _ ## M ## _end(NS ## builder_t *B)\ -{ T ## _ref_t ref = T ## _end(B);\ - return ref ? N ## _ ## M ## _add(B, ref) : -1; }\ -static inline int N ## _ ## M ## _create(NS ## builder_t *B __ ## T ##_formal_args)\ -{ T ## _ref_t ref = T ## _create(B __ ## T ## _call_args);\ - return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; }\ -static inline int N ## _ ## M ## _clone(NS ## builder_t *B, T ## _table_t t)\ -{ T ## _ref_t ref = T ## _clone(B, t);\ - return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; } - -/* M is the union value name and T is its type, i.e. the qualified name. */ -#define __flatbuffers_build_union_struct_value_field(NS, N, NU, M, T)\ -static inline int N ## _ ## M ## _add(NS ## builder_t *B, T ## _ref_t ref)\ -{ return N ## _add(B, NU ## _as_ ## M (ref)); }\ -static inline T ## _t *N ## _ ## M ## _start(NS ## builder_t *B)\ -{ return T ## _start(B); }\ -static inline int N ## _ ## M ## _end(NS ## builder_t *B)\ -{ T ## _ref_t ref = T ## _end(B);\ - return ref ? N ## _ ## M ## _add(B, ref) : -1; }\ -static inline int N ## _ ## M ## _create(NS ## builder_t *B __ ## T ##_formal_args)\ -{ T ## _ref_t ref = T ## _create(B __ ## T ## _call_args);\ - return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; }\ -static inline int N ## _ ## M ## _end_pe(NS ## builder_t *B)\ -{ T ## _ref_t ref = T ## _end_pe(B);\ - return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; }\ -static inline int N ## _ ## M ## _clone(NS ## builder_t *B, T ## _struct_t p)\ -{ T ## _ref_t ref = T ## _clone(B, p);\ - return ref ? N ## _add(B, NU ## _as_ ## M(ref)) : -1; } -#define __flatbuffers_build_union_string_value_field(NS, N, NU, M)\ -static inline int N ## _ ## M ## _add(NS ## builder_t *B, NS ## string_ref_t ref)\ -{ return N ## _add(B, NU ## _as_ ## M (ref)); }\ -__flatbuffers_build_string_field_ops(NS, N ## _ ## M) - -/* NS: common namespace, ID: table field id (not offset), TN: name of type T, TT: name of table type - * S: sizeof of scalar type, A: alignment of type T, default value V of type T. */ -#define __flatbuffers_build_scalar_field(ID, NS, N, TN, T, S, A, V, TT)\ -static inline int N ## _add(NS ## builder_t *B, const T v)\ -{ T *_p; if (v == V) return 0; if (!(_p = (T *)flatcc_builder_table_add(B, ID, S, A))) return -1;\ - TN ## _assign_to_pe(_p, v); return 0; }\ -static inline int N ## _force_add(NS ## builder_t *B, const T v)\ -{ T *_p; if (!(_p = (T *)flatcc_builder_table_add(B, ID, S, A))) return -1;\ - TN ## _assign_to_pe(_p, v); return 0; }\ -/* Clone does not skip default values and expects pe endian content. */\ -static inline int N ## _clone(NS ## builder_t *B, const T *p)\ -{ return 0 == flatcc_builder_table_add_copy(B, ID, p, S, A) ? -1 : 0; }\ -/* Transferring a missing field is a nop success with 0 as result. */\ -static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ -{ const T *_p = N ## _get_ptr(t); return _p ? N ## _clone(B, _p) : 0; } - -#define __flatbuffers_build_struct_field(ID, NS, N, TN, S, A, TT)\ -static inline TN ## _t *N ## _start(NS ## builder_t *B)\ -{ return (TN ## _t *)flatcc_builder_table_add(B, ID, S, A); }\ -static inline int N ## _end(NS ## builder_t *B)\ -{ if (!NS ## is_native_pe()) { TN ## _to_pe((TN ## _t *)flatcc_builder_table_edit(B, S)); } return 0; }\ -static inline int N ## _end_pe(NS ## builder_t *B) { return 0; }\ -static inline int N ## _create(NS ## builder_t *B __ ## TN ## _formal_args)\ -{ TN ## _t *_p = N ## _start(B); if (!_p) return 0; TN ##_assign_to_pe(_p __ ## TN ## _call_args);\ - return 0; }\ -static inline int N ## _add(NS ## builder_t *B, const TN ## _t *p)\ -{ TN ## _t *_p = N ## _start(B); if (!_p) return -1; TN ##_copy_to_pe(_p, p); return 0; }\ -static inline int N ## _clone(NS ## builder_t *B, TN ## _struct_t p)\ -{ return 0 == flatcc_builder_table_add_copy(B, ID, p, S, A) ? -1 : 0; }\ -static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ -{ TN ## _struct_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } - -#define __flatbuffers_build_vector_field(ID, NS, N, TN, T, TT)\ -static inline int N ## _add(NS ## builder_t *B, TN ## _vec_ref_t ref)\ -{ TN ## _vec_ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ? ((*_p = ref), 0) : -1; }\ -static inline int N ## _start(NS ## builder_t *B)\ -{ return TN ## _vec_start(B); }\ -static inline int N ## _end_pe(NS ## builder_t *B)\ -{ return N ## _add(B, TN ## _vec_end_pe(B)); }\ -static inline int N ## _end(NS ## builder_t *B)\ -{ return N ## _add(B, TN ## _vec_end(B)); }\ -static inline int N ## _create_pe(NS ## builder_t *B, T *data, size_t len)\ -{ return N ## _add(B, TN ## _vec_create_pe(B, data, len)); }\ -static inline int N ## _create(NS ## builder_t *B, T *data, size_t len)\ -{ return N ## _add(B, TN ## _vec_create(B, data, len)); }\ -static inline int N ## _slice(NS ## builder_t *B, TN ## _vec_t vec, size_t index, size_t len)\ -{ return N ## _add(B, TN ## _vec_slice(B, vec, index, len)); }\ -static inline int N ## _clone(NS ## builder_t *B, TN ## _vec_t vec)\ -{ return N ## _add(B, TN ## _vec_clone(B, vec)); }\ -static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ -{ TN ## _vec_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; }\ -__flatbuffers_build_vector_ops(NS, N, N, TN, T)\ - -#define __flatbuffers_build_offset_vector_field(ID, NS, N, TN, TT)\ -static inline int N ## _add(NS ## builder_t *B, TN ## _vec_ref_t ref)\ -{ TN ## _vec_ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ? ((*_p = ref), 0) : -1; }\ -static inline int N ## _start(NS ## builder_t *B)\ -{ return flatcc_builder_start_offset_vector(B); }\ -static inline int N ## _end(NS ## builder_t *B)\ -{ return N ## _add(B, flatcc_builder_end_offset_vector(B)); }\ -static inline int N ## _create(NS ## builder_t *B, const TN ## _ref_t *data, size_t len)\ -{ return N ## _add(B, flatcc_builder_create_offset_vector(B, data, len)); }\ -__flatbuffers_build_offset_vector_ops(NS, N, N, TN)\ -static inline int N ## _clone(NS ## builder_t *B, TN ## _vec_t vec)\ -{ return N ## _add(B, TN ## _vec_clone(B, vec)); }\ -static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ -{ TN ## _vec_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } - -/* depends on N ## _add which differs for union member fields and ordinary fields */\ -#define __flatbuffers_build_string_field_ops(NS, N)\ -static inline int N ## _start(NS ## builder_t *B)\ -{ return flatcc_builder_start_string(B); }\ -static inline int N ## _end(NS ## builder_t *B)\ -{ return N ## _add(B, flatcc_builder_end_string(B)); }\ -static inline int N ## _create(NS ## builder_t *B, const char *s, size_t len)\ -{ return N ## _add(B, flatcc_builder_create_string(B, s, len)); }\ -static inline int N ## _create_str(NS ## builder_t *B, const char *s)\ -{ return N ## _add(B, flatcc_builder_create_string_str(B, s)); }\ -static inline int N ## _create_strn(NS ## builder_t *B, const char *s, size_t max_len)\ -{ return N ## _add(B, flatcc_builder_create_string_strn(B, s, max_len)); }\ -static inline int N ## _clone(NS ## builder_t *B, NS ## string_t string)\ -{ return N ## _add(B, NS ## string_clone(B, string)); }\ -static inline int N ## _slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ -{ return N ## _add(B, NS ## string_slice(B, string, index, len)); }\ -__flatbuffers_build_string_ops(NS, N) - -#define __flatbuffers_build_string_field(ID, NS, N, TT)\ -static inline int N ## _add(NS ## builder_t *B, NS ## string_ref_t ref)\ -{ NS ## string_ref_t *_p; return (ref && (_p = flatcc_builder_table_add_offset(B, ID))) ? ((*_p = ref), 0) : -1; }\ -__flatbuffers_build_string_field_ops(NS, N)\ -static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ -{ NS ## string_t _p = N ## _get(t); return _p ? N ## _clone(B, _p) : 0; } - -#define __flatbuffers_build_table_vector_field(ID, NS, N, TN, TT)\ -__flatbuffers_build_offset_vector_field(ID, NS, N, TN, TT)\ -__flatbuffers_build_table_vector_ops(NS, N, TN) - -#define __flatbuffers_build_union_vector_field(ID, NS, N, TN, TT)\ -static inline int N ## _add(NS ## builder_t *B, TN ## _union_vec_ref_t uvref)\ -{ NS ## vec_ref_t *_p; if (!uvref.type || !uvref.value) return uvref.type == uvref.value ? 0 : -1;\ - if (!(_p = flatcc_builder_table_add_offset(B, ID - 1))) return -1; *_p = uvref.type;\ - if (!(_p = flatcc_builder_table_add_offset(B, ID))) return -1; *_p = uvref.value; return 0; }\ -static inline int N ## _start(NS ## builder_t *B)\ -{ return flatcc_builder_start_union_vector(B); }\ -static inline int N ## _end(NS ## builder_t *B)\ -{ return N ## _add(B, flatcc_builder_end_union_vector(B)); }\ -static inline int N ## _create(NS ## builder_t *B, const TN ## _union_ref_t *data, size_t len)\ -{ return N ## _add(B, flatcc_builder_create_union_vector(B, data, len)); }\ -__flatbuffers_build_union_vector_ops(NS, N, N, TN)\ -static inline int N ## _clone(NS ## builder_t *B, TN ## _union_vec_t vec)\ -{ return N ## _add(B, TN ## _vec_clone(B, vec)); }\ -static inline int N ## _pick(NS ## builder_t *B, TT ## _table_t t)\ -{ TN ## _union_vec_t _p = N ## _union(t); return _p.type ? N ## _clone(B, _p) : 0; } - -#define __flatbuffers_build_union_table_vector_value_field(NS, N, NU, M, T)\ -static inline int N ## _ ## M ## _push_start(NS ## builder_t *B)\ -{ return T ## _start(B); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_end(NS ## builder_t *B)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M (T ## _end(B))); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push(NS ## builder_t *B, T ## _ref_t ref)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M (ref)); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_create(NS ## builder_t *B __ ## T ##_formal_args)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _create(B __ ## T ## _call_args))); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_clone(NS ## builder_t *B, T ## _table_t t)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _clone(B, t))); } - -#define __flatbuffers_build_union_struct_vector_value_field(NS, N, NU, M, T)\ -static inline T ## _t *N ## _ ## M ## _push_start(NS ## builder_t *B)\ -{ return T ## _start(B); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_end(NS ## builder_t *B)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M (T ## _end(B))); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push(NS ## builder_t *B, T ## _ref_t ref)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M (ref)); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_create(NS ## builder_t *B __ ## T ##_formal_args)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _create(B __ ## T ## _call_args))); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_clone(NS ## builder_t *B, T ## _struct_t p)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M(T ## _clone(B, p))); } - -#define __flatbuffers_build_union_string_vector_value_field(NS, N, NU, M)\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push(NS ## builder_t *B, NS ## string_ref_t ref)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M (ref)); }\ -static inline int N ## _ ## M ## _push_start(NS ## builder_t *B)\ -{ return NS ## string_start(B); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_end(NS ## builder_t *B)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_end(B))); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_create(NS ## builder_t *B, const char *s, size_t len)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_create(B, s, len))); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_create_str(NS ## builder_t *B, const char *s)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_create_str(B, s))); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_create_strn(NS ## builder_t *B, const char *s, size_t max_len)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_create_strn(B, s, max_len))); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_clone(NS ## builder_t *B, NS ## string_t string)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_clone(B, string))); }\ -static inline NU ## _union_ref_t *N ## _ ## M ## _push_slice(NS ## builder_t *B, NS ## string_t string, size_t index, size_t len)\ -{ return NU ## _vec_push(B, NU ## _as_ ## M(NS ## string_slice(B, string, index, len))); } - -#define __flatbuffers_build_string_vector_field(ID, NS, N, TT)\ -__flatbuffers_build_offset_vector_field(ID, NS, N, NS ## string, TT)\ -__flatbuffers_build_string_vector_ops(NS, N) - -#define __flatbuffers_uint8_formal_args , uint8_t v0 -#define __flatbuffers_uint8_call_args , v0 -#define __flatbuffers_int8_formal_args , int8_t v0 -#define __flatbuffers_int8_call_args , v0 -#define __flatbuffers_bool_formal_args , flatbuffers_bool_t v0 -#define __flatbuffers_bool_call_args , v0 -#define __flatbuffers_uint16_formal_args , uint16_t v0 -#define __flatbuffers_uint16_call_args , v0 -#define __flatbuffers_uint32_formal_args , uint32_t v0 -#define __flatbuffers_uint32_call_args , v0 -#define __flatbuffers_uint64_formal_args , uint64_t v0 -#define __flatbuffers_uint64_call_args , v0 -#define __flatbuffers_int16_formal_args , int16_t v0 -#define __flatbuffers_int16_call_args , v0 -#define __flatbuffers_int32_formal_args , int32_t v0 -#define __flatbuffers_int32_call_args , v0 -#define __flatbuffers_int64_formal_args , int64_t v0 -#define __flatbuffers_int64_call_args , v0 -#define __flatbuffers_float_formal_args , float v0 -#define __flatbuffers_float_call_args , v0 -#define __flatbuffers_double_formal_args , double v0 -#define __flatbuffers_double_call_args , v0 - -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint8, uint8_t) -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int8, int8_t) -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_bool, flatbuffers_bool_t) -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint16, uint16_t) -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint32, uint32_t) -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_uint64, uint64_t) -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int16, int16_t) -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int32, int32_t) -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_int64, int64_t) -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_float, float) -__flatbuffers_build_scalar(flatbuffers_, flatbuffers_double, double) - -__flatbuffers_build_string(flatbuffers_) - -__flatbuffers_build_buffer(flatbuffers_) -#include "flatcc/flatcc_epilogue.h" -#endif /* FLATBUFFERS_COMMON_BUILDER_H */ diff --git a/common/src/flatbuffers_common_reader.h b/common/src/flatbuffers_common_reader.h deleted file mode 100644 index 529ecd8d9..000000000 --- a/common/src/flatbuffers_common_reader.h +++ /dev/null @@ -1,512 +0,0 @@ -#ifndef FLATBUFFERS_COMMON_READER_H -#define FLATBUFFERS_COMMON_READER_H - -/* Generated by flatcc 0.5.3-pre FlatBuffers schema compiler for C by dvide.com */ - -/* Common FlatBuffers read functionality for C. */ - -#include "flatcc/flatcc_prologue.h" -#include "flatcc/flatcc_flatbuffers.h" - - -#define __flatbuffers_read_scalar_at_byteoffset(N, p, o) N ## _read_from_pe((uint8_t *)(p) + (o)) -#define __flatbuffers_read_scalar(N, p) N ## _read_from_pe(p) -#define __flatbuffers_read_vt(ID, offset, t)\ -flatbuffers_voffset_t offset = 0;\ -{ flatbuffers_voffset_t id__tmp, *vt__tmp;\ - assert(t != 0 && "null pointer table access");\ - id__tmp = ID;\ - vt__tmp = (flatbuffers_voffset_t *)((uint8_t *)(t) -\ - __flatbuffers_soffset_read_from_pe(t));\ - if (__flatbuffers_voffset_read_from_pe(vt__tmp) >= sizeof(vt__tmp[0]) * (id__tmp + 3)) {\ - offset = __flatbuffers_voffset_read_from_pe(vt__tmp + id__tmp + 2);\ - }\ -} -#define __flatbuffers_field_present(ID, t) { __flatbuffers_read_vt(ID, offset__tmp, t) return offset__tmp != 0; } -#define __flatbuffers_scalar_field(T, ID, t)\ -{\ - __flatbuffers_read_vt(ID, offset__tmp, t)\ - if (offset__tmp) {\ - return (const T *)((uint8_t *)(t) + offset__tmp);\ - }\ - return 0;\ -} -#define __flatbuffers_define_scalar_field(ID, N, NK, TK, T, V)\ -static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ -{ __flatbuffers_read_vt(ID, offset__tmp, t__tmp)\ - return offset__tmp ? __flatbuffers_read_scalar_at_byteoffset(TK, t__tmp, offset__tmp) : V;\ -}\ -static inline T N ## _ ## NK(N ## _table_t t__tmp)\ -{ __flatbuffers_read_vt(ID, offset__tmp, t__tmp)\ - return offset__tmp ? __flatbuffers_read_scalar_at_byteoffset(TK, t__tmp, offset__tmp) : V;\ -}\ -static inline const T *N ## _ ## NK ## _get_ptr(N ## _table_t t__tmp)\ -__flatbuffers_scalar_field(T, ID, t__tmp)\ -static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ -__flatbuffers_field_present(ID, t__tmp)\ -__flatbuffers_define_scan_by_scalar_field(N, NK, T) -#define __flatbuffers_struct_field(T, ID, t, r)\ -{\ - __flatbuffers_read_vt(ID, offset__tmp, t)\ - if (offset__tmp) {\ - return (T)((uint8_t *)(t) + offset__tmp);\ - }\ - assert(!(r) && "required field missing");\ - return 0;\ -} -#define __flatbuffers_offset_field(T, ID, t, r, adjust)\ -{\ - flatbuffers_uoffset_t *elem__tmp;\ - __flatbuffers_read_vt(ID, offset__tmp, t)\ - if (offset__tmp) {\ - elem__tmp = (flatbuffers_uoffset_t *)((uint8_t *)(t) + offset__tmp);\ - /* Add sizeof so C api can have raw access past header field. */\ - return (T)((uint8_t *)(elem__tmp) + adjust +\ - __flatbuffers_uoffset_read_from_pe(elem__tmp));\ - }\ - assert(!(r) && "required field missing");\ - return 0;\ -} -#define __flatbuffers_vector_field(T, ID, t, r) __flatbuffers_offset_field(T, ID, t, r, sizeof(flatbuffers_uoffset_t)) -#define __flatbuffers_table_field(T, ID, t, r) __flatbuffers_offset_field(T, ID, t, r, 0) -#define __flatbuffers_define_struct_field(ID, N, NK, T, r)\ -static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ -__flatbuffers_struct_field(T, ID, t__tmp, r)\ -static inline T N ## _ ## NK(N ## _table_t t__tmp)\ -__flatbuffers_struct_field(T, ID, t__tmp, r)\ -static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ -__flatbuffers_field_present(ID, t__tmp) -#define __flatbuffers_define_vector_field(ID, N, NK, T, r)\ -static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ -__flatbuffers_vector_field(T, ID, t__tmp, r)\ -static inline T N ## _ ## NK(N ## _table_t t__tmp)\ -__flatbuffers_vector_field(T, ID, t__tmp, r)\ -static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ -__flatbuffers_field_present(ID, t__tmp) -#define __flatbuffers_define_table_field(ID, N, NK, T, r)\ -static inline T N ## _ ## NK ## _get(N ## _table_t t__tmp)\ -__flatbuffers_table_field(T, ID, t__tmp, r)\ -static inline T N ## _ ## NK(N ## _table_t t__tmp)\ -__flatbuffers_table_field(T, ID, t__tmp, r)\ -static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ -__flatbuffers_field_present(ID, t__tmp) -#define __flatbuffers_define_string_field(ID, N, NK, r)\ -static inline flatbuffers_string_t N ## _ ## NK ## _get(N ## _table_t t__tmp)\ -__flatbuffers_vector_field(flatbuffers_string_t, ID, t__tmp, r)\ -static inline flatbuffers_string_t N ## _ ## NK(N ## _table_t t__tmp)\ -__flatbuffers_vector_field(flatbuffers_string_t, ID, t__tmp, r)\ -static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ -__flatbuffers_field_present(ID, t__tmp)\ -__flatbuffers_define_scan_by_string_field(N, NK) -#define __flatbuffers_vec_len(vec)\ -{ return (vec) ? (size_t)__flatbuffers_uoffset_read_from_pe((flatbuffers_uoffset_t *)vec - 1) : 0; } -#define __flatbuffers_string_len(s) __flatbuffers_vec_len(s) -static inline size_t flatbuffers_vec_len(const void *vec) -__flatbuffers_vec_len(vec) -#define __flatbuffers_scalar_vec_at(N, vec, i)\ -{ assert(flatbuffers_vec_len(vec) > (i) && "index out of range");\ - return __flatbuffers_read_scalar(N, &(vec)[i]); } -#define __flatbuffers_struct_vec_at(vec, i)\ -{ assert(flatbuffers_vec_len(vec) > (i) && "index out of range"); return (vec) + (i); } -/* `adjust` skips past the header for string vectors. */ -#define __flatbuffers_offset_vec_at(T, vec, i, adjust)\ -{ const flatbuffers_uoffset_t *elem__tmp = (vec) + (i);\ - assert(flatbuffers_vec_len(vec) > (i) && "index out of range");\ - return (T)((uint8_t *)(elem__tmp) + (size_t)__flatbuffers_uoffset_read_from_pe(elem__tmp) + (adjust)); } -#define __flatbuffers_define_scalar_vec_len(N)\ -static inline size_t N ## _vec_len(N ##_vec_t vec__tmp)\ -{ return flatbuffers_vec_len(vec__tmp); } -#define __flatbuffers_define_scalar_vec_at(N, T) \ -static inline T N ## _vec_at(N ## _vec_t vec__tmp, size_t i__tmp)\ -__flatbuffers_scalar_vec_at(N, vec__tmp, i__tmp) -typedef const char *flatbuffers_string_t; -static inline size_t flatbuffers_string_len(flatbuffers_string_t s) -__flatbuffers_string_len(s) -typedef const flatbuffers_uoffset_t *flatbuffers_string_vec_t; -typedef flatbuffers_uoffset_t *flatbuffers_string_mutable_vec_t; -static inline size_t flatbuffers_string_vec_len(flatbuffers_string_vec_t vec) -__flatbuffers_vec_len(vec) -static inline flatbuffers_string_t flatbuffers_string_vec_at(flatbuffers_string_vec_t vec, size_t i) -__flatbuffers_offset_vec_at(flatbuffers_string_t, vec, i, sizeof(vec[0])) -typedef const void *flatbuffers_generic_t; -static inline flatbuffers_string_t flatbuffers_string_cast_from_generic(const flatbuffers_generic_t p) -{ return p ? ((const char *)p) + __flatbuffers_uoffset__size() : 0; } -typedef const flatbuffers_uoffset_t *flatbuffers_generic_vec_t; -typedef flatbuffers_uoffset_t *flatbuffers_generic_table_mutable_vec_t; -static inline size_t flatbuffers_generic_vec_len(flatbuffers_generic_vec_t vec) -__flatbuffers_vec_len(vec) -static inline flatbuffers_generic_t flatbuffers_generic_vec_at(flatbuffers_generic_vec_t vec, size_t i) -__flatbuffers_offset_vec_at(flatbuffers_generic_t, vec, i, 0) -static inline flatbuffers_generic_t flatbuffers_generic_vec_at_as_string(flatbuffers_generic_vec_t vec, size_t i) -__flatbuffers_offset_vec_at(flatbuffers_generic_t, vec, i, sizeof(vec[0])) -typedef struct flatbuffers_union { - flatbuffers_union_type_t type; - flatbuffers_generic_t value; -} flatbuffers_union_t; -typedef struct flatbuffers_union_vec { - const flatbuffers_union_type_t *type; - const flatbuffers_uoffset_t *value; -} flatbuffers_union_vec_t; -#define __flatbuffers_union_type_field(ID, t)\ -{\ - __flatbuffers_read_vt(ID, offset__tmp, t)\ - return offset__tmp ? __flatbuffers_read_scalar_at_byteoffset(__flatbuffers_utype, t, offset__tmp) : 0;\ -} -static inline flatbuffers_string_t flatbuffers_string_cast_from_union(const flatbuffers_union_t u__tmp)\ -{ return flatbuffers_string_cast_from_generic(u__tmp.value); } -#define __flatbuffers_define_union_field(NS, ID, N, NK, T, r)\ -static inline T ## _union_type_t N ## _ ## NK ## _type_get(N ## _table_t t__tmp)\ -__## NS ## union_type_field(((ID) - 1), t__tmp)\ -static inline NS ## generic_t N ## _ ## NK ## _get(N ## _table_t t__tmp)\ -__## NS ## table_field(NS ## generic_t, ID, t__tmp, r)\ -static inline T ## _union_type_t N ## _ ## NK ## _type(N ## _table_t t__tmp)\ -__## NS ## union_type_field(((ID) - 1), t__tmp)\ -static inline NS ## generic_t N ## _ ## NK(N ## _table_t t__tmp)\ -__## NS ## table_field(NS ## generic_t, ID, t__tmp, r)\ -static inline int N ## _ ## NK ## _is_present(N ## _table_t t__tmp)\ -__## NS ## field_present(ID, t__tmp)\ -static inline T ## _union_t N ## _ ## NK ## _union(N ## _table_t t__tmp)\ -{ T ## _union_t u__tmp = { 0, 0 }; u__tmp.type = N ## _ ## NK ## _type_get(t__tmp);\ - if (u__tmp.type == 0) return u__tmp; u__tmp.value = N ## _ ## NK ## _get(t__tmp); return u__tmp; }\ -static inline NS ## string_t N ## _ ## NK ## _as_string(N ## _table_t t__tmp)\ -{ return NS ## string_cast_from_generic(N ## _ ## NK ## _get(t__tmp)); }\ - -#define __flatbuffers_define_union_vector_ops(NS, T)\ -static inline size_t T ## _union_vec_len(T ## _union_vec_t uv__tmp)\ -{ return NS ## vec_len(uv__tmp.type); }\ -static inline T ## _union_t T ## _union_vec_at(T ## _union_vec_t uv__tmp, size_t i__tmp)\ -{ T ## _union_t u__tmp = { 0, 0 }; size_t n__tmp = NS ## vec_len(uv__tmp.type);\ - assert(n__tmp > (i__tmp) && "index out of range"); u__tmp.type = uv__tmp.type[i__tmp];\ - /* Unknown type is treated as NONE for schema evolution. */\ - if (u__tmp.type == 0) return u__tmp;\ - u__tmp.value = NS ## generic_vec_at(uv__tmp.value, i__tmp); return u__tmp; }\ -static inline NS ## string_t T ## _union_vec_at_as_string(T ## _union_vec_t uv__tmp, size_t i__tmp)\ -{ return (NS ## string_t) NS ## generic_vec_at_as_string(uv__tmp.value, i__tmp); }\ - -#define __flatbuffers_define_union_vector(NS, T)\ -typedef NS ## union_vec_t T ## _union_vec_t;\ -__## NS ## define_union_vector_ops(NS, T) -#define __flatbuffers_define_union(NS, T)\ -typedef NS ## union_t T ## _union_t;\ -__## NS ## define_union_vector(NS, T) -#define __flatbuffers_define_union_vector_field(NS, ID, N, NK, T, r)\ -__## NS ## define_vector_field(ID - 1, N, NK ## _type, T ## _vec_t, r)\ -__## NS ## define_vector_field(ID, N, NK, flatbuffers_generic_vec_t, r)\ -static inline T ## _union_vec_t N ## _ ## NK ## _union(N ## _table_t t__tmp)\ -{ T ## _union_vec_t uv__tmp; uv__tmp.type = N ## _ ## NK ## _type_get(t__tmp);\ - uv__tmp.value = N ## _ ## NK(t__tmp);\ - assert(NS ## vec_len(uv__tmp.type) == NS ## vec_len(uv__tmp.value)\ - && "union vector type length mismatch"); return uv__tmp; } -#include -static size_t flatbuffers_not_found = (size_t)-1; -static size_t flatbuffers_end = (size_t)-1; -#define __flatbuffers_identity(n) (n) -#define __flatbuffers_min(a, b) ((a) < (b) ? (a) : (b)) -/* Subtraction doesn't work for unsigned types. */ -#define __flatbuffers_scalar_cmp(x, y, n) ((x) < (y) ? -1 : (x) > (y)) -static inline int __flatbuffers_string_n_cmp(flatbuffers_string_t v, const char *s, size_t n) -{ size_t nv = flatbuffers_string_len(v); int x = strncmp(v, s, nv < n ? nv : n); - return x != 0 ? x : nv < n ? -1 : nv > n; } -/* `n` arg unused, but needed by string find macro expansion. */ -static inline int __flatbuffers_string_cmp(flatbuffers_string_t v, const char *s, size_t n) { (void)n; return strcmp(v, s); } -/* A = identity if searching scalar vectors rather than key fields. */ -/* Returns lowest matching index or not_found. */ -#define __flatbuffers_find_by_field(A, V, E, L, K, Kn, T, D)\ -{ T v__tmp; size_t a__tmp = 0, b__tmp, m__tmp; if (!(b__tmp = L(V))) { return flatbuffers_not_found; }\ - --b__tmp;\ - while (a__tmp < b__tmp) {\ - m__tmp = a__tmp + ((b__tmp - a__tmp) >> 1);\ - v__tmp = A(E(V, m__tmp));\ - if ((D(v__tmp, (K), (Kn))) < 0) {\ - a__tmp = m__tmp + 1;\ - } else {\ - b__tmp = m__tmp;\ - }\ - }\ - if (a__tmp == b__tmp) {\ - v__tmp = A(E(V, a__tmp));\ - if (D(v__tmp, (K), (Kn)) == 0) {\ - return a__tmp;\ - }\ - }\ - return flatbuffers_not_found;\ -} -#define __flatbuffers_find_by_scalar_field(A, V, E, L, K, T)\ -__flatbuffers_find_by_field(A, V, E, L, K, 0, T, __flatbuffers_scalar_cmp) -#define __flatbuffers_find_by_string_field(A, V, E, L, K)\ -__flatbuffers_find_by_field(A, V, E, L, K, 0, flatbuffers_string_t, __flatbuffers_string_cmp) -#define __flatbuffers_find_by_string_n_field(A, V, E, L, K, Kn)\ -__flatbuffers_find_by_field(A, V, E, L, K, Kn, flatbuffers_string_t, __flatbuffers_string_n_cmp) -#define __flatbuffers_define_find_by_scalar_field(N, NK, TK)\ -static inline size_t N ## _vec_find_by_ ## NK(N ## _vec_t vec__tmp, TK key__tmp)\ -__flatbuffers_find_by_scalar_field(N ## _ ## NK, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, TK) -#define __flatbuffers_define_scalar_find(N, T)\ -static inline size_t N ## _vec_find(N ## _vec_t vec__tmp, T key__tmp)\ -__flatbuffers_find_by_scalar_field(__flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T) -#define __flatbuffers_define_find_by_string_field(N, NK) \ -/* Note: find only works on vectors sorted by this field. */\ -static inline size_t N ## _vec_find_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp)\ -__flatbuffers_find_by_string_field(N ## _ ## NK, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ -static inline size_t N ## _vec_find_n_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp, int n__tmp)\ -__flatbuffers_find_by_string_n_field(N ## _ ## NK, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp) -#define __flatbuffers_define_default_find_by_scalar_field(N, NK, TK)\ -static inline size_t N ## _vec_find(N ## _vec_t vec__tmp, TK key__tmp)\ -{ return N ## _vec_find_by_ ## NK(vec__tmp, key__tmp); } -#define __flatbuffers_define_default_find_by_string_field(N, NK) \ -static inline size_t N ## _vec_find(N ## _vec_t vec__tmp, const char *s__tmp)\ -{ return N ## _vec_find_by_ ## NK(vec__tmp, s__tmp); }\ -static inline size_t N ## _vec_find_n(N ## _vec_t vec__tmp, const char *s__tmp, int n__tmp)\ -{ return N ## _vec_find_n_by_ ## NK(vec__tmp, s__tmp, n__tmp); } -/* A = identity if searching scalar vectors rather than key fields. */ -/* Returns lowest matching index or not_found. */ -#define __flatbuffers_scan_by_field(b, e, A, V, E, L, K, Kn, T, D)\ -{ T v__tmp; size_t i__tmp;\ - for (i__tmp = b; i__tmp < e; ++i__tmp) {\ - v__tmp = A(E(V, i__tmp));\ - if (D(v__tmp, (K), (Kn)) == 0) {\ - return i__tmp;\ - }\ - }\ - return flatbuffers_not_found;\ -} -#define __flatbuffers_rscan_by_field(b, e, A, V, E, L, K, Kn, T, D)\ -{ T v__tmp; size_t i__tmp = e;\ - while (i__tmp-- > b) {\ - v__tmp = A(E(V, i__tmp));\ - if (D(v__tmp, (K), (Kn)) == 0) {\ - return i__tmp;\ - }\ - }\ - return flatbuffers_not_found;\ -} -#define __flatbuffers_scan_by_scalar_field(b, e, A, V, E, L, K, T)\ -__flatbuffers_scan_by_field(b, e, A, V, E, L, K, 0, T, __flatbuffers_scalar_cmp) -#define __flatbuffers_scan_by_string_field(b, e, A, V, E, L, K)\ -__flatbuffers_scan_by_field(b, e, A, V, E, L, K, 0, flatbuffers_string_t, __flatbuffers_string_cmp) -#define __flatbuffers_scan_by_string_n_field(b, e, A, V, E, L, K, Kn)\ -__flatbuffers_scan_by_field(b, e, A, V, E, L, K, Kn, flatbuffers_string_t, __flatbuffers_string_n_cmp) -#define __flatbuffers_rscan_by_scalar_field(b, e, A, V, E, L, K, T)\ -__flatbuffers_rscan_by_field(b, e, A, V, E, L, K, 0, T, __flatbuffers_scalar_cmp) -#define __flatbuffers_rscan_by_string_field(b, e, A, V, E, L, K)\ -__flatbuffers_rscan_by_field(b, e, A, V, E, L, K, 0, flatbuffers_string_t, __flatbuffers_string_cmp) -#define __flatbuffers_rscan_by_string_n_field(b, e, A, V, E, L, K, Kn)\ -__flatbuffers_rscan_by_field(b, e, A, V, E, L, K, Kn, flatbuffers_string_t, __flatbuffers_string_n_cmp) -#define __flatbuffers_define_scan_by_scalar_field(N, NK, T)\ -static inline size_t N ## _vec_scan_by_ ## NK(N ## _vec_t vec__tmp, T key__tmp)\ -__flatbuffers_scan_by_scalar_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ -static inline size_t N ## _vec_scan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ -__flatbuffers_scan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ -static inline size_t N ## _vec_rscan_by_ ## NK(N ## _vec_t vec__tmp, T key__tmp)\ -__flatbuffers_rscan_by_scalar_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ -static inline size_t N ## _vec_rscan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ -__flatbuffers_rscan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T) -#define __flatbuffers_define_scalar_scan(N, T)\ -static inline size_t N ## _vec_scan(N ## _vec_t vec__tmp, T key__tmp)\ -__flatbuffers_scan_by_scalar_field(0, N ## _vec_len(vec__tmp), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ -static inline size_t N ## _vec_scan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ -__flatbuffers_scan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ -static inline size_t N ## _vec_rscan(N ## _vec_t vec__tmp, T key__tmp)\ -__flatbuffers_rscan_by_scalar_field(0, N ## _vec_len(vec__tmp), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T)\ -static inline size_t N ## _vec_rscan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, T key__tmp)\ -__flatbuffers_rscan_by_scalar_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), __flatbuffers_identity, vec__tmp, N ## _vec_at, N ## _vec_len, key__tmp, T) -#define __flatbuffers_define_scan_by_string_field(N, NK) \ -static inline size_t N ## _vec_scan_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp)\ -__flatbuffers_scan_by_string_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ -static inline size_t N ## _vec_scan_n_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp, int n__tmp)\ -__flatbuffers_scan_by_string_n_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp)\ -static inline size_t N ## _vec_scan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ -__flatbuffers_scan_by_string_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ -static inline size_t N ## _vec_scan_ex_n_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, int n__tmp)\ -__flatbuffers_scan_by_string_n_field(begin__tmp, __flatbuffers_min( end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp)\ -static inline size_t N ## _vec_rscan_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp)\ -__flatbuffers_rscan_by_string_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ -static inline size_t N ## _vec_rscan_n_by_ ## NK(N ## _vec_t vec__tmp, const char *s__tmp, int n__tmp)\ -__flatbuffers_rscan_by_string_n_field(0, N ## _vec_len(vec__tmp), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp)\ -static inline size_t N ## _vec_rscan_ex_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ -__flatbuffers_rscan_by_string_field(begin__tmp, __flatbuffers_min(end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp)\ -static inline size_t N ## _vec_rscan_ex_n_by_ ## NK(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, int n__tmp)\ -__flatbuffers_rscan_by_string_n_field(begin__tmp, __flatbuffers_min( end__tmp, N ## _vec_len(vec__tmp)), N ## _ ## NK ## _get, vec__tmp, N ## _vec_at, N ## _vec_len, s__tmp, n__tmp) -#define __flatbuffers_define_default_scan_by_scalar_field(N, NK, TK)\ -static inline size_t N ## _vec_scan(N ## _vec_t vec__tmp, TK key__tmp)\ -{ return N ## _vec_scan_by_ ## NK(vec__tmp, key__tmp); }\ -static inline size_t N ## _vec_scan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, TK key__tmp)\ -{ return N ## _vec_scan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, key__tmp); }\ -static inline size_t N ## _vec_rscan(N ## _vec_t vec__tmp, TK key__tmp)\ -{ return N ## _vec_rscan_by_ ## NK(vec__tmp, key__tmp); }\ -static inline size_t N ## _vec_rscan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, TK key__tmp)\ -{ return N ## _vec_rscan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, key__tmp); } -#define __flatbuffers_define_default_scan_by_string_field(N, NK) \ -static inline size_t N ## _vec_scan(N ## _vec_t vec__tmp, const char *s__tmp)\ -{ return N ## _vec_scan_by_ ## NK(vec__tmp, s__tmp); }\ -static inline size_t N ## _vec_scan_n(N ## _vec_t vec__tmp, const char *s__tmp, int n__tmp)\ -{ return N ## _vec_scan_n_by_ ## NK(vec__tmp, s__tmp, n__tmp); }\ -static inline size_t N ## _vec_scan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ -{ return N ## _vec_scan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp); }\ -static inline size_t N ## _vec_scan_ex_n(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, int n__tmp)\ -{ return N ## _vec_scan_ex_n_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp, n__tmp); }\ -static inline size_t N ## _vec_rscan(N ## _vec_t vec__tmp, const char *s__tmp)\ -{ return N ## _vec_rscan_by_ ## NK(vec__tmp, s__tmp); }\ -static inline size_t N ## _vec_rscan_n(N ## _vec_t vec__tmp, const char *s__tmp, int n__tmp)\ -{ return N ## _vec_rscan_n_by_ ## NK(vec__tmp, s__tmp, n__tmp); }\ -static inline size_t N ## _vec_rscan_ex(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp)\ -{ return N ## _vec_rscan_ex_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp); }\ -static inline size_t N ## _vec_rscan_ex_n(N ## _vec_t vec__tmp, size_t begin__tmp, size_t end__tmp, const char *s__tmp, int n__tmp)\ -{ return N ## _vec_rscan_ex_n_by_ ## NK(vec__tmp, begin__tmp, end__tmp, s__tmp, n__tmp); } -#define __flatbuffers_heap_sort(N, X, A, E, L, TK, TE, D, S)\ -static inline void __ ## N ## X ## __heap_sift_down(\ - N ## _mutable_vec_t vec__tmp, size_t start__tmp, size_t end__tmp)\ -{ size_t child__tmp, root__tmp; TK v1__tmp, v2__tmp, vroot__tmp;\ - root__tmp = start__tmp;\ - while ((root__tmp << 1) <= end__tmp) {\ - child__tmp = root__tmp << 1;\ - if (child__tmp < end__tmp) {\ - v1__tmp = A(E(vec__tmp, child__tmp));\ - v2__tmp = A(E(vec__tmp, child__tmp + 1));\ - if (D(v1__tmp, v2__tmp) < 0) {\ - child__tmp++;\ - }\ - }\ - vroot__tmp = A(E(vec__tmp, root__tmp));\ - v1__tmp = A(E(vec__tmp, child__tmp));\ - if (D(vroot__tmp, v1__tmp) < 0) {\ - S(vec__tmp, root__tmp, child__tmp, TE);\ - root__tmp = child__tmp;\ - } else {\ - return;\ - }\ - }\ -}\ -static inline void __ ## N ## X ## __heap_sort(N ## _mutable_vec_t vec__tmp)\ -{ size_t start__tmp, end__tmp, size__tmp;\ - size__tmp = L(vec__tmp); if (size__tmp == 0) return; end__tmp = size__tmp - 1; start__tmp = size__tmp >> 1;\ - do { __ ## N ## X ## __heap_sift_down(vec__tmp, start__tmp, end__tmp); } while (start__tmp--);\ - while (end__tmp > 0) { \ - S(vec__tmp, 0, end__tmp, TE);\ - __ ## N ## X ## __heap_sift_down(vec__tmp, 0, --end__tmp); } } -#define __flatbuffers_define_sort_by_field(N, NK, TK, TE, D, S)\ - __flatbuffers_heap_sort(N, _sort_by_ ## NK, N ## _ ## NK ## _get, N ## _vec_at, N ## _vec_len, TK, TE, D, S)\ -static inline void N ## _vec_sort_by_ ## NK(N ## _mutable_vec_t vec__tmp)\ -{ __ ## N ## _sort_by_ ## NK ## __heap_sort(vec__tmp); } -#define __flatbuffers_define_sort(N, TK, TE, D, S)\ -__flatbuffers_heap_sort(N, , __flatbuffers_identity, N ## _vec_at, N ## _vec_len, TK, TE, D, S)\ -static inline void N ## _vec_sort(N ## _mutable_vec_t vec__tmp) { __ ## N ## __heap_sort(vec__tmp); } -#define __flatbuffers_scalar_diff(x, y) ((x) < (y) ? -1 : (x) > (y)) -#define __flatbuffers_string_diff(x, y) __flatbuffers_string_n_cmp((x), (const char *)(y), flatbuffers_string_len(y)) -#define __flatbuffers_scalar_swap(vec, a, b, TE) { TE x__tmp = vec[b]; vec[b] = vec[a]; vec[a] = x__tmp; } -#define __flatbuffers_string_swap(vec, a, b, TE)\ -{ TE ta__tmp, tb__tmp, d__tmp;\ - d__tmp = (TE)((a - b) * sizeof(vec[0]));\ - ta__tmp = __flatbuffers_uoffset_read_from_pe(vec + b) - d__tmp;\ - tb__tmp = __flatbuffers_uoffset_read_from_pe(vec + a) + d__tmp;\ - __flatbuffers_uoffset_write_to_pe(vec + a, ta__tmp);\ - __flatbuffers_uoffset_write_to_pe(vec + b, tb__tmp); } -#define __flatbuffers_define_sort_by_scalar_field(N, NK, TK, TE)\ - __flatbuffers_define_sort_by_field(N, NK, TK, TE, __flatbuffers_scalar_diff, __flatbuffers_scalar_swap) -#define __flatbuffers_define_sort_by_string_field(N, NK)\ - __flatbuffers_define_sort_by_field(N, NK, flatbuffers_string_t, flatbuffers_uoffset_t, __flatbuffers_string_diff, __flatbuffers_string_swap) -#define __flatbuffers_define_scalar_sort(N, T) __flatbuffers_define_sort(N, T, T, __flatbuffers_scalar_diff, __flatbuffers_scalar_swap) -#define __flatbuffers_define_string_sort() __flatbuffers_define_sort(flatbuffers_string, flatbuffers_string_t, flatbuffers_uoffset_t, __flatbuffers_string_diff, __flatbuffers_string_swap) -#define __flatbuffers_define_scalar_vector(N, T)\ -typedef const T *N ## _vec_t;\ -typedef T *N ## _mutable_vec_t;\ -__flatbuffers_define_scalar_vec_len(N)\ -__flatbuffers_define_scalar_vec_at(N, T)\ -__flatbuffers_define_scalar_find(N, T)\ -__flatbuffers_define_scalar_scan(N, T)\ -__flatbuffers_define_scalar_sort(N, T) - -#define __flatbuffers_define_integer_type(N, T, W)\ -__flatcc_define_integer_accessors(N, T, W, flatbuffers_endian)\ -__flatbuffers_define_scalar_vector(N, T) -__flatbuffers_define_scalar_vector(flatbuffers_bool, flatbuffers_bool_t) -__flatbuffers_define_scalar_vector(flatbuffers_uint8, uint8_t) -__flatbuffers_define_scalar_vector(flatbuffers_int8, int8_t) -__flatbuffers_define_scalar_vector(flatbuffers_uint16, uint16_t) -__flatbuffers_define_scalar_vector(flatbuffers_int16, int16_t) -__flatbuffers_define_scalar_vector(flatbuffers_uint32, uint32_t) -__flatbuffers_define_scalar_vector(flatbuffers_int32, int32_t) -__flatbuffers_define_scalar_vector(flatbuffers_uint64, uint64_t) -__flatbuffers_define_scalar_vector(flatbuffers_int64, int64_t) -__flatbuffers_define_scalar_vector(flatbuffers_float, float) -__flatbuffers_define_scalar_vector(flatbuffers_double, double) -__flatbuffers_define_scalar_vector(flatbuffers_union_type, flatbuffers_union_type_t) -static inline size_t flatbuffers_string_vec_find(flatbuffers_string_vec_t vec, const char *s) -__flatbuffers_find_by_string_field(__flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) -static inline size_t flatbuffers_string_vec_find_n(flatbuffers_string_vec_t vec, const char *s, size_t n) -__flatbuffers_find_by_string_n_field(__flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) -static inline size_t flatbuffers_string_vec_scan(flatbuffers_string_vec_t vec, const char *s) -__flatbuffers_scan_by_string_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) -static inline size_t flatbuffers_string_vec_scan_n(flatbuffers_string_vec_t vec, const char *s, size_t n) -__flatbuffers_scan_by_string_n_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) -static inline size_t flatbuffers_string_vec_scan_ex(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s) -__flatbuffers_scan_by_string_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) -static inline size_t flatbuffers_string_vec_scan_ex_n(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s, size_t n) -__flatbuffers_scan_by_string_n_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) -static inline size_t flatbuffers_string_vec_rscan(flatbuffers_string_vec_t vec, const char *s) -__flatbuffers_rscan_by_string_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) -static inline size_t flatbuffers_string_vec_rscan_n(flatbuffers_string_vec_t vec, const char *s, size_t n) -__flatbuffers_rscan_by_string_n_field(0, flatbuffers_string_vec_len(vec), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) -static inline size_t flatbuffers_string_vec_rscan_ex(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s) -__flatbuffers_rscan_by_string_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s) -static inline size_t flatbuffers_string_vec_rscan_ex_n(flatbuffers_string_vec_t vec, size_t begin, size_t end, const char *s, size_t n) -__flatbuffers_rscan_by_string_n_field(begin, __flatbuffers_min(end, flatbuffers_string_vec_len(vec)), __flatbuffers_identity, vec, flatbuffers_string_vec_at, flatbuffers_string_vec_len, s, n) -__flatbuffers_define_string_sort() -#define __flatbuffers_define_struct_scalar_field(N, NK, TK, T)\ -static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp)\ -{ return t__tmp ? __flatbuffers_read_scalar(TK, &(t__tmp->NK)) : 0; }\ -static inline const T *N ## _ ## NK ## _get_ptr(N ## _struct_t t__tmp)\ -{ return t__tmp ? &(t__tmp->NK) : 0; }\ -static inline T N ## _ ## NK (N ## _struct_t t__tmp)\ -{ return t__tmp ? __flatbuffers_read_scalar(TK, &(t__tmp->NK)) : 0; }\ -__flatbuffers_define_scan_by_scalar_field(N, NK, T) -#define __flatbuffers_define_struct_struct_field(N, NK, T)\ -static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp) { return t__tmp ? &(t__tmp->NK) : 0; }\ -static inline T N ## _ ## NK (N ## _struct_t t__tmp) { return t__tmp ? &(t__tmp->NK) : 0; } -/* If fid is null, the function returns true without testing as buffer is not expected to have any id. */ -static inline int flatbuffers_has_identifier(const void *buffer, const char *fid) -{ flatbuffers_thash_t id, id2 = 0; if (fid == 0) { return 1; }; - id2 = flatbuffers_type_hash_from_string(fid); - id = __flatbuffers_thash_read_from_pe(((flatbuffers_uoffset_t *)buffer) + 1); - return id2 == 0 || id == id2; } -static inline int flatbuffers_has_type_hash(const void *buffer, flatbuffers_thash_t thash) -{ return thash == 0 || (__flatbuffers_thash_read_from_pe((flatbuffers_uoffset_t *)buffer + 1) == thash); } - -static inline flatbuffers_thash_t flatbuffers_get_type_hash(const void *buffer) -{ return __flatbuffers_thash_read_from_pe((flatbuffers_uoffset_t *)buffer + 1); } - -#define flatbuffers_verify_endian() flatbuffers_has_identifier("\x00\x00\x00\x00" "1234", "1234") -static inline void *flatbuffers_read_size_prefix(void *b, size_t *size_out) -{ if (size_out) { *size_out = (size_t)__flatbuffers_uoffset_read_from_pe(b); } - return (uint8_t *)b + sizeof(flatbuffers_uoffset_t); } -/* Null file identifier accepts anything, otherwise fid should be 4 characters. */ -#define __flatbuffers_read_root(T, K, buffer, fid)\ - ((!buffer || !flatbuffers_has_identifier(buffer, fid)) ? 0 :\ - ((T ## _ ## K ## t)(((uint8_t *)buffer) +\ - __flatbuffers_uoffset_read_from_pe(buffer)))) -#define __flatbuffers_read_typed_root(T, K, buffer, thash)\ - ((!buffer || !flatbuffers_has_type_hash(buffer, thash)) ? 0 :\ - ((T ## _ ## K ## t)(((uint8_t *)buffer) +\ - __flatbuffers_uoffset_read_from_pe(buffer)))) -#define __flatbuffers_nested_buffer_as_root(C, N, T, K)\ -static inline T ## _ ## K ## t C ## _ ## N ## _as_root_with_identifier(C ## _ ## table_t t__tmp, const char *fid__tmp)\ -{ const uint8_t *buffer__tmp = C ## _ ## N(t__tmp); return __flatbuffers_read_root(T, K, buffer__tmp, fid__tmp); }\ -static inline T ## _ ## K ## t C ## _ ## N ## _as_typed_root(C ## _ ## table_t t__tmp)\ -{ const uint8_t *buffer__tmp = C ## _ ## N(t__tmp); return __flatbuffers_read_root(T, K, buffer__tmp, C ## _ ## type_identifier); }\ -static inline T ## _ ## K ## t C ## _ ## N ## _as_root(C ## _ ## table_t t__tmp)\ -{ const char *fid__tmp = T ## _identifier;\ - const uint8_t *buffer__tmp = C ## _ ## N(t__tmp); return __flatbuffers_read_root(T, K, buffer__tmp, fid__tmp); } -#define __flatbuffers_buffer_as_root(N, K)\ -static inline N ## _ ## K ## t N ## _as_root_with_identifier(const void *buffer__tmp, const char *fid__tmp)\ -{ return __flatbuffers_read_root(N, K, buffer__tmp, fid__tmp); }\ -static inline N ## _ ## K ## t N ## _as_root_with_type_hash(const void *buffer__tmp, flatbuffers_thash_t thash__tmp)\ -{ return __flatbuffers_read_typed_root(N, K, buffer__tmp, thash__tmp); }\ -static inline N ## _ ## K ## t N ## _as_root(const void *buffer__tmp)\ -{ const char *fid__tmp = N ## _identifier;\ - return __flatbuffers_read_root(N, K, buffer__tmp, fid__tmp); }\ -static inline N ## _ ## K ## t N ## _as_typed_root(const void *buffer__tmp)\ -{ return __flatbuffers_read_typed_root(N, K, buffer__tmp, N ## _type_hash); } -#define __flatbuffers_struct_as_root(N) __flatbuffers_buffer_as_root(N, struct_) -#define __flatbuffers_table_as_root(N) __flatbuffers_buffer_as_root(N, table_) - -#include "flatcc/flatcc_epilogue.h" -#endif /* FLATBUFFERS_COMMON_H */ diff --git a/common/src/ucr_read_builder.h b/common/src/ucr_read_builder.h deleted file mode 100644 index 5b347277a..000000000 --- a/common/src/ucr_read_builder.h +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef UCR_READ_BUILDER_H -#define UCR_READ_BUILDER_H - -/* Generated by flatcc 0.5.3-pre FlatBuffers schema compiler for C by dvide.com */ - -#ifndef UCR_READ_READER_H -#include "ucr_read_reader.h" -#endif -#ifndef FLATBUFFERS_COMMON_BUILDER_H -#include "flatbuffers_common_builder.h" -#endif -#include "flatcc/flatcc_prologue.h" -#ifndef flatbuffers_identifier -#define flatbuffers_identifier 0 -#endif -#ifndef flatbuffers_extension -#define flatbuffers_extension ".bin" -#endif - -#define __unifyfs_Extent_formal_args , uint32_t v0, uint64_t v1, uint64_t v2 -#define __unifyfs_Extent_call_args , v0, v1, v2 -static inline unifyfs_Extent_t *unifyfs_Extent_assign(unifyfs_Extent_t *p, uint32_t v0, uint64_t v1, uint64_t v2) -{ p->fid = v0; p->offset = v1; p->length = v2; - return p; } -static inline unifyfs_Extent_t *unifyfs_Extent_copy(unifyfs_Extent_t *p, const unifyfs_Extent_t *p2) -{ p->fid = p2->fid; p->offset = p2->offset; p->length = p2->length; - return p; } -static inline unifyfs_Extent_t *unifyfs_Extent_assign_to_pe(unifyfs_Extent_t *p, uint32_t v0, uint64_t v1, uint64_t v2) -{ flatbuffers_uint32_assign_to_pe(&p->fid, v0); flatbuffers_uint64_assign_to_pe(&p->offset, v1); flatbuffers_uint64_assign_to_pe(&p->length, v2); - return p; } -static inline unifyfs_Extent_t *unifyfs_Extent_copy_to_pe(unifyfs_Extent_t *p, const unifyfs_Extent_t *p2) -{ flatbuffers_uint32_copy_to_pe(&p->fid, &p2->fid); flatbuffers_uint64_copy_to_pe(&p->offset, &p2->offset); flatbuffers_uint64_copy_to_pe(&p->length, &p2->length); - return p; } -static inline unifyfs_Extent_t *unifyfs_Extent_assign_from_pe(unifyfs_Extent_t *p, uint32_t v0, uint64_t v1, uint64_t v2) -{ flatbuffers_uint32_assign_from_pe(&p->fid, v0); flatbuffers_uint64_assign_from_pe(&p->offset, v1); flatbuffers_uint64_assign_from_pe(&p->length, v2); - return p; } -static inline unifyfs_Extent_t *unifyfs_Extent_copy_from_pe(unifyfs_Extent_t *p, const unifyfs_Extent_t *p2) -{ flatbuffers_uint32_copy_from_pe(&p->fid, &p2->fid); flatbuffers_uint64_copy_from_pe(&p->offset, &p2->offset); flatbuffers_uint64_copy_from_pe(&p->length, &p2->length); - return p; } -__flatbuffers_build_struct(flatbuffers_, unifyfs_Extent, 24, 8, unifyfs_Extent_identifier, unifyfs_Extent_type_identifier) - -static const flatbuffers_voffset_t __unifyfs_ReadRequest_required[] = { 0 }; -typedef flatbuffers_ref_t unifyfs_ReadRequest_ref_t; -static unifyfs_ReadRequest_ref_t unifyfs_ReadRequest_clone(flatbuffers_builder_t *B, unifyfs_ReadRequest_table_t t); -__flatbuffers_build_table(flatbuffers_, unifyfs_ReadRequest, 1) - -#define __unifyfs_ReadRequest_formal_args , unifyfs_Extent_vec_ref_t v0 -#define __unifyfs_ReadRequest_call_args , v0 -static inline unifyfs_ReadRequest_ref_t unifyfs_ReadRequest_create(flatbuffers_builder_t *B __unifyfs_ReadRequest_formal_args); -__flatbuffers_build_table_prolog(flatbuffers_, unifyfs_ReadRequest, unifyfs_ReadRequest_identifier, unifyfs_ReadRequest_type_identifier) - -__flatbuffers_build_vector_field(0, flatbuffers_, unifyfs_ReadRequest_extents, unifyfs_Extent, unifyfs_Extent_t, unifyfs_ReadRequest) - -static inline unifyfs_ReadRequest_ref_t unifyfs_ReadRequest_create(flatbuffers_builder_t *B __unifyfs_ReadRequest_formal_args) -{ - if (unifyfs_ReadRequest_start(B) - || unifyfs_ReadRequest_extents_add(B, v0)) { - return 0; - } - return unifyfs_ReadRequest_end(B); -} - -static unifyfs_ReadRequest_ref_t unifyfs_ReadRequest_clone(flatbuffers_builder_t *B, unifyfs_ReadRequest_table_t t) -{ - __flatbuffers_memoize_begin(B, t); - if (unifyfs_ReadRequest_start(B) - || unifyfs_ReadRequest_extents_pick(B, t)) { - return 0; - } - __flatbuffers_memoize_end(B, t, unifyfs_ReadRequest_end(B)); -} - -#include "flatcc/flatcc_epilogue.h" -#endif /* UCR_READ_BUILDER_H */ diff --git a/common/src/ucr_read_reader.h b/common/src/ucr_read_reader.h deleted file mode 100644 index eb26b9d5f..000000000 --- a/common/src/ucr_read_reader.h +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef UCR_READ_READER_H -#define UCR_READ_READER_H - -/* Generated by flatcc 0.5.3-pre FlatBuffers schema compiler for C by dvide.com */ - -#ifndef FLATBUFFERS_COMMON_READER_H -#include "flatbuffers_common_reader.h" -#endif -#include "flatcc/flatcc_flatbuffers.h" -#ifndef __alignas_is_defined -#include -#endif -#include "flatcc/flatcc_prologue.h" -#ifndef flatbuffers_identifier -#define flatbuffers_identifier 0 -#endif -#ifndef flatbuffers_extension -#define flatbuffers_extension ".bin" -#endif - -typedef struct unifyfs_Extent unifyfs_Extent_t; -typedef const unifyfs_Extent_t *unifyfs_Extent_struct_t; -typedef unifyfs_Extent_t *unifyfs_Extent_mutable_struct_t; -typedef const unifyfs_Extent_t *unifyfs_Extent_vec_t; -typedef unifyfs_Extent_t *unifyfs_Extent_mutable_vec_t; - -typedef const struct unifyfs_ReadRequest_table *unifyfs_ReadRequest_table_t; -typedef const flatbuffers_uoffset_t *unifyfs_ReadRequest_vec_t; -typedef flatbuffers_uoffset_t *unifyfs_ReadRequest_mutable_vec_t; -#ifndef unifyfs_Extent_identifier -#define unifyfs_Extent_identifier flatbuffers_identifier -#endif -#define unifyfs_Extent_type_hash ((flatbuffers_thash_t)0xfe153735) -#define unifyfs_Extent_type_identifier "\x35\x37\x15\xfe" -#ifndef unifyfs_ReadRequest_identifier -#define unifyfs_ReadRequest_identifier flatbuffers_identifier -#endif -#define unifyfs_ReadRequest_type_hash ((flatbuffers_thash_t)0x70b2f5ee) -#define unifyfs_ReadRequest_type_identifier "\xee\xf5\xb2\x70" - - -struct unifyfs_Extent { - alignas(8) uint32_t fid; - alignas(8) uint64_t offset; - alignas(8) uint64_t length; -}; -static_assert(sizeof(unifyfs_Extent_t) == 24, "struct size mismatch"); - -static inline const unifyfs_Extent_t *unifyfs_Extent__const_ptr_add(const unifyfs_Extent_t *p, size_t i) { return p + i; } -static inline unifyfs_Extent_t *unifyfs_Extent__ptr_add(unifyfs_Extent_t *p, size_t i) { return p + i; } -static inline unifyfs_Extent_struct_t unifyfs_Extent_vec_at(unifyfs_Extent_vec_t vec, size_t i) -__flatbuffers_struct_vec_at(vec, i) -static inline size_t unifyfs_Extent__size() { return 24; } -static inline size_t unifyfs_Extent_vec_len(unifyfs_Extent_vec_t vec) -__flatbuffers_vec_len(vec) -__flatbuffers_struct_as_root(unifyfs_Extent) - -__flatbuffers_define_struct_scalar_field(unifyfs_Extent, fid, flatbuffers_uint32, uint32_t) -__flatbuffers_define_struct_scalar_field(unifyfs_Extent, offset, flatbuffers_uint64, uint64_t) -__flatbuffers_define_struct_scalar_field(unifyfs_Extent, length, flatbuffers_uint64, uint64_t) - - -struct unifyfs_ReadRequest_table { uint8_t unused__; }; - -static inline size_t unifyfs_ReadRequest_vec_len(unifyfs_ReadRequest_vec_t vec) -__flatbuffers_vec_len(vec) -static inline unifyfs_ReadRequest_table_t unifyfs_ReadRequest_vec_at(unifyfs_ReadRequest_vec_t vec, size_t i) -__flatbuffers_offset_vec_at(unifyfs_ReadRequest_table_t, vec, i, 0) -__flatbuffers_table_as_root(unifyfs_ReadRequest) - -__flatbuffers_define_vector_field(0, unifyfs_ReadRequest, extents, unifyfs_Extent_vec_t, 0) - -#include "flatcc/flatcc_epilogue.h" -#endif /* UCR_READ_READER_H */ diff --git a/common/src/unifyfs_meta.h b/common/src/unifyfs_meta.h index f1fa59e76..35e070406 100644 --- a/common/src/unifyfs_meta.h +++ b/common/src/unifyfs_meta.h @@ -33,6 +33,14 @@ typedef struct { int rank; } name_rank_pair_t; +/* generic file extent */ +typedef struct { + size_t offset; + size_t length; + int gfid; +} unifyfs_extent_t; + + /* write-log metadata index structure */ typedef struct { off_t file_pos; /* start offset of data in file */ diff --git a/configure.ac b/configure.ac index af9106edd..666d92647 100755 --- a/configure.ac +++ b/configure.ac @@ -163,7 +163,6 @@ UNIFYFS_AC_GOTCHA UNIFYFS_AC_SPATH UNIFYFS_AC_MARGO -UNIFYFS_AC_FLATCC # openssl for md5 checksum UNIFYFS_AC_OPENSSL diff --git a/docs/build-intercept.rst b/docs/build-intercept.rst index 822911016..566104975 100644 --- a/docs/build-intercept.rst +++ b/docs/build-intercept.rst @@ -159,7 +159,6 @@ UnifyFS dependencies can then be installed. .. code-block:: Bash - $ spack install flatcc $ spack install gotcha $ spack install leveldb $ spack install margo ^mercury+bmi @@ -186,7 +185,6 @@ manually build UnifyFS from inside the source code directory. .. code-block:: Bash - $ spack load flatcc $ spack load gotcha $ spack load leveldb $ spack load mercury @@ -214,7 +212,7 @@ from the `UnifyFS repository `_. Build the Dependencies ********************** -UnifyFS requires MPI, LevelDB, GOTCHA, FlatCC, Margo and OpenSSL. +UnifyFS requires MPI, LevelDB, GOTCHA, Margo and OpenSSL. References to these dependencies can be found on our :doc:`` page. A `bootstrap.sh `_ script @@ -236,7 +234,7 @@ this: $ export PKG_CONFIG_PATH=path/to/mercury/lib/pkgconfig:path/to/argobots/lib/pkgconfig:path/to/margo/lib/pkgconfig $ ./autogen.sh - $ ./configure --prefix=/path/to/install --with-gotcha=/path/to/gotcha --with-leveldb=/path/to/leveldb --with-flatcc=/path/to/flatcc + $ ./configure --prefix=/path/to/install --with-gotcha=/path/to/gotcha --with-leveldb=/path/to/leveldb $ make $ make install diff --git a/docs/dependencies.rst b/docs/dependencies.rst index 6c8799a57..85b6cbc4b 100644 --- a/docs/dependencies.rst +++ b/docs/dependencies.rst @@ -6,8 +6,6 @@ UnifyFS Dependencies - `leveldb `_ version 1.22 -- `flatcc `_ version 0.5.3 - - `Margo `_ version 0.4.3 and its dependencies: - `Argobots `_ version 1.0 diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index 790834399..ede17bfd4 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -84,21 +84,18 @@ if HAVE_FORTRAN test_ftn_flags = $(AM_FCFLAGS) $(MPI_FFLAGS) \ -I$(top_srcdir)/client/src -I$(top_srcdir)/common/src test_ftn_ldadd = $(top_builddir)/client/src/libunifyfsf.la -lrt -lm $(FCLIBS) -test_ftn_ldflags = $(AM_LDFLAGS) $(MPI_FLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) +test_ftn_ldflags = $(AM_LDFLAGS) $(MPI_FLDFLAGS) endif test_gotcha_ldadd = $(top_builddir)/client/src/libunifyfs_gotcha.la -lrt -lm -test_gotcha_ldflags = $(AM_LDFLAGS) $(MPI_CLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) +test_gotcha_ldflags = $(AM_LDFLAGS) $(MPI_CLDFLAGS) test_posix_cppflags = $(AM_CPPFLAGS) $(MPI_CFLAGS) -DDISABLE_UNIFYFS test_posix_ldadd = -lrt -lm test_posix_ldflags = $(AM_LDFLAGS) $(MPI_CLDFLAGS) test_static_ldadd = $(top_builddir)/client/src/libunifyfs.la -lrt -lm -test_static_ldflags = -static $(CP_WRAPPERS) $(AM_LDFLAGS) $(MPI_CLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) +test_static_ldflags = -static $(CP_WRAPPERS) $(AM_LDFLAGS) $(MPI_CLDFLAGS) # Per-target flags begin here diff --git a/m4/flatcc.m4 b/m4/flatcc.m4 deleted file mode 100644 index 65596a194..000000000 --- a/m4/flatcc.m4 +++ /dev/null @@ -1,29 +0,0 @@ -AC_DEFUN([UNIFYFS_AC_FLATCC], [ - # preserve state of flags - FLATCC_OLD_CFLAGS=$CFLAGS - FLATCC_OLD_LDFLAGS=$LDFLAGS - - AC_ARG_VAR([FLATCC_ROOT],[Set the path to the LevelDB installation directory]) - - AS_IF([test -n "$FLATCC_ROOT"],[ - AC_MSG_NOTICE([FLATCC_ROOT is set, checking for flatcc in $FLATCC_ROOT]) - FLATCC_CFLAGS="-I${FLATCC_ROOT}/include" - FLATCC_LDFLAGS="-L${FLATCC_ROOT}/lib" - CFLAGS="$CFLAGS ${FLATCC_CFLAGS}" - LDFLAGS="$LDFLAGS ${FLATCC_LDFLAGS}" - ],[]) - - AC_CHECK_LIB([flatcc], [flatcc_create_context], - [FLATCC_LIBS="-lflatcc" - AC_SUBST(FLATCC_CFLAGS) - AC_SUBST(FLATCC_LDFLAGS) - AC_SUBST(FLATCC_LIBS) - ], - [AC_MSG_ERROR([couldn't find a suitable libflatcc, use FLATCC_ROOT to set the path to the flatcc installation directory.])], - [] - ) - - # restore flags - CFLAGS=$FLATCC_OLD_CFLAGS - LDFLAGS=$FLATCC_OLD_LDFLAGS -]) diff --git a/m4/leveldb.m4 b/m4/leveldb.m4 index 1b08ce5dd..b4f02daa5 100644 --- a/m4/leveldb.m4 +++ b/m4/leveldb.m4 @@ -19,7 +19,7 @@ AC_DEFUN([UNIFYFS_AC_LEVELDB], [ AC_SUBST(LEVELDB_LDFLAGS) AC_SUBST(LEVELDB_LIBS) ], - [AC_MSG_ERROR([couldn't find a suitable libleveldb, use LEVELDB_ROOT to set the path to the flatcc installation directory.])], + [AC_MSG_ERROR([couldn't find a suitable libleveldb, use LEVELDB_ROOT to set the path to the installation directory.])], [] ) diff --git a/server/src/Makefile.am b/server/src/Makefile.am index ca8217b3d..5996b46f6 100644 --- a/server/src/Makefile.am +++ b/server/src/Makefile.am @@ -21,8 +21,7 @@ unifyfsd_SOURCES = unifyfs_server.c unifyfsd_LDFLAGS = -static \ $(LEVELDB_LDFLAGS) \ - $(MARGO_LDFLAGS) \ - $(FLATCC_LDFLAGS) + $(MARGO_LDFLAGS) unifyfsd_LDADD = \ libunifyfsd.a \ @@ -31,7 +30,6 @@ unifyfsd_LDADD = \ $(LEVELDB_LIBS) \ $(MPI_CLDFLAGS) \ $(MARGO_LIBS) \ - $(FLATCC_LIBS) \ -lpthread -lm -lstdc++ -lrt AM_CPPFLAGS = \ @@ -46,7 +44,6 @@ AM_CFLAGS = -Wall -Werror \ $(MPI_CFLAGS) \ $(MERCURY_CFLAGS) \ $(ARGOBOTS_CFLAGS) \ - $(MARGO_CFLAGS) \ - $(FLATCC_CFLAGS) + $(MARGO_CFLAGS) CLEANFILES = $(bin_PROGRAMS) diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index 0adf16d5d..3ec752a28 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -33,7 +33,6 @@ // common headers #include "unifyfs_client_rpcs.h" -#include "ucr_read_builder.h" // server headers #include "unifyfs_global.h" @@ -94,7 +93,6 @@ int meta_init_store(unifyfs_cfg_t* cfg) { int rc, ratio; MPI_Comm comm = MPI_COMM_WORLD; - size_t path_len; long svr_ratio, range_sz; struct stat ss; char db_path[UNIFYFS_MAX_FILENAME] = {0}; diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 10f1d6ee3..caa0cf7c1 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -46,7 +46,6 @@ // margo rpcs #include "unifyfs_server_rpcs.h" #include "margo_server.h" -#include "ucr_read_builder.h" #define RM_LOCK(rm) \ @@ -63,18 +62,18 @@ do { \ /* One request manager thread is created for each client that a - * delegator serves. The main thread of the delegator assigns - * work to the request manager thread to retrieve data and send + * server serves. The margo rpc handler thread(s) assign work + * to the request manager thread to retrieve data and send * it back to the client. * * To start, given a read request from the client (via rpc) - * the handler function on the main delegator first queries the + * the handler function on the main server first queries the * key/value store using the given file id and byte range to obtain * the meta data on the physical location of the file data. This - * meta data provides the host delegator rank, the app/client - * ids that specify the log file on the remote delegator, the + * meta data provides the host server rank, the app/client + * ids that specify the log file on the remote server, the * offset within the log file and the length of data. The rpc - * handler function sorts the meta data by host delegator rank, + * handler function sorts the meta data by host server rank, * generates read requests, and inserts those into a list on a * data structure shared with the request manager (del_req_set). * @@ -106,8 +105,8 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) return NULL; } - /* initialize lock for shared data structures between - * main thread and request delegator thread */ + /* initialize lock for shared data structures of the + * request manager */ pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); @@ -120,8 +119,8 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) return NULL; } - /* initailize condition variable to flow control - * work between main thread and request delegator thread */ + /* initialize condition variable to synchronize work + * notifications for the request manager thread */ rc = pthread_cond_init(&(thrd_ctrl->thrd_cond), NULL); if (rc != 0) { LOGERR("pthread_cond_init failed for request " @@ -158,7 +157,7 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) return thrd_ctrl; } -/* order keyvals by gfid, then host delegator rank */ +/* order keyvals by gfid, then host server rank */ static int compare_kv_gfid_rank(const void* a, const void* b) { const unifyfs_keyval_t* kv_a = a; @@ -311,18 +310,18 @@ static void signal_new_requests(reqmgr_thrd_t* thrd_ctrl) /* wake up the request manager thread for the requesting client */ if (!thrd_ctrl->has_waiting_delegator) { - /* delegator thread is not waiting, but we are in critical - * section, we just added requests so we must wait for delegator + /* reqmgr thread is not waiting, but we are in critical + * section, we just added requests so we must wait for reqmgr * to signal us that it's reached the critical section before * we escape so we don't overwrite these requests before it * has had a chance to process them */ thrd_ctrl->has_waiting_dispatcher = 1; pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); - /* delegator thread has signaled us that it's now waiting */ + /* reqmgr thread has signaled us that it's now waiting */ thrd_ctrl->has_waiting_dispatcher = 0; } - /* have a delegator thread waiting on condition variable, + /* have a reqmgr thread waiting on condition variable, * signal it to begin processing the requests we just added */ pthread_cond_signal(&thrd_ctrl->thrd_cond); } @@ -333,7 +332,7 @@ static void signal_new_responses(reqmgr_thrd_t* thrd_ctrl) /* wake up the request manager thread */ if (thrd_ctrl->has_waiting_delegator) { - /* have a delegator thread waiting on condition variable, + /* have a reqmgr thread waiting on condition variable, * signal it to begin processing the responses we just added */ pthread_cond_signal(&thrd_ctrl->thrd_cond); } @@ -346,9 +345,6 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, int num_vals, unifyfs_keyval_t* keyvals) { - /* get app id for this request batch */ - int app_id = thrd_ctrl->app_id; - /* allocate read request structures */ chunk_read_req_t* all_chunk_reads = (chunk_read_req_t*) calloc((size_t)num_vals, sizeof(chunk_read_req_t)); @@ -372,11 +368,11 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, int prev_del = -1; int num_del = 0; for (i = 0; i < num_vals; i++) { - /* get target delegator for this request */ + /* get target server for this request */ int curr_del = keyvals[i].val.delegator_rank; - /* if target delegator is different from last target, - * increment our delegator count */ + /* if target server is different from last target, + * increment our server count */ if ((prev_del == -1) || (curr_del != prev_del)) { num_del++; } @@ -410,25 +406,25 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, remote_chunk_reads_t* reads = rdreq->remote_reads; /* iterate over write index values again and now create - * per-delegator chunk-reads info, for each delegator + * per-server chunk-reads info, for each server * that we'll request data from, this totals up the number * of read requests and total read data size from that - * delegator */ + * server */ prev_del = -1; size_t del_data_sz = 0; for (i = 0; i < num_vals; i++) { - /* get target delegator for this request */ + /* get target server for this request */ int curr_del = keyvals[i].val.delegator_rank; - /* if target delegator is different from last target, + /* if target server is different from last target, * close out the total number of bytes for the last - * delegator, note this assumes our write index values are - * sorted by delegator rank */ + * server, note this assumes our write index values are + * sorted by server rank */ if ((prev_del != -1) && (curr_del != prev_del)) { - /* record total data for previous delegator */ + /* record total data for previous server */ reads->total_sz = del_data_sz; - /* advance to read request for next delegator */ + /* advance to read request for next server */ reads += 1; /* reset our running tally of bytes to 0 */ @@ -436,11 +432,11 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, } prev_del = curr_del; - /* update total read data size for current delegator */ + /* update total read data size for current server */ del_data_sz += keyvals[i].val.len; - /* if this is the first read request for this delegator, - * initialize fields on the per-delegator read request + /* if this is the first read request for this server, + * initialize fields on the per-server read request * structure */ if (0 == reads->num_chunks) { /* TODO: let's describe what these fields are for */ @@ -451,11 +447,11 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, } /* increment number of read requests we're sending - * to this delegator */ + * to this server */ reads->num_chunks++; } - /* record total data size for final delegator (if any), + /* record total data size for final server (if any), * would have missed doing this in the above loop */ if (num_vals > 0) { reads->total_sz = del_data_sz; @@ -792,7 +788,7 @@ static int truncate_rewrite_keys( size_t newlen = (size_t)(filesize - first_offset); /* for the value, we store the log position, the length, - * the host server (delegator rank), the mount point id + * the host server (delegator_rank), the mount point id * (app id), and the client id (rank) */ unifyfs_val_t* val = unifyfs_vals[count]; val->addr = kv->val.addr; @@ -1081,7 +1077,7 @@ static int submit_read_request(reqmgr_thrd_t* thrd_ctrl, int num_keys, } /* if we get more than one write index entry - * sort them by file id and then by delegator rank */ + * sort them by file id and then by server rank */ if (num_vals > 1) { qsort(keyvals, (size_t)num_vals, sizeof(unifyfs_keyval_t), compare_kv_gfid_rank); @@ -1257,7 +1253,7 @@ static int split_index( /* read function for one requested extent, * called from rpc handler to fill shared data structures - * with read requests to be handled by the delegator thread + * with read requests to be handled by the reqmgr thread. * returns before requests are handled */ int rm_cmd_read( @@ -1346,19 +1342,17 @@ int rm_cmd_mread( reqmgr_thrd_t* thrd_ctrl = client->reqmgr; /* get the locations of all the read requests from the key-value store */ - unifyfs_ReadRequest_table_t readRequest = - unifyfs_ReadRequest_as_root(reqbuf); - unifyfs_Extent_vec_t extents = unifyfs_ReadRequest_extents(readRequest); - size_t extents_len = unifyfs_Extent_vec_len(extents); - assert(extents_len == req_num); + unifyfs_extent_t* read_reqs = (unifyfs_extent_t*)reqbuf; /* count up number of slices these request cover */ - int j; + int i; size_t slices = 0; - for (j = 0; j < req_num; j++) { + unifyfs_extent_t* req; + for (i = 0; i < req_num; i++) { /* get offset and length of next request */ - size_t off = unifyfs_Extent_offset(unifyfs_Extent_vec_at(extents, j)); - size_t len = unifyfs_Extent_length(unifyfs_Extent_vec_at(extents, j)); + req = read_reqs + i; + size_t off = req->offset; + size_t len = req->length; /* add in number of slices this request needs */ slices += num_slices(off, len); @@ -1383,11 +1377,12 @@ int rm_cmd_mread( /* we need to create a single server_read_req_t structure even with * multiple gfids. */ int num_keys = 0; - for (j = 0; j < req_num; j++) { + for (i = 0; i < req_num; i++) { /* get the file id for this request */ - int gfid = unifyfs_Extent_fid(unifyfs_Extent_vec_at(extents, j)); - size_t off = unifyfs_Extent_offset(unifyfs_Extent_vec_at(extents, j)); - size_t len = unifyfs_Extent_length(unifyfs_Extent_vec_at(extents, j)); + req = read_reqs + i; + int gfid = req->gfid; + size_t off = req->offset; + size_t len = req->length; LOGDBG("gfid:%d, offset:%zu, length:%zu", gfid, off, len); /* Generate a pair of keys for each read request, representing @@ -1429,10 +1424,10 @@ int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl) /* grab the lock */ RM_LOCK(thrd_ctrl); - /* if delegator thread is not waiting in critical + /* if reqmgr thread is not waiting in critical * section, let's wait on it to come back */ if (!thrd_ctrl->has_waiting_delegator) { - /* delegator thread is not in critical section, + /* reqmgr thread is not in critical section, * tell it we've got something and signal it */ thrd_ctrl->has_waiting_dispatcher = 1; pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); @@ -1441,16 +1436,16 @@ int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl) thrd_ctrl->has_waiting_dispatcher = 0; } - /* inform delegator thread that it's time to exit */ + /* inform reqmgr thread that it's time to exit */ thrd_ctrl->exit_flag = 1; - /* signal delegator thread */ + /* signal reqmgr thread */ pthread_cond_signal(&thrd_ctrl->thrd_cond); /* release the lock */ RM_UNLOCK(thrd_ctrl); - /* wait for delegator thread to exit */ + /* wait for reqmgr thread to exit */ int rc = pthread_join(thrd_ctrl->thrd, NULL); if (0 == rc) { pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); @@ -1599,7 +1594,7 @@ int rm_cmd_sync(int app_id, int client_id) * These functions define the logic of the request manager thread ***********************/ -/* pack the chunk read requests for a single remote delegator. +/* pack the chunk read requests for a single remote server. * * @param req_msg_buf: request buffer used for packing * @param req_num: number of read requests @@ -1666,7 +1661,7 @@ static int rm_request_remote_chunks(reqmgr_thrd_t* thrd_ctrl) debug_print_read_req(req); if (req->status == READREQ_READY) { req->status = READREQ_STARTED; - /* iterate over each delegator we need to send requests to */ + /* iterate over each server we need to send requests to */ remote_chunk_reads_t* remote_reads; size_t packed_sz; for (j = 0; j < req->num_remote_reads; j++) { @@ -1676,7 +1671,7 @@ static int rm_request_remote_chunks(reqmgr_thrd_t* thrd_ctrl) /* pack requests into send buffer, get packed size */ packed_sz = rm_pack_chunk_requests(sendbuf, remote_reads); - /* get rank of target delegator */ + /* get rank of target server */ int del_rank = remote_reads->rank; /* send requests */ @@ -1724,14 +1719,14 @@ static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) server_read_req_t* req = thrd_ctrl->read_reqs + i; if ((req->num_remote_reads > 0) && (req->status == READREQ_STARTED)) { - /* iterate over each delegator we need to send requests to */ + /* iterate over each server we need to send requests to */ remote_chunk_reads_t* rcr; for (j = 0; j < req->num_remote_reads; j++) { rcr = req->remote_reads + j; if (NULL == rcr->resp) { continue; } - LOGDBG("found read req %d responses from delegator %d", + LOGDBG("found read req %d responses from server %d", i, rcr->rank); rc = rm_handle_chunk_read_responses(thrd_ctrl, req, rcr); if (rc != (int)UNIFYFS_SUCCESS) { @@ -1845,7 +1840,7 @@ int rm_post_chunk_read_responses(int app_id, } if (NULL != del_reads) { - LOGDBG("posting chunk responses for req %d from delegator %d", + LOGDBG("posting chunk responses for req %d from server %d", req_id, src_rank); del_reads->resp = (chunk_read_resp_t*)resp_buf; if (del_reads->num_chunks != num_chks) { @@ -1880,7 +1875,7 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, { // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked - int errcode, gfid, i, num_chks, rc, thrd_id; + int errcode, gfid, i, num_chks, rc; int ret = (int)UNIFYFS_SUCCESS; chunk_read_resp_t* responses = NULL; shm_context* client_shm = NULL; @@ -1909,7 +1904,7 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, rdreq->req_ndx); ret = (int32_t)EINVAL; } else if (0 == del_reads->total_sz) { - LOGERR("empty chunk read response from delegator %d", del_reads->rank); + LOGERR("empty chunk read response from server %d", del_reads->rank); ret = (int32_t)EINVAL; } else { LOGDBG("handling chunk read responses from server %d: " diff --git a/t/Makefile.am b/t/Makefile.am index 45f3c758d..8002659bc 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -69,8 +69,7 @@ test_ldadd = \ $(top_builddir)/t/lib/libtap.la \ $(top_builddir)/t/lib/libtestutil.la \ $(top_builddir)/client/src/libunifyfs_gotcha.la \ - $(MPI_CLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) + $(MPI_CLDFLAGS) test_static_ldadd = \ $(top_builddir)/t/lib/libtap.la \ @@ -80,8 +79,7 @@ test_static_ldadd = \ test_static_ldflags = \ -static $(AM_LDFLAGS) \ $(CP_WRAPPERS) \ - $(MPI_CLDFLAGS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) + $(MPI_CLDFLAGS) test_metadata_ldadd = \ $(top_builddir)/t/lib/libtap.la \ @@ -94,7 +92,6 @@ test_metadata_ldadd = \ $(MERCURY_LDFLAGS) $(MERCURY_LIBS) \ $(ARGOBOTS_LDFLAGS) $(ARGOBOTS_LIBS) \ $(MARGO_LDFLAGS) $(MARGO_LIBS) \ - $(FLATCC_LDFLAGS) $(FLATCC_LIBS) \ -lpthread -lm -lstdc++ -lrt test_common_cppflags = \ diff --git a/t/ci/001-setup.sh b/t/ci/001-setup.sh index 9e60aaa30..a284e4da1 100755 --- a/t/ci/001-setup.sh +++ b/t/ci/001-setup.sh @@ -140,7 +140,7 @@ fi # don't fail out if [[ -n $(which spack 2>/dev/null) ]]; then loaded_modules=$(module list 2>&1) - modules="gotcha leveldb flatcc argobots mercury margo" + modules="gotcha leveldb argobots mercury margo" for mod in $modules; do if ! [[ $(echo "$loaded_modules" | fgrep "$mod") ]]; then echo "$errmsg $mod not detected. Please 'spack load $mod'" diff --git a/util/unifyfs-stage/src/Makefile.am b/util/unifyfs-stage/src/Makefile.am index 704ad8639..00e671a96 100644 --- a/util/unifyfs-stage/src/Makefile.am +++ b/util/unifyfs-stage/src/Makefile.am @@ -13,8 +13,7 @@ unifyfs_stage_CPPFLAGS = $(AM_CPPFLAGS) $(MPI_CFLAGS) \ unifyfs_stage_LDADD = $(top_builddir)/client/src/libunifyfs.la -lrt -lm unifyfs_stage_LDFLAGS = -static $(CP_WRAPPERS) $(AM_LDFLAGS) \ - $(MPI_CLDFLAGS) $(FLATCC_LDFLAGS) $(FLATCC_LIBS) \ - $(OPENSSL_LIBS) + $(MPI_CLDFLAGS) $(OPENSSL_LIBS) AM_CFLAGS = -Wall -Werror From 869d5b5d9595c9762c24ac797bc5427be84b7198 Mon Sep 17 00:00:00 2001 From: Craig P Steffen Date: Mon, 13 Jul 2020 14:38:24 -0400 Subject: [PATCH 151/168] document manifest file, ex. & use Documents the manifest file format for the transfer API "transfer-stage" binary, and by extension, the "unify" command when using the stage-in or stage-out options. Describes the transfer file format and gives a few example lines. --- docs/start-stop.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/start-stop.rst b/docs/start-stop.rst index 98cb9c32b..ac9e894a9 100644 --- a/docs/start-stop.rst +++ b/docs/start-stop.rst @@ -117,3 +117,29 @@ with a README documenting the installation instructions, is available within the source repository at ``util/scripts/lsfcsm``. Support for the SLURM resource manager is under development. + +----------------------------------------------- + Stage-in and Stage-out Manifest File Format +----------------------------------------------- + +The manifest file contains one or more file copy requests. +Each line in the manifest corresponds to one file copy request, +and contains both the source and destination file paths. Currently, +directory copies are not supported. + +Each line is formatted as . +If either of the filenames +contain whitespace or special characters, then both filenames should +be surrounded by double-quote characters (") (ASCII character 34 decimal). +The double-quote character and the linefeed end-of-line character are forbidden +in any filenames used in a unifyfs manifest file, but any other +characters are allowed, including control characters. +If a filename contains any characters that might be misinterpreted, then +enclosing the filename in double-quotes is always +a safe thing to do. + +Here is an example of a valid stage-in manifest file: + +``/scratch/users/me/input_data/input_1.dat /unifyfs/input/input_1.dat`` +``/home/users/me/configuration/run_12345.conf /unifyfs/config/run_12345.conf`` +``"/home/users/me/file with space.dat" "/unifyfs/file with space.dat"`` From 03a48535bf5eba108fec142ee8b56b98af9c71f8 Mon Sep 17 00:00:00 2001 From: CamStan Date: Mon, 10 Aug 2020 11:53:03 -0700 Subject: [PATCH 152/168] Update CI testing environment directories This updates several environment variables in the CI environemnt to fix bugs that showed up when testing on different systems. .gitlab-ci.yml - Remove workflow rules for now to get all pipelines to build - Remove CI related environment variables that are already set up in the Gitlab CI UI 001-setup.sh - Change the default CI_PROJDIR envar to the directory containing the current source to avoid issues with symlinks. - Set the SHARNESS_TEST_DIRECTORY to the current CI_DIR to avoid issues with symlinks when sharness.sh is sourced. - Make sure the temporary directory for UnifyFS files exists on each node ci-functions.sh - If unifyfsd.err exists, cat it before removing - Update which files are cleaned up --- .gitlab-ci.yml | 14 -------------- t/ci/001-setup.sh | 8 +++++++- t/ci/ci-functions.sh | 9 ++++++--- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 370f9ec95..92867a639 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,16 +6,6 @@ stages: - test-unit - test-integ -# Rules to determine when a pipeline will be generated. If none of these rules -# match, no pipeline will be generated. -workflow: - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - - if: '$CI_PIPELINE_SOURCE == "web"' - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "dev"' - - if: '$CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "dev"' - - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "dev"' - ##### System Templates ##### # Generic system templates used to contruct the final jobs on specific @@ -88,10 +78,6 @@ workflow: # full details. .integ-test-template: stage: test-integ - variables: - UNIFYFS_INSTALL: "$CI_PROJECT_DIR/unifyfs-install" - CI_PROJDIR: "$CI_PROJECT_DIR" - CI_NPROCS: "$NPROCS" script: - cd t/ci && prove -v RUN_CI_TESTS.sh diff --git a/t/ci/001-setup.sh b/t/ci/001-setup.sh index a284e4da1..f31898842 100755 --- a/t/ci/001-setup.sh +++ b/t/ci/001-setup.sh @@ -72,7 +72,6 @@ done [[ -z $infomsg ]] && infomsg="-- UNIFYFS JOB INFO:" [[ -z $errmsg ]] && errmsg="!!!! UNIFYFS JOB ERROR:" -export CI_PROJDIR=${CI_PROJDIR:-$HOME} export TMPDIR=${TMPDIR:-/tmp} export SYSTEM_NAME=$(echo $(hostname) | sed -r 's/(^[[:alpha:]]*)(.*)/\1/') @@ -83,8 +82,14 @@ export SYSTEM_NAME=$(echo $(hostname) | sed -r 's/(^[[:alpha:]]*)(.*)/\1/') echo "$infomsg Setting up sharness" CI_DIR=${CI_DIR:-$(dirname "$(readlink -fm $BASH_SOURCE)")} SHARNESS_DIR="$(dirname "$CI_DIR")" +UNIFYFS_SOURCE_DIR="$(dirname "$SHARNESS_DIR")" +export CI_PROJDIR=${CI_PROJDIR:-"$(dirname "$UNIFYFS_SOURCE_DIR")"} echo "$infomsg CI_DIR: $CI_DIR" echo "$infomsg SHARNESS_DIR: $SHARNESS_DIR" +echo "$infomsg UNIFYFS_SOURCE_DIR: $UNIFYFS_SOURCE_DIR" +echo "$infomsg CI_PROJDIR: $CI_PROJDIR" + +SHARNESS_TEST_DIRECTORY=${SHARNESS_TEST_DIRECTORY:-$CI_DIR} source ${SHARNESS_DIR}/sharness.sh source $SHARNESS_DIR/sharness.d/02-functions.sh source $CI_DIR/ci-functions.sh @@ -196,6 +201,7 @@ export UNIFYFS_DAEMONIZE=${UNIFYFS_DAEMONIZE:-off} # temp nlt=${TMPDIR}/unifyfs.${USER}.${SYSTEM_NAME}.${JOB_ID} export CI_TEMP_DIR=${CI_TEMP_DIR:-$nlt} +$JOB_RUN_ONCE_PER_NODE mkdir -p $CI_TEMP_DIR export UNIFYFS_RUNSTATE_DIR=${UNIFYFS_RUNSTATE_DIR:-$CI_TEMP_DIR} export UNIFYFS_META_DB_PATH=${UNIFYFS_META_DB_PATH:-$CI_TEMP_DIR} echo "$infomsg UNIFYFS_RUNSTATE_DIR set as $UNIFYFS_RUNSTATE_DIR" diff --git a/t/ci/ci-functions.sh b/t/ci/ci-functions.sh index 1a820ee41..c42672ee8 100755 --- a/t/ci/ci-functions.sh +++ b/t/ci/ci-functions.sh @@ -439,11 +439,14 @@ cleanup_hosts() pdsh -w $l_hl 'test -f /dev/shm/svr_id && /bin/cat /dev/shm/svr_id' pdsh -w $l_hl 'test -f /dev/shm/unifyfsd_id && /bin/cat \ /dev/shm/unifyfsd_id' + pdsh -w $l_hl 'test -f /tmp/unifyfsd.err.* && /bin/cat \ + /tmp/unifyfsd.err.*' pdsh -w $l_hl '/bin/rm -rfv /tmp/na_sm /tmp/*unifyfs* /var/tmp/*unifyfs* \ /dev/shm/unifyfsd_id /dev/shm/svr_id /dev/shm/*na_sm* \ - "'${UNIFYFS_SPILLOVER_DATA_DIR}'"/spill*.log \ - "'${UNIFYFS_SPILLOVER_META_DIR}'"/spill*.log \ - /dev/shm/*-recv-* /dev/shm/*-req-* /dev/shm/*-super-*' + /dev/shm/logio_mem* \ + "'${UNIFYFS_LOGIO_SPILL_DIR}'"/spill*.log \ + /dev/shm/*-recv-* /dev/shm/*-req-* /dev/shm/*-super-* \ + "'$CI_TEMP_DIR'"' # Reset capturing all output exec 1>&3 2>&4 From fd04652ff7e5004578b00bf022d6b843d4a5e8e1 Mon Sep 17 00:00:00 2001 From: CamStan Date: Wed, 7 Oct 2020 20:06:26 -0400 Subject: [PATCH 153/168] Update unit tests to run via MPI job launch In order to run tests on Ascent, `make check` needs to be launched using the MPI job launch command available on the system. This updates the unit testing suites to each run using the available MPI job launch command. As a result, the `make check` should always be run from within a single-node allocation and with the systems available MPI job launch command (e.g., `jsrun -r1 -n1 make check`). This also skips the `dies_ok` tests for now, as at least two of them hang on two of the four systems tested on. This seems to have started recently and make work again after a future system update. Alternatively, these tests could be rewrittin if we don't want UnifyFS to also segfault in these cases, or the tests could simply be removed. Updated the unit testing docs to reflect these changes. Updated .gitlab-ci.yml to launch the unit tests properly. --- .gitlab-ci.yml | 4 +++- docs/testing.rst | 32 +++++++++++++++++++++++++++----- t/0100-sysio-gotcha.t | 2 +- t/0200-stdio-gotcha.t | 2 +- t/0500-sysio-static.t | 2 +- t/0600-stdio-static.t | 2 +- t/0700-unifyfs-stage-full.t | 4 ++-- t/9005-unifyfs-unmount.t | 2 +- t/9100-metadata-api.t | 2 +- t/9300-unifyfs-stage-isolated.t | 2 +- t/server/metadata_suite.c | 4 ++++ t/sharness.d/00-test-env.sh | 2 +- t/std/fseek-ftell.c | 2 ++ t/std/fwrite-fread.c | 2 ++ t/std/size.c | 2 +- 15 files changed, 49 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92867a639..8a598785c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,7 @@ stages: .slurm-single-node-template: variables: + JOB_LAUNCH_COMMAND: "srun -N1 -n1" LLNL_SLURM_SCHEDULER_PARAMETERS: "-N 1 -p pbatch -t $UNIT_WALL_TIME -J unifyfs-unit-tests" .slurm-multi-node-template: @@ -33,6 +34,7 @@ stages: .lsf-single-node-template: variables: + JOB_LAUNCH_COMMAND: "jsrun -r1 -n1" LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -q pbatch -W $UNIT_WALL_TIME -J unifyfs-unit-tests" SCHEDULER_PARAMETERS: "-nnodes 1 -P $PROJECT_ID -W $UNIT_WALL_TIME -J unifyfs-unit-tests" @@ -69,7 +71,7 @@ stages: .unit-test-template: stage: test-unit script: - - cd unifyfs-build/t && make check + - cd unifyfs-build/t && $JOB_LAUNCH_COMMAND make check after_script: - rm -rf /tmp/unify* /tmp/tmp.* /tmp/mdhim* /tmp/na_sm | true diff --git a/docs/testing.rst b/docs/testing.rst index d235b7cc8..0911cf0c2 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -282,11 +282,33 @@ in (i.e., testing a wrapper that doesn't have any tests yet): Running the Tests ***************** -To manually run the UnifyFS test suite, simply run ``make check`` from your -build/t directory. If changes are made to existing files in the test suite, the -tests can be run again by simply doing ``make clean`` followed by ``make -check``. Individual tests may be run by hand. The test ``0001-setup.t`` should -normally be run first to start the UnifyFS daemon. +To manually run the UnifyFS unit test suite, simply run ``make check`` from the +inside the t/ directory of wherever you built UnifyFS. E.g., if you built in a +separate build/ directory, then do: + +.. code-block:: BASH + + $ cd build/t + $ make check + +If on a system where jobs are launched on a separate compute node, then use your +systems local MPI job launch command to run the unit tests: + +.. code-block:: BASH + + $ cd build/t + $ srun -N1 -n1 make check + +If changes are made to existing files in the test suite, the tests can be run +again by simply doing ``make clean`` followed by another ``make check``. + +Individual tests may be run by hand. The test *0001-setup.t* should +normally be run first to start the UnifyFS daemon. E.g., to run just the +*0100-sysio-gotcha.t* tests, do: + +.. code-block:: BASH + + $ make check TESTS='0001-setup.t 0100-sysio-gotcha.t 9010-stop-unifyfsd.t 9999-cleanup.t' .. note:: diff --git a/t/0100-sysio-gotcha.t b/t/0100-sysio-gotcha.t index 9a2795f5c..762121fab 100755 --- a/t/0100-sysio-gotcha.t +++ b/t/0100-sysio-gotcha.t @@ -5,4 +5,4 @@ # . $(dirname $0)/sharness.d/00-test-env.sh . $(dirname $0)/sharness.d/01-unifyfs-settings.sh -$UNIFYFS_BUILD_DIR/t/sys/sysio-gotcha.t +$JOB_RUN_COMMAND $UNIFYFS_BUILD_DIR/t/sys/sysio-gotcha.t diff --git a/t/0200-stdio-gotcha.t b/t/0200-stdio-gotcha.t index df7cad09f..b1fb030f3 100755 --- a/t/0200-stdio-gotcha.t +++ b/t/0200-stdio-gotcha.t @@ -5,4 +5,4 @@ # . $(dirname $0)/sharness.d/00-test-env.sh . $(dirname $0)/sharness.d/01-unifyfs-settings.sh -$UNIFYFS_BUILD_DIR/t/std/stdio-gotcha.t +$JOB_RUN_COMMAND $UNIFYFS_BUILD_DIR/t/std/stdio-gotcha.t diff --git a/t/0500-sysio-static.t b/t/0500-sysio-static.t index 3772ab8ac..b5a422a26 100755 --- a/t/0500-sysio-static.t +++ b/t/0500-sysio-static.t @@ -5,4 +5,4 @@ # . $(dirname $0)/sharness.d/00-test-env.sh . $(dirname $0)/sharness.d/01-unifyfs-settings.sh -$UNIFYFS_BUILD_DIR/t/sys/sysio-static.t +$JOB_RUN_COMMAND $UNIFYFS_BUILD_DIR/t/sys/sysio-static.t diff --git a/t/0600-stdio-static.t b/t/0600-stdio-static.t index e1322b635..83ff853c1 100755 --- a/t/0600-stdio-static.t +++ b/t/0600-stdio-static.t @@ -5,4 +5,4 @@ # . $(dirname $0)/sharness.d/00-test-env.sh . $(dirname $0)/sharness.d/01-unifyfs-settings.sh -$UNIFYFS_BUILD_DIR/t/std/stdio-static.t +$JOB_RUN_COMMAND $UNIFYFS_BUILD_DIR/t/std/stdio-static.t diff --git a/t/0700-unifyfs-stage-full.t b/t/0700-unifyfs-stage-full.t index 5448b7d94..de8540ba6 100755 --- a/t/0700-unifyfs-stage-full.t +++ b/t/0700-unifyfs-stage-full.t @@ -49,9 +49,9 @@ test_expect_success "target directory is empty" ' test_dir_is_empty ${UNIFYFS_TEST_TMPDIR}/stage_destination_0700 ' -${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage -m ${UNIFYFS_TEST_MOUNT} ${UNIFYFS_TEST_TMPDIR}/config_0700/test_IN.manifest > ${UNIFYFS_TEST_TMPDIR}/config_0700/stage_IN_output.OUT 2>&1 +$JOB_RUN_COMMAND ${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage -m ${UNIFYFS_TEST_MOUNT} ${UNIFYFS_TEST_TMPDIR}/config_0700/test_IN.manifest > ${UNIFYFS_TEST_TMPDIR}/config_0700/stage_IN_output.OUT 2>&1 -${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage -m ${UNIFYFS_TEST_MOUNT} ${UNIFYFS_TEST_TMPDIR}/config_0700/test_OUT.manifest > ${UNIFYFS_TEST_TMPDIR}/config_0700/stage_OUT_output.OUT 2>&1 +$JOB_RUN_COMMAND ${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage -m ${UNIFYFS_TEST_MOUNT} ${UNIFYFS_TEST_TMPDIR}/config_0700/test_OUT.manifest > ${UNIFYFS_TEST_TMPDIR}/config_0700/stage_OUT_output.OUT 2>&1 test_expect_success "input file has been staged to output" ' test_path_is_file ${UNIFYFS_TEST_TMPDIR}/stage_destination_0700/destination_0700.file diff --git a/t/9005-unifyfs-unmount.t b/t/9005-unifyfs-unmount.t index 31e5bdd7f..25d673752 100755 --- a/t/9005-unifyfs-unmount.t +++ b/t/9005-unifyfs-unmount.t @@ -5,4 +5,4 @@ # . $(dirname $0)/sharness.d/00-test-env.sh . $(dirname $0)/sharness.d/01-unifyfs-settings.sh -$UNIFYFS_BUILD_DIR/t/unifyfs_unmount.t +$JOB_RUN_COMMAND $UNIFYFS_BUILD_DIR/t/unifyfs_unmount.t diff --git a/t/9100-metadata-api.t b/t/9100-metadata-api.t index f02ea7f51..4396a4238 100755 --- a/t/9100-metadata-api.t +++ b/t/9100-metadata-api.t @@ -13,4 +13,4 @@ test_description="Test Metadata API" export UNIFYFS_META_DB_PATH=$UNIFYFS_TEST_TMPDIR/meta2 mkdir -p $UNIFYFS_META_DB_PATH -$UNIFYFS_BUILD_DIR/t/server/metadata.t +$JOB_RUN_COMMAND $UNIFYFS_BUILD_DIR/t/server/metadata.t diff --git a/t/9300-unifyfs-stage-isolated.t b/t/9300-unifyfs-stage-isolated.t index f6696ae05..007853999 100755 --- a/t/9300-unifyfs-stage-isolated.t +++ b/t/9300-unifyfs-stage-isolated.t @@ -49,7 +49,7 @@ test_expect_success "target directory is empty" ' test_dir_is_empty ${UNIFYFS_TEST_TMPDIR}/stage_destination_9300 ' -${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage -N ${UNIFYFS_TEST_TMPDIR}/config_9300/test_INOUT.manifest > ${UNIFYFS_TEST_TMPDIR}/config_9300/stage_INOUT_output.OUT 2>&1 +$JOB_RUN_COMMAND ${SHARNESS_BUILD_DIRECTORY}/util/unifyfs-stage/src/unifyfs-stage -N ${UNIFYFS_TEST_TMPDIR}/config_9300/test_INOUT.manifest > ${UNIFYFS_TEST_TMPDIR}/config_9300/stage_INOUT_output.OUT 2>&1 test_expect_success "input file has been staged to output" ' test_path_is_file ${UNIFYFS_TEST_TMPDIR}/stage_destination_9300/destination_9300.file diff --git a/t/server/metadata_suite.c b/t/server/metadata_suite.c index 20ef8ff1e..75ca51e77 100644 --- a/t/server/metadata_suite.c +++ b/t/server/metadata_suite.c @@ -1,3 +1,4 @@ +#include #include "unifyfs_configurator.h" #include "unifyfs_metadata.h" @@ -10,6 +11,8 @@ int main(int argc, char* argv[]) unifyfs_cfg_t server_cfg; int rc; + MPI_Init(&argc, &argv); + /* get the configuration */ rc = unifyfs_config_init(&server_cfg, argc, argv); if (rc != 0) { @@ -41,6 +44,7 @@ int main(int argc, char* argv[]) // shutdown the metadata service meta_sanitize(); + MPI_Finalize(); // finish the testing // needs to be last call diff --git a/t/sharness.d/00-test-env.sh b/t/sharness.d/00-test-env.sh index 566473019..6efd504a0 100644 --- a/t/sharness.d/00-test-env.sh +++ b/t/sharness.d/00-test-env.sh @@ -27,7 +27,7 @@ if test -n "$(which jsrun 2>/dev/null)"; then elif test -n "$(which srun 2>/dev/null)"; then JOB_RUN_COMMAND="srun -n1 -N1" elif test -n "$(which mpirun 2>/dev/null)"; then - JOB_RUN_COMMAND="mpirun -wd $UNIFYFS_BUILD_DIR -np 1" + JOB_RUN_COMMAND="mpirun -np 1" fi if test -z "$JOB_RUN_COMMAND"; then echo >&2 "Failed to find a suitable parallel job launcher" diff --git a/t/std/fseek-ftell.c b/t/std/fseek-ftell.c index 15df13b4d..a27adb914 100644 --- a/t/std/fseek-ftell.c +++ b/t/std/fseek-ftell.c @@ -40,6 +40,7 @@ int fseek_ftell_test(char* unifyfs_root) /* Create a random file at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); + skip(1, 3, "causing a hang on some architectures. Try after future update"); /* fseek on bad file stream should fail with errno=EBADF */ dies_ok({ fseek(fp, 0, SEEK_SET); }, "%s:%d fseek on bad file stream segfaults: %s", @@ -54,6 +55,7 @@ int fseek_ftell_test(char* unifyfs_root) /* rewind on non-open file stream should fail with errno=EBADF */ dies_ok({ rewind(fp); }, "%s:%d rewind on bad file stream segfaults: %s", __FILE__, __LINE__, strerror(errno)); + end_skip; /* Open a file and write to it to test fseek() */ fp = fopen(path, "w"); diff --git a/t/std/fwrite-fread.c b/t/std/fwrite-fread.c index 4809634db..125e41811 100644 --- a/t/std/fwrite-fread.c +++ b/t/std/fwrite-fread.c @@ -39,6 +39,7 @@ int fwrite_fread_test(char* unifyfs_root) testutil_rand_path(path, sizeof(path), unifyfs_root); + skip(1, 4, "causing a hang on some architectures. Try after future update"); /* fwrite to bad FILE stream should segfault */ dies_ok({ FILE* p = fopen("/tmp/fakefile", "r"); fwrite("hello world", 12, 1, p); }, @@ -59,6 +60,7 @@ int fwrite_fread_test(char* unifyfs_root) dies_ok({ int rc = feof(fp); ok(rc > 0); }, "%s:%d feof() on bad FILE stream segfaults: %s", __FILE__, __LINE__, strerror(errno)); + end_skip; /* Write "hello world" to a file */ fp = fopen(path, "w"); diff --git a/t/std/size.c b/t/std/size.c index 1ff91cccd..03faf5fe5 100644 --- a/t/std/size.c +++ b/t/std/size.c @@ -161,7 +161,7 @@ int size_test(char* unifyfs_root) ok(fclose(fp) == 0, "%s:%d fclose(): %s", __FILE__, __LINE__, strerror(errno)); - diag("Starting file size and fwrite/fread with append tests"); + diag("Finished file size and fwrite/fread with append tests"); return 0; } From 4a966a26e190ba25b56da1151ba845da1823e1dd Mon Sep 17 00:00:00 2001 From: CamStan Date: Thu, 8 Oct 2020 18:05:28 -0700 Subject: [PATCH 154/168] Update deprecated attribute in packages.yaml Spack has changed how their packages.yaml file is defined, causing the "paths" attribute to become deprecated. This updates the packages.yaml file in our .travis.yml to get rid of the warning messages in the Travis CI builds. --- .travis.yml | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a4113bea..bdf1c7596 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,28 +33,34 @@ before_install: mpi: [openmpi] autoconf: buildable: False - paths: - autoconf@2.69: /usr + externals: + - spec: "autoconf@2.69" + prefix: /usr automake: buildable: False - paths: - automake@1.15.1: /usr + externals: + - spec: "automake@1.15.1" + prefix: /usr cmake: buildable: False - paths: - cmake@3.12.4: /usr/local/cmake-3.12.4 + externals: + - spec: "cmake@3.12.4" + prefix: /usr/local/cmake-3.12.4 libtool: buildable: False - paths: - libtool@2.4.6: /usr + externals: + - spec: "libtool@2.4.6" + prefix: /usr m4: buildable: False - paths: - m4@1.4.18: /usr + externals: + - spec: "m4@1.4.18" + prefix: /usr openmpi: buildable: False - paths: - openmpi@2.1.1: /usr + externals: + - spec: "openmpi@2.1.1" + prefix: /usr EOF install: From 948dec9f445babe664e0e45dca9546029bc87596 Mon Sep 17 00:00:00 2001 From: CamStan Date: Fri, 30 Oct 2020 16:14:26 -0700 Subject: [PATCH 155/168] Add CI job template to clone to alternate location By default, when a Gitlab runner starts, it clones the repo to a default location before any of the job scripts are executed. This default location is pre-defined in the `CI_PROJECT_DIR` envar. This PR sets up an init stage template which can optionally be run before the build stage. This init stage will change to a desired directory and clone the repo again before executing any subsequent jobs. The `WORKING_DIR` envar needs to be defined in the desired jobs in order to create, clone, build, and run any tests from the new location. This PR sets up the Ascent jobs to use this new template and defines the `WORKING_DIR` envar using the `WORKING_DIR_BASE` envar. This envar is set, and can be changed, from the Gitlab web UI. For jobs not needing to change locations, if the `WORKING_DIR` envar is not defined, it gets defined as `CI_PROJECT_DIR` during the default `before_script` step of the running job. --- .gitlab-ci.yml | 22 +++++++++++++++++++++- .gitlab/ascent.yml | 11 +++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8a598785c..c98d46719 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ # keyword, we prevent the testing stages from blocking, in favor of a # DAG. stages: + - init - build - test-unit - test-integ @@ -45,6 +46,20 @@ stages: ##### Job Templates ##### +# Only use this template in a pre-build job if needing to clone and +# run subsequent jobs from a non-default location. +# The WORKING_DIR envar needs to be defined in the job variables. +# +# The before_script section here overrides the default before_script +# for jobs using this template. +.init-template: + stage: init + before_script: + - mkdir -pv $WORKING_DIR + - cd $WORKING_DIR + script: + - git clone -b ${CI_COMMIT_BRANCH} --depth=1 ${CI_REPOSITORY_URL} source + # Build script used by each system. The CC and FC variables are set in # the specific job scripts and evaluated in the before_script in order # to customize which compiler will be used for each job. @@ -56,7 +71,7 @@ stages: script: - ./autogen.sh - mkdir -p unifyfs-build unifyfs-install && cd unifyfs-build - - ../configure CC=$CC_PATH FC=$FC_PATH --prefix=$CI_PROJECT_DIR/unifyfs-install --enable-fortran --disable-silent-rules + - ../configure CC=$CC_PATH FC=$FC_PATH --prefix=${WORKING_DIR}/source/unifyfs-install --enable-fortran --disable-silent-rules - make V=1 - make V=1 install needs: [] @@ -92,12 +107,17 @@ stages: # prior to running when new compilers or architectures need to be # tested. # +# For jobs running in the not-default location, change directories +# to the WORKING_DIR directory. Otherwise, set WORKING_DIR to be the +# CI_PROJECT_DIR for the build step. +# # The COMPILER, CC_PATH, and FC_PATH variables are evaluated here. Set # them in their specific job scripts. # SPACK_COMPILER and SPACK_ARCH are then set to load the matching # dependencies for the desired compiler. before_script: - which spack || ((cd $HOME/spack && git describe) && . $HOME/spack/share/spack/setup-env.sh) + - [[ -d $WORKING_DIR ]] && cd ${WORKING_DIR}/source || export WORKING_DIR=${CI_PROJECT_DIR} - module load $COMPILER - CC_PATH=$($CC_COMMAND) - FC_PATH=$($FC_COMMAND) diff --git a/.gitlab/ascent.yml b/.gitlab/ascent.yml index b5a051901..9a31dce95 100644 --- a/.gitlab/ascent.yml +++ b/.gitlab/ascent.yml @@ -1,8 +1,15 @@ ##### Ascent Templates ##### +# The WORKING_DIR envar is defined to allow the init job to clone the +# git repo to a different location than the default. Subsequent jobs +# will then `cd` to this directory during their before_script stage. +# The WORKING_DIR_BASE envar is definied in the Gitlab UI. +# # The RUN_ASCENT variable can be toggled in the Gitlab interface to # toggle whether jobs should be run on this system. .ascent-template: + variables: + WORKING_DIR: ${WORKING_DIR_BASE}/${CI_PIPELINE_ID} extends: .base-template rules: - if: '$RUN_ASCENT != "ON"' @@ -19,12 +26,16 @@ ##### All Ascent Jobs ##### +ascent-gcc-4_8_5-init: + extends: [.ascent-shell-template, .init-template] + ascent-gcc-4_8_5-build: variables: COMPILER: gcc/4.8.5 CC_COMMAND: "which gcc" FC_COMMAND: "which gfortran" extends: [.ascent-shell-template, .build-template] + needs: ["ascent-gcc-4_8_5-init"] ascent-gcc-4_8_5-unit-test: variables: From c3a27c3b32a5834591dc4dd57ca7e47bfa5bcba3 Mon Sep 17 00:00:00 2001 From: CamStan Date: Wed, 4 Nov 2020 09:56:47 -0800 Subject: [PATCH 156/168] Fix gitlab ci check for existing working dir Change ternary operation to if/else as Gitlab CI lint doesn't like the prior. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c98d46719..6ba59646c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -117,7 +117,7 @@ stages: # dependencies for the desired compiler. before_script: - which spack || ((cd $HOME/spack && git describe) && . $HOME/spack/share/spack/setup-env.sh) - - [[ -d $WORKING_DIR ]] && cd ${WORKING_DIR}/source || export WORKING_DIR=${CI_PROJECT_DIR} + - if [[ -d $WORKING_DIR ]]; then cd ${WORKING_DIR}/source; else export WORKING_DIR=${CI_PROJECT_DIR}; fi - module load $COMPILER - CC_PATH=$($CC_COMMAND) - FC_PATH=$($FC_COMMAND) From b7ec79882b2bfade2da61d8b90e3a888ceedb13a Mon Sep 17 00:00:00 2001 From: CamStan Date: Wed, 4 Nov 2020 16:02:40 -0800 Subject: [PATCH 157/168] Correct paths for ci builds using default location Previous commit got tests working on Ascent, but broke the artifact upload paths for jobs using the default CI_PROJECT_DIR location. This commit should fix that and allow the CI jobs to work in both cases. --- .gitlab-ci.yml | 8 ++++---- .gitlab/ascent.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ba59646c..784e69e6f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,10 +55,10 @@ stages: .init-template: stage: init before_script: - - mkdir -pv $WORKING_DIR + - mkdir -v $WORKING_DIR - cd $WORKING_DIR script: - - git clone -b ${CI_COMMIT_BRANCH} --depth=1 ${CI_REPOSITORY_URL} source + - git clone -b ${CI_COMMIT_BRANCH} --depth=1 ${CI_REPOSITORY_URL} $WORKING_DIR # Build script used by each system. The CC and FC variables are set in # the specific job scripts and evaluated in the before_script in order @@ -71,7 +71,7 @@ stages: script: - ./autogen.sh - mkdir -p unifyfs-build unifyfs-install && cd unifyfs-build - - ../configure CC=$CC_PATH FC=$FC_PATH --prefix=${WORKING_DIR}/source/unifyfs-install --enable-fortran --disable-silent-rules + - ../configure CC=$CC_PATH FC=$FC_PATH --prefix=${WORKING_DIR}/unifyfs-install --enable-fortran --disable-silent-rules - make V=1 - make V=1 install needs: [] @@ -117,7 +117,7 @@ stages: # dependencies for the desired compiler. before_script: - which spack || ((cd $HOME/spack && git describe) && . $HOME/spack/share/spack/setup-env.sh) - - if [[ -d $WORKING_DIR ]]; then cd ${WORKING_DIR}/source; else export WORKING_DIR=${CI_PROJECT_DIR}; fi + - if [[ -d $WORKING_DIR ]]; then cd ${WORKING_DIR}; else export WORKING_DIR=${CI_PROJECT_DIR}; fi - module load $COMPILER - CC_PATH=$($CC_COMMAND) - FC_PATH=$($FC_COMMAND) diff --git a/.gitlab/ascent.yml b/.gitlab/ascent.yml index 9a31dce95..d997f6f9b 100644 --- a/.gitlab/ascent.yml +++ b/.gitlab/ascent.yml @@ -9,7 +9,7 @@ # toggle whether jobs should be run on this system. .ascent-template: variables: - WORKING_DIR: ${WORKING_DIR_BASE}/${CI_PIPELINE_ID} + WORKING_DIR: ${WORKING_DIR_BASE}/${CI_PIPELINE_ID}/source extends: .base-template rules: - if: '$RUN_ASCENT != "ON"' From 968c50dd6d377771994a2ca04a1516af62822dc7 Mon Sep 17 00:00:00 2001 From: CamStan Date: Thu, 5 Nov 2020 15:47:30 -0800 Subject: [PATCH 158/168] Final bugfix for ascent gitlab ci No sure why I removed this in my last commmit. Hopefully the last bugfix needed. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 784e69e6f..577f2b668 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,7 +55,7 @@ stages: .init-template: stage: init before_script: - - mkdir -v $WORKING_DIR + - mkdir -pv $WORKING_DIR - cd $WORKING_DIR script: - git clone -b ${CI_COMMIT_BRANCH} --depth=1 ${CI_REPOSITORY_URL} $WORKING_DIR From 353a485d44aec6c45b8b81bf43136dce3820f0fa Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Wed, 12 Aug 2020 17:41:02 -0400 Subject: [PATCH 159/168] Use tree-based group rpcs for file operations This commit squashes all changes from the margotreenew branch. A list of notable updates follows. * The implementation of file operations within the server is now similar to the Linux VFS layer, where a new unifyfs_fops struct contains the function pointers for the various operations such as metaget, metaset, sync, truncate, laminate, read, etc. See server/src/unifyfs_fops.h * The new distributed metadata implementation of file operations that leverages tree-based margo group rpcs is defined in server/src/unifyfs_fops_rpc.c . This implementation introduces the unifyfs_inode structure that keeps track of the metadata for a particular file identified by gfid. A unifyfs_inode_tree structure maps gfids to their corresponding inode. Currently, one server, the "owner", maintains the metadata for a given file until it is laminated, and all writes that have been synced are visible to readers. Upon lamination or truncation, updated file metadata is broadcast to the other servers. After lamination, all servers can handle metadata queries locally. The tree-based rpc logic is defined in unifyfs_tree.[ch] and unifyfs_group_rpc.[ch]. * The previous MDHIM-based support for file operations is maintained by server/src/unifyfs_fops_mdhim.c, which can be selected at configure time using '--enable-mdhim'. Note it may not be currently in a usable state. * The example programs have been updated to enable additional functionality test capabilities: - testutil now supports options to truncate the target file before or after writes - new sysio examples were added to test create/open, truncate, and unlink * Includes various bug fixes for problems encountered during testing: - single server startup hangs when using the unifyfs CLI tool - race condition on server pidfile creation - truncate filesize issues due to client caching of extents - lio_listio detection during configure - use argobots mutex for svcmgr state synchronization - use separate argobots mutex for reqmgr request locking - handle EINTR during posix_fallocate - stat file time fixes - segfault in remote read creation - remove logio spill files on server exit - avoid double free on client->reqmgr - improve errno handling in unit tests - fix chdir unit test errno expected value * The support for available Mercury NA plugins for internode RPC and RMA has been improved to allow for both OFI and BMI, and the currently available plugin is determined by examining the NA configuration (na_config.h). * Finally, the test suite has been updated to account for new test passes where things had previously been marked as skipped or todo TEST_CHECKPATCH_SKIP_FILES="client/src/unifyfsf.h" TEST_CHECKPATCH_SKIP_FILES+=",common/src/unifyfs_configurator.c" TEST_CHECKPATCH_SKIP_FILES+=",common/src/unifyfs_configurator.h" --- client/src/Makefile.am | 2 - client/src/margo_client.c | 352 ++- client/src/margo_client.h | 21 +- client/src/pmpi_wrappers.c | 4 +- client/src/pmpi_wrappers.h | 4 +- client/src/unifyfs-dirops.c | 4 +- client/src/unifyfs-dirops.h | 5 +- client/src/unifyfs-fixed.c | 37 +- client/src/unifyfs-fixed.h | 3 + client/src/unifyfs-internal.h | 21 +- client/src/unifyfs-stdio.c | 4 +- client/src/unifyfs-stdio.h | 4 +- client/src/unifyfs-sysio.c | 164 +- client/src/unifyfs-sysio.h | 4 +- client/src/unifyfs.c | 248 +- client/src/unifyfs.h | 32 +- client/src/unifyfsf.c | 4 +- client/src/unifyfsf.h | 4 +- common/src/Makefile.am | 11 +- {server => common}/src/arraylist.c | 4 +- {server => common}/src/arraylist.h | 4 +- common/src/cm_enumerator.c | 4 +- common/src/cm_enumerator.h | 4 +- common/src/rm_enumerator.c | 4 +- common/src/rm_enumerator.h | 4 +- common/src/seg_tree.c | 124 +- common/src/seg_tree.h | 27 + {client => common}/src/unifyfs-stack.c | 4 +- {client => common}/src/unifyfs-stack.h | 4 +- common/src/unifyfs_client_rpcs.h | 63 +- common/src/unifyfs_configurator.c | 4 +- common/src/unifyfs_configurator.h | 5 +- common/src/unifyfs_const.h | 17 +- common/src/unifyfs_keyval.c | 4 +- common/src/unifyfs_keyval.h | 4 +- common/src/unifyfs_log.c | 4 +- common/src/unifyfs_log.h | 5 +- common/src/unifyfs_logio.c | 21 +- common/src/unifyfs_logio.h | 11 +- common/src/unifyfs_meta.c | 12 + common/src/unifyfs_meta.h | 177 +- common/src/unifyfs_misc.h | 17 + common/src/unifyfs_rc.c | 4 +- common/src/unifyfs_rc.h | 4 +- common/src/unifyfs_rpc_types.h | 45 + common/src/unifyfs_rpc_util.c | 4 +- common/src/unifyfs_rpc_util.h | 4 +- common/src/unifyfs_server_rpcs.h | 167 +- common/src/unifyfs_shm.c | 28 +- common/src/unifyfs_shm.h | 4 +- configure.ac | 15 +- docs/testing.rst | 21 +- examples/src/Makefile.am | 42 +- examples/src/app-btio.c | 4 +- examples/src/app-hdf5-create.c | 5 +- examples/src/app-hdf5-writeread.c | 5 +- examples/src/app-mpiio.c | 5 +- examples/src/app-tileio.c | 4 +- examples/src/checkpoint-restart.c | 4 +- examples/src/chmod.c | 22 +- examples/src/multi-write.c | 9 +- examples/src/read-data.c | 3 +- examples/src/read.c | 4 +- examples/src/simul.c | 4 +- examples/src/size.c | 35 +- examples/src/sysio-cp.c | 5 +- examples/src/sysio-dir.c | 5 +- examples/src/sysio-open.c | 204 ++ examples/src/sysio-read.c | 5 +- examples/src/sysio-stat.c | 53 +- examples/src/sysio-truncate.c | 255 ++ examples/src/sysio-unlink.c | 160 + examples/src/sysio-write.c | 25 +- examples/src/sysio-writeread.c | 4 +- examples/src/sysio-writeread2.c | 4 +- examples/src/testlib.h | 5 +- examples/src/testutil.c | 71 +- examples/src/testutil.h | 43 +- examples/src/testutil_rdwr.h | 37 +- examples/src/transfer.c | 5 +- examples/src/write.c | 4 +- examples/src/writeread.c | 40 +- examples/src/writeread.f90 | 16 +- meta/src/Makefile.am | 2 + meta/src/ds_leveldb.h | 2 +- meta/src/partitioner.c | 2 +- meta/src/range_server.c | 2 +- server/src/Makefile.am | 80 +- server/src/extent_tree.c | 705 +++++ server/src/extent_tree.h | 188 ++ server/src/margo_server.c | 118 +- server/src/margo_server.h | 30 +- server/src/unifyfs_cmd_handler.c | 766 +++-- server/src/unifyfs_fops.h | 190 ++ server/src/unifyfs_fops_mdhim.c | 1197 ++++++++ server/src/unifyfs_fops_rpc.c | 377 +++ server/src/unifyfs_global.h | 23 +- server/src/unifyfs_group_rpc.c | 1147 +++++++ server/src/unifyfs_group_rpc.h | 78 + server/src/unifyfs_inode.c | 664 ++++ server/src/unifyfs_inode.h | 225 ++ server/src/unifyfs_inode_tree.c | 215 ++ server/src/unifyfs_inode_tree.h | 178 ++ ...fs_metadata.c => unifyfs_metadata_mdhim.c} | 51 +- ...fs_metadata.h => unifyfs_metadata_mdhim.h} | 90 +- server/src/unifyfs_p2p_rpc.c | 982 ++++++ server/src/unifyfs_p2p_rpc.h | 116 + server/src/unifyfs_request_manager.c | 2657 ++++++----------- server/src/unifyfs_request_manager.h | 116 +- server/src/unifyfs_server.c | 29 +- server/src/unifyfs_server_pid.c | 110 +- server/src/unifyfs_service_manager.c | 418 +-- server/src/unifyfs_service_manager.h | 8 +- server/src/unifyfs_tree.c | 138 + server/src/unifyfs_tree.h | 44 + t/9100-metadata-api.t | 16 - t/Makefile.am | 34 - t/common/seg_tree_test.c | 18 + t/lib/testutil.c | 4 +- t/server/metadata_suite.c | 54 - t/server/metadata_suite.h | 28 - t/server/unifyfs_meta_get_test.c | 42 - t/std/fflush.c | 3 +- t/std/fopen-fclose.c | 62 +- t/std/fseek-ftell.c | 274 +- t/std/fwrite-fread.c | 244 +- t/sys/chdir.c | 280 +- t/sys/creat-close.c | 47 +- t/sys/creat64.c | 17 +- t/sys/lseek.c | 242 +- t/sys/mkdir-rmdir.c | 133 +- t/sys/open.c | 41 +- t/sys/open64.c | 24 +- t/sys/unlink.c | 221 +- t/sys/write-read.c | 262 +- .../src/unifyfs-stage-transfer.c | 2 +- util/unifyfs/src/unifyfs-rm.c | 11 +- 137 files changed, 11939 insertions(+), 3906 deletions(-) rename {server => common}/src/arraylist.c (97%) rename {server => common}/src/arraylist.h (93%) rename {client => common}/src/unifyfs-stack.c (97%) rename {client => common}/src/unifyfs-stack.h (95%) create mode 100644 common/src/unifyfs_rpc_types.h create mode 100644 examples/src/sysio-open.c create mode 100644 examples/src/sysio-truncate.c create mode 100644 examples/src/sysio-unlink.c create mode 100644 server/src/extent_tree.c create mode 100644 server/src/extent_tree.h create mode 100644 server/src/unifyfs_fops.h create mode 100644 server/src/unifyfs_fops_mdhim.c create mode 100644 server/src/unifyfs_fops_rpc.c create mode 100644 server/src/unifyfs_group_rpc.c create mode 100644 server/src/unifyfs_group_rpc.h create mode 100644 server/src/unifyfs_inode.c create mode 100644 server/src/unifyfs_inode.h create mode 100644 server/src/unifyfs_inode_tree.c create mode 100644 server/src/unifyfs_inode_tree.h rename server/src/{unifyfs_metadata.c => unifyfs_metadata_mdhim.c} (95%) rename server/src/{unifyfs_metadata.h => unifyfs_metadata_mdhim.h} (72%) create mode 100644 server/src/unifyfs_p2p_rpc.c create mode 100644 server/src/unifyfs_p2p_rpc.h create mode 100644 server/src/unifyfs_tree.c create mode 100644 server/src/unifyfs_tree.h delete mode 100755 t/9100-metadata-api.t delete mode 100644 t/server/metadata_suite.c delete mode 100644 t/server/metadata_suite.h delete mode 100644 t/server/unifyfs_meta_get_test.c diff --git a/client/src/Makefile.am b/client/src/Makefile.am index 9df435752..ee975cdb0 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -50,8 +50,6 @@ CLIENT_COMMON_SOURCES = \ unifyfs-fixed.c \ unifyfs-fixed.h \ unifyfs-internal.h \ - unifyfs-stack.c \ - unifyfs-stack.h \ unifyfs-stdio.c \ unifyfs-stdio.h \ unifyfs-sysio.c \ diff --git a/client/src/margo_client.c b/client/src/margo_client.c index 2a0188f6b..a8bf18588 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + /************************************************************************** * margo_client.c - Implements the client-server RPC calls (shared-memory) **************************************************************************/ @@ -35,7 +49,7 @@ static void register_client_rpcs(client_rpc_context_t* ctx) CLIENT_REGISTER_RPC(truncate); CLIENT_REGISTER_RPC(unlink); CLIENT_REGISTER_RPC(laminate); - CLIENT_REGISTER_RPC(sync); + CLIENT_REGISTER_RPC(fsync); CLIENT_REGISTER_RPC(read); CLIENT_REGISTER_RPC(mread); @@ -168,10 +182,11 @@ static hg_handle_t create_handle(hg_id_t id) client_rpc_context_t* ctx = client_rpc_context; /* create handle for specified rpc */ - hg_handle_t handle; + hg_handle_t handle = HG_HANDLE_NULL; hg_return_t hret = margo_create(ctx->mid, ctx->svr_addr, id, &handle); - assert(hret == HG_SUCCESS); - + if (hret != HG_SUCCESS) { + LOGERR("margo_create() failed"); + } return handle; } @@ -193,24 +208,32 @@ int invoke_client_attach_rpc(void) /* call rpc function */ LOGDBG("invoking the attach rpc function in client"); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* decode response */ + int ret; unifyfs_attach_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } - /* free memory on input struct */ + /* free resources */ + margo_destroy(handle); if (NULL != in.logio_spill_dir) { free((void*)in.logio_spill_dir); } - /* free resources */ - margo_free_output(handle, &out); - margo_destroy(handle); - return (int)ret; + return ret; } /* invokes the mount rpc function */ @@ -234,32 +257,43 @@ int invoke_client_mount_rpc(void) /* call rpc function */ LOGDBG("invoking the mount rpc function in client"); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* free memory on input struct */ free((void*)in.mount_prefix); free((void*)in.client_addr_str); /* decode response */ + int ret; unifyfs_mount_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); - - /* get assigned client id, and verify app_id */ - unifyfs_client_id = (int) out.client_id; - int srvr_app_id = (int) out.app_id; - if (unifyfs_app_id != srvr_app_id) { - LOGWARN("mismatch on app_id - using %d, server returned %d", - unifyfs_app_id, srvr_app_id); + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + if (ret == (int)UNIFYFS_SUCCESS) { + /* get assigned client id, and verify app_id */ + unifyfs_client_id = (int) out.client_id; + int srvr_app_id = (int) out.app_id; + if (unifyfs_app_id != srvr_app_id) { + LOGWARN("mismatch on app_id - using %d, server returned %d", + unifyfs_app_id, srvr_app_id); + } + LOGDBG("My client id is %d", unifyfs_client_id); + } + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; } - LOGDBG("My client id is %d", unifyfs_client_id); /* free resources */ - margo_free_output(handle, &out); margo_destroy(handle); - return (int)ret; + + return ret; } /* function invokes the unmount rpc */ @@ -281,19 +315,29 @@ int invoke_client_unmount_rpc(void) /* call rpc function */ LOGDBG("invoking the unmount rpc function in client"); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* decode response */ + int ret; unifyfs_unmount_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } /* free resources */ - margo_free_output(handle, &out); margo_destroy(handle); - return (int)ret; + + return ret; } /* @@ -307,7 +351,8 @@ int invoke_client_unmount_rpc(void) * * f_meta: The metadata values to update. */ -int invoke_client_metaset_rpc(int create, unifyfs_file_attr_t* f_meta) +int invoke_client_metaset_rpc(unifyfs_file_attr_op_e attr_op, + unifyfs_file_attr_t* f_meta) { /* check that we have initialized margo */ if (NULL == client_rpc_context) { @@ -319,34 +364,38 @@ int invoke_client_metaset_rpc(int create, unifyfs_file_attr_t* f_meta) /* fill in input struct */ unifyfs_metaset_in_t in; - in.create = (int32_t) create; - in.gfid = (int32_t) f_meta->gfid; - in.filename = f_meta->filename; - in.mode = f_meta->mode; - in.uid = f_meta->uid; - in.gid = f_meta->gid; - in.size = f_meta->size; - in.atime = f_meta->atime; - in.mtime = f_meta->mtime; - in.ctime = f_meta->ctime; - in.is_laminated = f_meta->is_laminated; + in.app_id = (int32_t) unifyfs_app_id; + in.client_id = (int32_t) unifyfs_client_id; + in.attr_op = (int32_t) attr_op; + memcpy(&(in.attr), f_meta, sizeof(*f_meta)); /* call rpc function */ - LOGDBG("invoking the metaset rpc function in client"); + LOGDBG("invoking the metaset rpc function in client - gfid:%d file:%s", + in.attr.gfid, in.attr.filename); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* decode response */ + int ret; unifyfs_metaset_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } /* free resources */ - margo_free_output(handle, &out); margo_destroy(handle); - return (int)ret; + + return ret; } /* invokes the client metaget rpc function */ @@ -362,39 +411,44 @@ int invoke_client_metaget_rpc(int gfid, unifyfs_file_attr_t* file_meta) /* fill in input struct */ unifyfs_metaget_in_t in; - in.gfid = (int32_t)gfid; + in.app_id = (int32_t) unifyfs_app_id; + in.client_id = (int32_t) unifyfs_client_id; + in.gfid = (int32_t)gfid; /* call rpc function */ LOGDBG("invoking the metaget rpc function in client"); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* decode response */ + int ret; unifyfs_metaget_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); - - if (ret == (int32_t)UNIFYFS_SUCCESS) { - /* fill in results */ - memset(file_meta, 0, sizeof(unifyfs_file_attr_t)); - strcpy(file_meta->filename, out.filename); - file_meta->gfid = gfid; - file_meta->mode = out.mode; - file_meta->uid = out.uid; - file_meta->gid = out.gid; - file_meta->size = out.size; - file_meta->atime = out.atime; - file_meta->mtime = out.mtime; - file_meta->ctime = out.ctime; - file_meta->is_laminated = out.is_laminated; + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + if (ret == (int)UNIFYFS_SUCCESS) { + /* fill in results */ + memset(file_meta, 0, sizeof(unifyfs_file_attr_t)); + *file_meta = out.attr; + if (NULL != out.attr.filename) { + file_meta->filename = strdup(out.attr.filename); + } + } + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; } /* free resources */ - margo_free_output(handle, &out); margo_destroy(handle); - return (int)ret; + + return ret; } /* invokes the client filesize rpc function */ @@ -417,22 +471,32 @@ int invoke_client_filesize_rpc(int gfid, size_t* outsize) /* call rpc function */ LOGDBG("invoking the filesize rpc function in client"); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* decode response */ + int ret; unifyfs_filesize_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); - - /* save output from function */ - *outsize = (size_t) out.filesize; + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + if (ret == (int)UNIFYFS_SUCCESS) { + *outsize = (size_t) out.filesize; + } + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } /* free resources */ - margo_free_output(handle, &out); margo_destroy(handle); - return (int)ret; + + return ret; } /* invokes the client truncate rpc function */ @@ -456,19 +520,29 @@ int invoke_client_truncate_rpc(int gfid, size_t filesize) /* call rpc function */ LOGDBG("invoking the truncate rpc function in client"); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* decode response */ + int ret; unifyfs_truncate_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } /* free resources */ - margo_free_output(handle, &out); margo_destroy(handle); - return (int)ret; + + return ret; } /* invokes the client unlink rpc function */ @@ -491,19 +565,29 @@ int invoke_client_unlink_rpc(int gfid) /* call rpc function */ LOGDBG("invoking the unlink rpc function in client"); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* decode response */ + int ret; unifyfs_unlink_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } /* free resources */ - margo_free_output(handle, &out); margo_destroy(handle); - return (int)ret; + + return ret; } /* invokes the client-to-server laminate rpc function */ @@ -526,23 +610,33 @@ int invoke_client_laminate_rpc(int gfid) /* call rpc function */ LOGDBG("invoking the laminate rpc function in client"); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* decode response */ + int ret; unifyfs_laminate_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } /* free resources */ - margo_free_output(handle, &out); margo_destroy(handle); - return (int)ret; + + return ret; } /* invokes the client sync rpc function */ -int invoke_client_sync_rpc(void) +int invoke_client_sync_rpc(int gfid) { /* check that we have initialized margo */ if (NULL == client_rpc_context) { @@ -550,29 +644,40 @@ int invoke_client_sync_rpc(void) } /* get handle to rpc function */ - hg_handle_t handle = create_handle(client_rpc_context->rpcs.sync_id); + hg_handle_t handle = create_handle(client_rpc_context->rpcs.fsync_id); /* fill in input struct */ - unifyfs_sync_in_t in; + unifyfs_fsync_in_t in; in.app_id = (int32_t) unifyfs_app_id; in.client_id = (int32_t) unifyfs_client_id; + in.gfid = (int32_t) gfid; /* call rpc function */ LOGDBG("invoking the sync rpc function in client"); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* decode response */ - unifyfs_sync_out_t out; + int ret; + unifyfs_fsync_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIu32, ret); + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } /* free resources */ - margo_free_output(handle, &out); margo_destroy(handle); - return (int)ret; + + return ret; } /* invokes the client read rpc function */ @@ -597,17 +702,26 @@ int invoke_client_read_rpc(int gfid, size_t offset, size_t length) /* call rpc function */ LOGDBG("invoking the read rpc function in client"); hg_return_t hret = margo_forward(handle, &in); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + margo_destroy(handle); + return UNIFYFS_ERROR_MARGO; + } /* decode response */ + int ret; unifyfs_read_out_t out; hret = margo_get_output(handle, &out); - assert(hret == HG_SUCCESS); - int32_t ret = out.ret; - LOGDBG("Got response ret=%" PRIi32, ret); + if (hret == HG_SUCCESS) { + LOGDBG("Got response ret=%" PRIi32, out.ret); + ret = (int) out.ret; + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } /* free resources */ - margo_free_output(handle, &out); margo_destroy(handle); return ret; @@ -630,7 +744,6 @@ int unifyfs_mread_rpc_status_check(unifyfs_mread_rpc_ctx_t* ctx) /* flag becomes 1 when rpc is complete (otherwise 0) */ if (flag) { unifyfs_mread_out_t out; - hg_return_t hret = margo_get_output(ctx->handle, &out); if (hret == HG_SUCCESS) { ctx->rpc_ret = out.ret; @@ -670,7 +783,9 @@ int invoke_client_mread_rpc(int read_count, size_t size, void* buffer, hg_return_t hret = margo_bulk_create( client_rpc_context->mid, 1, &buffer, &size, HG_BULK_READ_ONLY, &in.bulk_handle); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + return UNIFYFS_ERROR_MARGO; + } /* fill in input struct */ in.app_id = (int32_t) unifyfs_app_id; @@ -685,7 +800,8 @@ int invoke_client_mread_rpc(int read_count, size_t size, void* buffer, ctx->handle = handle; ctx->req = req; } else { - ret = UNIFYFS_FAILURE; + LOGERR("margo_iforward() failed"); + ret = UNIFYFS_ERROR_MARGO; } /* margo_iforward serializes all data before returning, and it's safe to diff --git a/client/src/margo_client.h b/client/src/margo_client.h index 7ea974b8b..1c53b6026 100644 --- a/client/src/margo_client.h +++ b/client/src/margo_client.h @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + #ifndef _MARGO_CLIENT_H #define _MARGO_CLIENT_H @@ -19,7 +33,7 @@ typedef struct ClientRpcIds { hg_id_t truncate_id; hg_id_t unlink_id; hg_id_t laminate_id; - hg_id_t sync_id; + hg_id_t fsync_id; hg_id_t read_id; hg_id_t mread_id; } client_rpcs_t; @@ -45,7 +59,8 @@ int invoke_client_mount_rpc(void); int invoke_client_unmount_rpc(void); -int invoke_client_metaset_rpc(int create, unifyfs_file_attr_t* f_meta); +int invoke_client_metaset_rpc(unifyfs_file_attr_op_e attr_op, + unifyfs_file_attr_t* f_meta); int invoke_client_metaget_rpc(int gfid, unifyfs_file_attr_t* f_meta); @@ -57,7 +72,7 @@ int invoke_client_unlink_rpc(int gfid); int invoke_client_laminate_rpc(int gfid); -int invoke_client_sync_rpc(void); +int invoke_client_sync_rpc(int gfid); int invoke_client_read_rpc(int gfid, size_t offset, size_t length); diff --git a/client/src/pmpi_wrappers.c b/client/src/pmpi_wrappers.c index e9f137434..c0e9c730b 100644 --- a/client/src/pmpi_wrappers.c +++ b/client/src/pmpi_wrappers.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/client/src/pmpi_wrappers.h b/client/src/pmpi_wrappers.h index 995e44bbe..d2066dd59 100644 --- a/client/src/pmpi_wrappers.h +++ b/client/src/pmpi_wrappers.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/client/src/unifyfs-dirops.c b/client/src/unifyfs-dirops.c index 2353c7eb4..2d0b06946 100644 --- a/client/src/unifyfs-dirops.c +++ b/client/src/unifyfs-dirops.c @@ -129,13 +129,13 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) * re-populate with the global data? */ if (!unifyfs_fid_is_dir(fid)) { - errno = EIO; + errno = ENOTDIR; return NULL; } } else { fid = unifyfs_fid_create_file(upath); if (fid < 0) { - errno = EIO; + errno = unifyfs_rc_errno(-fid); return NULL; } diff --git a/client/src/unifyfs-dirops.h b/client/src/unifyfs-dirops.h index 8f239461c..876360391 100644 --- a/client/src/unifyfs-dirops.h +++ b/client/src/unifyfs-dirops.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2017, 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2017, 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #ifndef __UNIFYFS_DIROPS_H #define __UNIFYFS_DIROPS_H diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index d5520b2d7..90c33f6be 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -149,6 +149,39 @@ off_t unifyfs_rewrite_index_from_seg_tree(unifyfs_filemeta_t* meta) return max_log_offset; } +/* + * Find any write extents that span or exceed truncation point and remove them. + * + * This function is called when we truncate a file and there are cached writes. + */ +int truncate_write_meta(unifyfs_filemeta_t* meta, off_t trunc_sz) +{ + if (0 == trunc_sz) { + /* All writes should be removed. Clear extents_sync */ + seg_tree_clear(&meta->extents_sync); + + if (unifyfs_local_extents) { + /* Clear the local extent cache too */ + seg_tree_clear(&meta->extents); + } + return UNIFYFS_SUCCESS; + } + + unsigned long trunc_off = (unsigned long) trunc_sz; + int rc = seg_tree_remove(&meta->extents_sync, trunc_off, ULONG_MAX); + if (unifyfs_local_extents) { + rc = seg_tree_remove(&meta->extents, trunc_off, ULONG_MAX); + } + if (rc) { + LOGERR("removal of write extents due to truncation failed"); + rc = UNIFYFS_FAILURE; + } else { + rc = UNIFYFS_SUCCESS; + } + return rc; +} + + /* * Sync all the write extents for the target file(s) to the server. * The target_fid identifies a specific file, or all files (-1). @@ -199,7 +232,7 @@ int unifyfs_sync(int target_fid) } /* tell the server to grab our new extents */ - ret = invoke_client_sync_rpc(); + ret = invoke_client_sync_rpc(meta->gfid); if (ret != UNIFYFS_SUCCESS) { /* something went wrong when trying to flush extents */ LOGERR("failed to flush write index to server for gfid=%d", @@ -222,10 +255,12 @@ int unifyfs_sync(int target_fid) off_t logio_shmem_size; unifyfs_logio_get_sizes(logio_ctx, &logio_shmem_size, NULL); if (max_log_offset >= logio_shmem_size) { + LOGDBG("before logio spill sync") ret = unifyfs_logio_sync(logio_ctx); if (ret != UNIFYFS_SUCCESS) { LOGERR("failed to sync logio data"); } + LOGDBG("after logio spill sync"); } return ret; diff --git a/client/src/unifyfs-fixed.h b/client/src/unifyfs-fixed.h index 8086973dc..8053e0e7c 100644 --- a/client/src/unifyfs-fixed.h +++ b/client/src/unifyfs-fixed.h @@ -48,6 +48,9 @@ /* rewrite client's shared memory index of file write extents */ off_t unifyfs_rewrite_index_from_seg_tree(unifyfs_filemeta_t* meta); +/* remove/truncate write extents in client metadata */ +int truncate_write_meta(unifyfs_filemeta_t* meta, off_t trunc_sz); + /* sync all writes for target file(s) with the server */ int unifyfs_sync(int target_fid); diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 3e68becdf..6329780cc 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -550,18 +550,15 @@ int unifyfs_fid_unlink(int fid); /* issue a set of read requests */ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count); -int unifyfs_set_global_file_meta_from_fid( - int fid, - int create); - -int unifyfs_set_global_file_meta( - int gfid, - int create, - unifyfs_file_attr_t* gfattr); - -int unifyfs_get_global_file_meta( - int gfid, - unifyfs_file_attr_t* gfattr); +int unifyfs_set_global_file_meta_from_fid(int fid, + unifyfs_file_attr_op_e op); + +int unifyfs_set_global_file_meta(int gfid, + unifyfs_file_attr_op_e op, + unifyfs_file_attr_t* gfattr); + +int unifyfs_get_global_file_meta(int gfid, + unifyfs_file_attr_t* gfattr); // These require types/structures defined above #include "unifyfs-fixed.h" diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index d1fc63208..7bc1fea79 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/client/src/unifyfs-stdio.h b/client/src/unifyfs-stdio.h index 447ea0634..e8bd68608 100644 --- a/client/src/unifyfs-stdio.h +++ b/client/src/unifyfs-stdio.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 47b9f2b85..b7d1ab249 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -539,18 +539,10 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) /* get file id for path name */ int fid = unifyfs_get_fid_from_path(upath); if (fid >= 0) { - /* before we truncate, sync any data cached this file id */ - int ret = unifyfs_fid_sync(fid); - if (ret != UNIFYFS_SUCCESS) { - /* sync failed for some reason, set errno and return error */ - errno = unifyfs_rc_errno(ret); - return -1; - } - /* got the file locally, use fid_truncate the file */ int rc = unifyfs_fid_truncate(fid, length); if (rc != UNIFYFS_SUCCESS) { - errno = EIO; + errno = unifyfs_rc_errno(rc); return -1; } } else { @@ -559,7 +551,7 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) int rc = invoke_client_truncate_rpc(gfid, length); if (rc != UNIFYFS_SUCCESS) { LOGDBG("truncate rpc failed %s in UNIFYFS", upath); - errno = EIO; + errno = unifyfs_rc_errno(rc); return -1; } } @@ -697,7 +689,7 @@ static int __stat(const char* path, struct stat* buf) if (fid != -1) { int sync_rc = unifyfs_fid_sync(fid); if (sync_rc != UNIFYFS_SUCCESS) { - errno = EIO; + errno = unifyfs_rc_errno(sync_rc); return -1; } } @@ -710,9 +702,10 @@ static int __stat(const char* path, struct stat* buf) /* get stat information for file */ unifyfs_file_attr_t fattr; + memset(&fattr, 0, sizeof(fattr)); int ret = unifyfs_get_meta_with_size(gfid, &fattr); if (ret != UNIFYFS_SUCCESS) { - errno = EIO; + errno = unifyfs_rc_errno(ret); return -1; } @@ -911,8 +904,7 @@ int UNIFYFS_WRAP(fstatfs)(int fd, struct statfs* fsbuf) /* * Read 'count' bytes info 'buf' from file starting at offset 'pos'. * - * Returns number of bytes actually read, or -1 on error, in which - * case errno will be set. + * Returns success or error code. */ int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* nread) { @@ -959,17 +951,17 @@ int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* nread) req.offset = (size_t) pos; req.length = count; req.nread = 0; - req.errcode = UNIFYFS_SUCCESS; + req.errcode = EINPROGRESS; req.buf = buf; /* execute read operation */ int ret = unifyfs_gfid_read_reqs(&req, 1); if (ret != UNIFYFS_SUCCESS) { /* failed to issue read operation */ - return EIO; + return ret; } else if (req.errcode != UNIFYFS_SUCCESS) { /* read executed, but failed */ - return EIO; + return req.errcode; } /* success, get number of bytes read from read request field */ @@ -985,7 +977,7 @@ int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* nread) * is ignored. Fills any gaps with zeros */ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count, - size_t* nwritten) + size_t* nwritten) { /* assume we'll fail, set bytes written to 0 as a clue */ *nwritten = 0; @@ -1019,43 +1011,50 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count, return write_rc; } -int UNIFYFS_WRAP(creat)(const char* path, mode_t mode) +static int unifyfs_create(char* upath, mode_t mode) { /* equivalent to open(path, O_WRONLY|O_CREAT|O_TRUNC, mode) */ - /* check whether we should intercept this path */ - char upath[UNIFYFS_MAX_FILENAME]; - if (unifyfs_intercept_path(path, upath)) { - /* TODO: handle relative paths using current working directory */ + /* create the file */ + int fid; + int flags = O_WRONLY | O_CREAT | O_TRUNC; + off_t pos; + int rc = unifyfs_fid_open(upath, flags, mode, &fid, &pos); + if (rc != UNIFYFS_SUCCESS) { + errno = unifyfs_rc_errno(rc); + return -1; + } - /* create the file */ - int fid; - off_t pos; - int rc = unifyfs_fid_open(upath, O_WRONLY | O_CREAT | O_TRUNC, mode, &fid, &pos); - if (rc != UNIFYFS_SUCCESS) { - errno = unifyfs_rc_errno(rc); - return -1; - } + /* allocate a free file descriptor value */ + int fd = unifyfs_stack_pop(unifyfs_fd_stack); + if (fd < 0) { + /* ran out of file descriptors */ + errno = EMFILE; + return -1; + } - /* allocate a free file descriptor value */ - int fd = unifyfs_stack_pop(unifyfs_fd_stack); - if (fd < 0) { - /* ran out of file descriptors */ - errno = EMFILE; - return -1; - } + /* set file id and file pointer, flags include O_WRONLY */ + unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); + filedesc->fid = fid; + filedesc->pos = pos; + filedesc->read = 0; + filedesc->write = 1; - /* set file id and file pointer, flags include O_WRONLY */ - unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); - filedesc->fid = fid; - filedesc->pos = pos; - filedesc->read = 0; - filedesc->write = 1; - LOGDBG("UNIFYFS_open generated fd %d for file %s", fd, upath); - /* don't conflict with active system fds that range from 0 - (fd_limit) */ - int ret = fd + unifyfs_fd_limit; - return ret; + /* don't conflict with active system fds that range from 0 - (fd_limit) */ + int ret = fd + unifyfs_fd_limit; + LOGDBG("using fds (internal=%d, external=%d) for fid %d file %s", + fd, ret, fid, upath); + return ret; +} + +int UNIFYFS_WRAP(creat)(const char* path, mode_t mode) +{ + /* check whether we should intercept this path */ + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { + /* TODO: handle relative paths using current working directory */ + return unifyfs_create(upath, mode); } else { MAP_OR_FAIL(creat); int ret = UNIFYFS_REAL(creat)(path, mode); @@ -1068,11 +1067,8 @@ int UNIFYFS_WRAP(creat64)(const char* path, mode_t mode) /* check whether we should intercept this path */ char upath[UNIFYFS_MAX_FILENAME]; if (unifyfs_intercept_path(path, upath)) { - /* ERROR: fn not yet supported */ - fprintf(stderr, "Function not yet supported @ %s:%d\n", - __FILE__, __LINE__); - errno = ENOTSUP; - return -1; + /* TODO: handle relative paths using current working directory */ + return unifyfs_create(upath, mode); } else { MAP_OR_FAIL(creat64); int ret = UNIFYFS_REAL(creat64)(path, mode); @@ -1122,11 +1118,12 @@ int UNIFYFS_WRAP(open)(const char* path, int flags, ...) || ((flags & O_RDWR) == O_RDWR); filedesc->write = ((flags & O_WRONLY) == O_WRONLY) || ((flags & O_RDWR) == O_RDWR); - filedesc->append = ((flags & O_APPEND)); - LOGDBG("UNIFYFS_open generated fd %d for file %s", fd, upath); + filedesc->append = (flags & O_APPEND); /* don't conflict with active system fds that range from 0 - (fd_limit) */ ret = fd + unifyfs_fd_limit; + LOGDBG("using fds (internal=%d, external=%d) for fid %d file %s", + fd, ret, fid, upath); return ret; } else { MAP_OR_FAIL(open); @@ -1264,6 +1261,7 @@ off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) case SEEK_END: /* seek to EOF + offset */ logical_eof = unifyfs_fid_logical_size(fid); + LOGDBG("fid=%d EOF is offset %zu", fid, (size_t)logical_eof); if (logical_eof + offset < 0) { /* offset is negative and will result in negative position */ errno = EINVAL; @@ -1274,6 +1272,7 @@ off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) case SEEK_DATA: /* Using fallback approach: always return offset */ logical_eof = unifyfs_fid_logical_size(fid); + LOGDBG("fid=%d EOF is offset %zu", fid, (size_t)logical_eof); if (offset < 0 || offset > logical_eof) { /* negative offset and offset beyond EOF are invalid */ errno = ENXIO; @@ -1284,6 +1283,7 @@ off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) case SEEK_HOLE: /* Using fallback approach: always return offset for EOF */ logical_eof = unifyfs_fid_logical_size(fid); + LOGDBG("fid=%d EOF is offset %zu", fid, (size_t)logical_eof); if (offset < 0 || offset > logical_eof) { /* negative offset and offset beyond EOF are invalid */ errno = ENXIO; @@ -1550,6 +1550,11 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], for (i = 0; i < nitems; i++) { cbp = aiocb_list[i]; fd = cbp->aio_fildes; + + /* LOGDBG("aiocb(fd=%d, op=%d, count=%zu, offset=%zu, buf=%p)", + * cbp->aio_fildes, cbp->aio_lio_opcode, cbp->aio_nbytes, + * cbp->aio_offset, cbp->aio_buf); */ + switch (cbp->aio_lio_opcode) { case LIO_WRITE: { ssize_t wret; @@ -1598,6 +1603,8 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], break; } default: // LIO_NOP + LOGDBG("lio_vec[%d] - unexpected LIO op %d", + i, cbp->aio_lio_opcode); break; } } @@ -1606,7 +1613,7 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], rc = unifyfs_gfid_read_reqs(reqs, reqcnt); if (rc != UNIFYFS_SUCCESS) { /* error reading data */ - ret = -1; + ret = rc; } /* update aiocb fields to record error status and return value */ @@ -1614,7 +1621,8 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], for (ndx = 0; ndx < nitems; ndx++) { cbp = aiocb_list[ndx]; if (cbp == reqs[i].aiocbp) { - AIOCB_ERROR_CODE(cbp) = reqs[i].errcode; + AIOCB_ERROR_CODE(cbp) = + unifyfs_rc_errno((unifyfs_rc)(reqs[i].errcode)); if (0 == reqs[i].errcode) { AIOCB_RETURN_VAL(cbp) = reqs[i].length; } @@ -1626,8 +1634,9 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], free(reqs); - if (-1 == ret) { - errno = EIO; + if (ret) { + errno = unifyfs_rc_errno(ret); + ret = -1; } return ret; } @@ -1658,7 +1667,7 @@ ssize_t UNIFYFS_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) req.offset = offset; req.length = count; req.nread = 0; - req.errcode = UNIFYFS_SUCCESS; + req.errcode = EINPROGRESS; req.buf = buf; /* execute read operation */ @@ -1666,11 +1675,11 @@ ssize_t UNIFYFS_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) int ret = unifyfs_gfid_read_reqs(&req, 1); if (ret != UNIFYFS_SUCCESS) { /* error reading data */ - errno = EIO; + errno = unifyfs_rc_errno(ret); retcount = -1; } else if (req.errcode != UNIFYFS_SUCCESS) { /* error reading data */ - errno = EIO; + errno = unifyfs_rc_errno((unifyfs_rc)req.errcode); retcount = -1; } else { /* read succeeded, get number of bytes from nread field */ @@ -1716,6 +1725,8 @@ ssize_t UNIFYFS_WRAP(pwrite)(int fd, const void* buf, size_t count, /* write data to file */ size_t bytes; + LOGDBG("pwrite - fd=%d offset=%zu count=%zu", + fd, (size_t)offset, count); int write_rc = unifyfs_fd_write(fd, offset, buf, count, &bytes); if (write_rc != UNIFYFS_SUCCESS) { errno = unifyfs_rc_errno(write_rc); @@ -1823,18 +1834,10 @@ int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) return -1; } - /* before we truncate, sync any data cached this file id */ - int ret = unifyfs_fid_sync(fid); - if (ret != UNIFYFS_SUCCESS) { - /* sync failed for some reason, set errno and return error */ - errno = unifyfs_rc_errno(ret); - return -1; - } - /* truncate the file */ int rc = unifyfs_fid_truncate(fid, length); if (rc != UNIFYFS_SUCCESS) { - errno = EIO; + errno = unifyfs_rc_errno(rc); return -1; } @@ -2082,7 +2085,7 @@ int UNIFYFS_WRAP(close)(int fd) /* close the file id */ int close_rc = unifyfs_fid_close(fid); if (close_rc != UNIFYFS_SUCCESS) { - errno = EIO; + errno = unifyfs_rc_errno(close_rc); return -1; } @@ -2129,9 +2132,6 @@ static int __chmod(int fid, mode_t mode) * get the global file id */ int gfid = unifyfs_gfid_from_fid(fid); - /* TODO: need to fetch global metadata in case - * another process has changed it */ - /* * If the chmod clears all the existing write bits, then it's a laminate. * @@ -2143,8 +2143,8 @@ static int __chmod(int fid, mode_t mode) /* We're laminating. */ ret = invoke_client_laminate_rpc(gfid); if (ret) { - LOGERR("chmod: couldn't get the global file size on laminate"); - errno = EIO; + LOGERR("laminate failed"); + errno = unifyfs_rc_errno(ret); return -1; } } @@ -2154,11 +2154,12 @@ static int __chmod(int fid, mode_t mode) meta->mode = meta->mode | mode; /* update the global meta data to reflect new permissions */ - ret = unifyfs_set_global_file_meta_from_fid(fid, 0); + unifyfs_file_attr_op_e op = UNIFYFS_FILE_ATTR_OP_CHMOD; + ret = unifyfs_set_global_file_meta_from_fid(fid, op); if (ret) { LOGERR("chmod: can't set global meta entry for %s (fid:%d)", path, fid); - errno = EIO; + errno = unifyfs_rc_errno(ret); return -1; } @@ -2168,10 +2169,13 @@ static int __chmod(int fid, mode_t mode) if (ret) { LOGERR("chmod: can't get global meta entry for %s (fid:%d)", path, fid); - errno = EIO; + errno = unifyfs_rc_errno(ret); return -1; } + LOGDBG("attributes from global metadata"); + debug_print_file_attr(&attr); + /* update global size of file from global metadata */ unifyfs_fid_update_file_meta(fid, &attr); diff --git a/client/src/unifyfs-sysio.h b/client/src/unifyfs-sysio.h index 67b359988..a6e93939e 100644 --- a/client/src/unifyfs-sysio.h +++ b/client/src/unifyfs-sysio.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 5521fce73..0fc40c0f0 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -539,6 +539,8 @@ static int fid_store_alloc(int fid) } return UNIFYFS_SUCCESS; + } else { + LOGERR("failed to get filemeta for fid=%d", fid); } return UNIFYFS_FAILURE; } @@ -943,7 +945,7 @@ static void service_local_reqs( } else { LOGERR("local log read failed for offset=%zu size=%zu", (size_t)log_offset, length); - req->errcode = EIO; + req->errcode = rc; } /* get the next element in the tree */ @@ -1093,66 +1095,63 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) read_rc = invoke_client_read_rpc(gfid, offset, length); } - /* bail out with error if we failed to even start the read */ - if (read_rc != UNIFYFS_SUCCESS) { - LOGERR("Failed to issue read RPC to server"); - if (reqs != NULL) { - free(reqs); + /* ENODATA means server has no extents matching request(s) */ + if (read_rc != ENODATA) { + /* bail out with error if we failed to even start the read */ + if (read_rc != UNIFYFS_SUCCESS) { + LOGERR("Failed to issue read RPC to server"); + if (reqs != NULL) { + free(reqs); + } + return read_rc; } - return read_rc; - } - - /* - * ToDo: Exception handling when some of the requests - * are missed - */ - /* spin waiting for read data to come back from the server, - * we process it in batches as it comes in, eventually the - * server will tell us it's sent us everything it can */ - int done = 0; - int rpc_done = 0; - while (!done) { - int tmp_rc = delegator_wait(); - if (tmp_rc != UNIFYFS_SUCCESS) { - rc = UNIFYFS_FAILURE; - done = 1; - } else { - tmp_rc = process_read_data(read_reqs, count, &done); + /* spin waiting for read data to come back from the server, + * we process it in batches as it comes in, eventually the + * server will tell us it's sent us everything it can */ + int done = 0; + int rpc_done = 0; + while (!done) { + int tmp_rc = delegator_wait(); if (tmp_rc != UNIFYFS_SUCCESS) { - LOGERR("failed to process data from server"); rc = UNIFYFS_FAILURE; + done = 1; + } else { + tmp_rc = process_read_data(read_reqs, count, &done); + if (tmp_rc != UNIFYFS_SUCCESS) { + LOGERR("failed to process data from server"); + rc = UNIFYFS_FAILURE; + } + delegator_signal(); } - delegator_signal(); - } - /* if this was mread, track the progress */ - if (count > 1 && !rpc_done) { - tmp_rc = unifyfs_mread_rpc_status_check(&mread_ctx); - if (tmp_rc < 0) { - LOGERR("failed to check the rpc progress"); - continue; - } + /* if this was mread, track the progress */ + if (count > 1 && !rpc_done) { + tmp_rc = unifyfs_mread_rpc_status_check(&mread_ctx); + if (tmp_rc < 0) { + LOGERR("failed to check the rpc progress"); + continue; + } - /* if we received a response from the server, check any errors. - * for any errors, we do not have to wait for the data anymore. */ - if (tmp_rc) { - LOGDBG("received rpc response from the server (ret=%d)", - mread_ctx.rpc_ret); + /* if we received a response from the server, check for errors. + * upon finding errors, do not wait anymore. */ + if (tmp_rc) { + LOGDBG("received rpc response from the server (ret=%d)", + mread_ctx.rpc_ret); - if (mread_ctx.rpc_ret != UNIFYFS_SUCCESS) { - LOGERR("mread rpc failed on server (ret=%d)", - mread_ctx.rpc_ret); - return UNIFYFS_FAILURE; - } + if (mread_ctx.rpc_ret != UNIFYFS_SUCCESS) { + LOGERR("mread rpc failed on server (ret=%d)", + mread_ctx.rpc_ret); + return UNIFYFS_FAILURE; + } - rpc_done = 1; + rpc_done = 1; + } } } + LOGDBG("fetched all data from server for %d requests", count); } - LOGDBG("fetched all data from server for %d requests", count); - /* got all of the data we'll get from the server, * check for short reads and whether those short * reads are from errors, holes, or the end of the file */ @@ -1206,6 +1205,8 @@ int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) } /* copy zeros into request buffer */ + LOGDBG("zero-filling hole at offset %zu of length %zu", + gap_start, gap_length); char* req_ptr = req->buf + req->nread; memset(req_ptr, 0, gap_length); @@ -1432,17 +1433,16 @@ int unifyfs_fid_update_file_meta(int fid, unifyfs_file_attr_t* gfattr) * * gfid: The global file id on which to set metadata. * - * create: If set to 1, attempt to create the file first. If the file - * already exists, then update its metadata with the values in - * gfattr. If set to 0, and the file does not exist, then - * the server will return an error. + * op: If set to FILE_ATTR_OP_CREATE, attempt to create the file first. + * If the file already exists, then update its metadata with the values + * from fid filemeta. If not creating and the file does not exist, + * then the server will return an error. * * gfattr: The metadata values to store. */ -int unifyfs_set_global_file_meta( - int gfid, /* file id to set meta data for */ - int create, /* whether to set size/laminated fields (1) or not (0) */ - unifyfs_file_attr_t* gfattr) /* meta data to store for file */ +int unifyfs_set_global_file_meta(int gfid, + unifyfs_file_attr_op_e attr_op, + unifyfs_file_attr_t* gfattr) { /* check that we have an input buffer */ if (NULL == gfattr) { @@ -1453,8 +1453,8 @@ int unifyfs_set_global_file_meta( * submitting this under */ gfattr->gfid = gfid; - /* submit file attributes to global key/value store */ - int ret = invoke_client_metaset_rpc(create, gfattr); + /* send file attributes to server */ + int ret = invoke_client_metaset_rpc(attr_op, gfattr); return ret; } @@ -1481,27 +1481,30 @@ int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) * * fid: The local file id on which to base global metadata values. * - * create: If set to 1, attempt to create the file first. If the file - * already exists, then update its metadata with the values in - * gfattr. If set to 0, and the file does not exist, then - * the server will return an error. + * op: If set to FILE_ATTR_OP_CREATE, attempt to create the file first. + * If the file already exists, then update its metadata with the values + * from fid filemeta. If not creating and the file does not exist, + * then the server will return an error. */ -int unifyfs_set_global_file_meta_from_fid(int fid, int create) +int unifyfs_set_global_file_meta_from_fid(int fid, unifyfs_file_attr_op_e op) { /* initialize an empty file attributes structure */ - unifyfs_file_attr_t fattr = {0}; + unifyfs_file_attr_t fattr; + unifyfs_file_attr_set_invalid(&fattr); /* lookup local metadata for file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); assert(meta != NULL); - /* copy our file name */ - const char* path = unifyfs_path_from_fid(fid); - sprintf(fattr.filename, "%s", path); + /* get file name */ + char* filename = (char*) unifyfs_path_from_fid(fid); /* set global file id */ fattr.gfid = meta->gfid; + LOGDBG("setting global file metadata for fid:%d gfid:%d path:%s", + fid, fattr.gfid, filename); + /* use current time for atime/mtime/ctime */ struct timespec tp = {0}; clock_gettime(CLOCK_REALTIME, &tp); @@ -1509,20 +1512,27 @@ int unifyfs_set_global_file_meta_from_fid(int fid, int create) fattr.mtime = tp; fattr.ctime = tp; - /* copy file mode bits and lamination flag */ + /* copy file mode bits */ fattr.mode = meta->mode; - /* these fields are set by server, except when we're creating a - * new file in which case, we should initialize them both to 0 */ - fattr.is_laminated = 0; - fattr.size = 0; + if (op == UNIFYFS_FILE_ATTR_OP_CREATE) { + /* these fields are set by server, except when we're creating a + * new file in which case we should initialize them both to 0 */ + fattr.is_laminated = 0; + fattr.size = 0; + + /* capture current uid and gid */ + fattr.uid = getuid(); + fattr.gid = getgid(); - /* capture current uid and gid */ - fattr.uid = getuid(); - fattr.gid = getgid(); + fattr.filename = filename; + } + + LOGDBG("using following attributes"); + debug_print_file_attr(&fattr); /* submit file attributes to global key/value store */ - int ret = unifyfs_set_global_file_meta(meta->gfid, create, &fattr); + int ret = unifyfs_set_global_file_meta(meta->gfid, op, &fattr); return ret; } @@ -1537,7 +1547,7 @@ int unifyfs_fid_alloc(void) if (fid < 0) { /* need to create a new file, but we can't */ LOGERR("unifyfs_stack_pop() failed (%d)", fid); - return -1; + return -EMFILE; } return fid; } @@ -1558,14 +1568,12 @@ int unifyfs_fid_create_file(const char* path) /* check that pathname is within bounds */ size_t pathlen = strlen(path) + 1; if (pathlen > UNIFYFS_MAX_FILENAME) { - return ENAMETOOLONG; + return -ENAMETOOLONG; } /* allocate an id for this file */ int fid = unifyfs_fid_alloc(); if (fid < 0) { - /* was there an error? if so, return it */ - errno = ENOSPC; return fid; } @@ -1619,7 +1627,7 @@ int unifyfs_fid_create_directory(const char* path) /* can't create if it already exists */ if (found_global) { - return (int) EEXIST; + return EEXIST; } if (found_local) { @@ -1636,17 +1644,16 @@ int unifyfs_fid_create_directory(const char* path) * deletes the global entry without checking any local used entries * in other processes. * - * we currently return EIO, and this needs to be addressed according to - * a consistency model this fs intance assumes. + * we currently return EEXIS, and this needs to be addressed according + * to a consistency model this fs intance assumes. */ - return EIO; + return EEXIST; } /* now, we need to create a new directory. */ fid = unifyfs_fid_create_file(path); if (fid < 0) { - /* FIXME: ENOSPC or EIO? */ - return EIO; + return -fid; } /* Set as directory */ @@ -1655,11 +1662,12 @@ int unifyfs_fid_create_directory(const char* path) meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; /* insert global meta data for directory */ - int ret = unifyfs_set_global_file_meta_from_fid(fid, 1); + unifyfs_file_attr_op_e op = UNIFYFS_FILE_ATTR_OP_CREATE; + int ret = unifyfs_set_global_file_meta_from_fid(fid, op); if (ret != UNIFYFS_SUCCESS) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); - return EIO; + return ret; } return UNIFYFS_SUCCESS; @@ -1701,6 +1709,7 @@ int unifyfs_fid_write( } } else { /* unknown storage type */ + LOGERR("unknown storage type for fid=%d", fid); rc = EIO; } @@ -1720,23 +1729,34 @@ int unifyfs_fid_truncate(int fid, off_t length) return EINVAL; } - /* determine file storage type */ - if (meta->storage == FILE_STORAGE_LOGIO) { - /* invoke truncate rpc */ - int gfid = unifyfs_gfid_from_fid(fid); - int rc = invoke_client_truncate_rpc(gfid, length); - if (rc != UNIFYFS_SUCCESS) { - return rc; - } - - /* truncate succeeded, update global size to - * reflect truncated size, note log size is not affected */ - meta->global_size = length; - } else { + if (meta->storage != FILE_STORAGE_LOGIO) { /* unknown storage type */ return EIO; } + /* remove/update writes past truncation size for this file id */ + int rc = truncate_write_meta(meta, length); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* truncate is a sync point */ + rc = unifyfs_fid_sync(fid); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* update global size in filemeta to reflect truncated size. + * note that log size is not affected */ + meta->global_size = length; + + /* invoke truncate rpc */ + int gfid = unifyfs_gfid_from_fid(fid); + rc = invoke_client_truncate_rpc(gfid, length); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + return UNIFYFS_SUCCESS; } @@ -1834,9 +1854,8 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, /* initialize local metadata for this file */ fid = unifyfs_fid_create_file(path); if (fid < 0) { - /* FIXME: ENFILE or EIO ? */ LOGERR("failed to create a new file %s", path); - return EIO; + return -fid; } /* initialize local storage for this file */ @@ -1844,7 +1863,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, if (ret != UNIFYFS_SUCCESS) { LOGERR("failed to allocate storage space for file %s (fid=%d)", path, fid); - return EIO; + return ret; } /* initialize global size of file from global metadata */ @@ -1860,7 +1879,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } if (!(flags & O_DIRECTORY) && unifyfs_fid_is_dir(fid)) { - return ENOTDIR; + return EISDIR; } /* update local metadata from global metadata */ @@ -1882,31 +1901,29 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, LOGERR("%s does not exist (O_CREAT not given).", path); return ENOENT; } - - LOGDBG("Creating a new entry for %s.", path); - LOGDBG("superblock addr = %p; free_fid_stack = %p; filelist = %p", - shm_super_ctx->addr, free_fid_stack, unifyfs_filelist); + LOGDBG("Creating a new entry for %s", path); /* allocate a file id slot for this new file */ fid = unifyfs_fid_create_file(path); if (fid < 0) { LOGERR("Failed to create new file %s", path); - return ENFILE; + return -fid; } /* initialize the storage for the file */ int store_rc = fid_store_alloc(fid); if (store_rc != UNIFYFS_SUCCESS) { LOGERR("Failed to create storage for file %s", path); - return EIO; + return store_rc; } /* insert file attribute for file in key-value store */ - ret = unifyfs_set_global_file_meta_from_fid(fid, 1); + unifyfs_file_attr_op_e op = UNIFYFS_FILE_ATTR_OP_CREATE; + ret = unifyfs_set_global_file_meta_from_fid(fid, op); if (ret != UNIFYFS_SUCCESS) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); - return EIO; + return ret; } } @@ -1916,9 +1933,6 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, /* return local file id and starting file position */ *outfid = fid; *outpos = pos; - - LOGDBG("UNIFYFS_open generated fd %d for file %s", fid, path); - return UNIFYFS_SUCCESS; } @@ -2408,7 +2422,7 @@ static int unifyfs_finalize(void) /* close spillover files */ if (NULL != logio_ctx) { - unifyfs_logio_close(logio_ctx); + unifyfs_logio_close(logio_ctx, 0); logio_ctx = NULL; } if (unifyfs_spillmetablock != -1) { diff --git a/client/src/unifyfs.h b/client/src/unifyfs.h index d03c37b4c..045943891 100644 --- a/client/src/unifyfs.h +++ b/client/src/unifyfs.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -12,34 +12,6 @@ * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -/* - * Copyright (c) 2013, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * code Written by - * Raghunath Rajachandrasekar - * Kathryn Mohror - * Adam Moody - * All rights reserved. - * This file is part of CRUISE. - * For details, see https://github.com/hpc/cruise - * Please also read this file LICENSE.CRUISE - */ - #ifndef UNIFYFS_H #define UNIFYFS_H diff --git a/client/src/unifyfsf.c b/client/src/unifyfsf.c index 1480b4845..07e405ff7 100644 --- a/client/src/unifyfsf.c +++ b/client/src/unifyfsf.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017-2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/client/src/unifyfsf.h b/client/src/unifyfsf.h index 645fb2ce7..364cf437f 100644 --- a/client/src/unifyfsf.h +++ b/client/src/unifyfsf.h @@ -1,7 +1,7 @@ -! Copyright (c) 2017, Lawrence Livermore National Security, LLC. +! Copyright (c) 2020, Lawrence Livermore National Security, LLC. ! Produced at the Lawrence Livermore National Laboratory. ! -! Copyright 2017-2019, UT-Battelle, LLC. +! Copyright 2020, UT-Battelle, LLC. ! ! LLNL-CODE-741539 ! All rights reserved. diff --git a/common/src/Makefile.am b/common/src/Makefile.am index 11f09771d..7d2588224 100644 --- a/common/src/Makefile.am +++ b/common/src/Makefile.am @@ -5,10 +5,12 @@ include_HEADERS = unifyfs_const.h unifyfs_rc.h libunifyfs_commondir = $(includedir) BASE_SRCS = \ - ini.h \ - ini.c \ + arraylist.h \ + arraylist.c \ cm_enumerator.h \ cm_enumerator.c \ + ini.h \ + ini.c \ rm_enumerator.h \ rm_enumerator.c \ seg_tree.h \ @@ -33,12 +35,15 @@ BASE_SRCS = \ unifyfs_misc.h \ unifyfs_rpc_util.h \ unifyfs_rpc_util.c \ + unifyfs_rpc_types.h \ unifyfs_client_rpcs.h \ unifyfs_server_rpcs.h \ unifyfs_rc.h \ unifyfs_rc.c \ unifyfs_shm.h \ - unifyfs_shm.c + unifyfs_shm.c \ + unifyfs-stack.h \ + unifyfs-stack.c OPT_FLAGS = OPT_LIBS = diff --git a/server/src/arraylist.c b/common/src/arraylist.c similarity index 97% rename from server/src/arraylist.c rename to common/src/arraylist.c index 68b057676..f1808d879 100644 --- a/server/src/arraylist.c +++ b/common/src/arraylist.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/server/src/arraylist.h b/common/src/arraylist.h similarity index 93% rename from server/src/arraylist.h rename to common/src/arraylist.h index b33737614..8bd3d05ad 100644 --- a/server/src/arraylist.h +++ b/common/src/arraylist.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/cm_enumerator.c b/common/src/cm_enumerator.c index 23e46c82e..6cb9c9f3c 100644 --- a/common/src/cm_enumerator.c +++ b/common/src/cm_enumerator.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/cm_enumerator.h b/common/src/cm_enumerator.h index 50b611b73..2f1744cfa 100644 --- a/common/src/cm_enumerator.h +++ b/common/src/cm_enumerator.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/rm_enumerator.c b/common/src/rm_enumerator.c index efd3a8922..d5641a0a2 100644 --- a/common/src/rm_enumerator.c +++ b/common/src/rm_enumerator.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/rm_enumerator.h b/common/src/rm_enumerator.h index 4019a5b55..f8bfddf0b 100644 --- a/common/src/rm_enumerator.h +++ b/common/src/rm_enumerator.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/seg_tree.c b/common/src/seg_tree.c index 2907c0a6c..c47e3e10d 100644 --- a/common/src/seg_tree.c +++ b/common/src/seg_tree.c @@ -1,27 +1,17 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. - * LLNL-CODE-741539 - * All rights reserved. * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: + * Copyright 2020, UT-Battelle, LLC. * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * LLNL-CODE-741539 + * All rights reserved. * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + /* * This file is a simple, thread-safe, segment tree implementation. The * segments in the tree are non-overlapping. Added segments overwrite the old @@ -30,13 +20,14 @@ #include #include +#include #include #include -#include #include #include "seg_tree.h" #include "tree.h" +#include "unifyfs_log.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) @@ -46,7 +37,7 @@ #define MAX(a, b) ((a) > (b) ? (a) : (b)) #endif -int +static int compare_func(struct seg_tree_node* node1, struct seg_tree_node* node2) { if (node1->start > node2->end) { @@ -69,7 +60,7 @@ int seg_tree_init(struct seg_tree* seg_tree) RB_INIT(&seg_tree->head); return 0; -}; +} /* * Remove and free all nodes in the seg_tree. @@ -77,7 +68,7 @@ int seg_tree_init(struct seg_tree* seg_tree) void seg_tree_destroy(struct seg_tree* seg_tree) { seg_tree_clear(seg_tree); -}; +} /* Allocate a node for the range tree. Free node with free() when finished */ static struct seg_tree_node* @@ -336,6 +327,80 @@ int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, return rc; } +/* + * Remove or truncate one or more entries from the range tree + * if they overlap [start, end]. + * + * Returns 0 on success, nonzero otherwise. + */ +int seg_tree_remove( + struct seg_tree* seg_tree, + unsigned long start, + unsigned long end) +{ + struct seg_tree_node* node; + + LOGDBG("removing extents overlapping [%lu, %lu]", start, end); + + seg_tree_wrlock(seg_tree); + node = seg_tree_find_nolock(seg_tree, start, end); + while (node != NULL) { + if (start <= node->start) { + if (node->end <= end) { + /* start <= node_s <= node_e <= end + * remove whole extent */ + LOGDBG("removing node [%lu, %lu]", node->start, node->end); + RB_REMOVE(inttree, &seg_tree->head, node); + free(node); + seg_tree->count--; + } else { + /* start <= node_s <= end < node_e + * update node start */ + LOGDBG("updating node start from %lu to %lu", + node->start, (end + 1)); + + node->ptr += (end + 1 - node->start); + node->start = end + 1; + } + } else if (node->start < start) { + if (node->end <= end) { + /* node_s < start <= node_e <= end + * truncate node */ + LOGDBG("updating node end from %lu to %lu", + node->end, (start - 1)); + node->end = start - 1; + } else { + /* node_s < start <= end < node_e + * extent spans entire region, split into two nodes + * representing before/after region */ + unsigned long a_end = node->end; + unsigned long a_start = end + 1; + unsigned long a_ptr = node->ptr + (a_start - node->start); + + /* truncate existing (before) node */ + LOGDBG("updating before node end from %lu to %lu", + node->end, (start - 1)); + node->end = start - 1; + + /* add new (after) node */ + LOGDBG("add after node [%lu, %lu]", a_start, a_end); + seg_tree_unlock(seg_tree); + int rc = seg_tree_add(seg_tree, a_start, a_end, a_ptr); + if (rc) { + LOGERR("seg_tree_add() failed when splitting"); + return rc; + } + seg_tree_wrlock(seg_tree); + } + } + /* keep looking for nodes that overlap target region */ + node = seg_tree_find_nolock(seg_tree, start, end); + } + seg_tree_unlock(seg_tree); + + return 0; +} + /* * Search tree for an entry that overlaps with given range of [start, end]. * Returns the first overlapping entry if found, which is the overlapping entry @@ -448,7 +513,10 @@ seg_tree_iter(struct seg_tree* seg_tree, struct seg_tree_node* start) void seg_tree_rdlock(struct seg_tree* seg_tree) { - assert(pthread_rwlock_rdlock(&seg_tree->rwlock) == 0); + int rc = pthread_rwlock_rdlock(&seg_tree->rwlock); + if (rc) { + LOGERR("pthread_rwlock_rdlock() failed - rc=%d", rc); + } } /* @@ -459,7 +527,10 @@ seg_tree_rdlock(struct seg_tree* seg_tree) void seg_tree_wrlock(struct seg_tree* seg_tree) { - assert(pthread_rwlock_wrlock(&seg_tree->rwlock) == 0); + int rc = pthread_rwlock_wrlock(&seg_tree->rwlock); + if (rc) { + LOGERR("pthread_rwlock_wrlock() failed - rc=%d", rc); + } } /* @@ -470,7 +541,10 @@ seg_tree_wrlock(struct seg_tree* seg_tree) void seg_tree_unlock(struct seg_tree* seg_tree) { - assert(pthread_rwlock_unlock(&seg_tree->rwlock) == 0); + int rc = pthread_rwlock_unlock(&seg_tree->rwlock); + if (rc) { + LOGERR("pthread_rwlock_unlock() failed - rc=%d", rc); + } } /* diff --git a/common/src/seg_tree.h b/common/src/seg_tree.h index 20507eae7..a29416036 100644 --- a/common/src/seg_tree.h +++ b/common/src/seg_tree.h @@ -1,5 +1,20 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + #ifndef __SEG_TREE_H__ #define __SEG_TREE_H__ + #include #include "tree.h" @@ -37,6 +52,18 @@ void seg_tree_destroy(struct seg_tree* seg_tree); int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, unsigned long end, unsigned long ptr); +/* + * Remove or truncate one or more entries from the range tree + * if they overlap [start, end]. + * + * Returns 0 on success, nonzero otherwise. + */ +int seg_tree_remove( + struct seg_tree* seg_tree, + unsigned long start, + unsigned long end +); + /* * Find the first seg_tree_node that falls in a [start, end] range. */ diff --git a/client/src/unifyfs-stack.c b/common/src/unifyfs-stack.c similarity index 97% rename from client/src/unifyfs-stack.c rename to common/src/unifyfs-stack.c index 8d40b36c9..9e8b647fd 100644 --- a/client/src/unifyfs-stack.c +++ b/common/src/unifyfs-stack.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/client/src/unifyfs-stack.h b/common/src/unifyfs-stack.h similarity index 95% rename from client/src/unifyfs-stack.h rename to common/src/unifyfs-stack.h index 3dd475e19..61ad93408 100644 --- a/client/src/unifyfs-stack.h +++ b/common/src/unifyfs-stack.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/unifyfs_client_rpcs.h b/common/src/unifyfs_client_rpcs.h index 900ec10b3..64649bae0 100644 --- a/common/src/unifyfs_client_rpcs.h +++ b/common/src/unifyfs_client_rpcs.h @@ -25,10 +25,28 @@ #include #include +#include "unifyfs_rpc_types.h" + #ifdef __cplusplus extern "C" { #endif +typedef enum { + UNIFYFS_CLIENT_RPC_INVALID = 0, + UNIFYFS_CLIENT_RPC_ATTACH, + UNIFYFS_CLIENT_RPC_FILESIZE, + UNIFYFS_CLIENT_RPC_LAMINATE, + UNIFYFS_CLIENT_RPC_METAGET, + UNIFYFS_CLIENT_RPC_METASET, + UNIFYFS_CLIENT_RPC_MOUNT, + UNIFYFS_CLIENT_RPC_MULTIREAD, + UNIFYFS_CLIENT_RPC_READ, + UNIFYFS_CLIENT_RPC_SYNC, + UNIFYFS_CLIENT_RPC_TRUNCATE, + UNIFYFS_CLIENT_RPC_UNLINK, + UNIFYFS_CLIENT_RPC_UNMOUNT +} client_rpc_e; + /* unifyfs_attach_rpc (client => server) * * initialize server access to client's shared memory and file state */ @@ -68,28 +86,15 @@ MERCURY_GEN_PROC(unifyfs_unmount_in_t, MERCURY_GEN_PROC(unifyfs_unmount_out_t, ((int32_t)(ret))) DECLARE_MARGO_RPC_HANDLER(unifyfs_unmount_rpc) -/* need to transfer timespec structs */ -typedef struct timespec sys_timespec_t; -MERCURY_GEN_STRUCT_PROC(sys_timespec_t, - ((uint64_t)(tv_sec)) - ((uint64_t)(tv_nsec))) - /* unifyfs_metaset_rpc (client => server) * * given a global file id and a file name, * record key/value entry for this file */ MERCURY_GEN_PROC(unifyfs_metaset_in_t, - ((int32_t)(create)) - ((hg_const_string_t)(filename)) - ((int32_t)(gfid)) - ((uint32_t)(mode)) - ((uint32_t)(uid)) - ((uint32_t)(gid)) - ((uint64_t)(size)) - ((sys_timespec_t)(atime)) - ((sys_timespec_t)(mtime)) - ((sys_timespec_t)(ctime)) - ((uint32_t)(is_laminated))) + ((int32_t)(app_id)) + ((int32_t)(client_id)) + ((int32_t)(attr_op)) + ((unifyfs_file_attr_t)(attr))) MERCURY_GEN_PROC(unifyfs_metaset_out_t, ((int32_t)(ret))) DECLARE_MARGO_RPC_HANDLER(unifyfs_metaset_rpc) @@ -98,31 +103,25 @@ DECLARE_MARGO_RPC_HANDLER(unifyfs_metaset_rpc) * returns file metadata including size and name * given a global file id */ MERCURY_GEN_PROC(unifyfs_metaget_in_t, + ((int32_t)(app_id)) + ((int32_t)(client_id)) ((int32_t)(gfid))) MERCURY_GEN_PROC(unifyfs_metaget_out_t, ((int32_t)(ret)) - ((hg_const_string_t)(filename)) - ((int32_t)(gfid)) - ((uint32_t)(mode)) - ((uint32_t)(uid)) - ((uint32_t)(gid)) - ((uint64_t)(size)) - ((sys_timespec_t)(atime)) - ((sys_timespec_t)(mtime)) - ((sys_timespec_t)(ctime)) - ((uint32_t)(is_laminated))) + ((unifyfs_file_attr_t)(attr))) DECLARE_MARGO_RPC_HANDLER(unifyfs_metaget_rpc) -/* unifyfs_sync_rpc (client => server) +/* unifyfs_fsync_rpc (client => server) * * given a client identified by (app_id, client_id) as input, read the write * extents for one or more of the client's files from the shared memory index * and update the global metadata for the file(s) */ -MERCURY_GEN_PROC(unifyfs_sync_in_t, +MERCURY_GEN_PROC(unifyfs_fsync_in_t, ((int32_t)(app_id)) - ((int32_t)(client_id))) -MERCURY_GEN_PROC(unifyfs_sync_out_t, ((int32_t)(ret))) -DECLARE_MARGO_RPC_HANDLER(unifyfs_sync_rpc) + ((int32_t)(client_id)) + ((int32_t)(gfid))) +MERCURY_GEN_PROC(unifyfs_fsync_out_t, ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_fsync_rpc) /* unifyfs_filesize_rpc (client => server) * diff --git a/common/src/unifyfs_configurator.c b/common/src/unifyfs_configurator.c index 3b7dbb1c0..ac5daaed1 100644 --- a/common/src/unifyfs_configurator.c +++ b/common/src/unifyfs_configurator.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index 86c21000a..ab136e13f 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -92,7 +92,6 @@ UNIFYFS_CFG(server, max_app_clients, INT, MAX_APP_CLIENTS, "maximum number of clients per application", NULL) \ UNIFYFS_CFG_CLI(sharedfs, dir, STRING, NULLSTRING, "shared file system directory", configurator_directory_check, 'S', "specify full path to directory to contain server shared files") \ - #ifdef __cplusplus extern "C" { #endif diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h index 72c7ae1cd..39f0eb599 100644 --- a/common/src/unifyfs_const.h +++ b/common/src/unifyfs_const.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -50,22 +50,19 @@ #define UNIFYFS_MAX_HOSTNAME 64 // Server - Request Manager +#define MAX_DATA_TX_SIZE (4 * MIB) /* data transfer size (to client) */ #define MAX_META_PER_SEND (4 * KIB) /* max read request count per server */ #define REQ_BUF_LEN (MAX_META_PER_SEND * 64) /* chunk read reqs buffer size */ #define SHM_WAIT_INTERVAL 1000 /* unit: ns */ #define RM_MAX_ACTIVE_REQUESTS 64 /* number of concurrent read requests */ -#define MAX_BULK_TX_SIZE (8 * MIB) /* bulk transfer size */ // Server - Service Manager -#define LARGE_BURSTY_DATA (512 * MIB) -#define MAX_BURSTY_INTERVAL 10000 /* unit: us */ -#define MIN_SLEEP_INTERVAL 100 /* unit: us */ -#define SLEEP_INTERVAL 500 /* unit: us */ -#define SLEEP_SLICE_PER_UNIT 50 /* unit: us */ +#define MIN_SLEEP_INTERVAL 50 /* unit: us */ // Server - General -#define MAX_NUM_APPS 64 /* max # apps/mountpoints supported */ -#define MAX_APP_CLIENTS 256 /* max # clients per application */ +#define MAX_BULK_TX_SIZE (8 * MIB) /* bulk transfer size (between servers) */ +#define MAX_NUM_APPS 64 /* max # apps/mountpoints supported */ +#define MAX_APP_CLIENTS 256 /* max # clients per application */ #define UNIFYFS_DEFAULT_INIT_TIMEOUT 120 /* server init timeout (seconds) */ #define UNIFYFSD_PID_FILENAME "unifyfsd.pids" #define UNIFYFS_STAGE_STATUS_FILENAME "unifyfs-stage.status" diff --git a/common/src/unifyfs_keyval.c b/common/src/unifyfs_keyval.c index 30f69b3d8..e73237e9b 100644 --- a/common/src/unifyfs_keyval.c +++ b/common/src/unifyfs_keyval.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/unifyfs_keyval.h b/common/src/unifyfs_keyval.h index 33f88b194..d56e884cd 100644 --- a/common/src/unifyfs_keyval.h +++ b/common/src/unifyfs_keyval.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/unifyfs_log.c b/common/src/unifyfs_log.c index fb639fb5c..478593807 100644 --- a/common/src/unifyfs_log.c +++ b/common/src/unifyfs_log.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/unifyfs_log.h b/common/src/unifyfs_log.h index 44549485d..ee0160d53 100644 --- a/common/src/unifyfs_log.h +++ b/common/src/unifyfs_log.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -77,6 +77,7 @@ pid_t unifyfs_gettid(void); #define LOGERR(...) LOG(LOG_ERR, __VA_ARGS__) #define LOGWARN(...) LOG(LOG_WARN, __VA_ARGS__) +#define LOGINFO(...) LOG(LOG_INFO, __VA_ARGS__) #define LOGDBG(...) LOG(LOG_DBG, __VA_ARGS__) /* open specified file as debug file stream, diff --git a/common/src/unifyfs_logio.c b/common/src/unifyfs_logio.c index 6bdeb2a73..648418c91 100644 --- a/common/src/unifyfs_logio.c +++ b/common/src/unifyfs_logio.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -166,6 +166,7 @@ int unifyfs_logio_init_server(const int app_id, } } + char spillfile[UNIFYFS_MAX_FILENAME]; void* spill_mapping = NULL; int spill_fd = -1; if (spill_size) { @@ -175,7 +176,6 @@ int unifyfs_logio_init_server(const int app_id, } /* open the spill over file */ - char spillfile[UNIFYFS_MAX_FILENAME]; snprintf(spillfile, sizeof(spillfile), LOGIO_SPILL_FMTSTR, spill_dir, app_id, client_id); spill_fd = get_spillfile(spillfile, spill_size); @@ -202,6 +202,9 @@ int unifyfs_logio_init_server(const int app_id, ctx->spill_hdr = spill_mapping; ctx->spill_fd = spill_fd; ctx->spill_sz = spill_size; + if (spill_size) { + ctx->spill_file = strdup(spillfile); + } *pctx = ctx; return UNIFYFS_SUCCESS; @@ -372,7 +375,8 @@ int unifyfs_logio_init_client(const int app_id, } /* Close logio context */ -int unifyfs_logio_close(logio_context* ctx) +int unifyfs_logio_close(logio_context* ctx, + int clean_spill) { if (NULL == ctx) { return EINVAL; @@ -408,6 +412,15 @@ int unifyfs_logio_close(logio_context* ctx) } ctx->spill_fd = -1; } + if (clean_spill && (ctx->spill_file != NULL)) { + rc = unlink(ctx->spill_file); + if (rc != 0) { + int err = errno; + LOGERR("Failed to unlink logio spill file %s (errno=%s)", + ctx->spill_file, strerror(err)); + } + free(ctx->spill_file); + } } /* free the context struct */ diff --git a/common/src/unifyfs_logio.h b/common/src/unifyfs_logio.h index 2cdd1e2a2..15364e3df 100644 --- a/common/src/unifyfs_logio.h +++ b/common/src/unifyfs_logio.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -28,6 +28,7 @@ extern "C" { typedef struct logio_context { shm_context* shmem; /* shmem region for memory storage */ void* spill_hdr; /* mmap() address for spillover file log header */ + char* spill_file; /* pathname of spillover file */ size_t spill_sz; /* size of spillover file */ int spill_fd; /* spillover file descriptor */ } logio_context; @@ -67,10 +68,12 @@ int unifyfs_logio_init_client(const int app_id, /** * Close logio context. * - * @param ctx pointer to logio context + * @param ctx pointer to logio context + * @param clean_spill set to non-zero to have server remove spill file * @return UNIFYFS_SUCCESS, or error code */ -int unifyfs_logio_close(logio_context* ctx); +int unifyfs_logio_close(logio_context* ctx, + int clean_spill); /** * Allocate write space from logio context. diff --git a/common/src/unifyfs_meta.c b/common/src/unifyfs_meta.c index 227745343..d1fa5adcf 100644 --- a/common/src/unifyfs_meta.c +++ b/common/src/unifyfs_meta.c @@ -18,6 +18,18 @@ #include "unifyfs_meta.h" +/* extent slice size used for metadata */ +size_t meta_slice_sz = META_DEFAULT_RANGE_SZ; + +/* calculate number of slices in an extent given by start offset and length */ +size_t meta_num_slices(size_t offset, size_t length) +{ + size_t start = offset / meta_slice_sz; + size_t end = (offset + length - 1) / meta_slice_sz; + size_t count = end - start + 1; + return count; +} + /** * Hash a file path to a 64-bit unsigned integer using MD5 * @param path absolute file path diff --git a/common/src/unifyfs_meta.h b/common/src/unifyfs_meta.h index 35e070406..24d5dc460 100644 --- a/common/src/unifyfs_meta.h +++ b/common/src/unifyfs_meta.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2018, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2018, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -15,18 +15,30 @@ #ifndef UNIFYFS_META_H #define UNIFYFS_META_H +#include +#include #include +#include +#include +#include #include #include #include -#include #include "unifyfs_const.h" +#include "unifyfs_log.h" #ifdef __cplusplus extern "C" { #endif + +/* extent slice size used for metadata */ +extern size_t meta_slice_sz; + +/* calculate number of slices in an extent given by start offset and length */ +size_t meta_num_slices(size_t offset, size_t length); + /* structure used to detect clients/servers colocated on a host */ typedef struct { char hostname[UNIFYFS_MAX_HOSTNAME]; @@ -40,7 +52,6 @@ typedef struct { int gfid; } unifyfs_extent_t; - /* write-log metadata index structure */ typedef struct { off_t file_pos; /* start offset of data in file */ @@ -51,8 +62,11 @@ typedef struct { /* UnifyFS file attributes */ typedef struct { + char* filename; int gfid; - char filename[UNIFYFS_MAX_FILENAME]; + + /* Set when the file is laminated */ + int is_laminated; /* essential stat fields */ uint32_t mode; /* st_mode bits */ @@ -62,9 +76,6 @@ typedef struct { struct timespec atime; struct timespec mtime; struct timespec ctime; - - /* Set when the file is laminated */ - uint32_t is_laminated; } unifyfs_file_attr_t; enum { @@ -74,10 +85,151 @@ enum { UNIFYFS_STAT_DEFAULT_DIR_MODE = S_IFDIR | 0755, }; +static inline +int unifyfs_file_attr_set_invalid(unifyfs_file_attr_t* attr) +{ + if (!attr) { + return EINVAL; + } + + memset(attr, 0, sizeof(*attr)); + attr->filename = NULL; + attr->gfid = -1; + attr->is_laminated = -1; + attr->mode = -1; + attr->uid = -1; + attr->gid = -1; + attr->size = (uint64_t) -1; + + return 0; +} + +static inline +void debug_print_file_attr(unifyfs_file_attr_t* attr) +{ + if (!attr) { + return; + } + LOGDBG("fileattr(%p) - gfid=%d filename=%s laminated=%d", + attr, attr->gfid, attr->filename, attr->is_laminated); + LOGDBG(" - sz=%zu mode=%o uid=%d gid=%d", + (size_t)attr->size, attr->mode, attr->uid, attr->gid); + LOGDBG(" - atime=%ld.%09ld ctime=%ld.%09ld mtime=%ld.%09ld", + attr->atime.tv_sec, attr->atime.tv_nsec, + attr->ctime.tv_sec, attr->ctime.tv_nsec, + attr->mtime.tv_sec, attr->mtime.tv_nsec); +} + +typedef enum { + UNIFYFS_FILE_ATTR_OP_INVALID = 0, + UNIFYFS_FILE_ATTR_OP_CHGRP, + UNIFYFS_FILE_ATTR_OP_CHMOD, + UNIFYFS_FILE_ATTR_OP_CHOWN, + UNIFYFS_FILE_ATTR_OP_CREATE, + UNIFYFS_FILE_ATTR_OP_DATA, + UNIFYFS_FILE_ATTR_OP_LAMINATE, + UNIFYFS_FILE_ATTR_OP_TRUNCATE +} unifyfs_file_attr_op_e; + +/* + * updates @dst with new values from @src. + * ignores fields from @src with negative values. + */ +static inline +int unifyfs_file_attr_update(int attr_op, + unifyfs_file_attr_t* dst, + unifyfs_file_attr_t* src) +{ + if (!dst || !src + || (attr_op == UNIFYFS_FILE_ATTR_OP_INVALID) + || (dst->gfid != src->gfid)) { + return EINVAL; + } + + LOGDBG("updating attributes for gfid=%d", dst->gfid); + + /* Update fields only with valid values and associated operation. + * invalid values are set by unifyfs_file_attr_set_invalid() above */ + + if ((src->mode != -1) && + ((attr_op == UNIFYFS_FILE_ATTR_OP_CHMOD) || + (attr_op == UNIFYFS_FILE_ATTR_OP_CREATE) || + (attr_op == UNIFYFS_FILE_ATTR_OP_LAMINATE))) { + LOGDBG("setting mode to %o", src->mode); + dst->mode = src->mode; + } + + if ((src->uid != -1) && + ((attr_op == UNIFYFS_FILE_ATTR_OP_CHOWN) || + (attr_op == UNIFYFS_FILE_ATTR_OP_CREATE))) { + dst->uid = src->uid; + } + + if ((src->gid != -1) && + ((attr_op == UNIFYFS_FILE_ATTR_OP_CHGRP) || + (attr_op == UNIFYFS_FILE_ATTR_OP_CREATE))) { + dst->gid = src->gid; + } + + if ((src->size != (uint64_t)-1) && + ((attr_op == UNIFYFS_FILE_ATTR_OP_CREATE) || + (attr_op == UNIFYFS_FILE_ATTR_OP_DATA) || + (attr_op == UNIFYFS_FILE_ATTR_OP_LAMINATE) || + (attr_op == UNIFYFS_FILE_ATTR_OP_TRUNCATE))) { + LOGDBG("setting attr.size to %" PRIu64, src->size); + dst->size = src->size; + } + + if ((src->atime.tv_sec != 0) && + (attr_op == UNIFYFS_FILE_ATTR_OP_CREATE)) { + LOGDBG("setting attr.atime to %d.%09ld", + (int)src->atime.tv_sec, src->atime.tv_nsec); + dst->atime = src->atime; + } + + if ((src->mtime.tv_sec != 0) && + ((attr_op == UNIFYFS_FILE_ATTR_OP_CREATE) || + (attr_op == UNIFYFS_FILE_ATTR_OP_DATA) || + (attr_op == UNIFYFS_FILE_ATTR_OP_LAMINATE) || + (attr_op == UNIFYFS_FILE_ATTR_OP_TRUNCATE))) { + LOGDBG("setting attr.mtime to %d.%09ld", + (int)src->mtime.tv_sec, src->mtime.tv_nsec); + dst->mtime = src->mtime; + } + + if ((src->ctime.tv_sec != 0) && + ((attr_op == UNIFYFS_FILE_ATTR_OP_CHGRP) || + (attr_op == UNIFYFS_FILE_ATTR_OP_CHMOD) || + (attr_op == UNIFYFS_FILE_ATTR_OP_CHOWN) || + (attr_op == UNIFYFS_FILE_ATTR_OP_CREATE) || + (attr_op == UNIFYFS_FILE_ATTR_OP_DATA) || + (attr_op == UNIFYFS_FILE_ATTR_OP_LAMINATE))) { + LOGDBG("setting attr.ctime to %d.%09ld", + (int)src->ctime.tv_sec, src->ctime.tv_nsec); + dst->ctime = src->ctime; + } + + if ((src->is_laminated != -1) && + ((attr_op == UNIFYFS_FILE_ATTR_OP_CREATE) || + (attr_op == UNIFYFS_FILE_ATTR_OP_LAMINATE))) { + LOGDBG("setting attr.is_laminated to %d", src->is_laminated); + dst->is_laminated = src->is_laminated; + } + + if (src->filename && !dst->filename) { + LOGDBG("setting attr.filename to %s", src->filename); + dst->filename = strdup(src->filename); + } + + return 0; +} + static inline void unifyfs_file_attr_to_stat(unifyfs_file_attr_t* fattr, struct stat* sb) { if (fattr && sb) { + debug_print_file_attr(fattr); + sb->st_dev = UNIFYFS_STAT_DEFAULT_DEV; sb->st_ino = fattr->gfid; sb->st_mode = fattr->mode; @@ -85,6 +237,9 @@ void unifyfs_file_attr_to_stat(unifyfs_file_attr_t* fattr, struct stat* sb) sb->st_gid = fattr->gid; sb->st_rdev = UNIFYFS_STAT_DEFAULT_DEV; sb->st_size = fattr->size; + + /* TODO: use cfg.logio_chunk_size here for st_blksize + * and report acutal chunks allocated for st_blocks */ sb->st_blksize = UNIFYFS_STAT_DEFAULT_BLKSIZE; sb->st_blocks = fattr->size / UNIFYFS_STAT_DEFAULT_BLKSIZE; if (fattr->size % UNIFYFS_STAT_DEFAULT_BLKSIZE > 0) { @@ -98,9 +253,9 @@ void unifyfs_file_attr_to_stat(unifyfs_file_attr_t* fattr, struct stat* sb) */ sb->st_nlink = fattr->is_laminated ? 1 : 0; - sb->st_atime = fattr->atime.tv_sec; - sb->st_mtime = fattr->mtime.tv_sec; - sb->st_ctime = fattr->ctime.tv_sec; + sb->st_atim = fattr->atime; + sb->st_mtim = fattr->mtime; + sb->st_ctim = fattr->ctime; } } diff --git a/common/src/unifyfs_misc.h b/common/src/unifyfs_misc.h index 5e6c0cdac..3678c2ec6 100644 --- a/common/src/unifyfs_misc.h +++ b/common/src/unifyfs_misc.h @@ -1,5 +1,22 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + #ifndef __UNIFYFS_MISC__ #define __UNIFYFS_MISC__ + size_t strlcpy(char* dest, const char* src, size_t size); + int scnprintf(char* buf, size_t size, const char* fmt, ...); + #endif diff --git a/common/src/unifyfs_rc.c b/common/src/unifyfs_rc.c index 83a92d868..5bb7f61ca 100644 --- a/common/src/unifyfs_rc.c +++ b/common/src/unifyfs_rc.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/unifyfs_rc.h b/common/src/unifyfs_rc.h index ed14d46e1..cf54ad904 100644 --- a/common/src/unifyfs_rc.h +++ b/common/src/unifyfs_rc.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/unifyfs_rpc_types.h b/common/src/unifyfs_rpc_types.h new file mode 100644 index 000000000..126ffdd3e --- /dev/null +++ b/common/src/unifyfs_rpc_types.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef __UNIFYFS_RPC_TYPES_H +#define __UNIFYFS_RPC_TYPES_H + +#include +#include +#include + +#include "unifyfs_meta.h" + +/* rpc encode/decode for timespec structs */ +typedef struct timespec sys_timespec_t; +MERCURY_GEN_STRUCT_PROC(sys_timespec_t, + ((uint64_t)(tv_sec)) + ((uint64_t)(tv_nsec)) +) + +/* rpc encode/decode for unifyfs_file_attr_t */ +MERCURY_GEN_STRUCT_PROC(unifyfs_file_attr_t, + ((int32_t)(gfid)) + ((int32_t)(is_laminated)) + ((uint32_t)(mode)) + ((uint32_t)(uid)) + ((uint32_t)(gid)) + ((hg_size_t)(size)) + ((sys_timespec_t)(atime)) + ((sys_timespec_t)(ctime)) + ((sys_timespec_t)(mtime)) + ((hg_const_string_t)(filename)) +) + +#endif /* __UNIFYFS_RPC_TYPES_H */ diff --git a/common/src/unifyfs_rpc_util.c b/common/src/unifyfs_rpc_util.c index 7a60725a9..5cc99bb4f 100644 --- a/common/src/unifyfs_rpc_util.c +++ b/common/src/unifyfs_rpc_util.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/unifyfs_rpc_util.h b/common/src/unifyfs_rpc_util.h index 8067ed5c6..e18a51025 100644 --- a/common/src/unifyfs_rpc_util.h +++ b/common/src/unifyfs_rpc_util.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2018, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2018, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/common/src/unifyfs_server_rpcs.h b/common/src/unifyfs_server_rpcs.h index 10b3244e9..349e1f11b 100644 --- a/common/src/unifyfs_server_rpcs.h +++ b/common/src/unifyfs_server_rpcs.h @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + #ifndef __UNIFYFS_SERVER_RPCS_H #define __UNIFYFS_SERVER_RPCS_H @@ -5,51 +19,29 @@ * Declarations for server-server margo RPCs */ +#include #include #include #include #include +#include "unifyfs_rpc_types.h" + #ifdef __cplusplus extern "C" { #endif -/* server_hello_rpc (server => server) - * - * say hello from one server to another */ -MERCURY_GEN_PROC(server_hello_in_t, - ((int32_t)(src_rank)) - ((hg_const_string_t)(message_str))) -MERCURY_GEN_PROC(server_hello_out_t, - ((int32_t)(ret))) -DECLARE_MARGO_RPC_HANDLER(server_hello_rpc) +/*---- Server Point-to-Point (p2p) RPCs ----*/ -/* server_pid_rpc (server => server, n:1) - * - * notify readiness with pid to the master server (rank:0) */ +/* Report server pid to rank 0 */ MERCURY_GEN_PROC(server_pid_in_t, ((int32_t)(rank)) ((int32_t)(pid))) MERCURY_GEN_PROC(server_pid_out_t, ((int32_t)(ret))) -DECLARE_MARGO_RPC_HANDLER(server_pid_handle_rpc) - -/* server_request_rpc (server => server) - * - * request from one server to another */ -MERCURY_GEN_PROC(server_request_in_t, - ((int32_t)(src_rank)) - ((int32_t)(req_id)) - ((int32_t)(req_tag)) - ((hg_size_t)(bulk_size)) - ((hg_bulk_t)(bulk_handle))) -MERCURY_GEN_PROC(server_request_out_t, - ((int32_t)(ret))) -DECLARE_MARGO_RPC_HANDLER(server_request_rpc) +DECLARE_MARGO_RPC_HANDLER(server_pid_rpc) -/* chunk_read_request_rpc (server => server) - * - * request for chunk reads from another server */ +/* Chunk read request */ MERCURY_GEN_PROC(chunk_read_request_in_t, ((int32_t)(src_rank)) ((int32_t)(app_id)) @@ -62,9 +54,7 @@ MERCURY_GEN_PROC(chunk_read_request_out_t, ((int32_t)(ret))) DECLARE_MARGO_RPC_HANDLER(chunk_read_request_rpc) -/* chunk_read_response_rpc (server => server) - * - * response to remote chunk reads request */ +/* Chunk read response */ MERCURY_GEN_PROC(chunk_read_response_in_t, ((int32_t)(src_rank)) ((int32_t)(app_id)) @@ -77,6 +67,119 @@ MERCURY_GEN_PROC(chunk_read_response_out_t, ((int32_t)(ret))) DECLARE_MARGO_RPC_HANDLER(chunk_read_response_rpc) +/* Add file extents at owner */ +MERCURY_GEN_PROC(add_extents_in_t, + ((int32_t)(src_rank)) + ((int32_t)(gfid)) + ((int32_t)(num_extents)) + ((hg_bulk_t)(extents))) +MERCURY_GEN_PROC(add_extents_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(add_extents_rpc) + +/* Find file extent locations by querying owner */ +MERCURY_GEN_PROC(find_extents_in_t, + ((int32_t)(src_rank)) + ((int32_t)(gfid)) + ((int32_t)(num_extents)) + ((hg_bulk_t)(extents))) +MERCURY_GEN_PROC(find_extents_out_t, + ((int32_t)(num_locations)) + ((hg_bulk_t)(locations)) + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(find_extents_rpc) + +/* Get file size from owner */ +MERCURY_GEN_PROC(filesize_in_t, + ((int32_t)(gfid))) +MERCURY_GEN_PROC(filesize_out_t, + ((hg_size_t)(filesize)) + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(filesize_rpc) + +/* Laminate file at owner */ +MERCURY_GEN_PROC(laminate_in_t, + ((int32_t)(gfid))) +MERCURY_GEN_PROC(laminate_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(laminate_rpc) + +/* Get file metadata from owner */ +MERCURY_GEN_PROC(metaget_in_t, + ((int32_t)(gfid))) +MERCURY_GEN_PROC(metaget_out_t, + ((unifyfs_file_attr_t)(attr)) + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(metaget_rpc) + +/* Set file metadata at owner */ +MERCURY_GEN_PROC(metaset_in_t, + ((int32_t)(gfid)) + ((int32_t)(fileop)) + ((unifyfs_file_attr_t)(attr))) +MERCURY_GEN_PROC(metaset_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(metaset_rpc) + +/* Truncate file at owner */ +MERCURY_GEN_PROC(truncate_in_t, + ((int32_t)(gfid)) + ((hg_size_t)(filesize))) +MERCURY_GEN_PROC(truncate_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(truncate_rpc) + +/*---- Collective RPCs ----*/ + +/* Broadcast file extents to all servers */ +MERCURY_GEN_PROC(extent_bcast_in_t, + ((int32_t)(root)) + ((int32_t)(gfid)) + ((int32_t)(num_extents)) + ((hg_bulk_t)(extents))) +MERCURY_GEN_PROC(extent_bcast_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(extent_bcast_rpc) + +/* Broadcast file metadata to all servers */ +MERCURY_GEN_PROC(fileattr_bcast_in_t, + ((int32_t)(root)) + ((int32_t)(gfid)) + ((int32_t)(attrop)) + ((unifyfs_file_attr_t)(attr))) +MERCURY_GEN_PROC(fileattr_bcast_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(fileattr_bcast_rpc) + +/* Broadcast laminated file metadata to all servers */ +MERCURY_GEN_PROC(laminate_bcast_in_t, + ((int32_t)(root)) + ((int32_t)(gfid)) + ((int32_t)(num_extents)) + ((unifyfs_file_attr_t)(attr)) + ((hg_bulk_t)(extents))) +MERCURY_GEN_PROC(laminate_bcast_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(laminate_bcast_rpc) + +/* Broadcast truncation point to all servers */ +MERCURY_GEN_PROC(truncate_bcast_in_t, + ((int32_t)(root)) + ((int32_t)(gfid)) + ((hg_size_t)(filesize))) +MERCURY_GEN_PROC(truncate_bcast_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(truncate_bcast_rpc) + +/* Broadcast unlink to all servers */ +MERCURY_GEN_PROC(unlink_bcast_in_t, + ((int32_t)(root)) + ((int32_t)(gfid))) +MERCURY_GEN_PROC(unlink_bcast_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unlink_bcast_rpc) + + #ifdef __cplusplus } // extern "C" #endif diff --git a/common/src/unifyfs_shm.c b/common/src/unifyfs_shm.c index 3903d95af..61fed37c5 100644 --- a/common/src/unifyfs_shm.c +++ b/common/src/unifyfs_shm.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #include #include @@ -46,15 +47,20 @@ shm_context* unifyfs_shm_alloc(const char* name, size_t size) /* set size of shared memory region */ #ifdef HAVE_POSIX_FALLOCATE - ret = posix_fallocate(fd, 0, size); - if (ret != 0) { - /* failed to set size shared memory */ - errno = ret; - LOGERR("posix_fallocate failed for %s (%s)", - name, strerror(errno)); - close(fd); - return NULL; - } + do { /* this loop handles syscall interruption for large allocations */ + int try_count = 0; + ret = posix_fallocate(fd, 0, size); + if (ret != 0) { + /* failed to set size shared memory */ + try_count++; + if ((ret != EINTR) || (try_count >= 5)) { + LOGERR("posix_fallocate failed for %s (%s)", + name, strerror(ret)); + close(fd); + return NULL; + } + } + } while (ret != 0); #else errno = 0; ret = ftruncate(fd, size); diff --git a/common/src/unifyfs_shm.h b/common/src/unifyfs_shm.h index f4f9ca530..4c593f210 100644 --- a/common/src/unifyfs_shm.h +++ b/common/src/unifyfs_shm.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2018, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2018, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/configure.ac b/configure.ac index 666d92647..c43897743 100755 --- a/configure.ac +++ b/configure.ac @@ -103,6 +103,15 @@ AS_IF([test "x$enable_pmi" = "xyes"],[ AM_CONDITIONAL([USE_PMI2], [false]) ]) +# MDHIM support build option +AC_ARG_ENABLE([mdhim],[AS_HELP_STRING([--enable-mdhim],[Enable MDHIM build options.])]) +AS_IF([test "x$enable_mdhim" = "xyes"],[ + AM_CONDITIONAL([USE_MDHIM], [true]) + UNIFYFS_AC_LEVELDB +],[ + AM_CONDITIONAL([USE_MDHIM], [false]) +]) + AC_ARG_WITH(pkgconfigdir, [AS_HELP_STRING([--with-pkgconfigdir=DIR],[pkgconfig file in DIR @<:@LIBDIR/pkgconfig@:>@])], [pkgconfigdir=$withval], @@ -153,15 +162,13 @@ AS_IF([test "$have_C_mpi" != "yes"], [] ) -# look for leveldb library, sets LEVELDB_CFLAGS/LDFLAGS/LIBS -UNIFYFS_AC_LEVELDB - # look for gotcha library, sets GOTCHA_INCLUDE, GOTCHA_LIB UNIFYFS_AC_GOTCHA # look for spath library, sets SPATH_CFLAGS, SPATH_LDFLAGS, SPATH_LIBS UNIFYFS_AC_SPATH +# look for margo library, sets MARGO_CFLAGS, MARGO_LIBS UNIFYFS_AC_MARGO # openssl for md5 checksum @@ -214,7 +221,7 @@ CP_WRAPPERS+=",-wrap,chmod" CP_WRAPPERS+=",-wrap,fchmod" OLD_LIBS=$LIBS -LIBS+="-lrt" +LIBS+=" -lrt" AC_CHECK_FUNCS(lio_listio,[ CP_WRAPPERS+=",-wrap,lio_listio" ], []) diff --git a/docs/testing.rst b/docs/testing.rst index 0911cf0c2..18e01e94e 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -172,19 +172,24 @@ Here are some examples of libtap tests: ok(somefunc() == 42, "%s:%d somefunc() returns 42", __FILE__, __LINE__); - Also, note that ``errno`` is only set when an error occurs and is never set - back to ``0`` implicitly. + Also note that ``errno`` is only set when an error occurs and is never set + back to ``0`` implicitly by the system. When testing for a failure and using ``errno`` as part of the test, - resetting ``errno`` after the test will prevent subsequent tests from - appearing to error. + setting ``errno = 0`` before the test will ensure a previous test error + will not affect the current test. In the following example, we also + assign ``errno`` to another variable ``err`` for use in constructing the + test message. This is needed because the ``ok()`` macro may use system + calls that set ``errno``. .. code-block:: C - :emphasize-lines: 4 - ok(somefunc() == -1 && errno == ENOTTY, - "%s:%d somefunc() should fail (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); + int err, rc; errno = 0; + rc = systemcall(); + err = errno; + ok(rc == -1 && err == ENOTTY, + "%s:%d systemcall() should fail (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); ------------ diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index ede17bfd4..33c40e54e 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -18,6 +18,9 @@ if HAVE_LD_WRAP sysio-dir-static \ sysio-stat-static \ sysio-cp-static \ + sysio-truncate-static \ + sysio-unlink-static \ + sysio-open-static \ app-mpiio-static \ app-btio-static \ app-tileio-static \ @@ -42,6 +45,9 @@ if HAVE_GOTCHA sysio-dir-gotcha \ sysio-stat-gotcha \ sysio-cp-gotcha \ + sysio-truncate-gotcha \ + sysio-unlink-gotcha \ + sysio-open-gotcha \ app-mpiio-gotcha \ app-btio-gotcha \ app-tileio-gotcha \ @@ -174,6 +180,36 @@ sysio_cp_static_CPPFLAGS = $(test_cppflags) sysio_cp_static_LDADD = $(test_static_ldadd) sysio_cp_static_LDFLAGS = $(test_static_ldflags) +sysio_truncate_gotcha_SOURCES = sysio-truncate.c +sysio_truncate_gotcha_CPPFLAGS = $(test_cppflags) +sysio_truncate_gotcha_LDADD = $(test_gotcha_ldadd) +sysio_truncate_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +sysio_truncate_static_SOURCES = sysio-truncate.c +sysio_truncate_static_CPPFLAGS = $(test_cppflags) +sysio_truncate_static_LDADD = $(test_static_ldadd) +sysio_truncate_static_LDFLAGS = $(test_static_ldflags) + +sysio_unlink_gotcha_SOURCES = sysio-unlink.c +sysio_unlink_gotcha_CPPFLAGS = $(test_cppflags) +sysio_unlink_gotcha_LDADD = $(test_gotcha_ldadd) +sysio_unlink_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +sysio_unlink_static_SOURCES = sysio-unlink.c +sysio_unlink_static_CPPFLAGS = $(test_cppflags) +sysio_unlink_static_LDADD = $(test_static_ldadd) +sysio_unlink_static_LDFLAGS = $(test_static_ldflags) + +sysio_open_gotcha_SOURCES = sysio-open.c +sysio_open_gotcha_CPPFLAGS = $(test_cppflags) +sysio_open_gotcha_LDADD = $(test_gotcha_ldadd) +sysio_open_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +sysio_open_static_SOURCES = sysio-open.c +sysio_open_static_CPPFLAGS = $(test_cppflags) +sysio_open_static_LDADD = $(test_static_ldadd) +sysio_open_static_LDFLAGS = $(test_static_ldflags) + cr_posix_SOURCES = checkpoint-restart.c cr_posix_CPPFLAGS = $(test_posix_cppflags) cr_posix_LDADD = $(test_posix_ldadd) @@ -219,17 +255,17 @@ write_static_CPPFLAGS = $(test_cppflags) write_static_LDADD = $(test_static_ldadd) write_static_LDFLAGS = $(test_static_ldflags) -writeread_posix_SOURCES = writeread.c +writeread_posix_SOURCES = writeread.c testutil.c writeread_posix_CPPFLAGS = $(test_posix_cppflags) writeread_posix_LDADD = $(test_posix_ldadd) writeread_posix_LDFLAGS = $(test_posix_ldflags) -writeread_gotcha_SOURCES = writeread.c +writeread_gotcha_SOURCES = writeread.c testutil.c writeread_gotcha_CPPFLAGS = $(test_cppflags) writeread_gotcha_LDADD = $(test_gotcha_ldadd) writeread_gotcha_LDFLAGS = $(test_gotcha_ldflags) -writeread_static_SOURCES = writeread.c +writeread_static_SOURCES = writeread.c testutil.c writeread_static_CPPFLAGS = $(test_cppflags) writeread_static_LDADD = $(test_static_ldadd) writeread_static_LDFLAGS = $(test_static_ldflags) diff --git a/examples/src/app-btio.c b/examples/src/app-btio.c index 57a3a39da..a238f38da 100644 --- a/examples/src/app-btio.c +++ b/examples/src/app-btio.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/examples/src/app-hdf5-create.c b/examples/src/app-hdf5-create.c index 2c21aba5e..86817b998 100644 --- a/examples/src/app-hdf5-create.c +++ b/examples/src/app-hdf5-create.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + /* * Copyright by The HDF Group. * Copyright by the Board of Trustees of the University of Illinois. diff --git a/examples/src/app-hdf5-writeread.c b/examples/src/app-hdf5-writeread.c index 952182e60..f2c371bd8 100644 --- a/examples/src/app-hdf5-writeread.c +++ b/examples/src/app-hdf5-writeread.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + /* * Copyright by The HDF Group. * Copyright by the Board of Trustees of the University of Illinois. diff --git a/examples/src/app-mpiio.c b/examples/src/app-mpiio.c index 33efbd8c1..8444134f9 100644 --- a/examples/src/app-mpiio.c +++ b/examples/src/app-mpiio.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #include #include diff --git a/examples/src/app-tileio.c b/examples/src/app-tileio.c index 681624be1..53640c140 100644 --- a/examples/src/app-tileio.c +++ b/examples/src/app-tileio.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/examples/src/checkpoint-restart.c b/examples/src/checkpoint-restart.c index 3c4ef7206..7099144fd 100644 --- a/examples/src/checkpoint-restart.c +++ b/examples/src/checkpoint-restart.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/examples/src/chmod.c b/examples/src/chmod.c index af6ad85d4..e8d0dd7c6 100644 --- a/examples/src/chmod.c +++ b/examples/src/chmod.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -10,7 +10,9 @@ * This is the license for UnifyFS. * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. - * + */ + +/* * Test chmod() and fchmod() * * Test description: @@ -23,6 +25,7 @@ * 7. Laminate it using fchmod() * 8. Check file is laminated after */ + #include "testutil.h" int do_test(test_cfg* cfg) @@ -32,7 +35,9 @@ int do_test(test_cfg* cfg) int fd; file = mktemp_cmd(cfg, "/unifyfs"); - assert(file); + if (NULL == file) { + return ENOMEM; + } test_print(cfg, "Create empty file %s", file); test_print(cfg, "Before lamination stat() is:"); @@ -87,9 +92,14 @@ int main(int argc, char* argv[]) fflush(NULL); return rc; } - do_test(cfg); + + rc = do_test(cfg); + if (rc) { + test_print(cfg, "ERROR - Test %s failed! rc=%d", argv[0], rc); + fflush(NULL); + } test_fini(cfg); - return 0; + return rc; } diff --git a/examples/src/multi-write.c b/examples/src/multi-write.c index 4c95cdb18..369f82499 100644 --- a/examples/src/multi-write.c +++ b/examples/src/multi-write.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -10,7 +10,9 @@ * This is the license for UnifyFS. * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. - * + */ + +/* * Test doing lots of writes to many open files and verify the data is written * correctly. This can be used to exercise bugs. * @@ -22,6 +24,7 @@ * 4. Read them back, and verify the portions that did get written match the * data from bigbuf[]. */ + #include #include #include diff --git a/examples/src/read-data.c b/examples/src/read-data.c index fa3c962c9..af0e66f61 100644 --- a/examples/src/read-data.c +++ b/examples/src/read-data.c @@ -2,7 +2,7 @@ * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + /* read-data: This program aims to test reading data from a file with arbitrary * offset and length. This program can run either interactively (only * specifying filename) or non-interactively (specifying filename with offset diff --git a/examples/src/read.c b/examples/src/read.c index ae5630ad3..0e94719e8 100644 --- a/examples/src/read.c +++ b/examples/src/read.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/examples/src/simul.c b/examples/src/simul.c index 49e0cf5a2..3b1c2fa85 100644 --- a/examples/src/simul.c +++ b/examples/src/simul.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/examples/src/size.c b/examples/src/size.c index e57db50aa..5cd677442 100644 --- a/examples/src/size.c +++ b/examples/src/size.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -10,7 +10,9 @@ * This is the license for UnifyFS. * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. - * + */ + +/* * Test file size functions * * Test description: @@ -19,6 +21,7 @@ * 3. Have the last rank laminate the file. * 4. Check the file size again. It should be the real, laminated, file size. */ + #include "testutil.h" int do_test(test_cfg* cfg) @@ -27,18 +30,19 @@ int do_test(test_cfg* cfg) int rank = cfg->rank; file = mktemp_cmd(cfg, "/unifyfs"); - assert(file); - - test_print(cfg, "I'm writing 1KB to %s at my offset at %ld", - file, rank * 1024); + if (NULL == file) { + return ENOMEM; + } + test_print(cfg, "I'm writing 1 KiB to %s at my offset at %ld", + file, rank * 1024); dd_cmd(cfg, "/dev/zero", file, 1024, 1, rank); test_print(cfg, "Stating the file"); stat_cmd(cfg, file); + test_print(cfg, "After writing, file size is %lu, apparent-size %lu", - du_cmd(cfg, file, 0), - du_cmd(cfg, file, 1)); + du_cmd(cfg, file, 0), du_cmd(cfg, file, 1)); /* sync our extents */ sync_cmd(cfg, file); @@ -51,9 +55,9 @@ int do_test(test_cfg* cfg) /* laminate by removing write bits */ chmod(file, 0444); /* set to read-only */ - test_print(cfg, "After lamination, file size is %lu", - du_cmd(cfg, file, 0)); + du_cmd(cfg, file, 0)); + test_print(cfg, "Stating the file"); stat_cmd(cfg, file); } @@ -72,9 +76,14 @@ int main(int argc, char* argv[]) fflush(NULL); return rc; } - do_test(cfg); + + rc = do_test(cfg); + if (rc) { + test_print(cfg, "ERROR - Test %s failed! rc=%d", argv[0], rc); + fflush(NULL); + } test_fini(cfg); - return 0; + return rc; } diff --git a/examples/src/sysio-cp.c b/examples/src/sysio-cp.c index 62693fed1..0c04bf1ec 100644 --- a/examples/src/sysio-cp.c +++ b/examples/src/sysio-cp.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #include #include diff --git a/examples/src/sysio-dir.c b/examples/src/sysio-dir.c index 868870821..5b37bf26d 100644 --- a/examples/src/sysio-dir.c +++ b/examples/src/sysio-dir.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #include #include diff --git a/examples/src/sysio-open.c b/examples/src/sysio-open.c new file mode 100644 index 000000000..f23d3da54 --- /dev/null +++ b/examples/src/sysio-open.c @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "testlib.h" + +static int fd; /* target file descriptor */ +static int standard; /* not mounting unifyfs when set */ + +static int rank; +static int total_ranks; + +static int create_rank; +static int open_rank; +static int debug; /* pause for attaching debugger */ +static int unmount; /* unmount unifyfs after running the test */ +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ +static char* filename = "testfile"; /* testfile name under mountpoint */ +static char targetfile[NAME_MAX]; /* target file name */ + +static struct option long_opts[] = { + { "create", 1, 0, 'c' }, + { "debug", 0, 0, 'd' }, + { "filename", 1, 0, 'f' }, + { "help", 0, 0, 'h' }, + { "mount", 1, 0, 'm' }, + { "open", 1, 0, 'o' }, + { "standard", 0, 0, 's' }, + { "unmount", 0, 0, 'u' }, + { 0, 0, 0, 0}, +}; + +static char* short_opts = "c:df:hm:o:su"; + +static const char* usage_str = + "\n" + "Usage: %s [options...]\n" + "\n" + "Available options:\n" + " -c, --create= create the file from \n" + " (default: 0)\n" + " -d, --debug pause before running test\n" + " (handy for attaching in debugger)\n" + " -f, --filename= target file name under mountpoint\n" + " (default: testfile)\n" + " -h, --help help message\n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" + " -o, --open= open file from after create\n" + " (default: 0)\n" + " -s, --standard do not use unifyfs but run standard I/O\n" + " -u, --unmount unmount the filesystem after test\n" + "\n"; + +static char* program; + +static void print_usage(void) +{ + test_print_once(rank, usage_str, program); + exit(0); +} + +int main(int argc, char** argv) +{ + int ret = 0; + int ch = 0; + int optidx = 2; + + program = basename(strdup(argv[0])); + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &total_ranks); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + while ((ch = getopt_long(argc, argv, + short_opts, long_opts, &optidx)) >= 0) { + switch (ch) { + case 'c': + create_rank = atoi(optarg); + break; + + case 'f': + filename = strdup(optarg); + break; + + case 'd': + debug = 1; + break; + + case 'm': + mountpoint = strdup(optarg); + break; + + case 'o': + open_rank = atoi(optarg); + break; + + case 's': + standard = 1; + break; + + case 'u': + unmount = 1; + break; + + case 'h': + default: + print_usage(); + break; + } + } + + if (static_linked(program) && standard) { + test_print_once(rank, "--standard, -s option only works when " + "dynamically linked."); + exit(-1); + } + + sprintf(targetfile, "%s/%s", mountpoint, filename); + + if (debug) { + test_pause(rank, "Attempting to mount"); + } + + if (!standard) { + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); + if (ret) { + test_print(rank, "unifyfs_mount failed (return = %d)", ret); + exit(-1); + } + } + + if ((create_rank < 0 || create_rank > total_ranks - 1) || + (open_rank < 0 || open_rank > total_ranks - 1)) { + test_print(rank, "please specify valid rank\n"); + exit(-1); + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == open_rank) { + fd = open(targetfile, O_CREAT|O_RDWR|O_TRUNC, 0600); + if (fd < 0) { + test_print(rank, "open failed (%d: %s)\n", + errno, strerror(errno)); + exit(-1); + } + + test_print(rank, "created file %s successfully\n", targetfile); + + close(fd); + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == open_rank) { + fd = open(targetfile, O_RDWR); + if (fd < 0) { + test_print(rank, "open failed (%d: %s)\n", + errno, strerror(errno)); + exit(-1); + } + + test_print(rank, "opened file %s successfully\n", targetfile); + + close(fd); + } + + if (!standard && unmount) { + unifyfs_unmount(); + } + + MPI_Finalize(); + + return ret; +} + diff --git a/examples/src/sysio-read.c b/examples/src/sysio-read.c index 1cacbf6f3..fedbc431a 100644 --- a/examples/src/sysio-read.c +++ b/examples/src/sysio-read.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #include #include diff --git a/examples/src/sysio-stat.c b/examples/src/sysio-stat.c index 7f0159c6f..14eee6055 100644 --- a/examples/src/sysio-stat.c +++ b/examples/src/sysio-stat.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #include #include @@ -40,7 +41,8 @@ static int debug; static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ static char* filename = "/unifyfs"; -static int unmount; /* unmount unifyfs after running the test */ +static int unmount; /* unmount unifyfs after running the test */ +static int testrank = -1; /* if negative, execute from all ranks */ #define FP_SPECIAL 1 @@ -108,15 +110,30 @@ static void dump_stat(int rank, const struct stat* sb) printf("Last status change: %s\n\n", ctime(&sb->st_ctime)); } +static void do_stat(int rank) +{ + int ret = 0; + struct stat sb; + + ret = stat(filename, &sb); + if (ret < 0) { + test_print(rank, "stat failed on \"%s\" (%d:%s)", + filename, errno, strerror(errno)); + } else { + dump_stat(rank, &sb); + } +} + static struct option const long_opts[] = { { "debug", 0, 0, 'd' }, { "help", 0, 0, 'h' }, { "mount", 1, 0, 'm' }, { "unmount", 0, 0, 'u' }, + { "rank", 1, 0, 'r' }, { 0, 0, 0, 0}, }; -static char* short_opts = "dhm:u"; +static char* short_opts = "dhm:ur:"; static const char* usage_str = "\n" @@ -129,6 +146,7 @@ static const char* usage_str = " -m, --mount= use for unifyfs\n" " (default: /unifyfs)\n" " -u, --unmount unmount the filesystem after test\n" + " -r, --rank= only test on rank \n" "\n"; static char* program; @@ -167,6 +185,10 @@ int main(int argc, char** argv) unmount = 1; break; + case 'r': + testrank = atoi(optarg); + break; + case 'h': default: print_usage(); @@ -178,6 +200,11 @@ int main(int argc, char** argv) print_usage(); } + if (testrank > total_ranks - 1) { + test_print(0, "Please specify a valid rank number."); + print_usage(); + } + filename = argv[optind]; if (debug) { @@ -192,11 +219,21 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); - ret = stat(filename, &sb); - if (ret < 0) { - test_print(rank, "stat failed on \"%s\"", filename); + if (testrank < 0) { /* execute from all ranks in order */ + int i = 0; + + for (i = 0; i < total_ranks; i++) { + if (rank == i) { + do_stat(rank); + } + + MPI_Barrier(MPI_COMM_WORLD); + } + } else { - dump_stat(rank, &sb); + if (rank == testrank) { + do_stat(rank); + } } MPI_Barrier(MPI_COMM_WORLD); diff --git a/examples/src/sysio-truncate.c b/examples/src/sysio-truncate.c new file mode 100644 index 000000000..782fc4d3d --- /dev/null +++ b/examples/src/sysio-truncate.c @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "testlib.h" + +static int rank; +static int total_ranks; +static int debug; + +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ +static char* filename = "/unifyfs"; +static int unmount; /* unmount unifyfs after running the test */ +static int testrank; +static off_t targetlen; + +#define FP_SPECIAL 1 + +static void dump_stat(int rank, const struct stat* sb) +{ + printf("## [RANK %d] %s\n", rank, filename); + printf("File type: "); + + switch (sb->st_mode & S_IFMT) { + case S_IFREG: + printf("regular file\n"); + break; + case S_IFDIR: + printf("directory\n"); + break; + case S_IFCHR: + printf("character device\n"); + break; + case S_IFBLK: + printf("block device\n"); + break; + case S_IFLNK: + printf("symbolic (soft) link\n"); + break; + case S_IFIFO: + printf("FIFO or pipe\n"); + break; + case S_IFSOCK: + printf("socket\n"); + break; + default: + printf("unknown file type?\n"); + break; + } + + printf("Device containing i-node: major=%ld minor=%ld\n", + (long) major(sb->st_dev), (long) minor(sb->st_dev)); + + printf("I-node number: %ld\n", (long) sb->st_ino); + + printf("Mode: %lo\n", + (unsigned long) sb->st_mode); + + if (sb->st_mode & (S_ISUID | S_ISGID | S_ISVTX)) { + printf(" special bits set: %s%s%s\n", + (sb->st_mode & S_ISUID) ? "set-UID " : "", + (sb->st_mode & S_ISGID) ? "set-GID " : "", + (sb->st_mode & S_ISVTX) ? "sticky " : ""); + } + + printf("Number of (hard) links: %ld\n", (long) sb->st_nlink); + + printf("Ownership: UID=%ld GID=%ld\n", + (long) sb->st_uid, (long) sb->st_gid); + + if (S_ISCHR(sb->st_mode) || S_ISBLK(sb->st_mode)) { + printf("Device number (st_rdev): major=%ld; minor=%ld\n", + (long) major(sb->st_rdev), (long) minor(sb->st_rdev)); + } + + printf("File size: %lld bytes\n", (long long) sb->st_size); + printf("Optimal I/O block size: %ld bytes\n", (long) sb->st_blksize); + printf("512B blocks allocated: %lld\n", (long long) sb->st_blocks); + + printf("Last file access: %s", ctime(&sb->st_atime)); + printf("Last file modification: %s", ctime(&sb->st_mtime)); + printf("Last status change: %s\n\n", ctime(&sb->st_ctime)); +} + +static struct option long_opts[] = { + { "debug", 0, 0, 'd' }, + { "help", 0, 0, 'h' }, + { "length", 1, 0, 'l' }, + { "mount", 1, 0, 'm' }, + { "unmount", 0, 0, 'u' }, + { "rank", 1, 0, 'r' }, + { 0, 0, 0, 0}, +}; + +static char* short_opts = "dhl:m:ur:"; + +static const char* usage_str = + "\n" + "Usage: %s [options...] \n" + "\n" + "Available options:\n" + " -d, --debug pause before running test\n" + " (handy for attaching in debugger)\n" + " -h, --help help message\n" + " -l, --length= truncate the file to \n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" + " -u, --unmount unmount the filesystem after test\n" + " -r, --rank= test on rank (default: 0)\n" + "\n"; + +static char* program; + +static void print_usage(void) +{ + test_print_once(rank, usage_str, program); + exit(0); +} + +int main(int argc, char** argv) +{ + int ret = 0; + int ch = 0; + int optidx = 0; + struct stat sb = { 0, }; + + program = basename(strdup(argv[0])); + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &total_ranks); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + while ((ch = getopt_long(argc, argv, + short_opts, long_opts, &optidx)) >= 0) { + switch (ch) { + case 'd': + debug = 1; + break; + + case 'l': + targetlen = strtoull(optarg, NULL, 0); + break; + + case 'm': + mountpoint = strdup(optarg); + break; + + case 'u': + unmount = 1; + break; + + case 'r': + testrank = atoi(optarg); + break; + + case 'h': + default: + print_usage(); + break; + } + } + + if (argc - optind != 1) { + print_usage(); + } + + if (testrank > total_ranks - 1) { + test_print(0, "Please specify a valid rank number."); + print_usage(); + } + + filename = argv[optind]; + + if (debug) { + test_pause(rank, "Attempting to mount"); + } + + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); + if (ret) { + test_print(rank, "unifyfs_mount failed (return = %d)", ret); + exit(-1); + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == testrank) { + /* try stat the file before truncate */ + ret = stat(filename, &sb); + if (ret < 0) { + test_print(rank, "stat failed on \"%s\"", filename); + } else { + test_print(rank, "## stat before truncate to %llu\n", + (unsigned long long) targetlen); + dump_stat(rank, &sb); + } + + ret = truncate(filename, targetlen); + if (ret < 0) { + test_print(rank, "truncate failed on \"%s\": (errno=%d: %s)", + filename, errno, strerror(errno)); + } + + /* try stat the file again after truncate */ + ret = stat(filename, &sb); + if (ret < 0) { + test_print(rank, "stat failed on \"%s\"", filename); + } else { + test_print(rank, "## stat after truncate to %llu\n", + (unsigned long long) targetlen); + dump_stat(rank, &sb); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (unmount) { + unifyfs_unmount(); + } + + MPI_Finalize(); + + return ret; +} + diff --git a/examples/src/sysio-unlink.c b/examples/src/sysio-unlink.c new file mode 100644 index 000000000..4de9a7d4d --- /dev/null +++ b/examples/src/sysio-unlink.c @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "testlib.h" + +static int rank; +static int total_ranks; +static int debug; + +static char* mountpoint = "/unifyfs"; /* unifyfs mountpoint */ +static char* filename = "/unifyfs"; +static int unmount; /* unmount unifyfs after running the test */ +static int testrank; + +static struct option long_opts[] = { + { "debug", 0, 0, 'd' }, + { "help", 0, 0, 'h' }, + { "mount", 1, 0, 'm' }, + { "unmount", 0, 0, 'u' }, + { "rank", 1, 0, 'r' }, + { 0, 0, 0, 0}, +}; + +static char* short_opts = "dhm:ur:"; + +static const char* usage_str = + "\n" + "Usage: %s [options...] \n" + "\n" + "Available options:\n" + " -d, --debug pause before running test\n" + " (handy for attaching in debugger)\n" + " -h, --help help message\n" + " -m, --mount= use for unifyfs\n" + " (default: /unifyfs)\n" + " -u, --unmount unmount the filesystem after test\n" + " -r, --rank= test on rank (default: 0)\n" + "\n"; + +static char* program; + +static void print_usage(void) +{ + test_print_once(rank, usage_str, program); + exit(0); +} + +int main(int argc, char** argv) +{ + int ret = 0; + int ch = 0; + int optidx = 0; + + program = basename(strdup(argv[0])); + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &total_ranks); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + while ((ch = getopt_long(argc, argv, + short_opts, long_opts, &optidx)) >= 0) { + switch (ch) { + case 'd': + debug = 1; + break; + + case 'm': + mountpoint = strdup(optarg); + break; + + case 'u': + unmount = 1; + break; + + case 'r': + testrank = atoi(optarg); + break; + + case 'h': + default: + print_usage(); + break; + } + } + + if (argc - optind != 1) { + print_usage(); + } + + if (testrank > total_ranks - 1) { + test_print(0, "Please specify a valid rank number."); + print_usage(); + } + + filename = argv[optind]; + + if (debug) { + test_pause(rank, "Attempting to mount"); + } + + ret = unifyfs_mount(mountpoint, rank, total_ranks, 0); + if (ret) { + test_print(rank, "unifyfs_mount failed (return = %d)", ret); + exit(-1); + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == testrank) { + ret = unlink(filename); + if (ret < 0) { + test_print(rank, "unlink failed on \"%s\" (%s)", + filename, strerror(errno)); + } + } + +out: + MPI_Barrier(MPI_COMM_WORLD); + + if (unmount) { + unifyfs_unmount(); + } + + MPI_Finalize(); + + return ret; +} + diff --git a/examples/src/sysio-write.c b/examples/src/sysio-write.c index 2b5dd31b1..133e4a3eb 100644 --- a/examples/src/sysio-write.c +++ b/examples/src/sysio-write.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #include #include @@ -331,10 +332,22 @@ int main(int argc, char** argv) sprintf(&targetfile[strlen(targetfile)], "-%d", rank); } - fd = open(targetfile, O_RDWR | O_CREAT | O_TRUNC, 0600); - if (fd < 0) { - test_print(rank, "open failed"); - exit(-1); + if (rank == 0) { + fd = open(targetfile, O_RDWR | O_CREAT | O_TRUNC, 0600); + if (fd < 0) { + test_print(rank, "open failed"); + exit(-1); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (rank != 0) { + fd = open(targetfile, O_RDWR, 0600); + if (fd < 0) { + test_print(rank, "open failed"); + exit(-1); + } } ret = do_write(); diff --git a/examples/src/sysio-writeread.c b/examples/src/sysio-writeread.c index 1d7e815cd..9285b0f02 100644 --- a/examples/src/sysio-writeread.c +++ b/examples/src/sysio-writeread.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/examples/src/sysio-writeread2.c b/examples/src/sysio-writeread2.c index cf03fce86..76cad03a5 100644 --- a/examples/src/sysio-writeread2.c +++ b/examples/src/sysio-writeread2.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/examples/src/testlib.h b/examples/src/testlib.h index 06c87c6ae..586cd6ab9 100644 --- a/examples/src/testlib.h +++ b/examples/src/testlib.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #ifndef __TESTLIB_H #define __TESTLIB_H diff --git a/examples/src/testutil.c b/examples/src/testutil.c index 50b21219b..048876505 100644 --- a/examples/src/testutil.c +++ b/examples/src/testutil.c @@ -1,15 +1,19 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include /* device major() and minor() macros */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include "testutil.h" /* Clone of the apsrintf(). See the standard asprintf() man page for details */ @@ -96,7 +100,9 @@ int dd_cmd(test_cfg* cfg, char* infile, char* outfile, unsigned long bs, return errno; } buf = calloc(sizeof(*buf), bs); - assert(buf); + if (NULL == buf) { + return errno; + } rc = fseek(outfp, seek * bs, SEEK_SET); if (rc) { @@ -270,13 +276,15 @@ int sync_cmd(test_cfg* cfg, char* filename) int stat_cmd(test_cfg* cfg, char* filename) { struct stat sb; + time_t timestamp; int rc; const char* typestr; char* tmp; + char datestr[32]; rc = stat(filename, &sb); if (rc) { - test_print(cfg, "Error stating %s: %s", filename, strerror(rc)); + test_print(cfg, "ERROR: stat(%s) - %s", filename, strerror(rc)); return rc; } @@ -312,10 +320,11 @@ int stat_cmd(test_cfg* cfg, char* filename) test_print(cfg, "%-26s%s", tmp, typestr); free(tmp); - test_print(cfg, "Device containing i-node: major=%ld minor=%ld", + test_print(cfg, "Device containing i-node: major=%ld minor=%ld", (long) major(sb.st_dev), (long) minor(sb.st_dev)); - test_print(cfg, "I-node number: %ld", (long) sb.st_ino); + test_print(cfg, "I-node number: %lu", + (unsigned long) sb.st_ino); test_print(cfg, "Mode: %lo", (unsigned long) sb.st_mode); @@ -327,7 +336,8 @@ int stat_cmd(test_cfg* cfg, char* filename) (sb.st_mode & S_ISVTX) ? "sticky " : ""); } - test_print(cfg, "Number of (hard) links: %ld", (long) sb.st_nlink); + test_print(cfg, "Number of (hard) links: %lu", + (unsigned long) sb.st_nlink); test_print(cfg, "Ownership: UID=%ld GID=%ld", (long) sb.st_uid, (long) sb.st_gid); @@ -337,12 +347,25 @@ int stat_cmd(test_cfg* cfg, char* filename) (long) major(sb.st_rdev), (long) minor(sb.st_rdev)); } - test_print(cfg, "File size: %lld bytes", - (long long) sb.st_size); - test_print(cfg, "Optimal I/O block size: %ld bytes", - (long) sb.st_blksize); - test_print(cfg, "Blocks allocated: %lld", (long long) sb.st_blocks); - test_print(cfg, "Last file access: %s", ctime(&sb.st_atime)); - test_print(cfg, "Last file modification: %s", ctime(&sb.st_mtime)); - test_print(cfg, "Last status change: %s", ctime(&sb.st_ctime)); + test_print(cfg, "File size: %llu bytes", + (unsigned long long) sb.st_size); + test_print(cfg, "Optimal I/O block size: %lu bytes", + (unsigned long) sb.st_blksize); + test_print(cfg, "Blocks allocated: %llu", + (unsigned long long) sb.st_blocks); + + memset(datestr, 0, sizeof(datestr)); + timestamp = sb.st_atime; + ctime_r(×tamp, datestr); + test_print(cfg, "Last file access: %s", datestr); + + memset(datestr, 0, sizeof(datestr)); + timestamp = sb.st_mtime; + ctime_r(×tamp, datestr); + test_print(cfg, "Last file modification: %s", datestr); + + memset(datestr, 0, sizeof(datestr)); + timestamp = sb.st_ctime; + ctime_r(×tamp, datestr); + test_print(cfg, "Last status change: %s", datestr); } diff --git a/examples/src/testutil.h b/examples/src/testutil.h index b646c4d50..daa981e1d 100644 --- a/examples/src/testutil.h +++ b/examples/src/testutil.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -32,6 +33,7 @@ #include #include #include +#include #include #include @@ -124,6 +126,8 @@ typedef struct { int io_pattern; /* N1 or NN */ int io_check; /* use lipsum to verify data */ int io_shuffle; /* read and write different extents */ + int pre_wr_trunc; /* truncate file before writing */ + int post_wr_trunc; /* truncate file after writing */ int use_aio; /* use asynchronous IO */ int use_lio; /* use lio_listio instead of read/write */ int use_mapio; /* use mmap instead of read/write */ @@ -136,6 +140,7 @@ typedef struct { uint64_t n_blocks; /* number of I/O blocks */ uint64_t block_sz; /* IO block size (multiple of chunk_sz) */ uint64_t chunk_sz; /* per-IO-op size */ + off_t trunc_size; /* file size for truncate */ /* target file info */ char* filename; @@ -205,6 +210,8 @@ void test_config_print(test_cfg* cfg) fprintf(stderr, "\t io_pattern = %s\n", io_pattern_str(cfg->io_pattern)); fprintf(stderr, "\t io_check = %d\n", cfg->io_check); fprintf(stderr, "\t io_shuffle = %d\n", cfg->io_shuffle); + fprintf(stderr, "\t pre_trunc = %d\n", cfg->pre_wr_trunc); + fprintf(stderr, "\t post_trunc = %d\n", cfg->post_wr_trunc); fprintf(stderr, "\t use_aio = %d\n", cfg->use_aio); fprintf(stderr, "\t use_lio = %d\n", cfg->use_lio); fprintf(stderr, "\t use_mapio = %d\n", cfg->use_mapio); @@ -217,6 +224,7 @@ void test_config_print(test_cfg* cfg) fprintf(stderr, "\t n_blocks = %" PRIu64 "\n", cfg->n_blocks); fprintf(stderr, "\t block_sz = %" PRIu64 "\n", cfg->block_sz); fprintf(stderr, "\t chunk_sz = %" PRIu64 "\n", cfg->chunk_sz); + fprintf(stderr, "\t truncate_sz = %lu\n", (unsigned long)cfg->trunc_size); fprintf(stderr, "\n-- Target File --\n"); fprintf(stderr, "\t filename = %s\n", cfg->filename); @@ -458,7 +466,7 @@ int test_is_static(const char* program) // common options for all tests -static const char* test_short_opts = "a:Ab:c:df:hkLm:MNn:p:PSUvVx"; +static const char* test_short_opts = "a:Ab:c:df:hkLm:MNn:p:PSt:T:UvVx"; static const struct option test_long_opts[] = { { "appid", 1, 0, 'a' }, @@ -476,6 +484,8 @@ static const struct option test_long_opts[] = { { "mapio", 0, 0, 'N' }, { "pattern", 1, 0, 'p' }, { "prdwr", 0, 0, 'P' }, + { "pre-truncate", 1, 0, 't' }, + { "post-truncate", 1, 0, 'T' }, { "stdio", 0, 0, 'S' }, { "disable-unifyfs", 0, 0, 'U' }, { "verbose", 0, 0, 'v' }, @@ -519,6 +529,10 @@ static const char* test_usage_str = " (default: off)\n" " -S, --stdio use fread|fwrite instead of read|write\n" " (default: off)\n" + " -t, --pre-truncate= truncate file to size (B) before writing\n" + " (default: off)\n" + " -T, --post-truncate= truncate file to size (B) after writing\n" + " (default: off)\n" " -U, --disable-unifyfs do not use UnifyFS\n" " (default: enable UnifyFS)\n" " -v, --verbose print verbose information\n" @@ -610,6 +624,16 @@ int test_process_argv(test_cfg* cfg, cfg->use_stdio = 1; break; + case 't': + cfg->pre_wr_trunc = 1; + cfg->trunc_size = (off_t) strtoul(optarg, NULL, 0); + break; + + case 'T': + cfg->post_wr_trunc = 1; + cfg->trunc_size = (off_t) strtoul(optarg, NULL, 0); + break; + case 'U': cfg->use_unifyfs = 0; break; @@ -669,6 +693,19 @@ int test_process_argv(test_cfg* cfg, } // exhaustive check of incompatible I/O modes + if (cfg->pre_wr_trunc || cfg->post_wr_trunc) { + if (cfg->pre_wr_trunc && cfg->post_wr_trunc) { + test_print_once(cfg, + "USAGE ERROR: choose --pre-truncate or --post-truncate"); + exit(-1); + } + if (cfg->use_mapio || cfg->use_stdio) { + test_print_once(cfg, + "USAGE ERROR: pre/post-truncate incompatible with " + "[--mapio, --stdio]"); + exit(-1); + } + } if (cfg->use_aio && (cfg->use_mapio || cfg->use_mpiio || cfg->use_prdwr || cfg->use_stdio || cfg->use_vecio)) { diff --git a/examples/src/testutil_rdwr.h b/examples/src/testutil_rdwr.h index 367f8d2c8..2f5d5ec79 100644 --- a/examples/src/testutil_rdwr.h +++ b/examples/src/testutil_rdwr.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -17,6 +17,15 @@ #include "testutil.h" + +static inline +void test_print_aiocb(test_cfg* cfg, struct aiocb* cbp) +{ + test_print(cfg, "aiocb(fd=%d, op=%d, count=%zu, offset=%zu, buf=%p)", + cbp->aio_fildes, cbp->aio_lio_opcode, cbp->aio_nbytes, + cbp->aio_offset, cbp->aio_buf); +} + /* -------- Write Helper Methods -------- */ static inline @@ -107,6 +116,7 @@ int issue_write_req_batch(test_cfg* cfg, size_t n_reqs, struct aiocb* reqs) struct aiocb* lio_vec[n_reqs]; for (i = 0; i < n_reqs; i++) { lio_vec[i] = reqs + i; + //test_print_aiocb(cfg, lio_vec[i]); } lio_mode = LIO_WAIT; if (cfg->use_aio) { @@ -187,6 +197,29 @@ int wait_write_req_batch(test_cfg* cfg, size_t n_reqs, struct aiocb* reqs) return ret; } +static inline +int write_truncate(test_cfg* cfg) +{ + int rc = 0; + + if (cfg->use_mpiio) { + MPI_Offset mpi_off = (MPI_Offset) cfg->trunc_size; + MPI_CHECK(cfg, (MPI_File_set_size(cfg->mpifh, mpi_off))); + } else { + if (cfg->rank == 0 || cfg->io_pattern == IO_PATTERN_NN) { + if (-1 != cfg->fd) { // ftruncate(2) + rc = ftruncate(cfg->fd, cfg->trunc_size); + if (-1 == rc) { + test_print(cfg, "ftruncate() failed"); + return -1; + } + } + } + } + + return rc; +} + static inline int write_sync(test_cfg* cfg) { diff --git a/examples/src/transfer.c b/examples/src/transfer.c index a01e36702..e31c297f0 100644 --- a/examples/src/transfer.c +++ b/examples/src/transfer.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #include #include diff --git a/examples/src/write.c b/examples/src/write.c index c89cd9177..e1f9743ae 100644 --- a/examples/src/write.c +++ b/examples/src/write.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. diff --git a/examples/src/writeread.c b/examples/src/writeread.c index 8a63d791b..e62cc576a 100644 --- a/examples/src/writeread.c +++ b/examples/src/writeread.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -166,6 +166,12 @@ size_t generate_read_reqs(test_cfg* cfg, char* dstbuf, * * cfg.io_check - when enabled, lipsum data is used when writing * the file and verified when reading. + * + * cfg.pre_wr_truncate - when enabled, truncate the file to specified + * size before writing. + * + * cfg.post_wr_truncate - when enabled, truncate the file to specified + * size after writing. */ int main(int argc, char* argv[]) @@ -173,7 +179,7 @@ int main(int argc, char* argv[]) char* wr_buf; char* rd_buf; char* target_file; - struct aiocb* reqs; + struct aiocb* reqs = NULL; size_t num_reqs = 0; int rc; @@ -229,6 +235,10 @@ int main(int argc, char* argv[]) timer_stop_barrier(cfg, &time_create); test_print_verbose_once(cfg, "DEBUG: finished create"); + if (cfg->pre_wr_trunc) { + write_truncate(cfg); + } + // generate write requests test_print_verbose_once(cfg, "DEBUG: generating write requests"); wr_buf = calloc(test_config.n_blocks, test_config.block_sz); @@ -263,19 +273,21 @@ int main(int argc, char* argv[]) timer_stop_barrier(cfg, &time_sync); test_print_verbose_once(cfg, "DEBUG: finished sync"); - // close file - // stat file pre-laminate timer_start_barrier(cfg, &time_stat_pre); stat_file(cfg, target_file); timer_stop_barrier(cfg, &time_stat_pre); - test_print_verbose_once(cfg, "DEBUG: finished stat pre"); + test_print_verbose_once(cfg, "DEBUG: finished stat pre-laminate"); - // stat file pre-laminate (again) - timer_start_barrier(cfg, &time_stat_pre2); - stat_file(cfg, target_file); - timer_stop_barrier(cfg, &time_stat_pre2); - test_print_verbose_once(cfg, "DEBUG: finished stat pre2"); + if (cfg->post_wr_trunc) { + write_truncate(cfg); + + // stat file post-truncate + timer_start_barrier(cfg, &time_stat_pre2); + stat_file(cfg, target_file); + timer_stop_barrier(cfg, &time_stat_pre2); + test_print_verbose_once(cfg, "DEBUG: finished stat pre2 (post trunc)"); + } // laminate timer_start_barrier(cfg, &time_laminate); @@ -288,17 +300,15 @@ int main(int argc, char* argv[]) // stat file post-laminate timer_start_barrier(cfg, &time_stat_post); - stat_file(cfg, target_file); + stat_cmd(cfg, target_file); timer_stop_barrier(cfg, &time_stat_post); - test_print_verbose_once(cfg, "DEBUG: finished stat post"); + test_print_verbose_once(cfg, "DEBUG: finished stat post-laminate"); // post-write cleanup free(wr_buf); free(reqs); reqs = NULL; - // open file - // generate read requests test_print_verbose_once(cfg, "DEBUG: generating read requests"); rd_buf = calloc(test_config.n_blocks, test_config.block_sz); diff --git a/examples/src/writeread.f90 b/examples/src/writeread.f90 index 3c98fa6aa..7ecae3092 100644 --- a/examples/src/writeread.f90 +++ b/examples/src/writeread.f90 @@ -1,4 +1,16 @@ - program write_read_F +! Copyright (c) 2020, Lawrence Livermore National Security, LLC. +! Produced at the Lawrence Livermore National Laboratory. +! +! Copyright 2020, UT-Battelle, LLC. +! +! LLNL-CODE-741539 +! All rights reserved. +! +! This is the license for UnifyFS. +! For details, see https://github.com/LLNL/UnifyFS. +! Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + +program write_read_F implicit none @@ -43,7 +55,7 @@ program write_read_F writeunit = mynod open(unit=writeunit,file=fname,form='unformatted',action='write') - + write(writeunit,iostat=ios) W1 close(writeunit) diff --git a/meta/src/Makefile.am b/meta/src/Makefile.am index 30d4422d7..dd094febe 100644 --- a/meta/src/Makefile.am +++ b/meta/src/Makefile.am @@ -1,4 +1,6 @@ +if USE_MDHIM noinst_LIBRARIES = libmdhim.a +endif libmdhim_a_SOURCES = Mlog2/mlog2.c \ Mlog2/mlog2.h \ diff --git a/meta/src/ds_leveldb.h b/meta/src/ds_leveldb.h index 124aeaca1..befef0c23 100644 --- a/meta/src/ds_leveldb.h +++ b/meta/src/ds_leveldb.h @@ -47,7 +47,7 @@ #include "partitioner.h" #include "data_store.h" -#include "unifyfs_metadata.h" +#include "unifyfs_metadata_mdhim.h" /* Function pointer for comparator in C */ typedef int (*mdhim_store_cmp_fn_t)(void* arg, const char* a, size_t alen, diff --git a/meta/src/partitioner.c b/meta/src/partitioner.c index bf637d4dd..6f6013b80 100644 --- a/meta/src/partitioner.c +++ b/meta/src/partitioner.c @@ -41,7 +41,7 @@ #include #include "partitioner.h" -#include "unifyfs_metadata.h" +#include "unifyfs_metadata_mdhim.h" struct timeval calslicestart, calsliceend; double calslicetime = 0; diff --git a/meta/src/range_server.c b/meta/src/range_server.c index 75739dd09..460f1f3c3 100644 --- a/meta/src/range_server.c +++ b/meta/src/range_server.c @@ -50,7 +50,7 @@ #include "mdhim_options.h" #include "partitioner.h" #include "range_server.h" -#include "unifyfs_metadata.h" +#include "unifyfs_metadata_mdhim.h" #include "uthash.h" int recv_counter = 0; diff --git a/server/src/Makefile.am b/server/src/Makefile.am index 5996b46f6..47f5116f3 100644 --- a/server/src/Makefile.am +++ b/server/src/Makefile.am @@ -1,49 +1,91 @@ +bin_PROGRAMS = unifyfsd + noinst_LIBRARIES = libunifyfsd.a +unifyfsd_SOURCES = unifyfs_server.c + libunifyfsd_a_SOURCES = \ - arraylist.c \ - arraylist.h \ + extent_tree.c \ + extent_tree.h \ margo_server.c \ margo_server.h \ unifyfs_cmd_handler.c \ + unifyfs_fops.h \ unifyfs_global.h \ - unifyfs_metadata.c \ - unifyfs_metadata.h \ + unifyfs_group_rpc.h \ + unifyfs_group_rpc.c \ + unifyfs_inode.h \ + unifyfs_inode.c \ + unifyfs_inode_tree.h \ + unifyfs_inode_tree.c \ + unifyfs_metadata_mdhim.h \ + unifyfs_p2p_rpc.h \ + unifyfs_p2p_rpc.c \ unifyfs_request_manager.c \ unifyfs_request_manager.h \ unifyfs_service_manager.c \ unifyfs_service_manager.h \ - unifyfs_server_pid.c + unifyfs_server_pid.c \ + unifyfs_tree.c \ + unifyfs_tree.h -bin_PROGRAMS = unifyfsd +OPT_CPP_FLAGS = +OPT_C_FLAGS = +OPT_LD_FLAGS = +OPT_LIBS = + +if USE_MDHIM + + libunifyfsd_a_SOURCES += \ + unifyfs_metadata_mdhim.c \ + unifyfs_fops_mdhim.c + + OPT_CPP_FLAGS += \ + -DUSE_MDHIM \ + -I$(top_srcdir)/meta/src \ + -I$(top_srcdir)/meta/src/uthash \ + -I$(top_srcdir)/meta/src/Mlog2 + + OPT_C_FLAGS += \ + $(LEVELDB_CFLAGS) \ + $(MPI_CFLAGS) + + OPT_LD_FLAGS += \ + $(LEVELDB_LDFLAGS) \ + $(MPI_CLDFLAGS) + + OPT_LIBS += \ + $(top_builddir)/meta/src/libmdhim.a \ + $(LEVELDB_LIBS) + +else # ! USE_MDHIM + + libunifyfsd_a_SOURCES += \ + unifyfs_fops_rpc.c + +endif # USE_MDHIM -unifyfsd_SOURCES = unifyfs_server.c unifyfsd_LDFLAGS = -static \ - $(LEVELDB_LDFLAGS) \ + $(OPT_LD_FLAGS) \ $(MARGO_LDFLAGS) unifyfsd_LDADD = \ libunifyfsd.a \ $(top_builddir)/common/src/libunifyfs_common.la \ - $(top_builddir)/meta/src/libmdhim.a \ - $(LEVELDB_LIBS) \ - $(MPI_CLDFLAGS) \ + $(OPT_LIBS) \ $(MARGO_LIBS) \ -lpthread -lm -lstdc++ -lrt AM_CPPFLAGS = \ + $(OPT_CPP_FLAGS) \ -I$(top_srcdir)/common/src \ - -I$(top_srcdir)/client/src \ - -I$(top_srcdir)/meta/src \ - -I$(top_srcdir)/meta/src/uthash \ - -I$(top_srcdir)/meta/src/Mlog2 + -I$(top_srcdir)/client/src AM_CFLAGS = -Wall -Werror \ - $(LEVELDB_CFLAGS) \ - $(MPI_CFLAGS) \ + $(OPT_C_FLAGS) \ + $(MARGO_CFLAGS) \ $(MERCURY_CFLAGS) \ - $(ARGOBOTS_CFLAGS) \ - $(MARGO_CFLAGS) + $(ARGOBOTS_CFLAGS) CLEANFILES = $(bin_PROGRAMS) diff --git a/server/src/extent_tree.c b/server/src/extent_tree.c new file mode 100644 index 000000000..c6e05d8bf --- /dev/null +++ b/server/src/extent_tree.c @@ -0,0 +1,705 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * This file is a simple, thread-safe, segment tree implementation. The + * segments in the tree are non-overlapping. Added segments overwrite the old + * segments in the tree. This is used to coalesce writes before an fsync. + */ + +#include +#include +#include +#include +#include + +#include "extent_tree.h" +#include "tree.h" +#include "unifyfs_metadata_mdhim.h" + +#undef MIN +#define MIN(a, b) (a < b ? a : b) +#undef MAX +#define MAX(a, b) (a > b ? a : b) + +int compare_func( + struct extent_tree_node* node1, + struct extent_tree_node* node2) +{ + if (node1->start > node2->end) { + return 1; + } else if (node1->end < node2->start) { + return -1; + } else { + return 0; + } +} + +RB_PROTOTYPE(inttree, extent_tree_node, entry, compare_func) +RB_GENERATE(inttree, extent_tree_node, entry, compare_func) + +/* Returns 0 on success, positive non-zero error code otherwise */ +int extent_tree_init(struct extent_tree* extent_tree) +{ + memset(extent_tree, 0, sizeof(*extent_tree)); + pthread_rwlock_init(&extent_tree->rwlock, NULL); + RB_INIT(&extent_tree->head); + return 0; +} + +/* + * Remove and free all nodes in the extent_tree. + */ +void extent_tree_destroy(struct extent_tree* extent_tree) +{ + extent_tree_clear(extent_tree); + pthread_rwlock_destroy(&extent_tree->rwlock); +} + +/* Allocate a node for the range tree. Free node with free() when finished */ +static struct extent_tree_node* extent_tree_node_alloc( + unsigned long start, /* logical starting offset of extent */ + unsigned long end, /* logical ending offset of extent */ + int svr_rank, /* rank of server hosting data */ + int app_id, /* application id (namespace) on server rank */ + int cli_id, /* client rank on server rank */ + unsigned long pos) /* physical offset of data in log */ +{ + /* allocate a new node structure */ + struct extent_tree_node* node = calloc(1, sizeof(*node)); + if (!node) { + return NULL; + } + + /* record logical range and physical offset */ + node->start = start; + node->end = end; + node->svr_rank = svr_rank; + node->app_id = app_id; + node->cli_id = cli_id; + node->pos = pos; + + return node; +} + +/* + * Given two start/end ranges, return a new range from start1/end1 that + * does not overlap start2/end2. The non-overlapping range is stored + * in new_start/new_end. If there are no non-overlapping ranges, + * return 1 from this function, else return 0. If there are two + * non-overlapping ranges, return the first one in new_start/new_end. + */ +static int get_non_overlapping_range( + unsigned long start1, unsigned long end1, + long start2, long end2, + long* new_start, long* new_end) +{ + /* this function is only called when we know that segment 1 + * and segment 2 overlap with each other, find first portion + * of segment 1 that does not overlap with segment 2, if any */ + if (start1 < start2) { + /* Segment 1 inlcudes a portion before segment 2 starts + * return start/end of that leading portion of segment 1 + * + * s1-------e1 + * s2--------e2 + * ---- non-overlap + */ + *new_start = start1; + *new_end = start2 - 1; + return 0; + } else if (end1 > end2) { + /* Segment 1 does not start before segment 2, + * but segment 1 extends past end of segment 2 + * return start/end of trailing portion of segment 1 + * + * s1-----e1 + * s2-------e2 + * --- non-overlap + */ + *new_start = end2 + 1; + *new_end = end1; + return 0; + } + + /* Segment 2 completely envelops segment 1 + * nothing left of segment 1 to return + * so return 1 to indicate this case + * + * s1-------e1 + * s2-------------e2 + */ + return 1; +} + +/* + * Add an entry to the range tree. Returns 0 on success, nonzero otherwise. + */ +int extent_tree_add( + struct extent_tree* extent_tree, /* tree to add new extent item */ + unsigned long start, /* logical starting offset of extent */ + unsigned long end, /* logical ending offset of extent */ + int svr_rank, /* rank of server hosting data */ + int app_id, /* application id (namespace) on server rank */ + int cli_id, /* client rank on server rank */ + unsigned long pos) /* physical offset of data in log */ +{ + /* assume we'll succeed */ + int rc = 0; + + /* Create node to define our new range */ + struct extent_tree_node* node = extent_tree_node_alloc( + start, end, svr_rank, app_id, cli_id, pos); + if (!node) { + return ENOMEM; + } + + /* lock the tree so we can modify it */ + extent_tree_wrlock(extent_tree); + + /* Try to insert our range into the RB tree. If it overlaps with any other + * range, then it is not inserted, and the overlapping range node is + * returned in 'overlap'. If 'overlap' is NULL, then there were no + * overlaps, and our range was successfully inserted. */ + struct extent_tree_node* overlap; + while ((overlap = RB_INSERT(inttree, &extent_tree->head, node))) { + /* Our range overlaps with another range (in 'overlap'). Is there any + * any part of 'overlap' that does not overlap our range? If so, + * delete the old 'overlap' and insert the smaller, non-overlapping + * range. */ + long new_start = 0; + long new_end = 0; + int ret = get_non_overlapping_range(overlap->start, overlap->end, + start, end, &new_start, &new_end); + if (ret) { + /* The new range we are adding completely covers the existing + * range in the tree defined in overlap. + * We can't find a non-overlapping range. + * Delete the existing range. */ + RB_REMOVE(inttree, &extent_tree->head, overlap); + free(overlap); + extent_tree->count--; + } else { + /* Part of the old range was non-overlapping. Split the old range + * into two ranges: one for the non-overlapping section, and one for + * the remaining section. The non-overlapping section gets + * inserted without issue. The remaining section will be processed + * on the next pass of this while() loop. */ + struct extent_tree_node* resized = extent_tree_node_alloc( + new_start, new_end, + overlap->svr_rank, overlap->app_id, overlap->cli_id, + overlap->pos + (new_start - overlap->start)); + if (!resized) { + /* failed to allocate memory for range node, + * bail out and release lock without further + * changing state of extent tree */ + free(node); + rc = ENOMEM; + goto release_add; + } + + /* if the non-overlapping part came from the front + * portion of the existing range, then there is a + * trailing portion of the existing range to add back + * to be considered again in the next loop iteration */ + struct extent_tree_node* remaining = NULL; + if (resized->end < overlap->end) { + /* There's still a remaining section after the non-overlapping + * part. Add it in. */ + remaining = extent_tree_node_alloc( + resized->end + 1, overlap->end, + overlap->svr_rank, overlap->app_id, overlap->cli_id, + overlap->pos + (resized->end + 1 - overlap->start)); + if (!remaining) { + /* failed to allocate memory for range node, + * bail out and release lock without further + * changing state of extent tree */ + free(node); + free(resized); + rc = ENOMEM; + goto release_add; + } + } + + /* Remove our old range and release it */ + RB_REMOVE(inttree, &extent_tree->head, overlap); + free(overlap); + extent_tree->count--; + + /* Insert the non-overlapping part of the new range */ + RB_INSERT(inttree, &extent_tree->head, resized); + extent_tree->count++; + + /* if we have a trailing portion, insert range for that, + * and increase our extent count since we just turned one + * range entry into two */ + if (remaining != NULL) { + RB_INSERT(inttree, &extent_tree->head, remaining); + extent_tree->count++; + } + } + } + + /* increment segment count in the tree for the + * new range we just added */ + extent_tree->count++; + + /* update max ending offset if end of new range + * we just inserted is larger */ + extent_tree->max = MAX(extent_tree->max, end); + + /* get temporary pointer to the node we just added */ + struct extent_tree_node* target = node; + + /* check whether we can coalesce new extent with any preceding extent */ + struct extent_tree_node* prev = RB_PREV( + inttree, &extent_tree->head, target); + if (prev != NULL && prev->end + 1 == target->start) { + /* found a extent that ends just before the new extent starts, + * check whether they are also contiguous in the log */ + unsigned long pos_end = prev->pos + (prev->end - prev->start + 1); + if (prev->svr_rank == target->svr_rank && + prev->cli_id == target->cli_id && + prev->app_id == target->app_id && + pos_end == target->pos) { + /* the preceding extent describes a log position adjacent to + * the extent we just added, so we can merge them, + * append entry to previous by extending end of previous */ + prev->end = target->end; + + /* delete new extent from the tree and free it */ + RB_REMOVE(inttree, &extent_tree->head, target); + free(target); + extent_tree->count--; + + /* update target to point at previous extent since we just + * merged our new extent into it */ + target = prev; + } + } + + /* check whether we can coalesce new extent with any trailing extent */ + struct extent_tree_node* next = RB_NEXT( + inttree, &extent_tree->head, target); + if (next != NULL && target->end + 1 == next->start) { + /* found a extent that starts just after the new extent ends, + * check whether they are also contiguous in the log */ + unsigned long pos_end = target->pos + (target->end - target->start + 1); + if (target->svr_rank == next->svr_rank && + target->cli_id == next->cli_id && + target->app_id == next->app_id && + pos_end == next->pos) { + /* the target extent describes a log position adjacent to + * the next extent, so we can merge them, + * append entry to target by extending end of to cover next */ + target->end = next->end; + + /* delete next extent from the tree and free it */ + RB_REMOVE(inttree, &extent_tree->head, next); + free(next); + extent_tree->count--; + } + } + +release_add: + + /* done modifying the tree */ + extent_tree_unlock(extent_tree); + + return rc; +} + +/* search tree for entry that overlaps with given start/end + * offsets, return first overlapping entry if found, NULL otherwise, + * assumes caller has lock on tree */ +struct extent_tree_node* extent_tree_find( + struct extent_tree* extent_tree, /* tree to search */ + unsigned long start, /* starting offset to search */ + unsigned long end) /* ending offset to search */ +{ + /* Create a range of just our starting byte offset */ + struct extent_tree_node* node = extent_tree_node_alloc( + start, start, 0, 0, 0, 0); + if (!node) { + return NULL; + } + + /* search tree for either a range that overlaps with + * the target range (starting byte), or otherwise the + * node for the next biggest starting byte */ + struct extent_tree_node* next = RB_NFIND( + inttree, &extent_tree->head, node); + + free(node); + + /* we may have found a node that doesn't include our starting + * byte offset, but it would be the range with the lowest + * starting offset after the target starting offset, check whether + * this overlaps our end offset */ + if (next && next->start <= end) { + return next; + } + + /* otherwise, there is not element that overlaps with the + * target range of [start, end] */ + return NULL; +} + +/* truncate extents to use new maximum, discards extent entries + * that exceed the new truncated size, and rewrites any entry + * that overlaps */ +int extent_tree_truncate( + struct extent_tree* tree, /* tree to truncate */ + unsigned long size) /* size to truncate extents to */ +{ + if (0 == size) { + extent_tree_clear(tree); + return 0; + } + + /* lock the tree for reading */ + extent_tree_wrlock(tree); + + /* lookup node with the extent that has the maximum offset */ + struct extent_tree_node* node = RB_MAX(inttree, &tree->head); + + /* iterate backwards until we find an extent below + * the truncated size */ + while (node != NULL && node->end >= size) { + /* found an extent whose ending offset is equal to or + * extends beyond the truncated size, + * check whether the full extent is beyond the truncated + * size or whether the new size falls within this extent */ + if (node->start >= size) { + /* the start offset is also beyond the truncated size, + * meaning the entire range is beyond the truncated size, + * get pointer to next previous extent in tree */ + struct extent_tree_node* oldnode = node; + node = RB_PREV(inttree, &tree->head, node); + + /* remove this node from the tree and release it */ + LOGDBG("removing node [%lu, %lu] due to truncate=%lu", + node->start, node->end, size); + RB_REMOVE(inttree, &tree->head, oldnode); + free(oldnode); + + /* decrement the number of extents in the tree */ + tree->count--; + } else { + /* the range of this node overlaps with the truncated size + * so just update its end to be the new size */ + node->end = size - 1; + break; + } + } + + /* update maximum offset in tree */ + if (node != NULL) { + /* got at least one extent left, update maximum field */ + tree->max = node->end; + } else { + /* no extents left in the tree, set max back to 0 */ + tree->max = 0; + } + + /* done reading the tree */ + extent_tree_unlock(tree); + + return 0; +} + +/* + * Given a range tree and a starting node, iterate though all the nodes + * in the tree, returning the next one each time. If start is NULL, then + * start with the first node in the tree. + * + * This is meant to be called in a loop, like: + * + * extent_tree_rdlock(extent_tree); + * + * struct extent_tree_node *node = NULL; + * while ((node = extent_tree_iter(extent_tree, node))) { + * printf("[%d-%d]", node->start, node->end); + * } + * + * extent_tree_unlock(extent_tree); + * + * Note: this function does no locking, and assumes you're properly locking + * and unlocking the extent_tree before doing the iteration (see + * extent_tree_rdlock()/extent_tree_wrlock()/extent_tree_unlock()). + */ +struct extent_tree_node* extent_tree_iter( + struct extent_tree* extent_tree, + struct extent_tree_node* start) +{ + struct extent_tree_node* next = NULL; + if (start == NULL) { + /* Initial case, no starting node */ + next = RB_MIN(inttree, &extent_tree->head); + return next; + } + + /* + * We were given a valid start node. Look it up to start our traversal + * from there. + */ + next = RB_FIND(inttree, &extent_tree->head, start); + if (!next) { + /* Some kind of error */ + return NULL; + } + + /* Look up our next node */ + next = RB_NEXT(inttree, &extent_tree->head, start); + + return next; +} + +/* + * Lock a extent_tree for reading. This should only be used for calling + * extent_tree_iter(). All the other extent_tree functions provide their + * own locking. + */ +void extent_tree_rdlock(struct extent_tree* extent_tree) +{ + int rc = pthread_rwlock_rdlock(&extent_tree->rwlock); + if (rc) { + LOGERR("pthread_rwlock_rdlock() failed - rc=%d", rc); + } +} + +/* + * Lock a extent_tree for read/write. This should only be used for calling + * extent_tree_iter(). All the other extent_tree functions provide their + * own locking. + */ +void extent_tree_wrlock(struct extent_tree* extent_tree) +{ + int rc = pthread_rwlock_wrlock(&extent_tree->rwlock); + if (rc) { + LOGERR("pthread_rwlock_wrlock() failed - rc=%d", rc); + } +} + +/* + * Unlock a extent_tree for read/write. This should only be used for calling + * extent_tree_iter(). All the other extent_tree functions provide their + * own locking. + */ +void extent_tree_unlock(struct extent_tree* extent_tree) +{ + int rc = pthread_rwlock_unlock(&extent_tree->rwlock); + if (rc) { + LOGERR("pthread_rwlock_unlock() failed - rc=%d", rc); + } +} + +/* + * Remove all nodes in extent_tree, but keep it initialized so you can + * extent_tree_add() to it. + */ +void extent_tree_clear(struct extent_tree* extent_tree) +{ + struct extent_tree_node* node = NULL; + struct extent_tree_node* oldnode = NULL; + + extent_tree_wrlock(extent_tree); + + if (RB_EMPTY(&extent_tree->head)) { + /* extent_tree is empty, nothing to do */ + extent_tree_unlock(extent_tree); + return; + } + + /* Remove and free each node in the tree */ + while ((node = extent_tree_iter(extent_tree, node))) { + if (oldnode) { + RB_REMOVE(inttree, &extent_tree->head, oldnode); + free(oldnode); + } + oldnode = node; + } + if (oldnode) { + RB_REMOVE(inttree, &extent_tree->head, oldnode); + free(oldnode); + } + + extent_tree->count = 0; + extent_tree->max = 0; + extent_tree_unlock(extent_tree); +} + +/* Return the number of segments in the segment tree */ +unsigned long extent_tree_count(struct extent_tree* extent_tree) +{ + extent_tree_rdlock(extent_tree); + unsigned long count = extent_tree->count; + extent_tree_unlock(extent_tree); + return count; +} + +/* Return the maximum ending logical offset in the tree */ +unsigned long extent_tree_max_offset(struct extent_tree* extent_tree) +{ + extent_tree_rdlock(extent_tree); + unsigned long max = extent_tree->max; + extent_tree_unlock(extent_tree); + return max; +} + +/* given an extent tree and starting and ending logical offsets, + * fill in key/value entries that overlap that range, returns at + * most max entries starting from lowest starting offset, + * sets outnum with actual number of entries returned */ +int extent_tree_span( + struct extent_tree* extent_tree, /* extent tree to search */ + int gfid, /* global file id we're looking in */ + unsigned long start, /* starting logical offset */ + unsigned long end, /* ending logical offset */ + int max, /* maximum number of key/vals to return */ + void* _keys, /* array of length max for output keys */ + void* _vals, /* array of length max for output values */ + int* outnum) /* number of entries returned */ +{ + unifyfs_key_t* keys = (unifyfs_key_t*) _keys; + unifyfs_val_t* vals = (unifyfs_val_t*) _vals; + + /* initialize output parameters */ + *outnum = 0; + + /* lock the tree for reading */ + extent_tree_rdlock(extent_tree); + + int count = 0; + struct extent_tree_node* next = extent_tree_find(extent_tree, start, end); + while (next != NULL && + next->start <= end && + count < max) { + /* got an entry that overlaps with given span */ + + /* fill in key */ + unifyfs_key_t* key = &keys[count]; + key->gfid = gfid; + key->offset = next->start; + + /* fill in value */ + unifyfs_val_t* val = &vals[count]; + val->addr = next->pos; + val->len = next->end - next->start + 1; + val->delegator_rank = next->svr_rank; + val->app_id = next->app_id; + val->rank = next->cli_id; + + /* increment the number of key/values we found */ + count++; + + /* get the next element in the tree */ + next = extent_tree_iter(extent_tree, next); + } + + /* return to user the number of key/values we set */ + *outnum = count; + + /* done reading the tree */ + extent_tree_unlock(extent_tree); + + return 0; +} + +static void chunk_req_from_extent( + unsigned long req_offset, + unsigned long req_len, + struct extent_tree_node* n, + chunk_read_req_t* chunk) +{ + unsigned long offset = n->start; + unsigned long nbytes = n->end - n->start + 1; + unsigned long log_offset = n->pos; + unsigned long last = req_offset + req_len - 1; + + if (offset < req_offset) { + unsigned long diff = req_offset - offset; + + offset = req_offset; + log_offset += diff; + nbytes -= diff; + } + + if (n->end > last) { + unsigned long diff = n->end - last; + nbytes -= diff; + } + + chunk->offset = offset; + chunk->nbytes = nbytes; + chunk->log_offset = log_offset; + chunk->rank = n->svr_rank; + chunk->log_client_id = n->cli_id; + chunk->log_app_id = n->app_id; +} + +int extent_tree_get_chunk_list( + struct extent_tree* extent_tree, /* extent tree to search */ + unsigned long offset, /* starting logical offset */ + unsigned long len, /* length of extent */ + unsigned int* n_chunks, /* [out] number of extents returned */ + chunk_read_req_t** chunks) /* [out] extent array */ +{ + int ret = 0; + unsigned int count = 0; + unsigned long end = offset + len - 1; + struct extent_tree_node* first = NULL; + struct extent_tree_node* next = NULL; + chunk_read_req_t* out_chunks = NULL; + chunk_read_req_t* current = NULL; + + extent_tree_rdlock(extent_tree); + + first = extent_tree_find(extent_tree, offset, end); + next = first; + while (next && next->start <= end) { + count++; + next = extent_tree_iter(extent_tree, next); + } + + *n_chunks = count; + if (0 == count) { + goto out_unlock; + } + + out_chunks = calloc(count, sizeof(*out_chunks)); + if (!out_chunks) { + ret = ENOMEM; + goto out_unlock; + } + + next = first; + current = out_chunks; + while (next && next->start <= end) { + /* trim out the extent so it does not include the data that is not + * requested */ + chunk_req_from_extent(offset, len, next, current); + + next = extent_tree_iter(extent_tree, next); + current += 1; + } + + *chunks = out_chunks; + +out_unlock: + extent_tree_unlock(extent_tree); + + return ret; +} + diff --git a/server/src/extent_tree.h b/server/src/extent_tree.h new file mode 100644 index 000000000..4c239784f --- /dev/null +++ b/server/src/extent_tree.h @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef __EXTENT_TREE_H__ +#define __EXTENT_TREE_H__ + +#include + +#include "tree.h" +#include "unifyfs_global.h" + +struct extent_tree_node { + RB_ENTRY(extent_tree_node) entry; + unsigned long start; /* starting logical offset of range */ + unsigned long end; /* ending logical offset of range */ + int svr_rank; /* rank of server hosting data */ + int app_id; /* application id (namespace) on server rank */ + int cli_id; /* client rank on server rank */ + unsigned long pos; /* physical offset of data in log */ +}; + +struct extent_tree { + RB_HEAD(inttree, extent_tree_node) head; + pthread_rwlock_t rwlock; + unsigned long count; /* number of segments stored in tree */ + unsigned long max; /* maximum logical offset value in the tree */ +}; + +/* Returns 0 on success, positive non-zero error code otherwise */ +int extent_tree_init(struct extent_tree* extent_tree); + +/* + * Remove all nodes in extent_tree, but keep it initialized so you can + * extent_tree_add() to it. + */ +void extent_tree_clear(struct extent_tree* extent_tree); + +/* + * Remove and free all nodes in the extent_tree. + */ +void extent_tree_destroy(struct extent_tree* extent_tree); + +/* + * Add an entry to the range tree. Returns 0 on success, nonzero otherwise. + */ +int extent_tree_add( + struct extent_tree* extent_tree, /* tree to add new extent item */ + unsigned long start, /* logical starting offset of extent */ + unsigned long end, /* logical ending offset of extent */ + int svr_rank, /* rank of server hosting data */ + int app_id, /* application id (namespace) on server rank */ + int cli_id, /* client rank on server rank */ + unsigned long pos /* physical offset of data in log */ +); + +/* search tree for entry that overlaps with given start/end + * offsets, return first overlapping entry if found, NULL otherwise, + * assumes caller has lock on tree */ +struct extent_tree_node* extent_tree_find( + struct extent_tree* extent_tree, /* tree to search */ + unsigned long start, /* starting offset to search */ + unsigned long end /* ending offset to search */ +); + +/* truncate extents to use new maximum, discards extent entries + * that exceed the new truncated size, and rewrites any entry + * that overlaps */ +int extent_tree_truncate( + struct extent_tree* extent_tree, /* tree to truncate */ + unsigned long size /* size to truncate extents to */ +); + +/* + * Given a range tree and a starting node, iterate though all the nodes + * in the tree, returning the next one each time. If start is NULL, then + * start with the first node in the tree. + * + * This is meant to be called in a loop, like: + * + * extent_tree_rdlock(extent_tree); + * + * struct extent_tree_node *node = NULL; + * while ((node = extent_tree_iter(extent_tree, node))) { + * printf("[%d-%d]", node->start, node->end); + * } + * + * extent_tree_unlock(extent_tree); + * + * Note: this function does no locking, and assumes you're properly locking + * and unlocking the extent_tree before doing the iteration (see + * extent_tree_rdlock()/extent_tree_wrlock()/extent_tree_unlock()). + */ +struct extent_tree_node* extent_tree_iter( + struct extent_tree* extent_tree, + struct extent_tree_node* start); + +/* Return the number of segments in the segment tree */ +unsigned long extent_tree_count(struct extent_tree* extent_tree); + +/* Return the maximum ending logical offset in the tree */ +unsigned long extent_tree_max_offset(struct extent_tree* extent_tree); + +/* + * Locking functions for use with extent_tree_iter(). They allow you to + * lock the tree to iterate over it: + * + * extent_tree_rdlock(&extent_tree); + * + * struct extent_tree_node *node = NULL; + * while ((node = extent_tree_iter(extent_tree, node))) { + * printf("[%d-%d]", node->start, node->end); + * } + * + * extent_tree_unlock(&extent_tree); + */ + +/* + * Lock a extent_tree for reading. This should only be used for calling + * extent_tree_iter(). All the other extent_tree functions provide their + * own locking. + */ +void extent_tree_rdlock(struct extent_tree* extent_tree); + +/* + * Lock a extent_tree for read/write. This should only be used for calling + * extent_tree_iter(). All the other extent_tree functions provide their + * own locking. + */ +void extent_tree_wrlock(struct extent_tree* extent_tree); + +/* + * Unlock a extent_tree for read/write. This should only be used for calling + * extent_tree_iter(). All the other extent_tree functions provide their + * own locking. + */ +void extent_tree_unlock(struct extent_tree* extent_tree); + +/* given an extent tree and starting and ending logical offsets, + * fill in key/value entries that overlap that range, returns at + * most max entries starting from lowest starting offset, + * sets outnum with actual number of entries returned */ +int extent_tree_span( + struct extent_tree* extent_tree, /* extent tree to search */ + int gfid, /* global file id we're looking in */ + unsigned long start, /* starting logical offset */ + unsigned long end, /* ending logical offset */ + int max, /* maximum number of key/vals to return */ + void* keys, /* array of length max for output keys */ + void* vals, /* array of length max for output values */ + int* outnum); /* number of entries returned */ + +int extent_tree_get_chunk_list( + struct extent_tree* extent_tree, /* extent tree to search */ + unsigned long offset, /* starting logical offset */ + unsigned long len, /* length of extent */ + unsigned int* n_chunks, /* [out] number of chunks returned */ + chunk_read_req_t** chunks); /* [out] extent array */ + +/* dump method for debugging extent trees */ +static inline +void extent_tree_dump(struct extent_tree* extent_tree) +{ + if (NULL == extent_tree) { + return; + } + + extent_tree_rdlock(extent_tree); + + struct extent_tree_node* node = NULL; + while ((node = extent_tree_iter(extent_tree, node))) { + LOGDBG("[%lu-%lu]", node->start, node->end); + } + + extent_tree_unlock(extent_tree); +} + +#endif /* __EXTENT_TREE_H__ */ diff --git a/server/src/margo_server.c b/server/src/margo_server.c index ead8a20f5..36e74cbfc 100644 --- a/server/src/margo_server.c +++ b/server/src/margo_server.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017-2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -21,15 +21,31 @@ // server headers #include "unifyfs_global.h" #include "margo_server.h" +#include "na_config.h" // from mercury include lib // global variables ServerRpcContext_t* unifyfsd_rpc_context; bool margo_use_tcp = true; bool margo_lazy_connect; // = false +int margo_client_server_pool_sz = 4; +int margo_server_server_pool_sz = 4; +int margo_use_progress_thread = 1; +#if defined(NA_HAS_SM) static const char* PROTOCOL_MARGO_SHM = "na+sm://"; -static const char* PROTOCOL_MARGO_VERBS = "ofi+verbs://"; -static const char* PROTOCOL_MARGO_TCP = "bmi+tcp://"; +#else +#error Required Mercury NA shared memory plugin not found (please enable 'SM') +#endif + +#if defined(NA_HAS_BMI) +static const char* PROTOCOL_MARGO_TCP = "bmi+tcp://"; +static const char* PROTOCOL_MARGO_RMA = "bmi+tcp://"; +#elif defined(NA_HAS_OFI) +static const char* PROTOCOL_MARGO_TCP = "ofi+tcp://"; +static const char* PROTOCOL_MARGO_RMA = "ofi+verbs://"; +#else +#error No supported Mercury NA plugin found (please use one of: 'BMI', 'OFI') +#endif /* setup_remote_target - Initializes the server-server margo target */ static margo_instance_id setup_remote_target(void) @@ -45,10 +61,11 @@ static margo_instance_id setup_remote_target(void) if (margo_use_tcp) { margo_protocol = PROTOCOL_MARGO_TCP; } else { - margo_protocol = PROTOCOL_MARGO_VERBS; + margo_protocol = PROTOCOL_MARGO_RMA; } - mid = margo_init(margo_protocol, MARGO_SERVER_MODE, 1, 4); + mid = margo_init(margo_protocol, MARGO_SERVER_MODE, + margo_use_progress_thread, margo_server_server_pool_sz); if (mid == MARGO_INSTANCE_NULL) { LOGERR("margo_init(%s)", margo_protocol); return mid; @@ -82,20 +99,10 @@ static margo_instance_id setup_remote_target(void) /* register server-server RPCs */ static void register_server_server_rpcs(margo_instance_id mid) { - unifyfsd_rpc_context->rpcs.hello_id = - MARGO_REGISTER(mid, "server_hello_rpc", - server_hello_in_t, server_hello_out_t, - server_hello_rpc); - unifyfsd_rpc_context->rpcs.server_pid_id = MARGO_REGISTER(mid, "server_pid_rpc", server_pid_in_t, server_pid_out_t, - server_pid_handle_rpc); - - unifyfsd_rpc_context->rpcs.request_id = - MARGO_REGISTER(mid, "server_request_rpc", - server_request_in_t, server_request_out_t, - server_request_rpc); + server_pid_rpc); unifyfsd_rpc_context->rpcs.chunk_read_request_id = MARGO_REGISTER(mid, "chunk_read_request_rpc", @@ -106,6 +113,66 @@ static void register_server_server_rpcs(margo_instance_id mid) MARGO_REGISTER(mid, "chunk_read_response_rpc", chunk_read_response_in_t, chunk_read_response_out_t, chunk_read_response_rpc); + + unifyfsd_rpc_context->rpcs.extent_add_id = + MARGO_REGISTER(mid, "add_extents_rpc", + add_extents_in_t, add_extents_out_t, + add_extents_rpc); + + unifyfsd_rpc_context->rpcs.extent_bcast_id = + MARGO_REGISTER(mid, "extent_bcast_rpc", + extent_bcast_in_t, extent_bcast_out_t, + extent_bcast_rpc); + + unifyfsd_rpc_context->rpcs.extent_lookup_id = + MARGO_REGISTER(mid, "find_extents_rpc", + find_extents_in_t, find_extents_out_t, + find_extents_rpc); + + unifyfsd_rpc_context->rpcs.fileattr_bcast_id = + MARGO_REGISTER(mid, "fileattr_bcast_rpc", + fileattr_bcast_in_t, fileattr_bcast_out_t, + fileattr_bcast_rpc); + + unifyfsd_rpc_context->rpcs.filesize_id = + MARGO_REGISTER(mid, "filesize_rpc", + filesize_in_t, filesize_out_t, + filesize_rpc); + + unifyfsd_rpc_context->rpcs.laminate_id = + MARGO_REGISTER(mid, "laminate_rpc", + laminate_in_t, laminate_out_t, + laminate_rpc); + + unifyfsd_rpc_context->rpcs.laminate_bcast_id = + MARGO_REGISTER(mid, "laminate_bcast_rpc", + laminate_bcast_in_t, laminate_bcast_out_t, + laminate_bcast_rpc); + + unifyfsd_rpc_context->rpcs.metaget_id = + MARGO_REGISTER(mid, "metaget_rpc", + metaget_in_t, metaget_out_t, + metaget_rpc); + + unifyfsd_rpc_context->rpcs.metaset_id = + MARGO_REGISTER(mid, "metaset_rpc", + metaset_in_t, metaset_out_t, + metaset_rpc); + + unifyfsd_rpc_context->rpcs.truncate_id = + MARGO_REGISTER(mid, "truncate_rpc", + truncate_in_t, truncate_out_t, + truncate_rpc); + + unifyfsd_rpc_context->rpcs.truncate_bcast_id = + MARGO_REGISTER(mid, "truncate_bcast_rpc", + truncate_bcast_in_t, truncate_bcast_out_t, + truncate_bcast_rpc); + + unifyfsd_rpc_context->rpcs.unlink_bcast_id = + MARGO_REGISTER(mid, "unlink_bcast_rpc", + unlink_bcast_in_t, unlink_bcast_out_t, + unlink_bcast_rpc); } /* setup_local_target - Initializes the client-server margo target */ @@ -118,7 +185,8 @@ static margo_instance_id setup_local_target(void) hg_size_t self_string_sz = sizeof(self_string); margo_instance_id mid; - mid = margo_init(PROTOCOL_MARGO_SHM, MARGO_SERVER_MODE, 1, -1); + mid = margo_init(PROTOCOL_MARGO_SHM, MARGO_SERVER_MODE, + margo_use_progress_thread, margo_client_server_pool_sz); if (mid == MARGO_INSTANCE_NULL) { LOGERR("margo_init(%s)", PROTOCOL_MARGO_SHM); return mid; @@ -172,9 +240,9 @@ static void register_client_server_rpcs(margo_instance_id mid) unifyfs_metaset_in_t, unifyfs_metaset_out_t, unifyfs_metaset_rpc); - MARGO_REGISTER(mid, "unifyfs_sync_rpc", - unifyfs_sync_in_t, unifyfs_sync_out_t, - unifyfs_sync_rpc); + MARGO_REGISTER(mid, "unifyfs_fsync_rpc", + unifyfs_fsync_in_t, unifyfs_fsync_out_t, + unifyfs_fsync_rpc); MARGO_REGISTER(mid, "unifyfs_filesize_rpc", unifyfs_filesize_in_t, unifyfs_filesize_out_t, @@ -214,13 +282,15 @@ int margo_server_rpc_init(void) if (NULL == unifyfsd_rpc_context) { /* create rpc server context */ unifyfsd_rpc_context = calloc(1, sizeof(ServerRpcContext_t)); - assert(unifyfsd_rpc_context); + if (NULL == unifyfsd_rpc_context) { + return ENOMEM; + } } margo_instance_id mid; mid = setup_local_target(); if (mid == MARGO_INSTANCE_NULL) { - rc = UNIFYFS_FAILURE; + rc = UNIFYFS_ERROR_MARGO; } else { unifyfsd_rpc_context->shm_mid = mid; register_client_server_rpcs(mid); @@ -228,7 +298,7 @@ int margo_server_rpc_init(void) mid = setup_remote_target(); if (mid == MARGO_INSTANCE_NULL) { - rc = UNIFYFS_FAILURE; + rc = UNIFYFS_ERROR_MARGO; } else { unifyfsd_rpc_context->svr_mid = mid; register_server_server_rpcs(mid); diff --git a/server/src/margo_server.h b/server/src/margo_server.h index 7e3ad7c81..d5ba79c5c 100644 --- a/server/src/margo_server.h +++ b/server/src/margo_server.h @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + #ifndef _MARGO_SERVER_H #define _MARGO_SERVER_H @@ -16,11 +30,21 @@ #include typedef struct ServerRpcIds { - hg_id_t hello_id; - hg_id_t server_pid_id; - hg_id_t request_id; hg_id_t chunk_read_request_id; hg_id_t chunk_read_response_id; + hg_id_t extent_add_id; + hg_id_t extent_bcast_id; + hg_id_t extent_lookup_id; + hg_id_t filesize_id; + hg_id_t laminate_id; + hg_id_t laminate_bcast_id; + hg_id_t metaget_id; + hg_id_t metaset_id; + hg_id_t fileattr_bcast_id; + hg_id_t server_pid_id; + hg_id_t truncate_id; + hg_id_t truncate_bcast_id; + hg_id_t unlink_bcast_id; } server_rpcs_t; typedef struct ServerRpcContext { diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index 72e22124f..164df9d1a 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017-2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -33,7 +33,7 @@ // server components #include "unifyfs_global.h" -#include "unifyfs_metadata.h" +#include "unifyfs_metadata_mdhim.h" #include "unifyfs_request_manager.h" // margo rpcs @@ -59,42 +59,49 @@ static void unifyfs_mount_rpc(hg_handle_t handle) { int ret = (int)UNIFYFS_SUCCESS; + int app_id = -1; + int client_id = -1; /* get input params */ unifyfs_mount_in_t in; hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); - - /* read app_id and client_id from input */ - int app_id = unifyfs_generate_gfid(in.mount_prefix); - int client_id = -1; - - /* lookup app_config for given app_id */ - app_config* app_cfg = get_application(app_id); - if (app_cfg == NULL) { - /* insert new app_config into our app_configs array */ - LOGDBG("creating new application for app_id=%d", app_id); - app_cfg = new_application(app_id); - if (NULL == app_cfg) { - ret = UNIFYFS_FAILURE; - } + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; } else { - LOGDBG("using existing app_config for app_id=%d", app_id); - } - - if (NULL != app_cfg) { - LOGDBG("creating new app client for %s", in.client_addr_str); - app_client* client = new_app_client(app_cfg, - in.client_addr_str, - in.dbg_rank); - if (NULL == client) { - LOGERR("failed to create new client for app_id=%d dbg_rank=%d", - app_id, (int)in.dbg_rank); - ret = (int)UNIFYFS_FAILURE; + /* read app_id and client_id from input */ + app_id = unifyfs_generate_gfid(in.mount_prefix); + + /* lookup app_config for given app_id */ + app_config* app_cfg = get_application(app_id); + if (app_cfg == NULL) { + /* insert new app_config into our app_configs array */ + LOGDBG("creating new application for app_id=%d", app_id); + app_cfg = new_application(app_id); + if (NULL == app_cfg) { + ret = UNIFYFS_FAILURE; + } } else { - client_id = client->client_id; - LOGDBG("created new application client %d:%d", app_id, client_id); + LOGDBG("using existing app_config for app_id=%d", app_id); } + + if (NULL != app_cfg) { + LOGDBG("creating new app client for %s", in.client_addr_str); + app_client* client = new_app_client(app_cfg, + in.client_addr_str, + in.dbg_rank); + if (NULL == client) { + LOGERR("failed to create new client for app_id=%d dbg_rank=%d", + app_id, (int)in.dbg_rank); + ret = (int)UNIFYFS_FAILURE; + } else { + client_id = client->client_id; + LOGDBG("created new application client %d:%d", + app_id, client_id); + } + } + + margo_free_input(handle, &in); } /* build output structure to return to caller */ @@ -105,10 +112,11 @@ static void unifyfs_mount_rpc(hg_handle_t handle) /* send output back to caller */ hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } /* free margo resources */ - margo_free_input(handle, &in); margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(unifyfs_mount_rpc) @@ -122,31 +130,36 @@ static void unifyfs_attach_rpc(hg_handle_t handle) /* get input params */ unifyfs_attach_in_t in; hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); - - /* read app_id and client_id from input */ - int app_id = in.app_id; - int client_id = in.client_id; - - /* lookup client structure and attach it */ - app_client* client = get_app_client(app_id, client_id); - if (NULL != client) { - LOGDBG("attaching client %d:%d", app_id, client_id); - ret = attach_app_client(client, - in.logio_spill_dir, - in.logio_spill_size, - in.logio_mem_size, - in.shmem_data_size, - in.shmem_super_size, - in.meta_offset, - in.meta_size); - if (ret != UNIFYFS_SUCCESS) { - LOGERR("attach_app_client() failed"); - } + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; } else { - LOGERR("client not found (app_id=%d, client_id=%d)", - app_id, client_id); - ret = (int)UNIFYFS_FAILURE; + /* read app_id and client_id from input */ + int app_id = in.app_id; + int client_id = in.client_id; + + /* lookup client structure and attach it */ + app_client* client = get_app_client(app_id, client_id); + if (NULL != client) { + LOGDBG("attaching client %d:%d", app_id, client_id); + ret = attach_app_client(client, + in.logio_spill_dir, + in.logio_spill_size, + in.logio_mem_size, + in.shmem_data_size, + in.shmem_super_size, + in.meta_offset, + in.meta_size); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("attach_app_client() failed"); + } + } else { + LOGERR("client not found (app_id=%d, client_id=%d)", + app_id, client_id); + ret = (int)UNIFYFS_FAILURE; + } + + margo_free_input(handle, &in); } /* build output structure to return to caller */ @@ -155,33 +168,40 @@ static void unifyfs_attach_rpc(hg_handle_t handle) /* send output back to caller */ hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } /* free margo resources */ - margo_free_input(handle, &in); margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(unifyfs_attach_rpc) static void unifyfs_unmount_rpc(hg_handle_t handle) { + int ret = UNIFYFS_SUCCESS; + /* get input params */ unifyfs_unmount_in_t in; hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); - - /* read app_id and client_id from input */ - int app_id = in.app_id; - int client_id = in.client_id; - - /* disconnect app client */ - int ret = UNIFYFS_SUCCESS; - app_client* clnt = get_app_client(app_id, client_id); - if (NULL != clnt) { - ret = disconnect_app_client(clnt); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; } else { - LOGERR("application client not found"); - ret = EINVAL; + /* read app_id and client_id from input */ + int app_id = in.app_id; + int client_id = in.client_id; + + /* disconnect app client */ + app_client* clnt = get_app_client(app_id, client_id); + if (NULL != clnt) { + ret = disconnect_app_client(clnt); + } else { + LOGERR("application client not found"); + ret = EINVAL; + } + + margo_free_input(handle, &in); } /* build output structure to return to caller */ @@ -190,10 +210,11 @@ static void unifyfs_unmount_rpc(hg_handle_t handle) /* send output back to caller */ hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } /* free margo resources */ - margo_free_input(handle, &in); margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(unifyfs_unmount_rpc) @@ -202,37 +223,59 @@ DEFINE_MARGO_RPC_HANDLER(unifyfs_unmount_rpc) * given a global file id */ static void unifyfs_metaget_rpc(hg_handle_t handle) { - /* get input params */ - unifyfs_metaget_in_t in; - hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); + int ret = UNIFYFS_SUCCESS; + hg_return_t hret; - /* given the global file id, look up file attributes - * from key/value store */ - unifyfs_file_attr_t attr_val; - int ret = unifyfs_get_file_attribute(in.gfid, &attr_val); + /* get input params */ + unifyfs_metaget_in_t* in = malloc(sizeof(*in)); + if (NULL == in) { + ret = ENOMEM; + } else { + hret = margo_get_input(handle, in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + client_rpc_req_t* req = malloc(sizeof(client_rpc_req_t)); + if (NULL == req) { + ret = ENOMEM; + } else { + unifyfs_fops_ctx_t ctx = { + .app_id = in->app_id, + .client_id = in->client_id, + }; + req->req_type = UNIFYFS_CLIENT_RPC_METAGET; + req->handle = handle; + req->input = (void*) in; + req->bulk_buf = NULL; + req->bulk_sz = 0; + ret = rm_submit_client_rpc_request(&ctx, req); + } + + if (ret != UNIFYFS_SUCCESS) { + margo_free_input(handle, in); + } + } + } - /* build our output values */ - unifyfs_metaget_out_t out; - out.gfid = attr_val.gfid; - out.mode = attr_val.mode; - out.uid = attr_val.uid; - out.gid = attr_val.gid; - out.size = attr_val.size; - out.atime = attr_val.atime; - out.mtime = attr_val.mtime; - out.ctime = attr_val.ctime; - out.filename = attr_val.filename; - out.is_laminated = attr_val.is_laminated; - out.ret = ret; + /* if we hit an error during request submission, respond with the error */ + if (ret != UNIFYFS_SUCCESS) { + if (NULL != in) { + free(in); + } - /* send output back to caller */ - hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + /* return to caller */ + unifyfs_metaget_out_t out; + out.ret = (int32_t) ret; + memset(&(out.attr), 0, sizeof(out.attr)); + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - /* free margo resources */ - margo_free_input(handle, &in); - margo_destroy(handle); + /* free margo resources */ + margo_destroy(handle); + } } DEFINE_MARGO_RPC_HANDLER(unifyfs_metaget_rpc) @@ -240,99 +283,178 @@ DEFINE_MARGO_RPC_HANDLER(unifyfs_metaget_rpc) * record key/value entry for this file */ static void unifyfs_metaset_rpc(hg_handle_t handle) { + int ret = UNIFYFS_SUCCESS; + hg_return_t hret; + /* get input params */ - unifyfs_metaset_in_t in; - hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); - - /* store file name for given global file id */ - unifyfs_file_attr_t fattr; - memset(&fattr, 0, sizeof(fattr)); - int create = (int) in.create; - fattr.gfid = in.gfid; - strlcpy(fattr.filename, in.filename, sizeof(fattr.filename)); - fattr.mode = in.mode; - fattr.uid = in.uid; - fattr.gid = in.gid; - fattr.size = in.size; - fattr.atime = in.atime; - fattr.mtime = in.mtime; - fattr.ctime = in.ctime; - fattr.is_laminated = in.is_laminated; - - /* if we're creating the file, - * we initialize both the size and laminate flags */ - int ret = unifyfs_set_file_attribute(create, create, &fattr); + unifyfs_metaset_in_t* in = malloc(sizeof(*in)); + if (NULL == in) { + ret = ENOMEM; + } else { + hret = margo_get_input(handle, in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + client_rpc_req_t* req = malloc(sizeof(client_rpc_req_t)); + if (NULL == req) { + ret = ENOMEM; + } else { + unifyfs_fops_ctx_t ctx = { + .app_id = in->app_id, + .client_id = in->client_id, + }; + req->req_type = UNIFYFS_CLIENT_RPC_METASET; + req->handle = handle; + req->input = (void*) in; + req->bulk_buf = NULL; + req->bulk_sz = 0; + ret = rm_submit_client_rpc_request(&ctx, req); + } + + if (ret != UNIFYFS_SUCCESS) { + margo_free_input(handle, in); + } + } + } - /* build our output values */ - unifyfs_metaset_out_t out; - out.ret = ret; + /* if we hit an error during request submission, respond with the error */ + if (ret != UNIFYFS_SUCCESS) { + if (NULL != in) { + free(in); + } - /* return to caller */ - hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + /* return to caller */ + unifyfs_metaset_out_t out; + out.ret = (int32_t) ret; + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - /* free margo resources */ - margo_free_input(handle, &in); - margo_destroy(handle); + /* free margo resources */ + margo_destroy(handle); + } } DEFINE_MARGO_RPC_HANDLER(unifyfs_metaset_rpc) -/* given a client identified by (app_id, client_id) as input, read the write - * extents for one or more of the client's files from the shared memory index - * and update the global metadata for the file(s) */ -static void unifyfs_sync_rpc(hg_handle_t handle) +/* given a global file id and client identified by (app_id, client_id) as + * input, read the write extents for the file from the shared memory index + * and update its global metadata */ +static void unifyfs_fsync_rpc(hg_handle_t handle) { - /* get input params */ - unifyfs_sync_in_t in; - hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); + int ret = UNIFYFS_SUCCESS; + hg_return_t hret; - /* read the write indices for all client files from shmem and - * propagate their extents to our global metadata */ - int ret = rm_cmd_sync(in.app_id, in.client_id); + /* get input params */ + unifyfs_fsync_in_t* in = malloc(sizeof(*in)); + if (NULL == in) { + ret = ENOMEM; + } else { + hret = margo_get_input(handle, in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + client_rpc_req_t* req = malloc(sizeof(client_rpc_req_t)); + if (NULL == req) { + ret = ENOMEM; + } else { + unifyfs_fops_ctx_t ctx = { + .app_id = in->app_id, + .client_id = in->client_id, + }; + req->req_type = UNIFYFS_CLIENT_RPC_SYNC; + req->handle = handle; + req->input = (void*) in; + req->bulk_buf = NULL; + req->bulk_sz = 0; + ret = rm_submit_client_rpc_request(&ctx, req); + } + + if (ret != UNIFYFS_SUCCESS) { + margo_free_input(handle, in); + } + } + } - /* build our output values */ - unifyfs_sync_out_t out; - out.ret = ret; + /* if we hit an error during request submission, respond with the error */ + if (ret != UNIFYFS_SUCCESS) { + if (NULL != in) { + free(in); + } - /* return to caller */ - hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + /* return to caller */ + unifyfs_fsync_out_t out; + out.ret = (int32_t) ret; + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - /* free margo resources */ - margo_free_input(handle, &in); - margo_destroy(handle); + /* free margo resources */ + margo_destroy(handle); + } } -DEFINE_MARGO_RPC_HANDLER(unifyfs_sync_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_fsync_rpc) /* given an app_id, client_id, global file id, * return current file size */ static void unifyfs_filesize_rpc(hg_handle_t handle) { - /* get input params */ - unifyfs_filesize_in_t in; - hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); + int ret = UNIFYFS_SUCCESS; + hg_return_t hret; - /* read data for a single read request from client, - * returns data to client through shared memory */ - size_t filesize = 0; - int ret = rm_cmd_filesize(in.app_id, in.client_id, - in.gfid, &filesize); + /* get input params */ + unifyfs_filesize_in_t* in = malloc(sizeof(*in)); + if (NULL == in) { + ret = ENOMEM; + } else { + hret = margo_get_input(handle, in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + client_rpc_req_t* req = malloc(sizeof(client_rpc_req_t)); + if (NULL == req) { + ret = ENOMEM; + } else { + unifyfs_fops_ctx_t ctx = { + .app_id = in->app_id, + .client_id = in->client_id, + }; + req->req_type = UNIFYFS_CLIENT_RPC_FILESIZE; + req->handle = handle; + req->input = (void*) in; + req->bulk_buf = NULL; + req->bulk_sz = 0; + ret = rm_submit_client_rpc_request(&ctx, req); + } + + if (ret != UNIFYFS_SUCCESS) { + margo_free_input(handle, in); + } + } + } - /* build our output values */ - unifyfs_filesize_out_t out; - out.ret = (int32_t) ret; - out.filesize = (hg_size_t) filesize; + /* if we hit an error during request submission, respond with the error */ + if (ret != UNIFYFS_SUCCESS) { + if (NULL != in) { + free(in); + } - /* return to caller */ - hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + /* return to caller */ + unifyfs_filesize_out_t out; + out.ret = (int32_t) ret; + out.filesize = (hg_size_t) 0; + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - /* free margo resources */ - margo_free_input(handle, &in); - margo_destroy(handle); + /* free margo resources */ + margo_destroy(handle); + } } DEFINE_MARGO_RPC_HANDLER(unifyfs_filesize_rpc) @@ -340,26 +462,58 @@ DEFINE_MARGO_RPC_HANDLER(unifyfs_filesize_rpc) * and file size, truncate file to that size */ static void unifyfs_truncate_rpc(hg_handle_t handle) { - /* get input params */ - unifyfs_truncate_in_t in; - hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); + int ret = UNIFYFS_SUCCESS; + hg_return_t hret; - /* truncate file to specified size */ - int ret = rm_cmd_truncate(in.app_id, in.client_id, - in.gfid, in.filesize); + /* get input params */ + unifyfs_truncate_in_t* in = malloc(sizeof(*in)); + if (NULL == in) { + ret = ENOMEM; + } else { + hret = margo_get_input(handle, in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + client_rpc_req_t* req = malloc(sizeof(client_rpc_req_t)); + if (NULL == req) { + ret = ENOMEM; + } else { + unifyfs_fops_ctx_t ctx = { + .app_id = in->app_id, + .client_id = in->client_id, + }; + req->req_type = UNIFYFS_CLIENT_RPC_TRUNCATE; + req->handle = handle; + req->input = (void*) in; + req->bulk_buf = NULL; + req->bulk_sz = 0; + ret = rm_submit_client_rpc_request(&ctx, req); + } + + if (ret != UNIFYFS_SUCCESS) { + margo_free_input(handle, in); + } + } + } - /* build our output values */ - unifyfs_truncate_out_t out; - out.ret = (int32_t) ret; + /* if we hit an error during request submission, respond with the error */ + if (ret != UNIFYFS_SUCCESS) { + if (NULL != in) { + free(in); + } - /* return to caller */ - hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + /* return to caller */ + unifyfs_truncate_out_t out; + out.ret = (int32_t) ret; + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - /* free margo resources */ - margo_free_input(handle, &in); - margo_destroy(handle); + /* free margo resources */ + margo_destroy(handle); + } } DEFINE_MARGO_RPC_HANDLER(unifyfs_truncate_rpc) @@ -367,25 +521,59 @@ DEFINE_MARGO_RPC_HANDLER(unifyfs_truncate_rpc) * remove file from system */ static void unifyfs_unlink_rpc(hg_handle_t handle) { + int ret = UNIFYFS_SUCCESS; + hg_return_t hret; + /* get input params */ - unifyfs_unlink_in_t in; - hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); + unifyfs_unlink_in_t* in = malloc(sizeof(*in)); + if (NULL == in) { + ret = ENOMEM; + } else { + hret = margo_get_input(handle, in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + client_rpc_req_t* req = malloc(sizeof(client_rpc_req_t)); + if (NULL == req) { + ret = ENOMEM; + } else { + unifyfs_fops_ctx_t ctx = { + .app_id = in->app_id, + .client_id = in->client_id, + }; + req->req_type = UNIFYFS_CLIENT_RPC_UNLINK; + req->handle = handle; + req->input = (void*) in; + req->bulk_buf = NULL; + req->bulk_sz = 0; + ret = rm_submit_client_rpc_request(&ctx, req); + } + + if (ret != UNIFYFS_SUCCESS) { + margo_free_input(handle, in); + } + } + } - /* truncate file to specified size */ - int ret = rm_cmd_unlink(in.app_id, in.client_id, in.gfid); + /* if we hit an error during request submission, respond with the error */ + if (ret != UNIFYFS_SUCCESS) { + if (NULL != in) { + free(in); + } - /* build our output values */ - unifyfs_unlink_out_t out; - out.ret = (int32_t) ret; + /* return to caller */ + unifyfs_unlink_out_t out; + out.ret = (int32_t) ret; + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - /* return to caller */ - hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + /* free margo resources */ + margo_destroy(handle); + } - /* free margo resources */ - margo_free_input(handle, &in); - margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(unifyfs_unlink_rpc) @@ -393,43 +581,87 @@ DEFINE_MARGO_RPC_HANDLER(unifyfs_unlink_rpc) * laminate file */ static void unifyfs_laminate_rpc(hg_handle_t handle) { + int ret = UNIFYFS_SUCCESS; + hg_return_t hret; + /* get input params */ - unifyfs_laminate_in_t in; - hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); + unifyfs_laminate_in_t* in = malloc(sizeof(*in)); + if (NULL == in) { + ret = ENOMEM; + } else { + hret = margo_get_input(handle, in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + client_rpc_req_t* req = malloc(sizeof(client_rpc_req_t)); + if (NULL == req) { + ret = ENOMEM; + } else { + unifyfs_fops_ctx_t ctx = { + .app_id = in->app_id, + .client_id = in->client_id, + }; + req->req_type = UNIFYFS_CLIENT_RPC_LAMINATE; + req->handle = handle; + req->input = (void*) in; + req->bulk_buf = NULL; + req->bulk_sz = 0; + ret = rm_submit_client_rpc_request(&ctx, req); + } + + if (ret != UNIFYFS_SUCCESS) { + margo_free_input(handle, in); + } + } + } - /* truncate file to specified size */ - int ret = rm_cmd_laminate(in.app_id, in.client_id, in.gfid); + /* if we hit an error during request submission, respond with the error */ + if (ret != UNIFYFS_SUCCESS) { + if (NULL != in) { + free(in); + } - /* build our output values */ - unifyfs_laminate_out_t out; - out.ret = (int32_t) ret; + /* return to caller */ + unifyfs_laminate_out_t out; + out.ret = (int32_t) ret; + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - /* return to caller */ - hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + /* free margo resources */ + margo_destroy(handle); + } - /* free margo resources */ - margo_free_input(handle, &in); - margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(unifyfs_laminate_rpc) /* given an app_id, client_id, global file id, an offset, and a length, - * initiate read operation to lookup and return data, + * initiate read operation to lookup and return data. * client synchronizes with server again later when data is available * to be copied into user buffers */ static void unifyfs_read_rpc(hg_handle_t handle) { + int ret = (int) UNIFYFS_SUCCESS; + /* get input params */ unifyfs_read_in_t in; hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); - - /* read data for a single read request from client, - * returns data to client through shared memory */ - int ret = rm_cmd_read(in.app_id, in.client_id, - in.gfid, in.offset, in.length); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* read data for a single read request from client, + * returns data to client through shared memory */ + unifyfs_fops_ctx_t ctx = { + .app_id = in.app_id, + .client_id = in.client_id, + }; + ret = unifyfs_fops_read(&ctx, in.gfid, in.offset, in.length); + + margo_free_input(handle, &in); + } /* build our output values */ unifyfs_read_out_t out; @@ -437,10 +669,11 @@ static void unifyfs_read_rpc(hg_handle_t handle) /* return to caller */ hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } /* free margo resources */ - margo_free_input(handle, &in); margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(unifyfs_read_rpc) @@ -452,36 +685,56 @@ DEFINE_MARGO_RPC_HANDLER(unifyfs_read_rpc) * to be copied into user buffers */ static void unifyfs_mread_rpc(hg_handle_t handle) { + int ret = (int) UNIFYFS_SUCCESS; + /* get input params */ unifyfs_mread_in_t in; hg_return_t hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); - - /* allocate buffer to hold array of read requests */ - hg_size_t size = in.bulk_size; - void* buffer = (void*)malloc(size); - assert(buffer); - - /* get pointer to mercury structures to set up bulk transfer */ - const struct hg_info* hgi = margo_get_info(handle); - assert(hgi); - margo_instance_id mid = margo_hg_info_get_instance(hgi); - assert(mid != MARGO_INSTANCE_NULL); - - /* register local target buffer for bulk access */ - hg_bulk_t bulk_handle; - hret = margo_bulk_create(mid, 1, &buffer, &size, - HG_BULK_WRITE_ONLY, &bulk_handle); - assert(hret == HG_SUCCESS); - - /* get list of read requests */ - hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, - in.bulk_handle, 0, bulk_handle, 0, size); - assert(hret == HG_SUCCESS); - - /* initiate read operations to fetch data for read requests */ - int ret = rm_cmd_mread(in.app_id, in.client_id, - in.read_count, buffer); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* allocate buffer to hold array of read requests */ + hg_size_t size = in.bulk_size; + void* buffer = malloc(size); + if (NULL == buffer) { + ret = ENOMEM; + } else { + /* get pointer to mercury structures to set up bulk transfer */ + const struct hg_info* hgi = margo_get_info(handle); + assert(hgi); + margo_instance_id mid = margo_hg_info_get_instance(hgi); + assert(mid != MARGO_INSTANCE_NULL); + + /* register local target buffer for bulk access */ + hg_bulk_t bulk_handle; + hret = margo_bulk_create(mid, 1, &buffer, &size, + HG_BULK_WRITE_ONLY, &bulk_handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* get list of read requests */ + hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, + in.bulk_handle, 0, bulk_handle, + 0, size); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_transfer() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* initiate read operations to fetch data */ + unifyfs_fops_ctx_t ctx = { + .app_id = in.app_id, + .client_id = in.client_id, + }; + ret = unifyfs_fops_mread(&ctx, in.read_count, buffer); + } + margo_bulk_free(bulk_handle); + } + free(buffer); + } + margo_free_input(handle, &in); + } /* build our output values */ unifyfs_mread_out_t out; @@ -489,12 +742,11 @@ static void unifyfs_mread_rpc(hg_handle_t handle) /* return to caller */ hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } /* free margo resources */ - margo_free_input(handle, &in); - margo_bulk_free(bulk_handle); - free(buffer); margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(unifyfs_mread_rpc) diff --git a/server/src/unifyfs_fops.h b/server/src/unifyfs_fops.h new file mode 100644 index 000000000..c57038a43 --- /dev/null +++ b/server/src/unifyfs_fops.h @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef __UNIFYFS_FOPS_H +#define __UNIFYFS_FOPS_H + +#include "unifyfs_configurator.h" +#include "unifyfs_log.h" +#include "unifyfs_meta.h" + +/* + * extra information that we need to pass for file operations. + */ +struct _unifyfs_fops_ctx { + int app_id; + int client_id; +}; +typedef struct _unifyfs_fops_ctx unifyfs_fops_ctx_t; + +typedef int (*unifyfs_fops_init_t)(unifyfs_cfg_t* cfg); + +typedef int (*unifyfs_fops_metaget_t)(unifyfs_fops_ctx_t* ctx, + int gfid, unifyfs_file_attr_t* attr); + +typedef int (*unifyfs_fops_metaset_t)(unifyfs_fops_ctx_t* ctx, + int gfid, int attr_op, + unifyfs_file_attr_t* attr); + +typedef int (*unifyfs_fops_fsync_t)(unifyfs_fops_ctx_t* ctx, int gfid); + +typedef int (*unifyfs_fops_filesize_t)(unifyfs_fops_ctx_t* ctx, + int gfid, size_t* filesize); + +typedef int (*unifyfs_fops_truncate_t)(unifyfs_fops_ctx_t* ctx, + int gfid, off_t len); + +typedef int (*unifyfs_fops_laminate_t)(unifyfs_fops_ctx_t* ctx, int gfid); + +typedef int (*unifyfs_fops_unlink_t)(unifyfs_fops_ctx_t* ctx, int gfid); + +typedef int (*unifyfs_fops_read_t)(unifyfs_fops_ctx_t* ctx, + int gfid, off_t offset, size_t len); + +typedef int (*unifyfs_fops_mread_t)(unifyfs_fops_ctx_t* ctx, + size_t n_req, void* req); + +struct unifyfs_fops { + const char* name; + unifyfs_fops_init_t init; + unifyfs_fops_metaget_t metaget; + unifyfs_fops_metaset_t metaset; + unifyfs_fops_fsync_t fsync; + unifyfs_fops_filesize_t filesize; + unifyfs_fops_truncate_t truncate; + unifyfs_fops_laminate_t laminate; + unifyfs_fops_unlink_t unlink; + unifyfs_fops_read_t read; + unifyfs_fops_mread_t mread; +}; + +/* available file operations. */ +extern struct unifyfs_fops* unifyfs_fops_impl; + +/* the one that is configured to be used: defined in unifyfs_server.c */ +extern struct unifyfs_fops* global_fops_tab; + +static inline int unifyfs_fops_init(unifyfs_cfg_t* cfg) +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_fops* fops = unifyfs_fops_impl; + + if (!fops) { + LOGERR("failed to get the file operation table"); + } + + if (fops->init) { + ret = fops->init(cfg); + if (ret) { + LOGERR("failed to initialize fops table (ret=%d)", ret); + return ret; + } + } + + global_fops_tab = fops; + + return ret; +} + +static inline int unifyfs_fops_metaget(unifyfs_fops_ctx_t* ctx, + int gfid, unifyfs_file_attr_t* attr) +{ + if (!global_fops_tab->metaget) { + return ENOSYS; + } + + return global_fops_tab->metaget(ctx, gfid, attr); +} + +static inline int unifyfs_fops_metaset(unifyfs_fops_ctx_t* ctx, + int gfid, int attr_op, + unifyfs_file_attr_t* attr) +{ + if (!global_fops_tab->metaset) { + return ENOSYS; + } + + return global_fops_tab->metaset(ctx, gfid, attr_op, attr); +} + +static inline int unifyfs_fops_fsync(unifyfs_fops_ctx_t* ctx, int gfid) +{ + if (!global_fops_tab->fsync) { + return ENOSYS; + } + + return global_fops_tab->fsync(ctx, gfid); +} + +static inline int unifyfs_fops_filesize(unifyfs_fops_ctx_t* ctx, + int gfid, size_t* filesize) +{ + if (!global_fops_tab->filesize) { + return ENOSYS; + } + + return global_fops_tab->filesize(ctx, gfid, filesize); +} + +static inline int unifyfs_fops_truncate(unifyfs_fops_ctx_t* ctx, + int gfid, off_t len) +{ + if (!global_fops_tab->truncate) { + return ENOSYS; + } + + return global_fops_tab->truncate(ctx, gfid, len); +} + +static inline int unifyfs_fops_laminate(unifyfs_fops_ctx_t* ctx, int gfid) +{ + if (!global_fops_tab->laminate) { + return ENOSYS; + } + + return global_fops_tab->laminate(ctx, gfid); +} + +static inline int unifyfs_fops_unlink(unifyfs_fops_ctx_t* ctx, int gfid) +{ + if (!global_fops_tab->unlink) { + return ENOSYS; + } + + return global_fops_tab->unlink(ctx, gfid); +} + +static inline int unifyfs_fops_read(unifyfs_fops_ctx_t* ctx, + int gfid, off_t offset, size_t len) +{ + if (!global_fops_tab->read) { + return ENOSYS; + } + + LOGDBG("redirecting fops_read (fops_tab: %s)", global_fops_tab->name); + + return global_fops_tab->read(ctx, gfid, offset, len); +} + +static inline int unifyfs_fops_mread(unifyfs_fops_ctx_t* ctx, + size_t n_req, void* req) +{ + if (!global_fops_tab->mread) { + return ENOSYS; + } + + return global_fops_tab->mread(ctx, n_req, req); +} + +#endif /* __UNIFYFS_FOPS_H */ diff --git a/server/src/unifyfs_fops_mdhim.c b/server/src/unifyfs_fops_mdhim.c new file mode 100644 index 000000000..c51c916fa --- /dev/null +++ b/server/src/unifyfs_fops_mdhim.c @@ -0,0 +1,1197 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "unifyfs_group_rpc.h" +#include "unifyfs_metadata_mdhim.h" +#include "unifyfs_request_manager.h" + +/* given an extent corresponding to a write index, create new key/value + * pairs for that extent, splitting into multiple keys at the slice + * range boundaries (meta_slice_sz), it returns the number of + * newly created key/values inserted into the given key and value + * arrays */ +static int split_index( + unifyfs_key_t** keys, /* list to add newly created keys into */ + unifyfs_val_t** vals, /* list to add newly created values into */ + int* keylens, /* list for size of each key */ + int* vallens, /* list for size of each value */ + int gfid, /* global file id of write */ + size_t offset, /* starting byte offset of extent */ + size_t length, /* number of bytes in extent */ + size_t log_offset, /* offset within data log */ + int server_rank, /* rank of server hosting data */ + int app_id, /* app_id holding data */ + int client_rank) /* client rank holding data */ +{ + /* offset of first byte in request */ + size_t pos = offset; + + /* offset of last byte in request */ + size_t last_offset = offset + length - 1; + + /* this will track the current offset within the log + * where the data starts, we advance it with each key + * we generate depending on the data associated with + * each key */ + size_t logpos = log_offset; + + /* iterate over slice ranges and generate a start/end + * pair of keys for each */ + int count = 0; + while (pos <= last_offset) { + /* compute offset for first byte in this slice */ + size_t start = pos; + + /* offset for last byte in this slice, + * assume that's the last byte of the same slice + * containing start, unless that happens to be + * beyond the last byte of the actual request */ + size_t start_slice = start / meta_slice_sz; + size_t end = (start_slice + 1) * meta_slice_sz - 1; + if (end > last_offset) { + end = last_offset; + } + + /* length of extent in this slice */ + size_t len = end - start + 1; + + /* create key to describe this log entry */ + unifyfs_key_t* k = keys[count]; + k->gfid = gfid; + k->offset = start; + keylens[count] = sizeof(unifyfs_key_t); + + /* create value to store address of data */ + unifyfs_val_t* v = vals[count]; + v->addr = logpos; + v->len = len; + v->app_id = app_id; + v->rank = client_rank; + v->delegator_rank = server_rank; + vallens[count] = sizeof(unifyfs_val_t); + + /* advance to next slot in key/value arrays */ + count++; + + /* advance offset into log */ + logpos += len; + + /* advance to first byte offset of next slice */ + pos = end + 1; + } + + /* return number of keys we generated */ + return count; +} + +/* given a global file id, an offset, and a length to read from that + * file, create keys needed to query MDHIM for location of data + * corresponding to that extent, returns the number of keys inserted + * into key array provided by caller */ +static int split_request( + unifyfs_key_t** keys, /* list to add newly created keys into */ + int* keylens, /* list to add byte size of each key */ + int gfid, /* target global file id to read from */ + size_t offset, /* starting offset of read */ + size_t length) /* number of bytes to read */ +{ + /* offset of first byte in request */ + size_t pos = offset; + + /* offset of last byte in request */ + size_t last_offset = offset + length - 1; + + /* iterate over slice ranges and generate a start/end + * pair of keys for each */ + int count = 0; + while (pos <= last_offset) { + /* compute offset for first byte in this segment */ + size_t start = pos; + + /* offset for last byte in this segment, + * assume that's the last byte of the same segment + * containing start, unless that happens to be + * beyond the last byte of the actual request */ + size_t start_slice = start / meta_slice_sz; + size_t end = (start_slice + 1) * meta_slice_sz - 1; + if (end > last_offset) { + end = last_offset; + } + + /* create key to describe first byte we'll read + * in this slice */ + keys[count]->gfid = gfid; + keys[count]->offset = start; + keylens[count] = sizeof(unifyfs_key_t); + count++; + + /* create key to describe last byte we'll read + * in this slice */ + keys[count]->gfid = gfid; + keys[count]->offset = end; + keylens[count] = sizeof(unifyfs_key_t); + count++; + + /* advance to first byte offset of next slice */ + pos = end + 1; + } + + /* return number of keys we generated */ + return count; +} + + +static int mdhim_init(unifyfs_cfg_t* cfg) +{ + int ret = 0; + + LOGDBG("initializing file operations.."); + + ret = meta_init_store(cfg); + if (ret) { + LOGERR("failed to initialize the meta kv store (ret=%d)", ret); + } + + return ret; +} + +static int mdhim_metaget(unifyfs_fops_ctx_t* ctx, + int gfid, unifyfs_file_attr_t* attr) +{ + return unifyfs_get_file_attribute(gfid, attr); +} + +static int mdhim_metaset(unifyfs_fops_ctx_t* ctx, + int gfid, int create, unifyfs_file_attr_t* attr) +{ + return unifyfs_set_file_attribute(create, create, attr); +} + +static int mdhim_fsync(unifyfs_fops_ctx_t* ctx, int gfid) +{ + size_t i; + + /* assume we'll succeed */ + int ret = (int)UNIFYFS_SUCCESS; + + /* get memory page size on this machine */ + int page_sz = getpagesize(); + + /* get application client */ + app_client* client = get_app_client(ctx->app_id, ctx->client_id); + if (NULL == client) { + return EINVAL; + } + + /* get pointer to superblock for this client and app */ + shm_context* super_ctx = client->shmem_super; + if (NULL == super_ctx) { + LOGERR("missing client superblock"); + return UNIFYFS_FAILURE; + } + char* superblk = (char*)(super_ctx->addr); + + /* get pointer to start of key/value region in superblock */ + char* meta = superblk + client->super_meta_offset; + + /* get number of file extent index values client has for us, + * stored as a size_t value in meta region of shared memory */ + size_t extent_num_entries = *(size_t*)(meta); + + /* indices are stored in the superblock shared memory + * created by the client, these are stored as index_t + * structs starting one page size offset into meta region */ + char* ptr_extents = meta + page_sz; + + if (extent_num_entries == 0) { + /* Nothing to do */ + return UNIFYFS_SUCCESS; + } + + unifyfs_index_t* meta_payload = (unifyfs_index_t*)(ptr_extents); + + /* total up number of key/value pairs we'll need for this + * set of index values */ + size_t slices = 0; + for (i = 0; i < extent_num_entries; i++) { + size_t offset = meta_payload[i].file_pos; + size_t length = meta_payload[i].length; + slices += meta_num_slices(offset, length); + } + if (slices >= UNIFYFS_MAX_SPLIT_CNT) { + LOGERR("Error allocating buffers"); + return ENOMEM; + } + + /* pointers to memory we'll dynamically allocate for file extents */ + unifyfs_key_t** keys = NULL; + unifyfs_val_t** vals = NULL; + int* key_lens = NULL; + int* val_lens = NULL; + + /* allocate storage for file extent key/values */ + /* TODO: possibly get this from memory pool */ + keys = alloc_key_array(slices); + vals = alloc_value_array(slices); + key_lens = calloc(slices, sizeof(int)); + val_lens = calloc(slices, sizeof(int)); + if ((NULL == keys) || + (NULL == vals) || + (NULL == key_lens) || + (NULL == val_lens)) { + LOGERR("failed to allocate memory for file extents"); + ret = ENOMEM; + goto mdhim_sync_exit; + } + + /* create file extent key/values for insertion into MDHIM */ + int count = 0; + for (i = 0; i < extent_num_entries; i++) { + /* get file offset, length, and log offset for this entry */ + unifyfs_index_t* meta = &meta_payload[i]; + assert(gfid == meta->gfid); + size_t offset = meta->file_pos; + size_t length = meta->length; + size_t logpos = meta->log_pos; + + /* split this entry at the offset boundaries */ + int used = split_index( + &keys[count], &vals[count], &key_lens[count], &val_lens[count], + gfid, offset, length, logpos, + glb_pmi_rank, ctx->app_id, ctx->client_id); + + /* count up the number of keys we used for this index */ + count += used; + } + + /* batch insert file extent key/values into MDHIM */ + ret = unifyfs_set_file_extents((int)count, + keys, key_lens, vals, val_lens); + if (ret != UNIFYFS_SUCCESS) { + /* TODO: need proper error handling */ + LOGERR("unifyfs_set_file_extents() failed"); + goto mdhim_sync_exit; + } + +mdhim_sync_exit: + /* clean up memory */ + if (NULL != keys) { + free_key_array(keys); + } + + if (NULL != vals) { + free_value_array(vals); + } + + if (NULL != key_lens) { + free(key_lens); + } + + if (NULL != val_lens) { + free(val_lens); + } + + return ret; +} + +static int mdhim_filesize(unifyfs_fops_ctx_t* ctx, int gfid, size_t* outsize) +{ + size_t filesize = 0; + int ret = unifyfs_invoke_filesize_rpc(gfid, &filesize); + if (ret) { + LOGERR("filesize rpc failed (ret=%d)", ret); + } else { + LOGDBG("filesize rpc returned %zu", filesize); + *outsize = filesize; + } + + unifyfs_file_attr_t attr = { 0, }; + mdhim_metaget(ctx, gfid, &attr); + + /* return greater of rpc value and mdhim metadata size */ + size_t asize = (size_t) attr.size; + if (asize > filesize) { + *outsize = asize; + } + + return ret; +} + +/* delete any key whose last byte is beyond the specified + * file size */ +static int truncate_delete_keys( + size_t filesize, /* new file size */ + int num, /* number of entries in keyvals */ + unifyfs_keyval_t* keyvals) /* list of existing key/values */ +{ + /* assume we'll succeed */ + int ret = (int) UNIFYFS_SUCCESS; + + /* pointers to memory we'll dynamically allocate for file extents */ + unifyfs_key_t** unifyfs_keys = NULL; + unifyfs_val_t** unifyfs_vals = NULL; + int* unifyfs_key_lens = NULL; + int* unifyfs_val_lens = NULL; + + /* in the worst case, we'll have to delete all existing keys */ + /* allocate storage for file extent key/values */ + /* TODO: possibly get this from memory pool */ + unifyfs_keys = alloc_key_array(num); + unifyfs_vals = alloc_value_array(num); + unifyfs_key_lens = calloc(num, sizeof(int)); + unifyfs_val_lens = calloc(num, sizeof(int)); + if ((NULL == unifyfs_keys) || + (NULL == unifyfs_vals) || + (NULL == unifyfs_key_lens) || + (NULL == unifyfs_val_lens)) { + LOGERR("failed to allocate memory for file extents"); + ret = ENOMEM; + goto truncate_delete_exit; + } + + /* counter for number of key/values we need to delete */ + int delete_count = 0; + + /* iterate over each key, and if this index extends beyond desired + * file size, create an entry to delete that key */ + int i; + for (i = 0; i < num; i++) { + /* get pointer to next key value pair */ + unifyfs_keyval_t* kv = &keyvals[i]; + + /* get last byte offset for this segment of the file */ + size_t last_offset = kv->key.offset + kv->val.len; + + /* if this segment extends beyond the new file size, + * we need to delete this index entry */ + if (last_offset > filesize) { + /* found an index that extends past end of desired + * file size, get next empty key entry from the pool */ + unifyfs_key_t* key = unifyfs_keys[delete_count]; + + /* define the key to be deleted */ + key->gfid = kv->key.gfid; + key->offset = kv->key.offset; + + /* MDHIM needs to know the byte size of each key and value */ + unifyfs_key_lens[delete_count] = sizeof(unifyfs_key_t); + //unifyfs_val_lens[delete_count] = sizeof(unifyfs_val_t); + + /* increment the number of keys we're deleting */ + delete_count++; + } + } + + /* batch delete file extent key/values from MDHIM */ + if (delete_count > 0) { + ret = unifyfs_delete_file_extents(delete_count, + unifyfs_keys, unifyfs_key_lens); + if (ret != UNIFYFS_SUCCESS) { + /* TODO: need proper error handling */ + LOGERR("unifyfs_delete_file_extents() failed"); + goto truncate_delete_exit; + } + } + +truncate_delete_exit: + /* clean up memory */ + + if (NULL != unifyfs_keys) { + free_key_array(unifyfs_keys); + } + + if (NULL != unifyfs_vals) { + free_value_array(unifyfs_vals); + } + + if (NULL != unifyfs_key_lens) { + free(unifyfs_key_lens); + } + + if (NULL != unifyfs_val_lens) { + free(unifyfs_val_lens); + } + + return ret; +} + +/* rewrite any key that overlaps with new file size, + * we assume the existing key has already been deleted */ +static int truncate_rewrite_keys( + size_t filesize, /* new file size */ + int num, /* number of entries in keyvals */ + unifyfs_keyval_t* keyvals) /* list of existing key/values */ +{ + /* assume we'll succeed */ + int ret = (int) UNIFYFS_SUCCESS; + + /* pointers to memory we'll dynamically allocate for file extents */ + unifyfs_key_t** unifyfs_keys = NULL; + unifyfs_val_t** unifyfs_vals = NULL; + int* unifyfs_key_lens = NULL; + int* unifyfs_val_lens = NULL; + + /* in the worst case, we'll have to rewrite all existing keys */ + /* allocate storage for file extent key/values */ + /* TODO: possibly get this from memory pool */ + unifyfs_keys = alloc_key_array(num); + unifyfs_vals = alloc_value_array(num); + unifyfs_key_lens = calloc(num, sizeof(int)); + unifyfs_val_lens = calloc(num, sizeof(int)); + if ((NULL == unifyfs_keys) || + (NULL == unifyfs_vals) || + (NULL == unifyfs_key_lens) || + (NULL == unifyfs_val_lens)) { + LOGERR("failed to allocate memory for file extents"); + ret = ENOMEM; + goto truncate_rewrite_exit; + } + + /* counter for number of key/values we need to rewrite */ + int count = 0; + + /* iterate over each key, and if this index starts before + * and ends after the desired file size, create an entry + * that ends at new file size */ + int i; + for (i = 0; i < num; i++) { + /* get pointer to next key value pair */ + unifyfs_keyval_t* kv = &keyvals[i]; + + /* get first byte offset for this segment of the file */ + size_t first_offset = kv->key.offset; + + /* get last byte offset for this segment of the file */ + size_t last_offset = kv->key.offset + kv->val.len; + + /* if this segment extends beyond the new file size, + * we need to rewrite this index entry */ + if (first_offset < filesize && + last_offset > filesize) { + /* found an index that overlaps end of desired + * file size, get next empty key entry from the pool */ + unifyfs_key_t* key = unifyfs_keys[count]; + + /* define the key to be rewritten */ + key->gfid = kv->key.gfid; + key->offset = kv->key.offset; + + /* compute new length of this entry */ + size_t newlen = (size_t)(filesize - first_offset); + + /* for the value, we store the log position, the length, + * the host server (delegator rank), the mount point id + * (app id), and the client id (rank) */ + unifyfs_val_t* val = unifyfs_vals[count]; + val->addr = kv->val.addr; + val->len = newlen; + val->delegator_rank = kv->val.delegator_rank; + val->app_id = kv->val.app_id; + val->rank = kv->val.rank; + + /* MDHIM needs to know the byte size of each key and value */ + unifyfs_key_lens[count] = sizeof(unifyfs_key_t); + unifyfs_val_lens[count] = sizeof(unifyfs_val_t); + + /* increment the number of keys we're deleting */ + count++; + } + } + + /* batch set file extent key/values from MDHIM */ + if (count > 0) { + ret = unifyfs_set_file_extents(count, + unifyfs_keys, unifyfs_key_lens, + unifyfs_vals, unifyfs_val_lens); + if (ret != UNIFYFS_SUCCESS) { + /* TODO: need proper error handling */ + LOGERR("unifyfs_set_file_extents() failed"); + goto truncate_rewrite_exit; + } + } + +truncate_rewrite_exit: + /* clean up memory */ + + if (NULL != unifyfs_keys) { + free_key_array(unifyfs_keys); + } + + if (NULL != unifyfs_vals) { + free_value_array(unifyfs_vals); + } + + if (NULL != unifyfs_key_lens) { + free(unifyfs_key_lens); + } + + if (NULL != unifyfs_val_lens) { + free(unifyfs_val_lens); + } + + return ret; +} + +static int mdhim_truncate(unifyfs_fops_ctx_t* ctx, int gfid, off_t len) +{ + size_t newsize = (size_t) len; + + /* set offset and length to request *all* key/value pairs + * for this file */ + size_t offset = 0; + + /* want to pick the highest integer offset value a file + * could have here */ + size_t length = (SIZE_MAX >> 1) - 1; + + /* get the locations of all the read requests from the + * key-value store*/ + unifyfs_key_t key1, key2; + + /* create key to describe first byte we'll read */ + key1.gfid = gfid; + key1.offset = offset; + + /* create key to describe last byte we'll read */ + key2.gfid = gfid; + key2.offset = offset + length - 1; + + /* set up input params to specify range lookup */ + unifyfs_key_t* unifyfs_keys[2] = {&key1, &key2}; + int key_lens[2] = {sizeof(unifyfs_key_t), sizeof(unifyfs_key_t)}; + + /* look up all entries in this range */ + int num_vals = 0; + unifyfs_keyval_t* keyvals = NULL; + int rc = unifyfs_get_file_extents(2, unifyfs_keys, key_lens, + &num_vals, &keyvals); + if (UNIFYFS_SUCCESS != rc) { + /* failed to look up extents, bail with error */ + return UNIFYFS_FAILURE; + } + + /* compute our file size by iterating over each file + * segment and taking the max logical offset */ + int i; + size_t filesize = 0; + for (i = 0; i < num_vals; i++) { + /* get pointer to next key value pair */ + unifyfs_keyval_t* kv = &keyvals[i]; + + /* get last byte offset for this segment of the file */ + size_t last_offset = kv->key.offset + kv->val.len; + + /* update our filesize if this offset is bigger than the current max */ + if (last_offset > filesize) { + filesize = last_offset; + } + } + + /* get filesize as recorded in metadata, which may be bigger if + * user issued an ftruncate on the file to extend it past the + * last write */ + size_t filesize_meta = filesize; + + /* given the global file id, look up file attributes + * from key/value store */ + unifyfs_file_attr_t fattr; + rc = unifyfs_get_file_attribute(gfid, &fattr); + if (rc == UNIFYFS_SUCCESS) { + /* found file attribute for this file, now get its size */ + filesize_meta = fattr.size; + } else { + /* failed to find file attributes for this file */ + goto truncate_exit; + } + + /* take maximum of last write and file size from metadata */ + if (filesize_meta > filesize) { + filesize = filesize_meta; + } + + /* may need to throw away and rewrite keys if shrinking file */ + if (newsize < filesize) { + /* delete any key that extends beyond new file size */ + rc = truncate_delete_keys(newsize, num_vals, keyvals); + if (rc != UNIFYFS_SUCCESS) { + goto truncate_exit; + } + + /* rewrite any key that overlaps new file size */ + rc = truncate_rewrite_keys(newsize, num_vals, keyvals); + if (rc != UNIFYFS_SUCCESS) { + goto truncate_exit; + } + } + + /* update file size field with latest size */ + fattr.size = newsize; + rc = unifyfs_set_file_attribute(1, 0, &fattr); + if (rc != UNIFYFS_SUCCESS) { + /* failed to update file attributes with new file size */ + goto truncate_exit; + } + + rc = unifyfs_invoke_truncate_rpc(gfid, newsize); + if (rc) { + LOGERR("truncate rpc failed"); + } + +truncate_exit: + + /* free off key/value buffer returned from get_file_extents */ + if (NULL != keyvals) { + free(keyvals); + keyvals = NULL; + } + + return rc; +} + +static int mdhim_laminate(unifyfs_fops_ctx_t* ctx, int gfid) +{ + int rc = UNIFYFS_SUCCESS; + + /* given the global file id, look up file attributes + * from key/value store */ + unifyfs_file_attr_t attr = { 0, }; + int ret = mdhim_metaget(ctx, gfid, &attr); + if (ret != UNIFYFS_SUCCESS) { + /* failed to find attributes for the file */ + return ret; + } + + /* if item is not a file, bail with error */ + mode_t mode = (mode_t) attr.mode; + if ((mode & S_IFMT) != S_IFREG) { + /* item is not a regular file */ + LOGERR("ERROR: only regular files can be laminated (gfid=%d)", gfid); + return EINVAL; + } + + /* lookup current file size */ + size_t filesize; + ret = mdhim_filesize(ctx, gfid, &filesize); + if (ret != UNIFYFS_SUCCESS) { + /* failed to get file size for file */ + LOGERR("lamination file size calculation failed (gfid=%d)", gfid); + return ret; + } + + /* update fields in metadata */ + attr.size = filesize; + attr.is_laminated = 1; + + /* update metadata, set size and laminate */ + rc = unifyfs_set_file_attribute(1, 1, &attr); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("lamination metadata update failed (gfid=%d)", gfid); + } + + return rc; +} + +static int mdhim_unlink(unifyfs_fops_ctx_t* ctx, int gfid) +{ + int rc = UNIFYFS_SUCCESS; + + /* given the global file id, look up file attributes + * from key/value store */ + unifyfs_file_attr_t attr; + int ret = unifyfs_get_file_attribute(gfid, &attr); + if (ret != UNIFYFS_SUCCESS) { + /* failed to find attributes for the file */ + return ret; + } + + /* if item is a file, call truncate to free space */ + mode_t mode = (mode_t) attr.mode; + if ((mode & S_IFMT) == S_IFREG) { + /* item is regular file, truncate to 0 */ + ret = mdhim_truncate(ctx, gfid, 0); + if (ret != UNIFYFS_SUCCESS) { + /* failed to delete write extents for file, + * let's leave the file attributes in place */ + return ret; + } + } + + /* delete metadata */ + ret = unifyfs_delete_file_attribute(gfid); + if (ret != UNIFYFS_SUCCESS) { + rc = ret; + } + + rc = unifyfs_invoke_unlink_rpc(gfid); + if (rc) { + LOGERR("unlink rpc failed (ret=%d)", rc); + } + + return rc; +} + + +/* given a set of input key pairs, where each pair describes the first + * and last byte offset of a data range, refer to our local extent map + * and generate keyval responses for any ranges covering data that is + * local to the server, generate new key pairs to describe remaining + * holes that will be queried against the global key/value store, + * the list of output keys, key lengths, and keyvals are allocated + * and returned to be freed by the caller */ +static int get_local_keyvals( + int num_keys, /* number of input keys */ + unifyfs_key_t** keys, /* list of input keys */ + int* keylens, /* list of input key lengths */ + int* out_global, /* number of output keys for server */ + unifyfs_key_t*** out_keys, /* list of output keys */ + int** out_keylens, /* list of output key lengths */ + int* num_keyvals, /* number of output keyvals from local data */ + unifyfs_keyval_t** keyvals) /* list of output keyvals */ +{ + /* initialize output parameters */ + *out_global = 0; + *out_keys = NULL; + *out_keylens = NULL; + *num_keyvals = 0; + *keyvals = NULL; + + /* allocate memory to copy key/value data */ + int max_keyvals = UNIFYFS_MAX_SPLIT_CNT; + unifyfs_keyval_t* kvs_local = (unifyfs_keyval_t*) calloc( + max_keyvals, sizeof(unifyfs_keyval_t)); + if (NULL == kvs_local) { + LOGERR("failed to allocate keyvals"); + return (int)UNIFYFS_ERROR_MDHIM; + } + + /* allocate memory to define remaining keys to + * search in global store */ + unifyfs_key_t** keys_global = alloc_key_array(max_keyvals); + if (NULL == keys_global) { + LOGERR("failed to allocate keys"); + free(kvs_local); + return (int)UNIFYFS_ERROR_MDHIM; + } + + /* allocate memory to define key lengths for remaining keys to + * search in global store */ + int* keylens_global = (int*) calloc(max_keyvals, sizeof(int)); + if (NULL == keylens_global) { + LOGERR("failed to allocate keylens"); + free_key_array(keys_global); + free(kvs_local); + return (int)UNIFYFS_ERROR_MDHIM; + } + + /* counters for the number of local keyvals we create and the + * number of keys we generate for the global key/value store */ + int count_global = 0; + int count_local = 0; + + int i; + for (i = 0; i < num_keys; i += 2) { + /* get next key pair that describe start and end offsets */ + unifyfs_key_t* k1 = keys[i+0]; + unifyfs_key_t* k2 = keys[i+1]; + + /* get gfid, start, and end offset of this pair */ + int gfid = k1->gfid; + size_t start = k1->offset; + size_t end = k2->offset; + + /* we'll define key/values in these temp arrays that correspond + * to extents we have locally */ + unifyfs_key_t tmpkeys[UNIFYFS_MAX_SPLIT_CNT]; + unifyfs_val_t tmpvals[UNIFYFS_MAX_SPLIT_CNT]; + + /* look up any entries we can find in our local extent map */ + int num_local = 0; + int ret = unifyfs_inode_span_extents(gfid, start, end, + UNIFYFS_MAX_SPLIT_CNT, tmpkeys, tmpvals, &num_local); + if (ret) { + LOGERR("failed to span extents (gfid=%d)", gfid); + // now what? + } + + /* iterate over local keys, create new keys to pass to server + * for any holes in our local extents */ + int j; + size_t nextstart = start; + for (j = 0; j < num_local; j++) { + /* get next key/value returned from local extent */ + unifyfs_key_t* k = &tmpkeys[j]; + unifyfs_val_t* v = &tmpvals[j]; + + /* if we have a gap in our data, + * we need to ask the global key/value store */ + if (nextstart < k->offset) { + /* we're missing a section of bytes, so create a key + * pair to search for this hole in the global key/value + * store */ + + /* check that we don't overflow the global array */ + if (count_global + 2 > max_keyvals) { + /* exhausted our space */ + free(keylens_global); + free_key_array(keys_global); + free(kvs_local); + return ENOMEM; + } + + /* first key is for starting offset of the hole, + * which is defined in next start */ + unifyfs_key_t* gk1 = keys_global[count_global]; + gk1->gfid = gfid; + gk1->offset = nextstart; + keylens_global[count_global] = sizeof(unifyfs_key_t); + count_global++; + + /* second key is for ending offset of the hole, + * which will be the offset of the byte that comes + * just before the offset of the current key */ + unifyfs_key_t* gk2 = keys_global[count_global]; + gk2->gfid = gfid; + gk2->offset = k->offset - 1; + keylens_global[count_global] = sizeof(unifyfs_key_t); + count_global++; + } else { + /* otherwise we have a local extent that matches, + * copy the corresponding key/value pair into the + * local output array */ + + /* check that we don't overflow the local array */ + if (count_local + 1 > max_keyvals) { + /* exhausted our space */ + free(keylens_global); + free_key_array(keys_global); + free(kvs_local); + return ENOMEM; + } + + /* create a key/value describing the + * current local extent */ + + /* get pointer to next key/val */ + unifyfs_keyval_t* kv = &kvs_local[count_local]; + + /* copy in the key and value generated from the call + * to tree_span into our array of local key/value pairs */ + memcpy(&kv->key, k, sizeof(unifyfs_key_t)); + memcpy(&kv->val, v, sizeof(unifyfs_val_t)); + + /* increase the number of keyvals we've found locally */ + count_local++; + } + + /* advance to start of next segment we're looking for */ + nextstart = k->offset + v->len; + } + + /* verify that we covered the full range, create a key pair + * to look in the global key/value store for any trailing hole */ + if (nextstart <= end) { + /* check that we don't overflow the global array */ + if (count_global + 2 > max_keyvals) { + /* exhausted our space */ + free(keylens_global); + free_key_array(keys_global); + free(kvs_local); + return ENOMEM; + } + + /* first key is for starting offset of the hole, + * which is defined in next start */ + unifyfs_key_t* gk1 = keys_global[count_global]; + gk1->gfid = gfid; + gk1->offset = nextstart; + keylens_global[count_global] = sizeof(unifyfs_key_t); + count_global++; + + /* second key is for ending offset of the hole */ + unifyfs_key_t* gk2 = keys_global[count_global]; + gk2->gfid = gfid; + gk2->offset = end; + keylens_global[count_global] = sizeof(unifyfs_key_t); + count_global++; + } + } + + /* set output values */ + *out_global = count_global; + *out_keys = keys_global; + *out_keylens = keylens_global; + *num_keyvals = count_local; + *keyvals = kvs_local; + + return UNIFYFS_SUCCESS; +} + +static int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, int gfid, + int app_id, int client_id, int num_keys, + unifyfs_key_t** keys, int* keylens) +{ + int rc = UNIFYFS_SUCCESS; + + int num_vals = 0; + unifyfs_keyval_t* keyvals = NULL; + + /* not using our local extent map, + * lookup all keys from global key/value store */ + rc = unifyfs_get_file_extents(num_keys, keys, keylens, + &num_vals, &keyvals); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to lookup keyvals from global key/val store"); + return rc; + } + + /* this is to maintain limits imposed in previous code + * that would throw fatal errors */ + if (num_vals >= UNIFYFS_MAX_SPLIT_CNT || + num_vals >= MAX_META_PER_SEND) { + LOGERR("too many key/values returned in range lookup"); + if (NULL != keyvals) { + free(keyvals); + keyvals = NULL; + } + return ENOMEM; + } + + if (UNIFYFS_SUCCESS != rc) { + /* failed to find any key / value pairs */ + rc = UNIFYFS_FAILURE; + } else { + /* if we get more than one write index entry + * sort them by file id and then by delegator rank */ + if (num_vals > 1) { + qsort(keyvals, (size_t)num_vals, sizeof(unifyfs_keyval_t), + unifyfs_keyval_compare); + } + + server_read_req_t* rdreq = rm_reserve_read_req(thrd_ctrl); + if (NULL == rdreq) { + rc = UNIFYFS_FAILURE; + } else { + rdreq->app_id = app_id; + rdreq->client_id = client_id; + /* TODO: rdreq->extent was removed + * rdreq->extent.gfid = gfid; + * rdreq->extent.errcode = EINPROGRESS; + */ + rc = rm_create_chunk_requests(thrd_ctrl, rdreq, + num_vals, keyvals); + if (rc != (int)UNIFYFS_SUCCESS) { + rm_release_read_req(thrd_ctrl, rdreq); + } + } + } + + /* free off key/value buffer returned from get_file_extents */ + if (NULL != keyvals) { + free(keyvals); + keyvals = NULL; + } + + return rc; +} + +static int mdhim_read(unifyfs_fops_ctx_t* ctx, + int gfid, off_t offset, size_t length) +{ + /* get application client */ + int app_id = ctx->app_id; + int client_id = ctx->client_id; + app_client* client = get_app_client(app_id, client_id); + if (NULL == client) { + return (int)UNIFYFS_FAILURE; + } + + /* get thread control structure */ + reqmgr_thrd_t* thrd_ctrl = client->reqmgr; + + /* get chunks corresponding to requested client read extent + * + * Generate a pair of keys for the read request, representing the start + * and end offset. MDHIM returns all key-value pairs that fall within + * the offset range. + * + * TODO: this is specific to the MDHIM in the source tree and not portable + * to other KV-stores. This needs to be revisited to utilize some + * other mechanism to retrieve all relevant key-value pairs from the + * KV-store. + */ + + /* count number of slices this range covers */ + size_t slices = meta_num_slices(offset, length); + if (slices >= UNIFYFS_MAX_SPLIT_CNT) { + LOGERR("Error allocating buffers"); + return ENOMEM; + } + + /* allocate key storage */ + size_t key_cnt = slices * 2; + unifyfs_key_t** keys = alloc_key_array(key_cnt); + int* key_lens = (int*) calloc(key_cnt, sizeof(int)); + if ((NULL == keys) || + (NULL == key_lens)) { + // this is a fatal error + // TODO: we need better error handling + LOGERR("Error allocating buffers"); + return ENOMEM; + } + + /* split range of read request at boundaries used for + * MDHIM range query */ + split_request(keys, key_lens, gfid, offset, length); + + /* queue up the read operations */ + int rc = create_gfid_chunk_reads(thrd_ctrl, gfid, app_id, client_id, + key_cnt, keys, key_lens); + + /* free memory allocated for key storage */ + free_key_array(keys); + free(key_lens); + + return rc; +} + +static int mdhim_mread(unifyfs_fops_ctx_t* ctx, size_t num_req, void* reqbuf) +{ + int rc = UNIFYFS_SUCCESS; + int app_id = ctx->app_id; + int client_id = ctx->client_id; + unifyfs_extent_t* req; + unifyfs_extent_t* reqs = (unifyfs_extent_t*)reqbuf; + + /* get application client */ + app_client* client = get_app_client(app_id, client_id); + if (NULL == client) { + return (int)UNIFYFS_FAILURE; + } + + /* get thread control structure */ + reqmgr_thrd_t* thrd_ctrl = client->reqmgr; + + /* count up number of slices these request cover */ + int i; + size_t slices = 0; + for (i = 0; i < num_req; i++) { + req = reqs + i; + + /* get offset and length of next request */ + size_t off = req->offset; + size_t len = req->length; + + /* add in number of slices this request needs */ + slices += meta_num_slices(off, len); + } + if (slices >= UNIFYFS_MAX_SPLIT_CNT) { + LOGERR("Error allocating buffers"); + return ENOMEM; + } + + /* allocate key storage */ + size_t key_cnt = slices * 2; + unifyfs_key_t** keys = alloc_key_array(key_cnt); + int* key_lens = (int*) calloc(key_cnt, sizeof(int)); + if ((NULL == keys) || + (NULL == key_lens)) { + // this is a fatal error + // TODO: we need better error handling + LOGERR("Error allocating buffers"); + return ENOMEM; + } + + /* get chunks corresponding to requested client read extents */ + int ret; + int num_keys = 0; + int last_gfid = -1; + for (i = 0; i < num_req; i++) { + req = reqs + i; + + /* get the file id for this request */ + int gfid = req->gfid; + + /* if we have switched to a different file, create chunk reads + * for the previous file */ + if (i && (gfid != last_gfid)) { + /* create requests for all extents of last_gfid */ + ret = create_gfid_chunk_reads(thrd_ctrl, last_gfid, + app_id, client_id, + num_keys, keys, key_lens); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("Error creating chunk reads for gfid=%d", last_gfid); + rc = ret; + } + + /* reset key counter for the current gfid */ + num_keys = 0; + } + + /* get offset and length of current read request */ + size_t off = req->offset; + size_t len = req->length; + LOGDBG("gfid:%d, offset:%zu, length:%zu", gfid, off, len); + + /* Generate a pair of keys for each read request, representing + * the start and end offsets. MDHIM returns all key-value pairs that + * fall within the offset range. + * + * TODO: this is specific to the MDHIM in the source tree and not + * portable to other KV-stores. This needs to be revisited to + * utilize some other mechanism to retrieve all relevant KV + * pairs from the KV-store. + */ + + /* split range of read request at boundaries used for + * MDHIM range query */ + int used = split_request(&keys[num_keys], &key_lens[num_keys], + gfid, off, len); + num_keys += used; + + /* keep track of the last gfid value that we processed */ + last_gfid = gfid; + } + + /* create requests for all extents of final gfid */ + ret = create_gfid_chunk_reads(thrd_ctrl, last_gfid, + app_id, client_id, + num_keys, keys, key_lens); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("Error creating chunk reads for gfid=%d", last_gfid); + rc = ret; + } + + /* free memory allocated for key storage */ + free_key_array(keys); + free(key_lens); + + return rc; +} + +static struct unifyfs_fops _fops_mdhim = { + .name = "mdhim", + .init = mdhim_init, + .metaget = mdhim_metaget, + .metaset = mdhim_metaset, + .fsync = mdhim_fsync, + .filesize = mdhim_filesize, + .truncate = mdhim_truncate, + .laminate = mdhim_laminate, + .unlink = mdhim_unlink, + .read = mdhim_read, + .mread = mdhim_mread, +}; + +struct unifyfs_fops* unifyfs_fops_impl = &_fops_mdhim; + diff --git a/server/src/unifyfs_fops_rpc.c b/server/src/unifyfs_fops_rpc.c new file mode 100644 index 000000000..0492e5885 --- /dev/null +++ b/server/src/unifyfs_fops_rpc.c @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "unifyfs_inode_tree.h" +#include "unifyfs_inode.h" +#include "unifyfs_group_rpc.h" +#include "unifyfs_p2p_rpc.h" +#include "unifyfs_request_manager.h" + + +static +int rpc_init(unifyfs_cfg_t* cfg) +{ + int ret = 0; + long range_sz = 0; + + LOGDBG("initializing file operations.."); + + ret = configurator_int_val(cfg->meta_range_size, &range_sz); + if (ret != 0) { + LOGERR("failed to read configuration (meta_range_size)"); + } + meta_slice_sz = (size_t) range_sz; + + return ret; +} + +static +int rpc_metaget(unifyfs_fops_ctx_t* ctx, + int gfid, + unifyfs_file_attr_t* attr) +{ + return unifyfs_invoke_metaget_rpc(gfid, attr); +} + +static +int rpc_metaset(unifyfs_fops_ctx_t* ctx, + int gfid, + int attr_op, + unifyfs_file_attr_t* attr) +{ + return unifyfs_invoke_metaset_rpc(gfid, attr_op, attr); +} + +/* + * sync rpc from client contains extents for a single gfid (file). + */ +static +int rpc_fsync(unifyfs_fops_ctx_t* ctx, + int gfid) +{ + size_t i; + + /* assume we'll succeed */ + int ret = UNIFYFS_SUCCESS; + + /* get memory page size on this machine */ + int page_sz = getpagesize(); + + /* get application client */ + app_client* client = get_app_client(ctx->app_id, ctx->client_id); + if (NULL == client) { + return EINVAL; + } + + /* get pointer to superblock for this client and app */ + shm_context* super_ctx = client->shmem_super; + if (NULL == super_ctx) { + LOGERR("missing client superblock"); + return UNIFYFS_FAILURE; + } + char* superblk = (char*)(super_ctx->addr); + + /* get pointer to start of key/value region in superblock */ + char* meta = superblk + client->super_meta_offset; + + /* get number of file extent index values client has for us, + * stored as a size_t value in meta region of shared memory */ + size_t num_extents = *(size_t*)(meta); + + /* indices are stored in the superblock shared memory + * created by the client, these are stored as index_t + * structs starting one page size offset into meta region + * + * Is it safe to assume that the index information in this superblock is + * not going to be modified by the client while we perform this operation? + */ + char* ptr_extents = meta + page_sz; + + if (num_extents == 0) { + return UNIFYFS_SUCCESS; /* Nothing to do */ + } + + unifyfs_index_t* meta_payload = (unifyfs_index_t*)(ptr_extents); + + struct extent_tree_node* extents = calloc(num_extents, sizeof(*extents)); + if (!extents) { + LOGERR("failed to allocate memory for local_extents"); + return ENOMEM; + } + + /* the sync rpc now contains extents from a single file/gfid */ + assert(gfid == meta_payload[0].gfid); + + for (i = 0; i < num_extents; i++) { + struct extent_tree_node* extent = &extents[i]; + unifyfs_index_t* meta = &meta_payload[i]; + + extent->start = meta->file_pos; + extent->end = (meta->file_pos + meta->length) - 1; + extent->svr_rank = glb_pmi_rank; + extent->app_id = ctx->app_id; + extent->cli_id = ctx->client_id; + extent->pos = meta->log_pos; + } + + /* update local inode state first */ + ret = unifyfs_inode_add_extents(gfid, num_extents, extents); + if (ret) { + LOGERR("failed to add local extents (gfid=%d, ret=%d)", gfid, ret); + return ret; + } + + /* then update owner inode state */ + ret = unifyfs_invoke_add_extents_rpc(gfid, num_extents, extents); + if (ret) { + LOGERR("failed to add extents (gfid=%d, ret=%d)", gfid, ret); + } + + return ret; +} + +static +int rpc_filesize(unifyfs_fops_ctx_t* ctx, + int gfid, + size_t* filesize) +{ + return unifyfs_invoke_filesize_rpc(gfid, filesize); +} + +static +int rpc_truncate(unifyfs_fops_ctx_t* ctx, + int gfid, + off_t len) +{ + return unifyfs_invoke_truncate_rpc(gfid, len); +} + +static +int rpc_laminate(unifyfs_fops_ctx_t* ctx, + int gfid) +{ + return unifyfs_invoke_laminate_rpc(gfid); +} + +static +int rpc_unlink(unifyfs_fops_ctx_t* ctx, + int gfid) +{ + return unifyfs_invoke_broadcast_unlink(gfid); +} + +static +int create_remote_read_requests(unsigned int n_chunks, + chunk_read_req_t* chunks, + unsigned int* outlen, + server_chunk_reads_t** out) +{ + int prev_rank = -1; + unsigned int num_server_reads = 0; + unsigned int i = 0; + server_chunk_reads_t* remote_reads = NULL; + server_chunk_reads_t* current = NULL; + chunk_read_req_t* pos = NULL; + + /* count how many servers we need to contact */ + for (i = 0; i < n_chunks; i++) { + chunk_read_req_t* curr_chunk = &chunks[i]; + int curr_rank = curr_chunk->rank; + if (curr_rank != prev_rank) { + num_server_reads++; + } + prev_rank = curr_rank; + } + + /* allocate and fill the per-server request data structure */ + remote_reads = (server_chunk_reads_t*) calloc(num_server_reads, + sizeof(*remote_reads)); + if (!remote_reads) { + LOGERR("failed to allocate memory for remote_reads"); + return ENOMEM; + } + + pos = chunks; + unsigned int processed = 0; + + LOGDBG("preparing remote read request for %u chunks (%d servers)", + n_chunks, num_server_reads); + + for (i = 0; i < num_server_reads; i++) { + int rank = pos->rank; + + current = &remote_reads[i]; + current->rank = rank; + current->reqs = pos; + + for ( ; processed < n_chunks; pos++) { + if (pos->rank != rank) { + break; + } + current->total_sz += pos->nbytes; + current->num_chunks++; + processed++; + } + + LOGDBG("%u/%u chunks processed: server %d (%u chunks, %zu bytes)", + processed, n_chunks, rank, + current->num_chunks, current->total_sz); + } + + *outlen = num_server_reads; + *out = remote_reads; + return UNIFYFS_SUCCESS; +} + +static +int submit_read_request(unifyfs_fops_ctx_t* ctx, + size_t count, + unifyfs_inode_extent_t* extents) +{ + if ((count == 0) || (NULL == extents)) { + return EINVAL; + } + + LOGDBG("handling read request (%zu chunk requests)", count); + + /* see if we have a valid app information */ + int app_id = ctx->app_id; + int client_id = ctx->client_id; + + /* get application client */ + app_client* client = get_app_client(app_id, client_id); + if (NULL == client) { + return UNIFYFS_FAILURE; + } + + /* group requested extents by gfid */ + int ret = UNIFYFS_SUCCESS; + int extent_ndx = 0; + while (extent_ndx < (int)count) { + int curr_ndx = extent_ndx; + int curr_gfid = extents[extent_ndx].gfid; + + /* get count of extents for current gfid */ + unsigned int curr_count = 0; + while ((curr_ndx < count) && (extents[curr_ndx].gfid == curr_gfid)) { + curr_count++; + curr_ndx++; + } + + unsigned int n_chunks = 0; + chunk_read_req_t* chunks = NULL; + int rc = unifyfs_invoke_find_extents_rpc(curr_gfid, curr_count, + extents + extent_ndx, + &n_chunks, &chunks); + if (rc) { + LOGERR("failed to find extent locations"); + return rc; + } + if (n_chunks > 0) { + /* prepare the read request requests */ + unsigned int n_remote_reads = 0; + server_chunk_reads_t* remote_reads = NULL; + rc = create_remote_read_requests(n_chunks, chunks, + &n_remote_reads, &remote_reads); + if (rc) { + LOGERR("failed to prepare the remote read requests"); + if (NULL != chunks) { + free(chunks); + } + return rc; + } + + /* fill the information of server_read_req_t and submit */ + server_read_req_t rdreq = { 0, }; + rdreq.app_id = app_id; + rdreq.client_id = client_id; + rdreq.chunks = chunks; + rdreq.num_server_reads = (int) n_remote_reads; + rdreq.remote_reads = remote_reads; + ret = rm_submit_read_request(&rdreq); + } else { + ret = ENODATA; + } + + /* advance to next group */ + extent_ndx += curr_count; + } + + return ret; +} + +static +int rpc_read(unifyfs_fops_ctx_t* ctx, + int gfid, + off_t offset, + size_t length) +{ + unifyfs_inode_extent_t chunk = { 0, }; + + chunk.gfid = gfid; + chunk.offset = offset; + chunk.length = length; + + return submit_read_request(ctx, 1, &chunk); +} + +static +int rpc_mread(unifyfs_fops_ctx_t* ctx, + size_t n_req, + void* read_reqs) +{ + int ret = UNIFYFS_SUCCESS; + int i = 0; + unifyfs_inode_extent_t* chunks = NULL; + unifyfs_extent_t* reqs = (unifyfs_extent_t*) read_reqs; + + chunks = calloc(n_req, sizeof(*chunks)); + if (!chunks) { + LOGERR("failed to allocate the chunk request"); + return ENOMEM; + } + + for (i = 0; i < (int)n_req; i++) { + unifyfs_inode_extent_t* ch = chunks + i; + unifyfs_extent_t* req = reqs + i; + ch->gfid = req->gfid; + ch->offset = req->offset; + ch->length = req->length; + } + + ret = submit_read_request(ctx, n_req, chunks); + + if (chunks) { + free(chunks); + chunks = NULL; + } + + return ret; +} + +static struct unifyfs_fops _fops_rpc = { + .name = "rpc", + .init = rpc_init, + .metaget = rpc_metaget, + .metaset = rpc_metaset, + .fsync = rpc_fsync, + .filesize = rpc_filesize, + .truncate = rpc_truncate, + .laminate = rpc_laminate, + .unlink = rpc_unlink, + .read = rpc_read, + .mread = rpc_mread, +}; + +struct unifyfs_fops* unifyfs_fops_impl = &_fops_rpc; diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index 9205d4f08..6b4bed1e4 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -41,6 +41,8 @@ #include #include #include +#include +#include // common headers #include "arraylist.h" @@ -49,14 +51,9 @@ #include "unifyfs_logio.h" #include "unifyfs_meta.h" #include "unifyfs_shm.h" - -#include -#include - -#if defined(UNIFYFSD_USE_MPI) -# include -#endif - +#include "unifyfs_fops.h" +#include "unifyfs_client_rpcs.h" +#include "unifyfs_server_rpcs.h" /* Some global variables/structures used throughout the server code */ @@ -79,6 +76,7 @@ typedef struct { extern server_info_t* glb_servers; /* array of server info structs */ extern size_t glb_num_servers; /* number of entries in glb_servers array */ +extern struct unifyfs_inode_tree* global_inode_tree; /* global inode tree */ /* defines commands for messages sent to service manager threads */ typedef enum { @@ -101,6 +99,7 @@ typedef struct { size_t log_offset; /* remote log offset */ int log_app_id; /* remote log application id */ int log_client_id; /* remote log client id */ + int rank; /* remote server rank who holds data */ } chunk_read_req_t; typedef struct { @@ -111,7 +110,7 @@ typedef struct { } chunk_read_resp_t; typedef struct { - int rank; /* remote delegator rank */ + int rank; /* server rank */ int rdreq_id; /* read-request id */ int app_id; /* app id of requesting client process */ int client_id; /* client id of requesting client process */ @@ -122,7 +121,7 @@ typedef struct { * @SM: received requests buffer */ chunk_read_resp_t* resp; /* @RM: received responses buffer * @SM: allocated responses buffer */ -} remote_chunk_reads_t; +} server_chunk_reads_t; typedef struct { size_t length; /* length of data to read */ diff --git a/server/src/unifyfs_group_rpc.c b/server/src/unifyfs_group_rpc.c new file mode 100644 index 000000000..5adc64d13 --- /dev/null +++ b/server/src/unifyfs_group_rpc.c @@ -0,0 +1,1147 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "unifyfs_global.h" +#include "unifyfs_tree.h" +#include "margo_server.h" +#include "unifyfs_server_rpcs.h" +#include "unifyfs_group_rpc.h" + +#ifndef UNIFYFS_BCAST_K_ARY +# define UNIFYFS_BCAST_K_ARY 2 +#endif + +/* server collective (coll) margo request structure */ +typedef struct { + margo_request request; + hg_handle_t handle; +} coll_request; + +/* helper method to initialize collective request rpc handle for child peer */ +static int get_request_handle(hg_id_t request_hgid, + int peer_rank, + coll_request* creq) +{ + int rc = UNIFYFS_SUCCESS; + + /* get address for specified server rank */ + hg_addr_t addr = glb_servers[peer_rank].margo_svr_addr; + + /* get handle to rpc function */ + hg_return_t hret = margo_create(unifyfsd_rpc_context->svr_mid, addr, + request_hgid, &(creq->handle)); + if (hret != HG_SUCCESS) { + LOGERR("failed to get handle for request(%p) to server %d", + creq, peer_rank); + rc = UNIFYFS_ERROR_MARGO; + } + + return rc; +} + +/* helper method to forward collective rpc request to one child */ +static int forward_request(void* input_ptr, + coll_request* creq) +{ + int rc = UNIFYFS_SUCCESS; + + /* call rpc function */ + hg_return_t hret = margo_iforward(creq->handle, input_ptr, + &(creq->request)); + if (hret != HG_SUCCESS) { + LOGERR("failed to forward request(%p)", creq); + rc = UNIFYFS_ERROR_MARGO; + } + + return rc; +} + +/* helper method to wait for collective rpc child request completion */ +static int wait_for_request(coll_request* creq) +{ + int rc = UNIFYFS_SUCCESS; + + /* call rpc function */ + hg_return_t hret = margo_wait(creq->request); + if (hret != HG_SUCCESS) { + LOGERR("wait on request(%p) failed", creq); + rc = UNIFYFS_ERROR_MARGO; + } + + return rc; +} + +/************************************************************************* + * Broadcast file extents metadata + *************************************************************************/ + + +/* file extents metadata broadcast rpc handler */ +static void extent_bcast_rpc(hg_handle_t handle) +{ + LOGDBG("MARGOTREE: extent bcast handler"); + + /* assume we'll succeed */ + int32_t ret = UNIFYFS_SUCCESS; + + /* get instance id */ + margo_instance_id mid = margo_hg_handle_get_instance(handle); + + /* get input params */ + extent_bcast_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* get root of tree and global file id to lookup filesize + * record tag calling process wants us to include in our + * later response */ + int gfid = (int) in.gfid; + int32_t num_extents = (int32_t) in.num_extents; + + /* allocate memory for extents */ + struct extent_tree_node* extents; + extents = calloc(num_extents, sizeof(struct extent_tree_node)); + + /* get client address */ + const struct hg_info* info = margo_get_info(handle); + hg_addr_t client_address = info->addr; + + /* expose local bulk buffer */ + hg_size_t buf_size = num_extents * sizeof(struct extent_tree_node); + hg_bulk_t extent_data; + void* datap = extents; + hret = margo_bulk_create(mid, 1, &datap, &buf_size, + HG_BULK_READWRITE, &extent_data); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + int i, rc; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.extent_bcast_id; + + /* create communication tree structure */ + unifyfs_tree_t bcast_tree; + unifyfs_tree_init(glb_pmi_rank, glb_pmi_size, in.root, + UNIFYFS_BCAST_K_ARY, &bcast_tree); + + /* initiate data transfer */ + margo_request bulk_request; + hret = margo_bulk_itransfer(mid, HG_BULK_PULL, client_address, + in.extents, 0, + extent_data, 0, + buf_size, + &bulk_request); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_itransfer() failed"); + ret = UNIFYFS_ERROR_MARGO; + } + + /* update input structure to point to local bulk handle */ + in.extents = extent_data; + + /* allocate memory for request objects + * TODO: possibly get this from memory pool */ + coll_request* requests = + calloc(bcast_tree.child_count, sizeof(*requests)); + if (NULL == requests) { + ret = ENOMEM; + } else { + /* allocate mercury handles for forwarding the request */ + for (i = 0; i < bcast_tree.child_count; i++) { + /* allocate handle for request to this child */ + int child = bcast_tree.child_ranks[i]; + get_request_handle(req_hgid, child, requests+i); + } + } + + /* wait for data transfer to finish */ + hret = margo_wait(bulk_request); + if (hret != HG_SUCCESS) { + LOGERR("margo_wait() for bulk transfer failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + LOGDBG("received %d extents (%zu bytes) from %d", + num_extents, (size_t)buf_size, (int)in.root); + + if (NULL != requests) { + /* forward request down the tree */ + for (i = 0; i < bcast_tree.child_count; i++) { + /* invoke filesize request rpc on child */ + rc = forward_request((void*)&in, requests+i); + } + } + + ret = unifyfs_inode_add_extents(gfid, num_extents, extents); + if (ret) { + LOGERR("add of remote extents failed (ret=%d)", ret); + // what do we do now? + } + LOGDBG("added %d extents (%zu bytes) from %d", + num_extents, (size_t)buf_size, (int)in.root); + + if (NULL != requests) { + /* wait for the requests to finish */ + coll_request* req; + for (i = 0; i < bcast_tree.child_count; i++) { + req = requests + i; + rc = wait_for_request(req); + if (rc == UNIFYFS_SUCCESS) { + /* get the output of the rpc */ + extent_bcast_out_t out; + hret = margo_get_output(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + int child_ret = (int) out.ret; + LOGDBG("MARGOTREE: extbcast child[%d] " + "response: %d", i, child_ret); + if (child_ret != UNIFYFS_SUCCESS) { + ret = child_ret; + } + margo_free_output(req->handle, &out); + } + margo_destroy(req->handle); + } else { + ret = rc; + } + } + free(requests); + } + } + /* free bulk data handle */ + margo_bulk_free(extent_data); + + /* release communication tree resources */ + unifyfs_tree_free(&bcast_tree); + } + margo_free_input(handle, &in); + } + + /* build our output values */ + extent_bcast_out_t out; + out.ret = ret; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + LOGDBG("MARGOTREE: extent bcast rpc handler - responded"); + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(extent_bcast_rpc) + +/* Forward the extent broadcast to all children and wait for responses */ +static +int extent_bcast_forward(const unifyfs_tree_t* broadcast_tree, + extent_bcast_in_t* in) +{ + LOGDBG("MARGOTREE: extent bcast forward"); + + /* get info for tree */ + int child_count = broadcast_tree->child_count; + if (0 == child_count) { + return UNIFYFS_SUCCESS; + } + + int* child_ranks = broadcast_tree->child_ranks; + + /* allocate memory for request objects + * TODO: possibly get this from memory pool */ + coll_request* requests = calloc(child_count, + sizeof(*requests)); + + /* forward request down the tree */ + int i, rc, ret; + coll_request* req; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.extent_bcast_id; + for (i = 0; i < child_count; i++) { + req = requests + i; + + /* allocate handle */ + rc = get_request_handle(req_hgid, child_ranks[i], req); + if (rc == UNIFYFS_SUCCESS) { + /* invoke extbcast request rpc on child */ + rc = forward_request((void*)in, req); + } else { + ret = rc; + } + } + + /* wait for the requests to finish */ + for (i = 0; i < child_count; i++) { + req = requests + i; + rc = wait_for_request(req); + if (rc == UNIFYFS_SUCCESS) { + LOGDBG("MARGOTREE: extent bcast - child[%d] responded", i); + /* get the output of the rpc */ + extent_bcast_out_t out; + hg_return_t hret = margo_get_output(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + int child_ret = out.ret; + if (child_ret != UNIFYFS_SUCCESS) { + ret = child_ret; + } + margo_free_output(req->handle, &out); + } + margo_destroy(req->handle); + } else { + ret = rc; + } + } + + return ret; +} + +/* Execute broadcast tree for extent metadata */ +int unifyfs_invoke_broadcast_extents_rpc(int gfid, unsigned int len, + struct extent_tree_node* extents) +{ + /* assuming success */ + int ret = UNIFYFS_SUCCESS; + + /* create communication tree */ + unifyfs_tree_t bcast_tree; + unifyfs_tree_init(glb_pmi_rank, glb_pmi_size, glb_pmi_rank, + UNIFYFS_BCAST_K_ARY, &bcast_tree); + + hg_size_t num_extents = len; + hg_size_t buf_size = num_extents * sizeof(*extents); + + LOGDBG("broadcasting %u extents for gfid=%d)", + len, gfid); + + /* create bulk data structure containing the extents + * NOTE: bulk data is always read only at the root of the broadcast tree */ + hg_bulk_t extents_bulk; + void* datap = (void*) extents; + hg_return_t hret = margo_bulk_create(unifyfsd_rpc_context->svr_mid, 1, + &datap, &buf_size, + HG_BULK_READ_ONLY, &extents_bulk); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* fill in input struct */ + extent_bcast_in_t in; + in.root = (int32_t)glb_pmi_rank; + in.gfid = gfid; + in.num_extents = num_extents; + in.extents = extents_bulk; + + extent_bcast_forward(&bcast_tree, &in); + + /* free bulk data handle */ + margo_bulk_free(extents_bulk); + } + + /* free tree resources and passed extents */ + unifyfs_tree_free(&bcast_tree); + free(extents); + + return ret; +} + +/************************************************************************* + * Broadcast file attributes and extents metadata due to laminate + *************************************************************************/ + +/* file extents metadata broadcast rpc handler */ +static void laminate_bcast_rpc(hg_handle_t handle) +{ + LOGDBG("MARGOTREE: laminate bcast handler"); + + int32_t ret; + + /* get instance id */ + margo_instance_id mid = margo_hg_handle_get_instance(handle); + + /* get input params */ + laminate_bcast_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* get root of tree and global file id to lookup filesize + * record tag calling process wants us to include in our + * later response */ + int gfid = (int) in.gfid; + size_t num_extents = (size_t) in.num_extents; + unifyfs_file_attr_t* fattr = &(in.attr); + + /* allocate memory for extents */ + struct extent_tree_node* extents; + extents = calloc(num_extents, sizeof(struct extent_tree_node)); + + /* get client address */ + const struct hg_info* info = margo_get_info(handle); + hg_addr_t client_address = info->addr; + + /* expose local bulk buffer */ + hg_size_t buf_size = num_extents * sizeof(struct extent_tree_node); + hg_bulk_t extent_data; + void* datap = extents; + hret = margo_bulk_create(mid, 1, &datap, &buf_size, + HG_BULK_READWRITE, &extent_data); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + int i, rc; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.laminate_bcast_id; + + /* create communication tree structure */ + unifyfs_tree_t bcast_tree; + unifyfs_tree_init(glb_pmi_rank, glb_pmi_size, in.root, + UNIFYFS_BCAST_K_ARY, &bcast_tree); + + /* initiate data transfer */ + margo_request bulk_request; + hret = margo_bulk_itransfer(mid, HG_BULK_PULL, + client_address, in.extents, 0, + extent_data, 0, + buf_size, &bulk_request); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_itransfer() failed"); + ret = UNIFYFS_ERROR_MARGO; + } + + /* allocate memory for request objects + * TODO: possibly get this from memory pool */ + coll_request* requests = + calloc(bcast_tree.child_count, sizeof(*requests)); + if (NULL == requests) { + ret = ENOMEM; + } else { + /* allocate mercury handles for forwarding the request */ + for (i = 0; i < bcast_tree.child_count; i++) { + /* allocate handle for request to this child */ + int child = bcast_tree.child_ranks[i]; + get_request_handle(req_hgid, child, requests+i); + } + } + + /* wait for data transfer to finish */ + hret = margo_wait(bulk_request); + if (hret != HG_SUCCESS) { + LOGERR("margo_wait() for bulk transfer failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + LOGDBG("laminating gfid=%d, received %zu extents from %d", + gfid, num_extents, (int)in.root); + + if (NULL != requests) { + /* update input structure to point to local bulk handle */ + in.extents = extent_data; + + /* forward request down the tree */ + for (i = 0; i < bcast_tree.child_count; i++) { + /* invoke filesize request rpc on child */ + rc = forward_request((void*)&in, requests+i); + } + } + + /* add the final set of extents */ + ret = unifyfs_inode_add_extents(gfid, num_extents, extents); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("laminate extents update failed (ret=%d)", ret); + } + + /* update attributes only after final extents added */ + ret = unifyfs_inode_metaset(gfid, + UNIFYFS_FILE_ATTR_OP_LAMINATE, + fattr); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("laminate attrs update failed (ret=%d)", ret); + } + + if (NULL != requests) { + /* wait for the requests to finish */ + coll_request* req; + for (i = 0; i < bcast_tree.child_count; i++) { + req = requests + i; + rc = wait_for_request(req); + if (rc == UNIFYFS_SUCCESS) { + /* get the output of the rpc */ + laminate_bcast_out_t out; + hret = margo_get_output(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + int child_ret = (int) out.ret; + LOGDBG("MARGOTREE: laminate child[%d] " + "response: %d", i, child_ret); + if (child_ret != UNIFYFS_SUCCESS) { + ret = child_ret; + } + margo_free_output(req->handle, &out); + } + margo_destroy(req->handle); + } else { + ret = rc; + } + } + free(requests); + } + } + /* free bulk data handle */ + margo_bulk_free(extent_data); + + /* release communication tree resources */ + unifyfs_tree_free(&bcast_tree); + } + margo_free_input(handle, &in); + } + + /* build our output values */ + laminate_bcast_out_t out; + out.ret = ret; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + LOGDBG("MARGOTREE: laminate bcast handler - responded"); + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(laminate_bcast_rpc) + +/* Forward the laminate broadcast to all children and wait for responses */ +static +int laminate_bcast_forward(const unifyfs_tree_t* broadcast_tree, + laminate_bcast_in_t* in) +{ + /* get info for tree */ + int* child_ranks = broadcast_tree->child_ranks; + int child_count = broadcast_tree->child_count; + if (0 == child_count) { + return UNIFYFS_SUCCESS; + } + + int gfid = (int) in->gfid; + LOGDBG("MARGOTREE: laminate bcast forward for gfid=%d", gfid); + + /* allocate memory for request objects + * TODO: possibly get this from memory pool */ + coll_request* requests = calloc(child_count, + sizeof(*requests)); + + /* forward request down the tree */ + int i, rc, ret; + coll_request* req; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.laminate_bcast_id; + for (i = 0; i < child_count; i++) { + req = requests + i; + + /* allocate handle */ + rc = get_request_handle(req_hgid, child_ranks[i], req); + if (rc == UNIFYFS_SUCCESS) { + /* invoke extbcast request rpc on child */ + rc = forward_request((void*)in, req); + } else { + ret = rc; + } + } + + /* wait for the requests to finish */ + for (i = 0; i < child_count; i++) { + req = requests + i; + rc = wait_for_request(req); + if (rc == UNIFYFS_SUCCESS) { + LOGDBG("MARGOTREE: laminate bcast - child[%d] responded", i); + /* get the output of the rpc */ + laminate_bcast_out_t out; + hg_return_t hret = margo_get_output(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + int child_ret = out.ret; + if (child_ret != UNIFYFS_SUCCESS) { + ret = child_ret; + } + margo_free_output(req->handle, &out); + } + margo_destroy(req->handle); + } else { + ret = rc; + } + } + + return ret; +} + +/* Execute broadcast tree for attributes and extent metadata due to laminate */ +int unifyfs_invoke_broadcast_laminate(int gfid) +{ + int ret; + + LOGDBG("broadcasting laminate for gfid=%d", gfid); + + /* get attributes and extents metadata */ + unifyfs_file_attr_t attrs; + ret = unifyfs_inode_metaget(gfid, &attrs); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("failed to get file attributes for gfid=%d", gfid); + return ret; + } + + size_t n_extents; + struct extent_tree_node* extents; + ret = unifyfs_inode_get_extents(gfid, &n_extents, &extents); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("failed to get extents for gfid=%d", gfid); + return ret; + } + + /* create bulk data structure containing the extents + * NOTE: bulk data is always read only at the root of the broadcast tree */ + hg_size_t num_extents = n_extents; + hg_size_t buf_size = num_extents * sizeof(*extents); + hg_bulk_t extents_bulk; + void* datap = (void*) extents; + hg_return_t hret = margo_bulk_create(unifyfsd_rpc_context->svr_mid, 1, + &datap, &buf_size, + HG_BULK_READ_ONLY, &extents_bulk); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* create broadcast communication tree */ + unifyfs_tree_t bcast_tree; + unifyfs_tree_init(glb_pmi_rank, glb_pmi_size, glb_pmi_rank, + UNIFYFS_BCAST_K_ARY, &bcast_tree); + + /* fill input struct and forward */ + laminate_bcast_in_t in; + in.root = (int32_t) glb_pmi_rank; + in.gfid = (int32_t) gfid; + in.attr = attrs; + in.num_extents = (int32_t) num_extents; + in.extents = extents_bulk; + laminate_bcast_forward(&bcast_tree, &in); + + /* free tree resources */ + unifyfs_tree_free(&bcast_tree); + + /* free bulk data handle */ + margo_bulk_free(extents_bulk); + } + + /* free extents array */ + free(extents); + + return ret; +} + + +/************************************************************************* + * Broadcast file truncation + *************************************************************************/ + +/* Forward the truncate broadcast to all children and wait for responses */ +static +int truncate_bcast_forward(const unifyfs_tree_t* broadcast_tree, + truncate_bcast_in_t* in) +{ + int i, rc, ret; + int gfid = (int) in->gfid; + size_t fsize = (size_t) in->filesize; + LOGDBG("MARGOTREE: truncate bcast forward - gfid=%d size=%zu", + gfid, fsize); + + /* apply truncation to local file state */ + ret = unifyfs_inode_truncate(gfid, (unsigned long)fsize); + if (ret) { + LOGERR("unifyfs_inode_truncate(gfid=%d, size=%zu) failed - ret=%d", + gfid, fsize, ret); + goto out; + } + + /* get info for tree */ + int child_count = broadcast_tree->child_count; + int* child_ranks = broadcast_tree->child_ranks; + if (child_count > 0) { + LOGDBG("MARGOTREE: sending truncate to %d children", + child_count); + + /* allocate memory for request objects + * TODO: possibly get this from memory pool */ + coll_request* requests = calloc(child_count, + sizeof(coll_request)); + if (!requests) { + ret = ENOMEM; + goto out; + } + + /* forward request down the tree */ + coll_request* req; + for (i = 0; i < child_count; i++) { + req = requests + i; + + /* get rank of this child */ + int child = child_ranks[i]; + LOGDBG("MARGOTREE: truncate child[%d] is rank %d - %s", + i, child, glb_servers[child].margo_svr_addr_str); + + /* allocate handle */ + rc = get_request_handle(unifyfsd_rpc_context->rpcs.truncate_id, + child, req); + if (rc == UNIFYFS_SUCCESS) { + /* invoke truncate request rpc on child */ + rc = forward_request((void*)in, req); + } else { + ret = rc; + } + } + + /* wait for the requests to finish */ + for (i = 0; i < child_count; i++) { + req = requests + i; + rc = wait_for_request(req); + if (rc == UNIFYFS_SUCCESS) { + /* get the output of the rpc */ + truncate_bcast_out_t out; + hg_return_t hret = margo_get_output(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + int child_ret = out.ret; + LOGDBG("MARGOTREE: truncate child[%d] response: ret=%d", + i, child_ret); + if (child_ret != UNIFYFS_SUCCESS) { + ret = child_ret; + } + margo_free_output(req->handle, &out); + } + margo_destroy(req->handle); + } else { + ret = rc; + } + } + + free(requests); + } + +out: + return ret; +} + +/* truncate broadcast rpc handler */ +static void truncate_bcast_rpc(hg_handle_t handle) +{ + LOGDBG("MARGOTREE: truncate bcast handler"); + + /* assume we'll succeed */ + int32_t ret = UNIFYFS_SUCCESS; + + /* get input params */ + truncate_bcast_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* create communication tree */ + unifyfs_tree_t bcast_tree; + unifyfs_tree_init(glb_pmi_rank, glb_pmi_size, in.root, + UNIFYFS_BCAST_K_ARY, &bcast_tree); + + ret = truncate_bcast_forward(&bcast_tree, &in); + + unifyfs_tree_free(&bcast_tree); + margo_free_input(handle, &in); + } + + /* build our output values */ + truncate_bcast_out_t out; + out.ret = ret; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(truncate_bcast_rpc) + +/* Execute broadcast tree for file truncate */ +int unifyfs_invoke_broadcast_truncate(int gfid, size_t filesize) +{ + LOGDBG("broadcasting truncate for gfid=%d filesize=%zu", + gfid, filesize); + + /* assuming success */ + int ret = UNIFYFS_SUCCESS; + + /* create communication tree */ + unifyfs_tree_t bcast_tree; + unifyfs_tree_init(glb_pmi_rank, glb_pmi_size, glb_pmi_rank, + UNIFYFS_BCAST_K_ARY, &bcast_tree); + + /* fill in input struct */ + truncate_bcast_in_t in; + in.root = (int32_t) glb_pmi_rank; + in.gfid = gfid; + in.filesize = filesize; + + ret = truncate_bcast_forward(&bcast_tree, &in); + if (ret) { + LOGERR("truncate_bcast_forward failed: (ret=%d)", ret); + } + + unifyfs_tree_free(&bcast_tree); + + return ret; +} + +/************************************************************************* + * Broadcast updates to file attributes + *************************************************************************/ + +/* Forward the fileattr broadcast to all children and wait for responses */ +static +int fileattr_bcast_forward(const unifyfs_tree_t* broadcast_tree, + fileattr_bcast_in_t* in) +{ + int i, rc, ret; + int gfid = (int) in->gfid; + + LOGDBG("MARGOTREE: fileattr bcast forward (gfid=%d)", gfid); + + /* set local metadata for target file */ + ret = unifyfs_inode_metaset(gfid, in->attrop, &in->attr); + if (ret) { + goto out; + } + + /* get info for tree */ + int child_count = broadcast_tree->child_count; + int* child_ranks = broadcast_tree->child_ranks; + if (child_count > 0) { + LOGDBG("MARGOTREE: %d: sending metaset to %d children", + glb_pmi_rank, child_count); + + /* allocate memory for request objects + * TODO: possibly get this from memory pool */ + coll_request* requests = calloc(child_count, + sizeof(coll_request)); + if (!requests) { + ret = ENOMEM; + goto out; + } + + /* forward request down the tree */ + coll_request* req; + hg_id_t hgid = unifyfsd_rpc_context->rpcs.fileattr_bcast_id; + for (i = 0; i < child_count; i++) { + req = requests + i; + + /* get rank of this child */ + int child = child_ranks[i]; + LOGDBG("MARGOTREE: metaset child[%d] is rank %d - %s", + i, child, glb_servers[child].margo_svr_addr_str); + + /* allocate handle */ + rc = get_request_handle(hgid, child, req); + if (rc == UNIFYFS_SUCCESS) { + /* invoke metaset request rpc on child */ + rc = forward_request((void*)in, req); + } else { + ret = rc; + } + } + + /* wait for the requests to finish */ + for (i = 0; i < child_count; i++) { + req = requests + i; + rc = wait_for_request(req); + if (rc == UNIFYFS_SUCCESS) { + /* get the output of the rpc */ + fileattr_bcast_out_t out; + hg_return_t hret = margo_get_output(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + int child_ret = out.ret; + LOGDBG("MARGOTREE: metaset child[%d] response: ret=%d", + i, child_ret); + if (child_ret != UNIFYFS_SUCCESS) { + ret = child_ret; + } + margo_free_output(req->handle, &out); + } + margo_destroy(req->handle); + } else { + ret = rc; + } + } + + free(requests); + } +out: + return ret; +} + +/* file attributes broadcast rpc handler */ +static void fileattr_bcast_rpc(hg_handle_t handle) +{ + LOGDBG("MARGOTREE: fileattr bcast handler"); + + /* assume we'll succeed */ + int32_t ret = UNIFYFS_SUCCESS; + + /* get input params */ + fileattr_bcast_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* create communication tree */ + unifyfs_tree_t bcast_tree; + unifyfs_tree_init(glb_pmi_rank, glb_pmi_size, in.root, + UNIFYFS_BCAST_K_ARY, &bcast_tree); + + ret = fileattr_bcast_forward(&bcast_tree, &in); + + unifyfs_tree_free(&bcast_tree); + margo_free_input(handle, &in); + } + + /* build our output values */ + fileattr_bcast_out_t out; + out.ret = ret; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(fileattr_bcast_rpc) + +/* Execute broadcast tree for file attributes update */ +int unifyfs_invoke_broadcast_fileattr(int gfid, + int attr_op, + unifyfs_file_attr_t* fattr) +{ + LOGDBG("broadcasting file attributes for gfid=%d", gfid); + + /* create communication tree */ + unifyfs_tree_t bcast_tree; + unifyfs_tree_init(glb_pmi_rank, glb_pmi_size, glb_pmi_rank, + UNIFYFS_BCAST_K_ARY, &bcast_tree); + + /* fill in input struct */ + fileattr_bcast_in_t in; + in.root = (int32_t) glb_pmi_rank; + in.gfid = gfid; + in.attrop = attr_op; + in.attr = *fattr; + + int ret = fileattr_bcast_forward(&bcast_tree, &in); + if (ret) { + LOGERR("fileattr_bcast_forward failed: (ret=%d)", ret); + } + + unifyfs_tree_free(&bcast_tree); + + return ret; +} + +/************************************************************************* + * Broadcast file unlink + *************************************************************************/ + +/* Forward the unlink broadcast to all children and wait for responses */ +static +int unlink_bcast_forward(const unifyfs_tree_t* broadcast_tree, + unlink_bcast_in_t* in) +{ + int i, rc, ret; + int gfid = (int) in->gfid; + + LOGDBG("MARGOTREE: unlink bcast forward (gfid=%d)", gfid); + + /* remove local file metadata */ + ret = unifyfs_inode_unlink(in->gfid); + if (ret) { + goto out; + } + + /* get info for tree */ + int child_count = broadcast_tree->child_count; + int* child_ranks = broadcast_tree->child_ranks; + if (child_count > 0) { + LOGDBG("MARGOTREE: %d: sending unlink to %d children", + glb_pmi_rank, child_count); + + /* allocate memory for request objects + * TODO: possibly get this from memory pool */ + coll_request* requests = calloc(child_count, + sizeof(coll_request)); + if (!requests) { + ret = ENOMEM; + goto out; + } + + /* forward request down the tree */ + coll_request* req; + hg_id_t hgid = unifyfsd_rpc_context->rpcs.unlink_bcast_id; + for (i = 0; i < child_count; i++) { + req = requests + i; + + /* get rank of this child */ + int child = child_ranks[i]; + LOGDBG("MARGOTREE: unlink child[%d] is rank %d - %s", + i, child, glb_servers[child].margo_svr_addr_str); + + /* allocate handle */ + rc = get_request_handle(hgid, child, req); + if (rc == UNIFYFS_SUCCESS) { + /* invoke unlink request rpc on child */ + rc = forward_request((void*)in, req); + } else { + ret = rc; + } + } + + /* wait for the requests to finish */ + for (i = 0; i < child_count; i++) { + req = requests + i; + rc = wait_for_request(req); + if (rc == UNIFYFS_SUCCESS) { + /* get the output of the rpc */ + unlink_bcast_out_t out; + hg_return_t hret = margo_get_output(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + int child_ret = out.ret; + LOGDBG("MARGOTREE: unlink child[%d] response: ret=%d", + i, child_ret); + if (child_ret != UNIFYFS_SUCCESS) { + ret = child_ret; + } + margo_free_output(req->handle, &out); + } + margo_destroy(req->handle); + } else { + ret = rc; + } + } + + free(requests); + } + +out: + return ret; +} + +/* unlink broacast rpc handler */ +static void unlink_bcast_rpc(hg_handle_t handle) +{ + LOGDBG("MARGOTREE: unlink bcast handler"); + + int32_t ret; + + /* get input params */ + unlink_bcast_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* create communication tree */ + unifyfs_tree_t bcast_tree; + unifyfs_tree_init(glb_pmi_rank, glb_pmi_size, in.root, + UNIFYFS_BCAST_K_ARY, &bcast_tree); + + ret = unlink_bcast_forward(&bcast_tree, &in); + + unifyfs_tree_free(&bcast_tree); + margo_free_input(handle, &in); + } + + /* build our output values */ + unlink_bcast_out_t out; + out.ret = ret; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(unlink_bcast_rpc) + +/* Execute broadcast tree for file unlink */ +int unifyfs_invoke_broadcast_unlink(int gfid) +{ + LOGDBG("broadcasting unlink for gfid=%d", gfid); + + /* create communication tree */ + unifyfs_tree_t bcast_tree; + unifyfs_tree_init(glb_pmi_rank, glb_pmi_size, glb_pmi_rank, + UNIFYFS_BCAST_K_ARY, &bcast_tree); + + /* fill in input struct */ + unlink_bcast_in_t in; + in.root = (int32_t) glb_pmi_rank; + in.gfid = (int32_t) gfid; + + int ret = unlink_bcast_forward(&bcast_tree, &in); + if (ret) { + LOGERR("unlink_bcast_forward failed: (ret=%d)", ret); + } + + unifyfs_tree_free(&bcast_tree); + + return ret; +} diff --git a/server/src/unifyfs_group_rpc.h b/server/src/unifyfs_group_rpc.h new file mode 100644 index 000000000..80db9f0d9 --- /dev/null +++ b/server/src/unifyfs_group_rpc.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef _UNIFYFS_GROUP_RPC_H +#define _UNIFYFS_GROUP_RPC_H + +#include "unifyfs_tree.h" +#include "unifyfs_inode.h" + +/* Collective Server RPCs */ + +/** + * @brief Broadcast file extents metadata to all servers + * + * @param gfid target file + * @param len length of file extents array + * @param extents array of extents to broadcast + * + * @return success|failure + */ +int unifyfs_invoke_broadcast_extents(int gfid, + unsigned int len, + struct extent_tree_node* extents); + +/** + * @brief Broadcast file attributes metadata to all servers + * + * @param gfid target file + * @param fileop file operation that triggered metadata update + * @param attr file attributes + * + * @return success|failure + */ +int unifyfs_invoke_broadcast_fileattr(int gfid, + int fileop, + unifyfs_file_attr_t* attr); + +/** + * @brief Broadcast file attributes and extent metadata to all servers + * + * @param gfid target file + * + * @return success|failure + */ +int unifyfs_invoke_broadcast_laminate(int gfid); + +/** + * @brief Truncate target file at all servers + * + * @param gfid target file + * @param filesize truncated file size + * + * @return success|failure + */ +int unifyfs_invoke_broadcast_truncate(int gfid, size_t filesize); + +/** + * @brief Unlink file at all servers + * + * @param gfid target file + * + * @return success|failure + */ +int unifyfs_invoke_broadcast_unlink(int gfid); + + +#endif // UNIFYFS_GROUP_RPC_H diff --git a/server/src/unifyfs_inode.c b/server/src/unifyfs_inode.c new file mode 100644 index 000000000..dc1e58230 --- /dev/null +++ b/server/src/unifyfs_inode.c @@ -0,0 +1,664 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include +#include +#include +#include +#include + +#include "unifyfs_inode.h" +#include "unifyfs_inode_tree.h" + +struct unifyfs_inode_tree _global_inode_tree; +struct unifyfs_inode_tree* global_inode_tree = &_global_inode_tree; + +static inline +struct unifyfs_inode* unifyfs_inode_alloc(int gfid, unifyfs_file_attr_t* attr) +{ + struct unifyfs_inode* ino = calloc(1, sizeof(*ino)); + + if (ino) { + ino->gfid = gfid; + ino->attr = *attr; + ino->attr.filename = strdup(attr->filename); + pthread_rwlock_init(&ino->rwlock, NULL); + ABT_mutex_create(&(ino->abt_sync)); + } + + return ino; +} + +static inline +int unifyfs_inode_destroy(struct unifyfs_inode* ino) +{ + int ret = UNIFYFS_SUCCESS; + + if (ino) { + if (NULL != ino->attr.filename) { + free(ino->attr.filename); + } + + if (NULL != ino->extents) { + extent_tree_destroy(ino->extents); + free(ino->extents); + } + + pthread_rwlock_destroy(&ino->rwlock); + free(ino); + } else { + ret = EINVAL; + } + + return ret; +} + +/** + * @brief read lock the inode for ro access. + * + * @param ino inode structure to get access + * + * @return 0 on success, errno otherwise + */ +static inline +int unifyfs_inode_rdlock(struct unifyfs_inode* ino) +{ + return pthread_rwlock_rdlock(&ino->rwlock); +} + +/** + * @brief write lock the inode for w+r access. + * + * @param ino inode structure to get access + * + * @return 0 on success, errno otherwise + */ +static inline +int unifyfs_inode_wrlock(struct unifyfs_inode* ino) +{ + return pthread_rwlock_wrlock(&ino->rwlock); +} + +/** + * @brief unlock the inode. + * + * @param ino inode structure to unlock + */ +static inline +void unifyfs_inode_unlock(struct unifyfs_inode* ino) +{ + pthread_rwlock_unlock(&ino->rwlock); +} + +int unifyfs_inode_create(int gfid, unifyfs_file_attr_t* attr) +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_inode* ino = NULL; + + if (!attr) { + return EINVAL; + } + + ino = unifyfs_inode_alloc(gfid, attr); + + unifyfs_inode_tree_wrlock(global_inode_tree); + { + ret = unifyfs_inode_tree_insert(global_inode_tree, ino); + } + unifyfs_inode_tree_unlock(global_inode_tree); + + if (ret) { + free(ino); + } + + return ret; +} + +int unifyfs_inode_update_attr(int gfid, int attr_op, + unifyfs_file_attr_t* attr) +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_inode* ino = NULL; + + if (!attr) { + return EINVAL; + } + + unifyfs_inode_tree_rdlock(global_inode_tree); + { + ino = unifyfs_inode_tree_search(global_inode_tree, gfid); + if (!ino) { + ret = ENOENT; + goto out_unlock_tree; + } + + unifyfs_inode_wrlock(ino); + { + unifyfs_file_attr_update(attr_op, &ino->attr, attr); + } + unifyfs_inode_unlock(ino); + } +out_unlock_tree: + unifyfs_inode_tree_unlock(global_inode_tree); + + return ret; +} + +int unifyfs_inode_metaset(int gfid, int attr_op, + unifyfs_file_attr_t* attr) +{ + int ret; + + if (attr_op == UNIFYFS_FILE_ATTR_OP_CREATE) { + ret = unifyfs_inode_create(gfid, attr); + } else { + ret = unifyfs_inode_update_attr(gfid, attr_op, attr); + } + + return ret; +} + +int unifyfs_inode_metaget(int gfid, unifyfs_file_attr_t* attr) +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_inode* ino = NULL; + + if (!global_inode_tree || !attr) { + return EINVAL; + } + + unifyfs_inode_tree_rdlock(global_inode_tree); + { + ino = unifyfs_inode_tree_search(global_inode_tree, gfid); + if (ino) { + *attr = ino->attr; + } else { + ret = ENOENT; + } + } + unifyfs_inode_tree_unlock(global_inode_tree); + + return ret; +} + +int unifyfs_inode_unlink(int gfid) +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_inode* ino = NULL; + + unifyfs_inode_tree_wrlock(global_inode_tree); + { + ret = unifyfs_inode_tree_remove(global_inode_tree, gfid, &ino); + } + unifyfs_inode_tree_unlock(global_inode_tree); + + if (ret) { + goto out; + } + + ret = unifyfs_inode_destroy(ino); +out: + return ret; +} + +int unifyfs_inode_truncate(int gfid, unsigned long size) +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_inode* ino = NULL; + + unifyfs_inode_tree_rdlock(global_inode_tree); + { + ino = unifyfs_inode_tree_search(global_inode_tree, gfid); + if (!ino) { + ret = ENOENT; + goto out_unlock_tree; + } + + unifyfs_inode_rdlock(ino); + { + if (ino->attr.is_laminated) { + LOGERR("cannot truncate a laminated file (gfid=%d)", gfid); + ret = EINVAL; + goto unlock_inode; + } + ino->attr.size = size; + + if (NULL != ino->extents) { + ret = extent_tree_truncate(ino->extents, size); + } + } +unlock_inode: + unifyfs_inode_unlock(ino); + } +out_unlock_tree: + unifyfs_inode_tree_unlock(global_inode_tree); + + return ret; +} + +static struct extent_tree* inode_get_extent_tree(struct unifyfs_inode* ino) +{ + struct extent_tree* tree = ino->extents; + + /* create one if it doesn't exist yet */ + if (!tree) { + tree = calloc(1, sizeof(*tree)); + + if (!tree) { + LOGERR("failed to allocate memory for extent tree"); + return NULL; + } + + extent_tree_init(tree); + + ino->extents = tree; + } + + return tree; +} + +int unifyfs_inode_add_extents(int gfid, int num_extents, + struct extent_tree_node* nodes) +{ + int ret = UNIFYFS_SUCCESS; + int i = 0; + struct unifyfs_inode* ino = NULL; + struct extent_tree* tree = NULL; + + unifyfs_inode_tree_rdlock(global_inode_tree); + { + ino = unifyfs_inode_tree_search(global_inode_tree, gfid); + if (!ino) { + ret = ENOENT; + goto out_unlock_tree; + } + + if (ino->attr.is_laminated) { + LOGERR("trying to add extents to a laminated file (gfid=%d)", + gfid); + ret = EINVAL; + goto out_unlock_tree; + } + + tree = inode_get_extent_tree(ino); + if (!tree) { /* failed to create one */ + ret = ENOMEM; + goto out_unlock_tree; + } + + for (i = 0; i < num_extents; i++) { + struct extent_tree_node* current = &nodes[i]; + + /* the output becomes too noisy with this: + * LOGDBG("new extent[%4d]: (%lu, %lu)", + * i, current->start, current->end); + */ + + ABT_mutex_lock(ino->abt_sync); + ret = extent_tree_add(tree, current->start, current->end, + current->svr_rank, current->app_id, + current->cli_id, current->pos); + ABT_mutex_unlock(ino->abt_sync); + if (ret) { + LOGERR("failed to add extents"); + goto out_unlock_tree; + } + } + + /* if the extent tree max offset is greater than the size we + * we currently have in the inode attributes, then update the + * inode size */ + unsigned long extent_sz = extent_tree_max_offset(ino->extents) + 1; + if ((uint64_t)extent_sz > ino->attr.size) { + unifyfs_inode_wrlock(ino); + ino->attr.size = extent_sz; + unifyfs_inode_unlock(ino); + } + + LOGINFO("added %d extents to inode (gfid=%d, filesize=%" PRIu64 ")", + num_extents, gfid, ino->attr.size); + } +out_unlock_tree: + unifyfs_inode_tree_unlock(global_inode_tree); + + return ret; +} + +int unifyfs_inode_get_filesize(int gfid, size_t* offset) +{ + int ret = UNIFYFS_SUCCESS; + size_t filesize = 0; + struct unifyfs_inode* ino = NULL; + + unifyfs_inode_tree_rdlock(global_inode_tree); + { + ino = unifyfs_inode_tree_search(global_inode_tree, gfid); + if (!ino) { + ret = ENOENT; + goto out_unlock_tree; + } + + unifyfs_inode_rdlock(ino); + { + /* the size is updated each time we add extents or truncate, + * so no need to recalculate */ + filesize = ino->attr.size; + } + unifyfs_inode_unlock(ino); + + *offset = filesize; + + LOGDBG("local file size (gfid=%d): %lu", gfid, filesize); + } +out_unlock_tree: + unifyfs_inode_tree_unlock(global_inode_tree); + + return ret; +} + +int unifyfs_inode_laminate(int gfid) +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_inode* ino = NULL; + + unifyfs_inode_tree_rdlock(global_inode_tree); + { + ino = unifyfs_inode_tree_search(global_inode_tree, gfid); + if (!ino) { + ret = ENOENT; + goto out_unlock_tree; + } + + unifyfs_inode_wrlock(ino); + { + ino->attr.is_laminated = 1; + } + unifyfs_inode_unlock(ino); + + LOGDBG("file laminated (gfid=%d)", gfid); + } +out_unlock_tree: + unifyfs_inode_tree_unlock(global_inode_tree); + + return ret; +} + +int unifyfs_inode_get_extents(int gfid, size_t* n, + struct extent_tree_node** nodes) +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_inode* ino = NULL; + + if (!n || !nodes) { + return EINVAL; + } + + unifyfs_inode_tree_rdlock(global_inode_tree); + { + ino = unifyfs_inode_tree_search(global_inode_tree, gfid); + if (!ino) { + ret = ENOENT; + goto out_unlock_tree; + } + + unifyfs_inode_rdlock(ino); + { + int i = 0; + struct extent_tree* tree = ino->extents; + size_t n_nodes = tree->count; + struct extent_tree_node* _nodes = calloc(n_nodes, sizeof(*_nodes)); + struct extent_tree_node* current = NULL; + + if (!_nodes) { + ret = ENOMEM; + goto out_unlock_inode; + } + + while (NULL != (current = extent_tree_iter(tree, current))) { + _nodes[i] = *current; + i++; + } + + *n = n_nodes; + *nodes = _nodes; + } +out_unlock_inode: + unifyfs_inode_unlock(ino); + } +out_unlock_tree: + unifyfs_inode_tree_unlock(global_inode_tree); + + return ret; +} + +int unifyfs_inode_get_extent_chunks(unifyfs_inode_extent_t* extent, + unsigned int* n_chunks, + chunk_read_req_t** chunks) +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_inode* ino = NULL; + int gfid = extent->gfid; + + unifyfs_inode_tree_rdlock(global_inode_tree); + { + ino = unifyfs_inode_tree_search(global_inode_tree, gfid); + if (!ino) { + ret = ENOENT; + goto out_unlock_tree; + } + + unifyfs_inode_rdlock(ino); + { + if (NULL != ino->extents) { + unsigned long offset = extent->offset; + unsigned long len = extent->length; + ret = extent_tree_get_chunk_list(ino->extents, offset, len, + n_chunks, chunks); + if (ret) { + LOGERR("failed to get chunks for gfid:%d, ret=%d", + gfid, ret); + } + } + } + unifyfs_inode_unlock(ino); + } +out_unlock_tree: + unifyfs_inode_tree_unlock(global_inode_tree); + + if (ret == UNIFYFS_SUCCESS) { + /* extent_tree_get_chunk_list does not populate the gfid field */ + for (unsigned int i = 0; i < *n_chunks; i++) { + (*chunks)[i].gfid = gfid; + } + } else { + *n_chunks = 0; + *chunks = NULL; + } + + return ret; +} + +static +int compare_chunk_read_reqs(const void* _c1, const void* _c2) +{ + chunk_read_req_t* c1 = (chunk_read_req_t*) _c1; + chunk_read_req_t* c2 = (chunk_read_req_t*) _c2; + + if (c1->rank > c2->rank) { + return 1; + } else if (c1->rank < c2->rank) { + return -1; + } else { + return 0; + } +} + + +int unifyfs_inode_resolve_extent_chunks(unsigned int n_extents, + unifyfs_inode_extent_t* extents, + unsigned int* n_locs, + chunk_read_req_t** chunklocs) +{ + int ret = UNIFYFS_SUCCESS; + unsigned int i = 0; + unsigned int j = 0; + unsigned int n_chunks = 0; + chunk_read_req_t* chunks = NULL; + unsigned int* n_resolved = NULL; + chunk_read_req_t** resolved = NULL; + + void* buf = calloc(n_extents, (sizeof(*n_resolved) + sizeof(*resolved))); + if (NULL == buf) { + LOGERR("failed to allocate memory"); + ret = ENOMEM; + goto out_fail; + } + + n_resolved = (unsigned int*) buf; + resolved = (chunk_read_req_t**) &n_resolved[n_extents]; + + /* resolve chunks addresses for all requests from inode tree */ + for (i = 0; i < n_extents; i++) { + unifyfs_inode_extent_t* current = &extents[i]; + + LOGDBG("resolving chunk request (gfid=%d, offset=%lu, length=%lu)", + current->gfid, current->offset, current->length); + + ret = unifyfs_inode_get_extent_chunks(current, + &n_resolved[i], &resolved[i]); + if (ret) { + LOGERR("failed to resolve the chunk request for chunk " + "[gfid=%d, offset=%lu, length=%zu] (ret=%d)", + current->gfid, current->offset, current->length, ret); + goto out_fail; + } + + n_chunks += n_resolved[i]; + } + + LOGDBG("resolved %d chunks for read request", n_chunks); + if (n_chunks > 0) { + /* store all chunks in a flat array */ + chunks = calloc(n_chunks, sizeof(*chunks)); + if (!chunks) { + LOGERR("failed to allocate memory for storing resolved chunks"); + ret = ENOMEM; + goto out_fail; + } + + chunk_read_req_t* pos = chunks; + for (i = 0; i < n_extents; i++) { + for (j = 0; j < n_resolved[i]; j++) { + *pos = resolved[i][j]; + pos++; + } + if (resolved[i]) { + free(resolved[i]); + } + } + + /* sort the requests based on server rank */ + qsort(chunks, n_chunks, sizeof(*chunks), compare_chunk_read_reqs); + + chunk_read_req_t* chk = chunks; + for (i = 0; i < n_chunks; i++, chk++) { + LOGDBG(" [%d] (offset=%lu, nbytes=%lu) @ (%d log(%d:%d:%lu))", + i, chk->offset, chk->nbytes, chk->rank, + chk->log_client_id, chk->log_app_id, chk->log_offset); + } + } + + *n_locs = n_chunks; + *chunklocs = chunks; + +out_fail: + if (ret != UNIFYFS_SUCCESS) { + if (chunks) { + free(chunks); + chunks = NULL; + } + } + + if (NULL != buf) { + free(buf); + } + + return ret; +} + +int unifyfs_inode_span_extents( + int gfid, /* global file id we're looking in */ + unsigned long start, /* starting logical offset */ + unsigned long end, /* ending logical offset */ + int max, /* maximum number of key/vals to return */ + void* keys, /* array of length max for output keys */ + void* vals, /* array of length max for output values */ + int* outnum) /* number of entries returned */ +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_inode* ino = NULL; + + unifyfs_inode_tree_rdlock(global_inode_tree); + { + ino = unifyfs_inode_tree_search(global_inode_tree, gfid); + if (!ino) { + ret = ENOENT; + goto out_unlock_tree; + } + + unifyfs_inode_rdlock(ino); + { + ret = extent_tree_span(ino->extents, gfid, start, end, + max, keys, vals, outnum); + if (ret) { + LOGERR("extent_tree_span failed (gfid=%d, ret=%d)", + gfid, ret); + } + } + unifyfs_inode_unlock(ino); + } +out_unlock_tree: + unifyfs_inode_tree_unlock(global_inode_tree); + + return ret; +} + +int unifyfs_inode_dump(int gfid) +{ + int ret = UNIFYFS_SUCCESS; + struct unifyfs_inode* ino = NULL; + + unifyfs_inode_tree_rdlock(global_inode_tree); + { + ino = unifyfs_inode_tree_search(global_inode_tree, gfid); + if (!ino) { + ret = ENOENT; + goto out_unlock_tree; + } + + unifyfs_inode_rdlock(ino); + { + LOGDBG("== inode (gfid=%d) ==\n", ino->gfid); + if (NULL != ino->extents) { + LOGDBG("extents:"); + extent_tree_dump(ino->extents); + } + } + unifyfs_inode_unlock(ino); + } +out_unlock_tree: + unifyfs_inode_tree_unlock(global_inode_tree); + + return ret; +} diff --git a/server/src/unifyfs_inode.h b/server/src/unifyfs_inode.h new file mode 100644 index 000000000..915fa6a6c --- /dev/null +++ b/server/src/unifyfs_inode.h @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef __UNIFYFS_INODE_H +#define __UNIFYFS_INODE_H + +#include +#include "tree.h" +#include "extent_tree.h" +#include "unifyfs_meta.h" +#include "unifyfs_global.h" + +/** + * @brief file extent descriptor + */ +struct unifyfs_inode_extent { + int gfid; + unsigned long offset; + unsigned long length; +}; +typedef struct unifyfs_inode_extent unifyfs_inode_extent_t; + +/** + * @brief file and directory inode structure. this holds: + */ +struct unifyfs_inode { + /* tree entry for global inode tree */ + RB_ENTRY(unifyfs_inode) inode_tree_entry; + + int gfid; /* global file identifier */ + unifyfs_file_attr_t attr; /* file attributes */ + struct extent_tree* extents; /* extent information */ + + pthread_rwlock_t rwlock; /* rwlock for pthread access */ + ABT_mutex abt_sync; /* mutex for argobots ULT access */ +}; + +/** + * @brief create a new inode with given parameters. The newly created inode + * will be inserted to the global inode tree (global_inode_tree). + * + * @param gfid global file identifier. + * @param attr attributes of the new file. + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_create(int gfid, unifyfs_file_attr_t* attr); + +/** + * @brief update the attributes of file with @gfid. The attributes are + * selectively updated with unifyfs_file_attr_update() function (see + * common/unifyfs_meta.h). + * + * @param gfid global file identifier + * @param attr new attributes + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_update_attr(int gfid, int attr_op, + unifyfs_file_attr_t* attr); + +/** + * @brief create a new or update an existing inode. + * + * @param gfid global file identifier + * @param create try to create a new inode if set + * @param attr file attributes + * + * @return 0 on success, errno otherwise + */ + +int unifyfs_inode_metaset(int gfid, int attr_op, + unifyfs_file_attr_t* attr); + +/** + * @brief read attributes for file with @gfid. + * + * @param gfid global file identifier + * @param attr [out] file attributes to be filled + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_metaget(int gfid, unifyfs_file_attr_t* attr); + +/** + * @brief unlink file with @gfid. this will remove the target file inode from + * the global inode tree. + * + * @param gfid global file identifier + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_unlink(int gfid); + +/** + * @brief truncate size of file with @gfid to @size. + * + * @param gfid global file identifier + * @param size new file size + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_truncate(int gfid, unsigned long size); + +/** + * @brief get the local extent array from the target inode + * + * @param gfid the global file identifier + * @param n the number of extents, set by this function + * @param nodes the pointer to the array of extents, caller should free this + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_get_extents(int gfid, size_t* n, + struct extent_tree_node** nodes); + +/** + * @brief add new extents to the inode + * + * @param gfid the global file identifier + * @param n the number of new extents in @nodes + * @param nodes an array of extents to be added + * + * @return + */ +int unifyfs_inode_add_extents(int gfid, int n, struct extent_tree_node* nodes); + +/** + * @brief get the maximum file size from the local extent tree of given file + * + * @param gfid global file identifier + * @param offset [out] file offset to be filled by this function + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_get_filesize(int gfid, size_t* offset); + +/** + * @brief set the given file as laminated + * + * @param gfid global file identifier + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_laminate(int gfid); + +/** + * @brief Get chunks for given file extent + * + * @param extent target file extent + * + * @param[out] n_chunks number of output chunk locations + * @param[out] chunks array of output chunk locations + * + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_inode_get_extent_chunks(unifyfs_inode_extent_t* extent, + unsigned int* n_chunks, + chunk_read_req_t** chunks); + +/** + * @brief Get chunk locations for an array of file extents + * + * @param n_extents number of input extents + * @param extents array or requested extents + * + * @param[out] n_locs number of output chunk locations + * @param[out] chunklocs array of output chunk locations + * + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_inode_resolve_extent_chunks(unsigned int n_extents, + unifyfs_inode_extent_t* extents, + unsigned int* n_locs, + chunk_read_req_t** chunklocs); + +/** + * @brief calls extents_tree_span, which will do: + * + * given an extent tree and starting and ending logical offsets, fill in + * key/value entries that overlap that range, returns at most max entries + * starting from lowest starting offset, sets outnum with actual number of + * entries returned + * + * @param gfid global file id + * @param start starting logical offset + * @param end ending logical offset + * @param max maximum number of key/vals to return + * @param keys array of length max for output keys + * @param vals array of length max for output values + * @param outnum number of entries returned + * + * @return + */ +int unifyfs_inode_span_extents( + int gfid, /* global file id we're looking in */ + unsigned long start, /* starting logical offset */ + unsigned long end, /* ending logical offset */ + int max, /* maximum number of key/vals to return */ + void* keys, /* array of length max for output keys */ + void* vals, /* array of length max for output values */ + int* outnum); /* number of entries returned */ + +/** + * @brief prints the inode information to the log stream + * + * @param gfid global file identifier + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_dump(int gfid); + +#endif /* __UNIFYFS_INODE_H */ + diff --git a/server/src/unifyfs_inode_tree.c b/server/src/unifyfs_inode_tree.c new file mode 100644 index 000000000..311a6a994 --- /dev/null +++ b/server/src/unifyfs_inode_tree.c @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include +#include +#include +#include +#include +#include + +#include "unifyfs_inode_tree.h" + +#undef MIN +#undef MAX +#define MIN(a, b) (a < b ? a : b) +#define MAX(a, b) (a > b ? a : b) + +static int unifyfs_inode_tree_compare_func( + struct unifyfs_inode* node1, + struct unifyfs_inode* node2) +{ + if (node1->gfid > node2->gfid) { + return 1; + } else if (node1->gfid < node2->gfid) { + return -1; + } else { + return 0; + } +} + +RB_PROTOTYPE( + rb_inode_tree, unifyfs_inode, + inode_tree_entry, unifyfs_inode_tree_compare_func) +RB_GENERATE( + rb_inode_tree, unifyfs_inode, + inode_tree_entry, unifyfs_inode_tree_compare_func) + +/* Returns 0 on success, positive non-zero error code otherwise */ +int unifyfs_inode_tree_init( + struct unifyfs_inode_tree* tree) +{ + int ret = 0; + + if (!tree) { + return EINVAL; + } + + memset(tree, 0, sizeof(*tree)); + ret = pthread_rwlock_init(&tree->rwlock, NULL); + RB_INIT(&tree->head); + + return ret; +} + +/* Remove and free all nodes in the unifyfs_inode_tree. */ +void unifyfs_inode_tree_destroy( + struct unifyfs_inode_tree* tree) +{ + if (tree) { + unifyfs_inode_tree_clear(tree); + pthread_rwlock_destroy(&tree->rwlock); + } +} + +int unifyfs_inode_tree_insert( + struct unifyfs_inode_tree* tree, /* tree on which to add new entry */ + struct unifyfs_inode* ino) /* initial file attribute */ +{ + int ret = 0; + struct unifyfs_inode* existing = NULL; + + if (!ino || (ino->gfid != ino->attr.gfid)) { + return EINVAL; + } + + /* check if the node already exists */ + existing = RB_FIND(rb_inode_tree, &tree->head, ino); + if (existing) { + return EEXIST; + } + + RB_INSERT(rb_inode_tree, &tree->head, ino); + + return ret; +} + +/* Search for and return entry for given gfid on specified tree. + * If not found, return NULL, assumes caller has lock on tree */ +struct unifyfs_inode* unifyfs_inode_tree_search( + struct unifyfs_inode_tree* tree, + int gfid) +{ + struct unifyfs_inode node = { .gfid = gfid, }; + + return RB_FIND(rb_inode_tree, &tree->head, &node); +} + +int unifyfs_inode_tree_remove( + struct unifyfs_inode_tree* tree, + int gfid, + struct unifyfs_inode** removed) +{ + int ret = 0; + struct unifyfs_inode* ino = NULL; + + ino = unifyfs_inode_tree_search(tree, gfid); + if (!ino) { + return ENOENT; + } + + RB_REMOVE(rb_inode_tree, &tree->head, ino); + + *removed = ino; + + return ret; +} + +/* + * Given a range tree and a starting node, iterate though all the nodes + * in the tree, returning the next one each time. If start is NULL, then + * start with the first node in the tree. + * + * This is meant to be called in a loop, like: + * + * gfid2ext_tree_rdlock(tree); + * + * struct unifyfs_inode *node = NULL; + * while ((node = gfid2ext_tree_iter(tree, node))) { + * printf("[%d-%d]", node->start, node->end); + * } + * + * gfid2ext_tree_unlock(tree); + * + * Note: this function does no locking, and assumes you're properly locking + * and unlocking the gfid2ext_tree before doing the iteration (see + * gfid2ext_tree_rdlock()/gfid2ext_tree_wrlock()/gfid2ext_tree_unlock()). + */ +struct unifyfs_inode* unifyfs_inode_tree_iter( + struct unifyfs_inode_tree* tree, + struct unifyfs_inode* start) +{ + struct unifyfs_inode* next = NULL; + if (start == NULL) { + /* Initial case, no starting node */ + next = RB_MIN(rb_inode_tree, &tree->head); + return next; + } + + /* + * We were given a valid start node. Look it up to start our traversal + * from there. + */ + next = RB_FIND(rb_inode_tree, &tree->head, start); + if (!next) { + /* Some kind of error */ + return NULL; + } + + /* Look up our next node */ + next = RB_NEXT(rb_inode_tree, &tree->head, start); + + return next; +} + +/* + * Remove all nodes in unifyfs_inode_tree, but keep it initialized so you can + * unifyfs_inode_tree_add() to it. + */ +void unifyfs_inode_tree_clear( + struct unifyfs_inode_tree* tree) +{ + struct unifyfs_inode* node = NULL; + struct unifyfs_inode* oldnode = NULL; + + unifyfs_inode_tree_wrlock(tree); + + if (RB_EMPTY(&tree->head)) { + /* unifyfs_inode_tree is empty, nothing to do */ + unifyfs_inode_tree_unlock(tree); + return; + } + + /* Remove and free each node in the tree */ + while ((node = unifyfs_inode_tree_iter(tree, node))) { + if (oldnode) { + RB_REMOVE(rb_inode_tree, &tree->head, oldnode); + if (oldnode->extents != NULL) { + extent_tree_destroy(oldnode->extents); + } + free(oldnode); + } + oldnode = node; + } + if (oldnode) { + RB_REMOVE(rb_inode_tree, &tree->head, oldnode); + if (oldnode->extents != NULL) { + extent_tree_destroy(oldnode->extents); + } + free(oldnode); + } + + unifyfs_inode_tree_unlock(tree); +} + diff --git a/server/src/unifyfs_inode_tree.h b/server/src/unifyfs_inode_tree.h new file mode 100644 index 000000000..8bec5d8d2 --- /dev/null +++ b/server/src/unifyfs_inode_tree.h @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef __UNIFYFS_INODE_TREE_H +#define __UNIFYFS_INODE_TREE_H + +#include +#include "tree.h" +#include "extent_tree.h" +#include "unifyfs_meta.h" +#include "unifyfs_inode.h" + +/* + * unifyfs_inode_tree: balanced binary tree (RB) for keeping active inodes. + * + * NOTE: except for unifyfs_inode_tree_destroy, none of the following functions + * perform locking itself, but the caller should accordingly lock/unlock using + * unifyfs_inode_tree_rdlock, unifyfs_inode_tree_wrlock, and + * unifyfs_inode_tree_unlock. + */ +struct unifyfs_inode_tree { + RB_HEAD(rb_inode_tree, unifyfs_inode) head; /** inode RB tree */ + pthread_rwlock_t rwlock; /** lock for accessing tree */ +}; + +/** + * @brief initialize the inode tree. + * + * @param tree the tree structure to be initialized. this should be allocated + * by the caller. + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_tree_init(struct unifyfs_inode_tree* tree); + +/** + * @brief Remove all nodes in unifyfs_inode_tree, but keep it initialized so + * you can unifyfs_inode_tree_add() to it. + * + * @param tree inode tree to remove + */ +void unifyfs_inode_tree_clear(struct unifyfs_inode_tree* tree); + +/** + * @brief Remove and free all nodes in the unifyfs_inode_tree. + * + * @param tree inode tree to remove + */ +void unifyfs_inode_tree_destroy(struct unifyfs_inode_tree* tree); + +/** + * @brief Insert a new inode to the tree. + * + * @param tree inode tree + * @param ino new inode to insert + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_tree_insert(struct unifyfs_inode_tree* tree, + struct unifyfs_inode* ino); + +/** + * @brief Remove an inode with @gfid. + * + * @param tree inode tree + * @param gfid global file identifier of the target inode + * @param removed [out] removed inode + * + * @return 0 on success, errno otherwise + */ +int unifyfs_inode_tree_remove(struct unifyfs_inode_tree* tree, + int gfid, struct unifyfs_inode** removed); + +/* Search for and return extents for given gfid on specified tree. + * If not found, return NULL, assumes caller has lock on tree */ +struct unifyfs_inode* unifyfs_inode_tree_search( + struct unifyfs_inode_tree* tree, /* tree to search */ + int gfid /* global file id to find */ +); + +/** + * @brief Iterate the inode tree. + * + * Given a range tree and a starting node, iterate though all the nodes + * in the tree, returning the next one each time. If start is NULL, then + * start with the first node in the tree. + * + * This is meant to be called in a loop, like: + * + * unifyfs_inode_tree_rdlock(tree); + * + * struct unifyfs_inode *node = NULL; + * while ((node = unifyfs_inode_tree_iter(tree, node))) { + * printf("[%d-%d]", node->start, node->end); + * } + * + * unifyfs_inode_tree_unlock(tree); + * + * Note: this function does no locking, and assumes you're properly locking + * and unlocking the unifyfs_inode_tree before doing the iteration; see: + * unifyfs_inode_tree_rdlock(), unifyfs_inode_tree_wrlock(), + * unifyfs_inode_tree_unlock(). + * + * @param tree inode tree to iterate + * @param start the starting node + * + * @return inode structure + */ +struct unifyfs_inode* unifyfs_inode_tree_iter(struct unifyfs_inode_tree* tree, + struct unifyfs_inode* start); + +/* + * Locking functions for use with unifyfs_inode_tree_iter(). They allow you to + * lock the tree to iterate over it: + * + * unifyfs_inode_tree_rdlock(&tree); + * + * struct unifyfs_inode *node = NULL; + * while ((node = unifyfs_inode_tree_iter(tree, node))) { + * printf("[%d-%d]", node->start, node->end); + * } + * + * unifyfs_inode_tree_unlock(&tree); + */ + +/** + * @brief Lock a unifyfs_inode_tree for reading. This should only be used for + * calling unifyfs_inode_tree_iter(). All the other unifyfs_inode_tree + * functions provide their own locking. + * + * @param tree inode tree + * + * @return 0 on success, errno otherwise + */ +static inline int unifyfs_inode_tree_rdlock(struct unifyfs_inode_tree* tree) +{ + return pthread_rwlock_rdlock(&tree->rwlock); +} + +/** + * @brief Lock a unifyfs_inode_tree for read/write. This should only be used + * for calling unifyfs_inode_tree_iter(). All the other unifyfs_inode_tree + * functions provide their own locking. + * + * @param tree inode tree + * + * @return 0 on success, errno otherwise + */ +static inline int unifyfs_inode_tree_wrlock(struct unifyfs_inode_tree* tree) +{ + return pthread_rwlock_wrlock(&tree->rwlock); +} + +/** + * @brief Unlock a unifyfs_inode_tree for read/write. This should only be used + * for calling unifyfs_inode_tree_iter(). All the other unifyfs_inode_tree + * functions provide their own locking. + * + * @param tree inode tree + */ +static inline void unifyfs_inode_tree_unlock(struct unifyfs_inode_tree* tree) +{ + pthread_rwlock_unlock(&tree->rwlock); +} + +#endif /* __UNIFYFS_INODE_TREE_H */ + diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata_mdhim.c similarity index 95% rename from server/src/unifyfs_metadata.c rename to server/src/unifyfs_metadata_mdhim.c index 3ec752a28..5ae6635a6 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata_mdhim.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017-2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -36,13 +36,12 @@ // server headers #include "unifyfs_global.h" -#include "unifyfs_metadata.h" +#include "unifyfs_metadata_mdhim.h" // MDHIM headers #include "indexes.h" #include "mdhim.h" - struct mdhim_t* md; /* we use two MDHIM indexes: @@ -52,23 +51,6 @@ struct mdhim_t* md; #define IDX_FILE_ATTR (1) struct index_t* unifyfs_indexes[2]; -size_t meta_slice_sz; - -void debug_log_key_val(const char* ctx, - unifyfs_key_t* key, - unifyfs_val_t* val) -{ - if ((key != NULL) && (val != NULL)) { - LOGDBG("@%s - key(gfid=%d, offset=%lu), " - "val(del=%d, len=%lu, addr=%lu, app=%d, rank=%d)", - ctx, key->gfid, key->offset, - val->delegator_rank, val->len, val->addr, - val->app_id, val->rank); - } else if (key != NULL) { - LOGDBG("@%s - key(gfid=%d, offset=%lu)", - ctx, key->gfid, key->offset); - } -} int unifyfs_key_compare(unifyfs_key_t* a, unifyfs_key_t* b) { @@ -88,6 +70,33 @@ int unifyfs_key_compare(unifyfs_key_t* a, unifyfs_key_t* b) } } +/* order keyvals by gfid, then host delegator rank */ +int unifyfs_keyval_compare(const void* a, const void* b) +{ + assert((NULL != a) && (NULL != b)); + + const unifyfs_keyval_t* kv_a = a; + const unifyfs_keyval_t* kv_b = b; + + int gfid_a = kv_a->key.gfid; + int gfid_b = kv_b->key.gfid; + if (gfid_a == gfid_b) { + int rank_a = kv_a->val.delegator_rank; + int rank_b = kv_b->val.delegator_rank; + if (rank_a == rank_b) { + return 0; + } else if (rank_a < rank_b) { + return -1; + } else { + return 1; + } + } else if (gfid_a < gfid_b) { + return -1; + } else { + return 1; + } +} + /* initialize the key-value store */ int meta_init_store(unifyfs_cfg_t* cfg) { diff --git a/server/src/unifyfs_metadata.h b/server/src/unifyfs_metadata_mdhim.h similarity index 72% rename from server/src/unifyfs_metadata.h rename to server/src/unifyfs_metadata_mdhim.h index 0aae5a75d..94ef193c7 100644 --- a/server/src/unifyfs_metadata.h +++ b/server/src/unifyfs_metadata_mdhim.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017-2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -27,17 +27,13 @@ * Please read https://github.com/llnl/burstfs/LICENSE for full license text. */ -#ifndef UNIFYFS_METADATA_H -#define UNIFYFS_METADATA_H +#ifndef UNIFYFS_METADATA_MDHIM_H +#define UNIFYFS_METADATA_MDHIM_H #include "unifyfs_configurator.h" -#include "unifyfs_global.h" - -#define MANIFEST_FILE_NAME "mdhim_manifest_" - +#include "unifyfs_log.h" +#include "unifyfs_meta.h" -/* extent slice size used for metadata */ -extern size_t meta_slice_sz; /* Key for file attributes */ typedef int fattr_key_t; @@ -79,10 +75,10 @@ typedef struct { } unifyfs_keyval_t; int unifyfs_key_compare(unifyfs_key_t* a, unifyfs_key_t* b); +int unifyfs_keyval_compare(const void* a, const void* b); -void debug_log_key_val(const char* ctx, - unifyfs_key_t* key, - unifyfs_val_t* val); +/* return number of slice ranges needed to cover range */ +size_t meta_num_slices(size_t offset, size_t length); int meta_sanitize(void); int meta_init_store(unifyfs_cfg_t* cfg); @@ -172,4 +168,70 @@ int unifyfs_set_file_extents(int num_entries, unifyfs_key_t** keys, int* key_lens, unifyfs_val_t** vals, int* val_lens); -#endif + + +static inline +unifyfs_key_t** alloc_key_array(int elems) +{ + int size = elems * (sizeof(unifyfs_key_t*) + sizeof(unifyfs_key_t)); + + void* mem_block = calloc(size, sizeof(char)); + + unifyfs_key_t** array_ptr = mem_block; + unifyfs_key_t* key_ptr = (unifyfs_key_t*)(array_ptr + elems); + + for (int i = 0; i < elems; i++) { + array_ptr[i] = &key_ptr[i]; + } + + return (unifyfs_key_t**)mem_block; +} + +static inline +unifyfs_val_t** alloc_value_array(int elems) +{ + int size = elems * (sizeof(unifyfs_val_t*) + sizeof(unifyfs_val_t)); + + void* mem_block = calloc(size, sizeof(char)); + + unifyfs_val_t** array_ptr = mem_block; + unifyfs_val_t* key_ptr = (unifyfs_val_t*)(array_ptr + elems); + + for (int i = 0; i < elems; i++) { + array_ptr[i] = &key_ptr[i]; + } + + return (unifyfs_val_t**)mem_block; +} + +static inline +void free_key_array(unifyfs_key_t** array) +{ + free(array); +} + +static inline +void free_value_array(unifyfs_val_t** array) +{ + free(array); +} + +static inline +void debug_log_key_val(const char* ctx, + unifyfs_key_t* key, + unifyfs_val_t* val) +{ + if ((key != NULL) && (val != NULL)) { + LOGDBG("@%s - key(gfid=%d, offset=%lu), " + "val(del=%d, len=%lu, addr=%lu, app=%d, rank=%d)", + ctx, key->gfid, key->offset, + val->delegator_rank, val->len, val->addr, + val->app_id, val->rank); + } else if (key != NULL) { + LOGDBG("@%s - key(gfid=%d, offset=%lu)", + ctx, key->gfid, key->offset); + } +} + + +#endif // UNIFYFS_METADATA_MDHIM_H diff --git a/server/src/unifyfs_p2p_rpc.c b/server/src/unifyfs_p2p_rpc.c new file mode 100644 index 000000000..92a1e553e --- /dev/null +++ b/server/src/unifyfs_p2p_rpc.c @@ -0,0 +1,982 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "unifyfs_global.h" +#include "margo_server.h" +#include "unifyfs_server_rpcs.h" +#include "unifyfs_p2p_rpc.h" +#include "unifyfs_group_rpc.h" + +/************************************************************************* + * Peer-to-peer RPC helper methods + *************************************************************************/ + +/* determine server responsible for maintaining target file's metadata */ +int hash_gfid_to_server(int gfid) +{ + return gfid % glb_pmi_size; +} + +/* server peer-to-peer (p2p) margo request structure */ +typedef struct { + margo_request request; + hg_addr_t peer; + hg_handle_t handle; +} p2p_request; + +/* helper method to initialize peer request rpc handle */ +static +int get_request_handle(hg_id_t request_hgid, + int peer_rank, + p2p_request* req) +{ + int rc = UNIFYFS_SUCCESS; + + /* get address for specified server rank */ + req->peer = glb_servers[peer_rank].margo_svr_addr; + + /* get handle to rpc function */ + hg_return_t hret = margo_create(unifyfsd_rpc_context->svr_mid, req->peer, + request_hgid, &(req->handle)); + if (hret != HG_SUCCESS) { + LOGERR("failed to get handle for p2p request(%p) to server %d", + req, peer_rank); + rc = UNIFYFS_ERROR_MARGO; + } + + return rc; +} + +/* helper method to forward peer rpc request */ +static +int forward_request(void* input_ptr, + p2p_request* req) +{ + int rc = UNIFYFS_SUCCESS; + + /* call rpc function */ + hg_return_t hret = margo_iforward(req->handle, input_ptr, + &(req->request)); + if (hret != HG_SUCCESS) { + LOGERR("failed to forward p2p request(%p)", req); + rc = UNIFYFS_ERROR_MARGO; + } + + return rc; +} + +/* helper method to wait for peer rpc request completion */ +static +int wait_for_request(p2p_request* req) +{ + int rc = UNIFYFS_SUCCESS; + + /* call rpc function */ + hg_return_t hret = margo_wait(req->request); + if (hret != HG_SUCCESS) { + LOGERR("wait on p2p request(%p) failed", req); + rc = UNIFYFS_ERROR_MARGO; + } + + return rc; +} + +/************************************************************************* + * File extents metadata update request + *************************************************************************/ + +/* Add extents rpc handler */ +static void add_extents_rpc(hg_handle_t handle) +{ + LOGDBG("add_extents rpc handler"); + + /* assume we'll succeed */ + int32_t ret = UNIFYFS_SUCCESS; + + const struct hg_info* hgi = margo_get_info(handle); + assert(hgi); + margo_instance_id mid = margo_hg_info_get_instance(hgi); + assert(mid != MARGO_INSTANCE_NULL); + + /* get input params */ + add_extents_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + int sender = in.src_rank; + int gfid = in.gfid; + size_t num_extents = (size_t) in.num_extents; + size_t bulk_sz = num_extents * sizeof(struct extent_tree_node); + + /* allocate memory for extents */ + void* extents_buf = malloc(bulk_sz); + if (NULL == extents_buf) { + LOGERR("allocation for bulk extents failed"); + ret = ENOMEM; + } else { + /* register local target buffer for bulk access */ + hg_bulk_t bulk_handle; + hret = margo_bulk_create(mid, 1, &extents_buf, &bulk_sz, + HG_BULK_WRITE_ONLY, &bulk_handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* get list of read requests */ + hret = margo_bulk_transfer(mid, HG_BULK_PULL, + hgi->addr, in.extents, 0, + bulk_handle, 0, + bulk_sz); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_transfer() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* store new extents */ + LOGDBG("received %zu extents for gfid=%d from %d", + num_extents, gfid, sender); + struct extent_tree_node* extents = extents_buf; + ret = unifyfs_inode_add_extents(gfid, num_extents, extents); + if (ret) { + LOGERR("failed to add extents from %d (ret=%d)", + sender, ret); + } + } + margo_bulk_free(bulk_handle); + } + free(extents_buf); + } + margo_free_input(handle, &in); + } + + /* build our output values */ + add_extents_out_t out; + out.ret = ret; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(add_extents_rpc) + +/* Add extents to target file */ +int unifyfs_invoke_add_extents_rpc(int gfid, + unsigned int num_extents, + struct extent_tree_node* extents) +{ + int owner_rank = hash_gfid_to_server(gfid); + if (owner_rank == glb_pmi_rank) { + /* I'm the owner, already did local add */ + return UNIFYFS_SUCCESS; + } + + /* forward request to file owner */ + p2p_request preq; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.extent_add_id; + int rc = get_request_handle(req_hgid, owner_rank, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* create a margo bulk transfer handle for extents array */ + hg_bulk_t bulk_handle; + void* buf = (void*) extents; + size_t buf_sz = (size_t)num_extents * sizeof(struct extent_tree_node); + hg_return_t hret = margo_bulk_create(unifyfsd_rpc_context->svr_mid, + 1, &buf, &buf_sz, + HG_BULK_READ_ONLY, &bulk_handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + return UNIFYFS_ERROR_MARGO; + } + + /* fill rpc input struct and forward request */ + add_extents_in_t in; + in.src_rank = (int32_t) glb_pmi_rank; + in.gfid = (int32_t) gfid; + in.num_extents = (int32_t) num_extents; + in.extents = bulk_handle; + rc = forward_request((void*)&in, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + margo_bulk_free(bulk_handle); + + /* wait for request completion */ + rc = wait_for_request(&preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* get the output of the rpc */ + int ret; + add_extents_out_t out; + hret = margo_get_output(preq.handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + ret = out.ret; + margo_free_output(preq.handle, &out); + } + margo_destroy(preq.handle); + + return ret; +} + +/************************************************************************* + * File extents metadata lookup request + *************************************************************************/ + +/* find extents rpc handler */ +static void find_extents_rpc(hg_handle_t handle) +{ + LOGDBG("find_extents rpc handler"); + + int32_t ret; + unsigned int num_chunks = 0; + chunk_read_req_t* chunk_locs = NULL; + + const struct hg_info* hgi = margo_get_info(handle); + assert(hgi); + margo_instance_id mid = margo_hg_info_get_instance(hgi); + assert(mid != MARGO_INSTANCE_NULL); + + /* get input params */ + find_extents_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + int sender = in.src_rank; + int gfid = in.gfid; + size_t num_extents = (size_t) in.num_extents; + size_t bulk_sz = num_extents * sizeof(unifyfs_inode_extent_t); + + /* allocate memory for extents */ + void* extents_buf = malloc(bulk_sz); + if (NULL == extents_buf) { + LOGERR("allocation for bulk extents failed"); + ret = ENOMEM; + } else { + /* register local target buffer for bulk access */ + hg_bulk_t bulk_req_handle; + hret = margo_bulk_create(mid, 1, &extents_buf, &bulk_sz, + HG_BULK_WRITE_ONLY, &bulk_req_handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* get list of read requests */ + hret = margo_bulk_transfer(mid, HG_BULK_PULL, + hgi->addr, in.extents, 0, + bulk_req_handle, 0, + bulk_sz); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_transfer() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* lookup requested extents */ + LOGDBG("received %zu extent lookups for gfid=%d from %d", + num_extents, gfid, sender); + unifyfs_inode_extent_t* extents = extents_buf; + ret = unifyfs_inode_resolve_extent_chunks(num_extents, + extents, + &num_chunks, + &chunk_locs); + if (ret) { + LOGERR("failed to find extents for %d (ret=%d)", + sender, ret); + } + } + margo_bulk_free(bulk_req_handle); + } + free(extents_buf); + } + margo_free_input(handle, &in); + } + + /* fill rpc response struct with output values */ + hg_bulk_t bulk_resp_handle; + find_extents_out_t out; + out.ret = ret; + out.num_locations = 0; + if (ret == UNIFYFS_SUCCESS) { + void* buf = (void*) chunk_locs; + size_t buf_sz = (size_t)num_chunks * sizeof(chunk_read_req_t); + hret = margo_bulk_create(mid, 1, &buf, &buf_sz, + HG_BULK_READ_ONLY, &bulk_resp_handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + out.num_locations = num_chunks; + out.locations = bulk_resp_handle; + } + } + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + if (out.num_locations) { + margo_bulk_free(bulk_resp_handle); + } + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(find_extents_rpc) + +/* Lookup extent locations for target file */ +int unifyfs_invoke_find_extents_rpc(int gfid, + unsigned int num_extents, + unifyfs_inode_extent_t* extents, + unsigned int* num_chunks, + chunk_read_req_t** chunks) +{ + if ((NULL == num_chunks) || (NULL == chunks)) { + return EINVAL; + } + *num_chunks = 0; + *chunks = NULL; + + int owner_rank = hash_gfid_to_server(gfid); + + /* do local inode metadata lookup to check for laminated */ + unifyfs_file_attr_t attrs; + int ret = unifyfs_inode_metaget(gfid, &attrs); + if (ret == UNIFYFS_SUCCESS) { + if (attrs.is_laminated || (owner_rank == glb_pmi_rank)) { + /* do local lookup */ + ret = unifyfs_inode_resolve_extent_chunks((size_t)num_extents, + extents, + num_chunks, chunks); + if (ret) { + LOGERR("failed to find extents for gfid=%d (ret=%d)", + gfid, ret); + } + return ret; + } + } + + /* forward request to file owner */ + p2p_request preq; + margo_instance_id mid = unifyfsd_rpc_context->svr_mid; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.extent_lookup_id; + int rc = get_request_handle(req_hgid, owner_rank, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* create a margo bulk transfer handle for extents array */ + hg_bulk_t bulk_req_handle; + void* buf = (void*) extents; + size_t buf_sz = (size_t)num_extents * sizeof(unifyfs_inode_extent_t); + hg_return_t hret = margo_bulk_create(mid, 1, &buf, &buf_sz, + HG_BULK_READ_ONLY, &bulk_req_handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + return UNIFYFS_ERROR_MARGO; + } + + /* fill rpc input struct and forward request */ + find_extents_in_t in; + in.src_rank = (int32_t) glb_pmi_rank; + in.gfid = (int32_t) gfid; + in.num_extents = (int32_t) num_extents; + in.extents = bulk_req_handle; + rc = forward_request((void*)&in, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + margo_bulk_free(bulk_req_handle); + + /* wait for request completion */ + rc = wait_for_request(&preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* get the output of the rpc */ + find_extents_out_t out; + hret = margo_get_output(preq.handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + ret = out.ret; + if (ret == UNIFYFS_SUCCESS) { + /* allocate local buffer for chunk locations */ + unsigned int n_chks = (unsigned int) out.num_locations; + buf_sz = (size_t)n_chks * sizeof(chunk_read_req_t); + buf = malloc(buf_sz); + if (NULL == buf) { + LOGERR("allocation for bulk locations failed"); + ret = ENOMEM; + } else { + /* create a margo bulk transfer handle for locations array */ + hg_bulk_t bulk_resp_handle; + hret = margo_bulk_create(mid, 1, &buf, &buf_sz, + HG_BULK_WRITE_ONLY, + &bulk_resp_handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* pull locations array */ + hret = margo_bulk_transfer(mid, HG_BULK_PULL, + preq.peer, out.locations, 0, + bulk_resp_handle, 0, + buf_sz); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_transfer() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* lookup requested extents */ + LOGDBG("received %u chunk locations for gfid=%d", + n_chks, gfid); + *chunks = (chunk_read_req_t*) buf; + *num_chunks = (unsigned int) n_chks; + } + margo_bulk_free(bulk_resp_handle); + } + } + } + margo_free_output(preq.handle, &out); + } + margo_destroy(preq.handle); + + return ret; +} + +/************************************************************************* + * File attributes request + *************************************************************************/ + +/* Metaget rpc handler */ +static void metaget_rpc(hg_handle_t handle) +{ + LOGDBG("metaget rpc handler"); + + int32_t ret; + + /* initialize invalid attributes */ + unifyfs_file_attr_t attrs; + unifyfs_file_attr_set_invalid(&attrs); + + /* get input params */ + metaget_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + ret = unifyfs_inode_metaget(in.gfid, &attrs); + margo_free_input(handle, &in); + } + + /* fill output values */ + metaget_out_t out; + out.ret = ret; + out.attr = attrs; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(metaget_rpc) + +/* Get file attributes for target file */ +int unifyfs_invoke_metaget_rpc(int gfid, + unifyfs_file_attr_t* attrs) +{ + if (NULL == attrs) { + return EINVAL; + } + + int owner_rank = hash_gfid_to_server(gfid); + + /* do local inode metadata lookup to check for laminated */ + int rc = unifyfs_inode_metaget(gfid, attrs); + if ((rc == UNIFYFS_SUCCESS) && (attrs->is_laminated)) { + /* if laminated, we already have final metadata locally */ + return UNIFYFS_SUCCESS; + } + if (owner_rank == glb_pmi_rank) { + return rc; + } + + int need_local_metadata = 0; + if (rc == ENOENT) { + /* inode_metaget above failed with ENOENT, need to create inode */ + need_local_metadata = 1; + } + + /* forward request to file owner */ + p2p_request preq; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.metaget_id; + rc = get_request_handle(req_hgid, owner_rank, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* fill rpc input struct and forward request */ + metaget_in_t in; + in.gfid = (int32_t)gfid; + rc = forward_request((void*)&in, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* wait for request completion */ + rc = wait_for_request(&preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* get the output of the rpc */ + int ret; + metaget_out_t out; + hg_return_t hret = margo_get_output(preq.handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + ret = out.ret; + if (ret == UNIFYFS_SUCCESS) { + *attrs = out.attr; + if (out.attr.filename != NULL) { + attrs->filename = strdup(out.attr.filename); + } + if (need_local_metadata) { + unifyfs_inode_metaset(gfid, UNIFYFS_FILE_ATTR_OP_CREATE, + attrs); + } + } + margo_free_output(preq.handle, &out); + } + margo_destroy(preq.handle); + + return ret; +} + +/************************************************************************* + * File size request + *************************************************************************/ + +/* Filesize rpc handler */ +static void filesize_rpc(hg_handle_t handle) +{ + LOGDBG("filesize rpc handler"); + + int32_t ret; + hg_size_t filesize = 0; + + /* get input params */ + filesize_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + ret = unifyfs_inode_get_filesize(in.gfid, &filesize); + margo_free_input(handle, &in); + } + + /* build our output values */ + filesize_out_t out; + out.ret = ret; + out.filesize = filesize; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(filesize_rpc) + +/* Get current global size for the target file */ +int unifyfs_invoke_filesize_rpc(int gfid, + size_t* filesize) +{ + if (NULL == filesize) { + return EINVAL; + } + + int owner_rank = hash_gfid_to_server(gfid); + + /* do local inode metadata lookup to check for laminated */ + unifyfs_file_attr_t attrs; + int rc = unifyfs_inode_metaget(gfid, &attrs); + if ((rc == UNIFYFS_SUCCESS) && (attrs.is_laminated)) { + /* if laminated, we already have final metadata stored locally */ + *filesize = (size_t) attrs.size; + return UNIFYFS_SUCCESS; + } + if (owner_rank == glb_pmi_rank) { + *filesize = (size_t) attrs.size; + return rc; + } + + /* forward request to file owner */ + p2p_request preq; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.filesize_id; + rc = get_request_handle(req_hgid, owner_rank, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* fill rpc input struct and forward request */ + filesize_in_t in; + in.gfid = (int32_t)gfid; + rc = forward_request((void*)&in, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* wait for request completion */ + rc = wait_for_request(&preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* get the output of the rpc */ + int ret; + filesize_out_t out; + hg_return_t hret = margo_get_output(preq.handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + ret = out.ret; + if (ret == UNIFYFS_SUCCESS) { + *filesize = (size_t) out.filesize; + } + margo_free_output(preq.handle, &out); + } + margo_destroy(preq.handle); + + return ret; +} + +/************************************************************************* + * File attributes update request + *************************************************************************/ + +/* Metaset rpc handler */ +static void metaset_rpc(hg_handle_t handle) +{ + LOGDBG("metaset rpc handler"); + + int32_t ret; + + /* get input params */ + metaset_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + unifyfs_file_attr_op_e attr_op = in.fileop; + ret = unifyfs_inode_metaset(in.gfid, attr_op, &(in.attr)); + margo_free_input(handle, &in); + } + + /* build our output values */ + metaset_out_t out; + out.ret = ret; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(metaset_rpc) + +/* Set metadata for target file */ +int unifyfs_invoke_metaset_rpc(int gfid, + int attr_op, + unifyfs_file_attr_t* attrs) +{ + if (NULL == attrs) { + return EINVAL; + } + + int owner_rank = hash_gfid_to_server(gfid); + if (owner_rank == glb_pmi_rank) { + /* I'm the owner, do local inode metadata update */ + return unifyfs_inode_metaset(gfid, attr_op, attrs); + } + + /* forward request to file owner */ + p2p_request preq; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.metaset_id; + int rc = get_request_handle(req_hgid, owner_rank, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* fill rpc input struct and forward request */ + metaset_in_t in; + in.gfid = (int32_t) gfid; + in.fileop = (int32_t) attr_op; + in.attr = *attrs; + rc = forward_request((void*)&in, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* wait for request completion */ + rc = wait_for_request(&preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* get the output of the rpc */ + int ret; + metaset_out_t out; + hg_return_t hret = margo_get_output(preq.handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + ret = out.ret; + margo_free_output(preq.handle, &out); + + /* if update at owner succeeded, do it locally */ + if (ret == UNIFYFS_SUCCESS) { + ret = unifyfs_inode_metaset(gfid, attr_op, attrs); + } + } + margo_destroy(preq.handle); + + return ret; +} + +/************************************************************************* + * File lamination request + *************************************************************************/ + +/* Laminate rpc handler */ +static void laminate_rpc(hg_handle_t handle) +{ + LOGDBG("laminate rpc handler"); + + int32_t ret; + + /* get input params */ + laminate_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + int gfid = (int) in.gfid; + margo_free_input(handle, &in); + + ret = unifyfs_inode_laminate(gfid); + if (ret == UNIFYFS_SUCCESS) { + /* tell the rest of the servers */ + ret = unifyfs_invoke_broadcast_laminate(gfid); + } + } + + /* build our output values */ + laminate_out_t out; + out.ret = ret; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(laminate_rpc) + +/* Laminate the target file */ +int unifyfs_invoke_laminate_rpc(int gfid) +{ + int ret; + int owner_rank = hash_gfid_to_server(gfid); + if (owner_rank == glb_pmi_rank) { + /* I'm the owner, do local inode metadata update */ + ret = unifyfs_inode_laminate(gfid); + if (ret == UNIFYFS_SUCCESS) { + /* tell the rest of the servers */ + ret = unifyfs_invoke_broadcast_laminate(gfid); + } + return ret; + } + + /* forward request to file owner */ + p2p_request preq; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.laminate_id; + int rc = get_request_handle(req_hgid, owner_rank, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* fill rpc input struct and forward request */ + laminate_in_t in; + in.gfid = (int32_t)gfid; + rc = forward_request((void*)&in, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* wait for request completion */ + rc = wait_for_request(&preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* get the output of the rpc */ + laminate_out_t out; + hg_return_t hret = margo_get_output(preq.handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + ret = out.ret; + margo_free_output(preq.handle, &out); + } + margo_destroy(preq.handle); + + return ret; +} + +/************************************************************************* + * File truncation request + *************************************************************************/ + +/* Truncate rpc handler */ +static void truncate_rpc(hg_handle_t handle) +{ + LOGDBG("truncate rpc handler"); + + int32_t ret; + + /* get input params */ + truncate_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + int gfid = (int) in.gfid; + size_t fsize = (size_t) in.filesize; + ret = unifyfs_invoke_broadcast_truncate(gfid, fsize); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("truncate(gfid=%d, size=%zu) broadcast failed", + gfid, fsize); + } + margo_free_input(handle, &in); + } + + /* build our output values */ + truncate_out_t out; + out.ret = ret; + + /* send output back to caller */ + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } + + /* free margo resources */ + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(truncate_rpc) + +/* Truncate the target file */ +int unifyfs_invoke_truncate_rpc(int gfid, + size_t filesize) +{ + int owner_rank = hash_gfid_to_server(gfid); + if (owner_rank == glb_pmi_rank) { + /* I'm the owner, start broadcast update. The local inode will be + * updated as part of this update. */ + return unifyfs_invoke_broadcast_truncate(gfid, filesize); + } + + /* forward request to file owner */ + p2p_request preq; + hg_id_t req_hgid = unifyfsd_rpc_context->rpcs.truncate_id; + int rc = get_request_handle(req_hgid, owner_rank, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* fill rpc input struct and forward request */ + truncate_in_t in; + in.gfid = (int32_t) gfid; + in.filesize = (hg_size_t) filesize; + rc = forward_request((void*)&in, &preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* wait for request completion */ + rc = wait_for_request(&preq); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } + + /* get the output of the rpc */ + int ret; + truncate_out_t out; + hg_return_t hret = margo_get_output(preq.handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* set return value */ + ret = out.ret; + margo_free_output(preq.handle, &out); + } + margo_destroy(preq.handle); + + return ret; +} diff --git a/server/src/unifyfs_p2p_rpc.h b/server/src/unifyfs_p2p_rpc.h new file mode 100644 index 000000000..9710a8ca4 --- /dev/null +++ b/server/src/unifyfs_p2p_rpc.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef _UNIFYFS_P2P_RPC_H +#define _UNIFYFS_P2P_RPC_H + +#include "unifyfs_global.h" +#include "extent_tree.h" +#include "unifyfs_inode.h" + + +/* Point-to-point Server RPCs */ + + +/* determine server responsible for maintaining target file's metadata */ +int hash_gfid_to_server(int gfid); + + +/** + * @brief Add new extents to target file + * + * @param gfid target file + * @param num_extents length of file extents array + * @param extents array of extents to add + * + * @return success|failure + */ +int unifyfs_invoke_add_extents_rpc(int gfid, + unsigned int num_extents, + struct extent_tree_node* extents); + +/** + * @brief Find location of extents for target file + * + * @param gfid target file + * @param num_extents length of file extents array + * @param extents array of extents to find + * + * @param[out] num_chunks number of chunk locations + * @param[out] chunks array of chunk locations for requested extents + * + * @return success|failure + */ +int unifyfs_invoke_find_extents_rpc(int gfid, + unsigned int num_extents, + unifyfs_inode_extent_t* extents, + unsigned int* num_chunks, + chunk_read_req_t** chunks); + +/** + * @brief Get file size for the target file + * + * @param gfid target file + * @param filesize pointer to size variable + * + * @return success|failure + */ +int unifyfs_invoke_filesize_rpc(int gfid, + size_t* filesize); + +/** + * @brief Laminate the target file + * + * @param gfid target file + * + * @return success|failure + */ +int unifyfs_invoke_laminate_rpc(int gfid); + +/** + * @brief Get metadata for target file + * + * @param gfid target file + * @param create flag indicating if this is a newly created file + * @param attr file attributes to update + * + * @return success|failure + */ +int unifyfs_invoke_metaget_rpc(int gfid, + unifyfs_file_attr_t* attrs); + +/** + * @brief Update metadata for target file + * + * @param gfid target file + * @param attr_op metadata operation that triggered update + * @param attr file attributes to update + * + * @return success|failure + */ +int unifyfs_invoke_metaset_rpc(int gfid, int attr_op, + unifyfs_file_attr_t* attrs); + +/** + * @brief Truncate target file + * + * @param gfid target file + * @param filesize truncated file size + * + * @return success|failure + */ +int unifyfs_invoke_truncate_rpc(int gfid, size_t filesize); + + +#endif // UNIFYFS_P2P_RPC_H diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index caa0cf7c1..1ffa869df 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017-2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -27,26 +27,22 @@ * Please read https://github.com/llnl/burstfs/LICENSE for full license text. */ -// system headers -#include -#include -#include -#include -#include - // general support #include "unifyfs_global.h" -#include "unifyfs_log.h" // server components +#include "unifyfs_inode_tree.h" +#include "unifyfs_metadata_mdhim.h" #include "unifyfs_request_manager.h" #include "unifyfs_service_manager.h" -#include "unifyfs_metadata.h" // margo rpcs +#include "unifyfs_group_rpc.h" #include "unifyfs_server_rpcs.h" #include "margo_server.h" +// system headers +#include // nanosleep #define RM_LOCK(rm) \ do { \ @@ -60,39 +56,39 @@ do { \ pthread_mutex_unlock(&(rm->thrd_lock)); \ } while (0) +#define RM_REQ_LOCK(rm) \ +do { \ + LOGDBG("locking RM[%d:%d] reqs", rm->app_id, rm->client_id); \ + ABT_mutex_lock(rm->reqs_sync); \ +} while (0) -/* One request manager thread is created for each client that a - * server serves. The margo rpc handler thread(s) assign work - * to the request manager thread to retrieve data and send - * it back to the client. - * - * To start, given a read request from the client (via rpc) - * the handler function on the main server first queries the - * key/value store using the given file id and byte range to obtain - * the meta data on the physical location of the file data. This - * meta data provides the host server rank, the app/client - * ids that specify the log file on the remote server, the - * offset within the log file and the length of data. The rpc - * handler function sorts the meta data by host server rank, - * generates read requests, and inserts those into a list on a - * data structure shared with the request manager (del_req_set). +#define RM_REQ_UNLOCK(rm) \ +do { \ + LOGDBG("unlocking RM[%d:%d] reqs", rm->app_id, rm->client_id); \ + ABT_mutex_unlock(rm->reqs_sync); \ +} while (0) + +/* One request manager thread is created for each client of the + * server. The margo rpc handler thread(s) assign work to the + * request manager thread to handle data and metadata operations. * - * The request manager thread coordinates with the main thread + * The request manager thread coordinates with other threads * using a lock and a condition variable to protect the shared data - * structure and impose flow control. When assigned work, the - * request manager thread packs and sends request messages to - * service manager threads on remote delegators via MPI send. - * It waits for data to be sent back, and unpacks the read replies - * in each message into a shared memory buffer for the client. - * When the shared memory is full or all data has been received, - * it signals the client process to process the read replies. - * It iterates with the client until all incoming read replies - * have been transferred. */ -/* create a request manager thread for the given app_id - * and client_id, returns pointer to thread control structure - * on success and NULL on failure */ - -/* Create Request Manager thread for application client */ + * structure and impose flow control. When assigned work, the + * request manager thread either handles the request directly, or + * forwards requests to remote servers. + * + * For read requests, the request manager waits for data chunk + * responses and places the data into a shared memory data buffer + * specific to the client. When the shared memory is full or all + * data has been received, the request manager signals the client + * to process the read replies. It iterates with the client until + * all incoming read replies have been transferred. */ + +/* Create a request manager thread for the application client + * corresponding to the given app_id and client_id. + * Returns pointer to thread control structure on success, or + * NULL on failure */ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) { /* allocate a new thread control structure */ @@ -131,6 +127,18 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) return NULL; } + /* create the argobots mutex for synchronizing access to reqs state */ + ABT_mutex_create(&(thrd_ctrl->reqs_sync)); + + /* allocate a list to track client rpc requests */ + thrd_ctrl->client_reqs = arraylist_create(); + if (thrd_ctrl->client_reqs == NULL) { + LOGERR("failed to allocate request manager client_reqs!"); + pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); + free(thrd_ctrl); + return NULL; + } + /* record app and client id this thread will be serving */ thrd_ctrl->app_id = app_id; thrd_ctrl->client_id = client_id; @@ -138,12 +146,12 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) /* initialize flow control flags */ thrd_ctrl->exit_flag = 0; thrd_ctrl->exited = 0; - thrd_ctrl->has_waiting_delegator = 0; + thrd_ctrl->waiting_for_work = 0; thrd_ctrl->has_waiting_dispatcher = 0; /* launch request manager thread */ rc = pthread_create(&(thrd_ctrl->thrd), NULL, - rm_delegate_request_thread, (void*)thrd_ctrl); + request_manager_thread, (void*)thrd_ctrl); if (rc != 0) { LOGERR("failed to create request manager thread for " "app_id=%d client_id=%d - rc=%d (%s)", @@ -157,85 +165,18 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) return thrd_ctrl; } -/* order keyvals by gfid, then host server rank */ -static int compare_kv_gfid_rank(const void* a, const void* b) -{ - const unifyfs_keyval_t* kv_a = a; - const unifyfs_keyval_t* kv_b = b; - - int gfid_a = kv_a->key.gfid; - int gfid_b = kv_b->key.gfid; - if (gfid_a == gfid_b) { - int rank_a = kv_a->val.delegator_rank; - int rank_b = kv_b->val.delegator_rank; - if (rank_a == rank_b) { - return 0; - } else if (rank_a < rank_b) { - return -1; - } else { - return 1; - } - } else if (gfid_a < gfid_b) { - return -1; - } else { - return 1; - } -} - -unifyfs_key_t** alloc_key_array(int elems) -{ - int size = elems * (sizeof(unifyfs_key_t*) + sizeof(unifyfs_key_t)); - - void* mem_block = calloc(size, sizeof(char)); - - unifyfs_key_t** array_ptr = mem_block; - unifyfs_key_t* key_ptr = (unifyfs_key_t*)(array_ptr + elems); - - for (int i = 0; i < elems; i++) { - array_ptr[i] = &key_ptr[i]; - } - - return (unifyfs_key_t**)mem_block; -} - -unifyfs_val_t** alloc_value_array(int elems) -{ - int size = elems * (sizeof(unifyfs_val_t*) + sizeof(unifyfs_val_t)); - - void* mem_block = calloc(size, sizeof(char)); - - unifyfs_val_t** array_ptr = mem_block; - unifyfs_val_t* key_ptr = (unifyfs_val_t*)(array_ptr + elems); - - for (int i = 0; i < elems; i++) { - array_ptr[i] = &key_ptr[i]; - } - - return (unifyfs_val_t**)mem_block; -} - -void free_key_array(unifyfs_key_t** array) -{ - free(array); -} - -void free_value_array(unifyfs_val_t** array) -{ - free(array); -} - static void debug_print_read_req(server_read_req_t* req) { if (NULL != req) { LOGDBG("server_read_req[%d] status=%d, num_remote=%d", - req->req_ndx, req->status, req->num_remote_reads); + req->req_ndx, req->status, req->num_server_reads); } } -static server_read_req_t* reserve_read_req(reqmgr_thrd_t* thrd_ctrl) +server_read_req_t* rm_reserve_read_req(reqmgr_thrd_t* thrd_ctrl) { server_read_req_t* rdreq = NULL; - RM_LOCK(thrd_ctrl); + RM_REQ_LOCK(thrd_ctrl); if (thrd_ctrl->num_read_reqs < RM_MAX_ACTIVE_REQUESTS) { if (thrd_ctrl->next_rdreq_ndx < (RM_MAX_ACTIVE_REQUESTS - 1)) { rdreq = thrd_ctrl->read_reqs + thrd_ctrl->next_rdreq_ndx; @@ -258,18 +199,17 @@ static server_read_req_t* reserve_read_req(reqmgr_thrd_t* thrd_ctrl) } else { LOGERR("maxed-out request manager read_reqs array!!"); } - RM_UNLOCK(thrd_ctrl); + RM_REQ_UNLOCK(thrd_ctrl); return rdreq; } -static int __release_read_req(reqmgr_thrd_t* thrd_ctrl, - server_read_req_t* rdreq) +static int release_read_req(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq) { - // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked - int rc = (int)UNIFYFS_SUCCESS; if (rdreq != NULL) { + RM_REQ_LOCK(thrd_ctrl); LOGDBG("releasing read req %d", rdreq->req_ndx); if (rdreq->req_ndx == (thrd_ctrl->next_rdreq_ndx - 1)) { thrd_ctrl->next_rdreq_ndx--; @@ -284,7 +224,7 @@ static int __release_read_req(reqmgr_thrd_t* thrd_ctrl, thrd_ctrl->num_read_reqs--; LOGDBG("after release (active=%d, next=%d)", thrd_ctrl->num_read_reqs, thrd_ctrl->next_rdreq_ndx); - debug_print_read_req(rdreq); + RM_REQ_UNLOCK(thrd_ctrl); } else { rc = EINVAL; LOGERR("NULL read_req"); @@ -293,58 +233,61 @@ static int __release_read_req(reqmgr_thrd_t* thrd_ctrl, return rc; } -static int release_read_req(reqmgr_thrd_t* thrd_ctrl, server_read_req_t* rdreq) +int rm_release_read_req(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq) { - int rc = (int)UNIFYFS_SUCCESS; - - RM_LOCK(thrd_ctrl); - rc = __release_read_req(thrd_ctrl, rdreq); - RM_UNLOCK(thrd_ctrl); - - return rc; + return release_read_req(thrd_ctrl, rdreq); } -static void signal_new_requests(reqmgr_thrd_t* thrd_ctrl) +static void signal_new_requests(reqmgr_thrd_t* reqmgr) { - // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked - - /* wake up the request manager thread for the requesting client */ - if (!thrd_ctrl->has_waiting_delegator) { - /* reqmgr thread is not waiting, but we are in critical - * section, we just added requests so we must wait for reqmgr - * to signal us that it's reached the critical section before - * we escape so we don't overwrite these requests before it - * has had a chance to process them */ - thrd_ctrl->has_waiting_dispatcher = 1; - pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); - - /* reqmgr thread has signaled us that it's now waiting */ - thrd_ctrl->has_waiting_dispatcher = 0; + RM_LOCK(reqmgr); + pid_t this_thread = unifyfs_gettid(); + if (this_thread != reqmgr->tid) { + /* wake up the request manager thread for the requesting client */ + if (!reqmgr->waiting_for_work) { + /* reqmgr thread is not waiting, but we are in critical + * section, we just added requests so we must wait for reqmgr + * to signal us that it's reached the critical section before + * we escape so we don't overwrite these requests before it + * has had a chance to process them */ + reqmgr->has_waiting_dispatcher = 1; + pthread_cond_wait(&reqmgr->thrd_cond, &reqmgr->thrd_lock); + + /* reqmgr thread has signaled us that it's now waiting */ + reqmgr->has_waiting_dispatcher = 0; + } + /* have a reqmgr thread waiting on condition variable, + * signal it to begin processing the requests we just added */ + pthread_cond_signal(&reqmgr->thrd_cond); } - /* have a reqmgr thread waiting on condition variable, - * signal it to begin processing the requests we just added */ - pthread_cond_signal(&thrd_ctrl->thrd_cond); + RM_UNLOCK(reqmgr); } -static void signal_new_responses(reqmgr_thrd_t* thrd_ctrl) +static void signal_new_responses(reqmgr_thrd_t* reqmgr) { - // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked - - /* wake up the request manager thread */ - if (thrd_ctrl->has_waiting_delegator) { - /* have a reqmgr thread waiting on condition variable, - * signal it to begin processing the responses we just added */ - pthread_cond_signal(&thrd_ctrl->thrd_cond); + RM_LOCK(reqmgr); + pid_t this_thread = unifyfs_gettid(); + if (this_thread != reqmgr->tid) { + /* wake up the request manager thread */ + if (reqmgr->waiting_for_work) { + /* have a reqmgr thread waiting on condition variable, + * signal it to begin processing the responses we just added */ + pthread_cond_signal(&reqmgr->thrd_cond); + } } + RM_UNLOCK(reqmgr); } /* issue remote chunk read requests for extent chunks * listed within keyvals */ -int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, - server_read_req_t* rdreq, - int num_vals, - unifyfs_keyval_t* keyvals) +int rm_create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq, + int num_vals, + unifyfs_keyval_t* keyvals) { + LOGDBG("creating chunk requests for rdreq %d", rdreq->req_ndx); + /* allocate read request structures */ chunk_read_req_t* all_chunk_reads = (chunk_read_req_t*) calloc((size_t)num_vals, sizeof(chunk_read_req_t)); @@ -352,17 +295,10 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, LOGERR("failed to allocate chunk-reads array"); return ENOMEM; } - - /* wait on lock before we attach new array to global variable */ - RM_LOCK(thrd_ctrl); - - LOGDBG("creating chunk requests for rdreq %d", rdreq->req_ndx); - - /* attach read request array to global request mananger struct */ rdreq->chunks = all_chunk_reads; /* iterate over write index values and create read requests - * for each one, also count up number of delegators that we'll + * for each one, also count up number of servers that we'll * forward read requests to */ int i; int prev_del = -1; @@ -393,17 +329,16 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, /* allocate per-delgator chunk-reads */ int num_dels = num_del; - rdreq->num_remote_reads = num_dels; - rdreq->remote_reads = (remote_chunk_reads_t*) - calloc((size_t)num_dels, sizeof(remote_chunk_reads_t)); + rdreq->num_server_reads = num_dels; + rdreq->remote_reads = (server_chunk_reads_t*) + calloc((size_t)num_dels, sizeof(server_chunk_reads_t)); if (NULL == rdreq->remote_reads) { LOGERR("failed to allocate remote-reads array"); - RM_UNLOCK(thrd_ctrl); return ENOMEM; } /* get pointer to start of chunk read request array */ - remote_chunk_reads_t* reads = rdreq->remote_reads; + server_chunk_reads_t* reads = rdreq->remote_reads; /* iterate over write index values again and now create * per-server chunk-reads info, for each server @@ -463,11 +398,51 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, /* wake up the request manager thread for the requesting client */ signal_new_requests(thrd_ctrl); - RM_UNLOCK(thrd_ctrl); - return UNIFYFS_SUCCESS; } +int rm_submit_read_request(server_read_req_t* req) +{ + int ret = UNIFYFS_SUCCESS; + int i = 0; + app_client* client = NULL; + reqmgr_thrd_t* thrd_ctrl = NULL; + server_read_req_t* rdreq = NULL; + + if (!req || !req->chunks || !req->remote_reads) { + return EINVAL; + } + + client = get_app_client(req->app_id, req->client_id); + if (NULL == client) { + return UNIFYFS_FAILURE; + } + + thrd_ctrl = client->reqmgr; + + rdreq = rm_reserve_read_req(thrd_ctrl); + if (!rdreq) { + LOGERR("failed to allocate a request"); + return UNIFYFS_FAILURE; + } + + rdreq->app_id = req->app_id; + rdreq->client_id = req->client_id; + rdreq->num_server_reads = req->num_server_reads; + rdreq->chunks = req->chunks; + rdreq->remote_reads = req->remote_reads; + + for (i = 0; i < rdreq->num_server_reads; i++) { + server_chunk_reads_t* read = &rdreq->remote_reads[i]; + read->rdreq_id = rdreq->req_ndx; + } + + rdreq->status = READREQ_READY; + signal_new_requests(thrd_ctrl); + + return ret; +} + /* signal the client process for it to start processing read * data in shared memory */ static int client_signal(shm_data_header* hdr, @@ -522,1459 +497,891 @@ static int client_wait(shm_data_header* hdr) return rc; } -/************************ - * These functions are called by the rpc handler to assign work - * to the request manager thread - ***********************/ - -/* given an app_id, client_id and global file id, - * compute and return file size for specified file - */ -int rm_cmd_filesize( - int app_id, /* app_id for requesting client */ - int client_id, /* client_id for requesting client */ - int gfid, /* global file id of read request */ - size_t* outsize) /* output file size */ +/* function called by main thread to instruct + * resource manager thread to exit, + * returns UNIFYFS_SUCCESS on success */ +int rm_request_exit(reqmgr_thrd_t* thrd_ctrl) { - /* initialize output file size to something deterministic, - * in case we drop out with an error */ - *outsize = 0; - - /* set offset and length to request *all* key/value pairs - * for this file */ - size_t offset = 0; - - /* want to pick the highest integer offset value a file - * could have here */ - size_t length = (SIZE_MAX >> 1) - 1; - - /* get the locations of all the read requests from the - * key-value store*/ - unifyfs_key_t key1, key2; - - /* create key to describe first byte we'll read */ - key1.gfid = gfid; - key1.offset = offset; - - /* create key to describe last byte we'll read */ - key2.gfid = gfid; - key2.offset = offset + length - 1; - - /* set up input params to specify range lookup */ - unifyfs_key_t* unifyfs_keys[2] = {&key1, &key2}; - int key_lens[2] = {sizeof(unifyfs_key_t), sizeof(unifyfs_key_t)}; - - /* look up all entries in this range */ - int num_vals = 0; - unifyfs_keyval_t* keyvals = NULL; - int rc = unifyfs_get_file_extents(2, unifyfs_keys, key_lens, - &num_vals, &keyvals); - if (UNIFYFS_SUCCESS != rc) { - /* failed to look up extents, bail with error */ - LOGERR("failed to retrieve extent metadata for gfid=%d", gfid); - return UNIFYFS_FAILURE; + if (thrd_ctrl->exited) { + /* already done */ + return UNIFYFS_SUCCESS; } - /* compute our file size by iterating over each file - * segment and taking the max logical offset */ - int i; - size_t filesize = 0; - for (i = 0; i < num_vals; i++) { - /* get pointer to next key value pair */ - unifyfs_keyval_t* kv = &keyvals[i]; + /* grab the lock */ + RM_LOCK(thrd_ctrl); - /* get last byte offset for this segment of the file */ - size_t last_offset = kv->key.offset + kv->val.len; + /* if reqmgr thread is not waiting in critical + * section, let's wait on it to come back */ + if (!thrd_ctrl->waiting_for_work) { + /* reqmgr thread is not in critical section, + * tell it we've got something and signal it */ + thrd_ctrl->has_waiting_dispatcher = 1; + pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); - /* update our filesize if this offset is bigger than the current max */ - if (last_offset > filesize) { - filesize = last_offset; - } + /* we're no longer waiting */ + thrd_ctrl->has_waiting_dispatcher = 0; } - /* free off key/value buffer returned from get_file_extents */ - if (NULL != keyvals) { - free(keyvals); - keyvals = NULL; - } + /* inform reqmgr thread that it's time to exit */ + thrd_ctrl->exit_flag = 1; - /* get filesize as recorded in metadata, which may be bigger if - * user issued an ftruncate on the file to extend it past the - * last write */ - size_t filesize_meta = filesize; + /* signal reqmgr thread */ + pthread_cond_signal(&thrd_ctrl->thrd_cond); - /* given the global file id, look up file attributes - * from key/value store */ - unifyfs_file_attr_t fattr; - int ret = unifyfs_get_file_attribute(gfid, &fattr); - if (ret == UNIFYFS_SUCCESS) { - /* found file attribute for this file, now get its size */ - filesize_meta = fattr.size; - } else { - /* failed to find file attributes for this file */ - LOGERR("failed to retrieve attributes for gfid=%d", gfid); - return UNIFYFS_FAILURE; - } + /* release the lock */ + RM_UNLOCK(thrd_ctrl); - /* take maximum of last write and file size from metadata */ - if (filesize_meta > filesize) { - filesize = filesize_meta; + /* wait for reqmgr thread to exit */ + int rc = pthread_join(thrd_ctrl->thrd, NULL); + if (0 == rc) { + pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); + pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); + thrd_ctrl->exited = 1; } - - *outsize = filesize; - return rc; + return UNIFYFS_SUCCESS; } -/* delete any key whose last byte is beyond the specified - * file size */ -static int truncate_delete_keys( - size_t filesize, /* new file size */ - int num, /* number of entries in keyvals */ - unifyfs_keyval_t* keyvals) /* list of existing key/values */ +/************************ + * These functions define the logic of the request manager thread + ***********************/ + +/* pack the chunk read requests for a single remote server. + * + * @param req_msg_buf: request buffer used for packing + * @param req_num: number of read requests + * @return size of packed buffer (or error code) + */ +static size_t rm_pack_chunk_requests(char* req_msg_buf, + server_chunk_reads_t* remote_reads) { - /* assume we'll succeed */ - int ret = (int) UNIFYFS_SUCCESS; - - /* pointers to memory we'll dynamically allocate for file extents */ - unifyfs_key_t** unifyfs_keys = NULL; - unifyfs_val_t** unifyfs_vals = NULL; - int* unifyfs_key_lens = NULL; - int* unifyfs_val_lens = NULL; - - /* in the worst case, we'll have to delete all existing keys */ - /* allocate storage for file extent key/values */ - /* TODO: possibly get this from memory pool */ - unifyfs_keys = alloc_key_array(num); - unifyfs_vals = alloc_value_array(num); - unifyfs_key_lens = calloc(num, sizeof(int)); - unifyfs_val_lens = calloc(num, sizeof(int)); - if ((NULL == unifyfs_keys) || - (NULL == unifyfs_vals) || - (NULL == unifyfs_key_lens) || - (NULL == unifyfs_val_lens)) { - LOGERR("failed to allocate memory for file extents"); - ret = ENOMEM; - goto truncate_delete_exit; - } - - /* counter for number of key/values we need to delete */ - int delete_count = 0; - - /* iterate over each key, and if this index extends beyond desired - * file size, create an entry to delete that key */ - int i; - for (i = 0; i < num; i++) { - /* get pointer to next key value pair */ - unifyfs_keyval_t* kv = &keyvals[i]; - - /* get last byte offset for this segment of the file */ - size_t last_offset = kv->key.offset + kv->val.len; - - /* if this segment extends beyond the new file size, - * we need to delete this index entry */ - if (last_offset > filesize) { - /* found an index that extends past end of desired - * file size, get next empty key entry from the pool */ - unifyfs_key_t* key = unifyfs_keys[delete_count]; - - /* define the key to be deleted */ - key->gfid = kv->key.gfid; - key->offset = kv->key.offset; - - /* MDHIM needs to know the byte size of each key and value */ - unifyfs_key_lens[delete_count] = sizeof(unifyfs_key_t); - //unifyfs_val_lens[delete_count] = sizeof(unifyfs_val_t); - - /* increment the number of keys we're deleting */ - delete_count++; - } - } + /* send format: + * (int) cmd - specifies type of message (SVC_CMD_RDREQ_CHK) + * (int) req_cnt - number of requests in message + * (size_t) total_sz - total number of bytes requested + * {sequence of chunk_read_req_t} */ + int req_cnt = remote_reads->num_chunks; + size_t reqs_sz = req_cnt * sizeof(chunk_read_req_t); + size_t packed_size = (2 * sizeof(int)) + sizeof(size_t) + reqs_sz; - /* batch delete file extent key/values from MDHIM */ - if (delete_count > 0) { - ret = unifyfs_delete_file_extents(delete_count, - unifyfs_keys, unifyfs_key_lens); - if (ret != UNIFYFS_SUCCESS) { - /* TODO: need proper error handling */ - LOGERR("unifyfs_delete_file_extents() failed"); - goto truncate_delete_exit; - } - } + assert(req_cnt <= MAX_META_PER_SEND); -truncate_delete_exit: - /* clean up memory */ + /* get pointer to start of send buffer */ + char* ptr = req_msg_buf; + memset(ptr, 0, packed_size); - if (NULL != unifyfs_keys) { - free_key_array(unifyfs_keys); - } + /* pack command */ + int cmd = (int)SVC_CMD_RDREQ_CHK; + *((int*)ptr) = cmd; + ptr += sizeof(int); - if (NULL != unifyfs_vals) { - free_value_array(unifyfs_vals); - } + /* pack request count */ + *((int*)ptr) = req_cnt; + ptr += sizeof(int); - if (NULL != unifyfs_key_lens) { - free(unifyfs_key_lens); - } + /* pack total requested data size */ + *((size_t*)ptr) = remote_reads->total_sz; + ptr += sizeof(size_t); - if (NULL != unifyfs_val_lens) { - free(unifyfs_val_lens); - } + /* copy requests into buffer */ + memcpy(ptr, remote_reads->reqs, reqs_sz); + ptr += reqs_sz; - return ret; + /* return number of bytes used to pack requests */ + return packed_size; } -/* rewrite any key that overlaps with new file size, - * we assume the existing key has already been deleted */ -static int truncate_rewrite_keys( - size_t filesize, /* new file size */ - int num, /* number of entries in keyvals */ - unifyfs_keyval_t* keyvals) /* list of existing key/values */ +/* send the chunk read requests to remote servers + * + * @param thrd_ctrl : reqmgr thread control structure + * @return success/error code + */ +static int rm_request_remote_chunks(reqmgr_thrd_t* thrd_ctrl) { - /* assume we'll succeed */ - int ret = (int) UNIFYFS_SUCCESS; - - /* pointers to memory we'll dynamically allocate for file extents */ - unifyfs_key_t** unifyfs_keys = NULL; - unifyfs_val_t** unifyfs_vals = NULL; - int* unifyfs_key_lens = NULL; - int* unifyfs_val_lens = NULL; - - /* in the worst case, we'll have to rewrite all existing keys */ - /* allocate storage for file extent key/values */ - /* TODO: possibly get this from memory pool */ - unifyfs_keys = alloc_key_array(num); - unifyfs_vals = alloc_value_array(num); - unifyfs_key_lens = calloc(num, sizeof(int)); - unifyfs_val_lens = calloc(num, sizeof(int)); - if ((NULL == unifyfs_keys) || - (NULL == unifyfs_vals) || - (NULL == unifyfs_key_lens) || - (NULL == unifyfs_val_lens)) { - LOGERR("failed to allocate memory for file extents"); - ret = ENOMEM; - goto truncate_rewrite_exit; - } - - /* counter for number of key/values we need to rewrite */ - int count = 0; - - /* iterate over each key, and if this index starts before - * and ends after the desired file size, create an entry - * that ends at new file size */ - int i; - for (i = 0; i < num; i++) { - /* get pointer to next key value pair */ - unifyfs_keyval_t* kv = &keyvals[i]; - - /* get first byte offset for this segment of the file */ - size_t first_offset = kv->key.offset; - - /* get last byte offset for this segment of the file */ - size_t last_offset = kv->key.offset + kv->val.len; - - /* if this segment extends beyond the new file size, - * we need to rewrite this index entry */ - if (first_offset < filesize && - last_offset > filesize) { - /* found an index that overlaps end of desired - * file size, get next empty key entry from the pool */ - unifyfs_key_t* key = unifyfs_keys[count]; - - /* define the key to be rewritten */ - key->gfid = kv->key.gfid; - key->offset = kv->key.offset; - - /* compute new length of this entry */ - size_t newlen = (size_t)(filesize - first_offset); - - /* for the value, we store the log position, the length, - * the host server (delegator_rank), the mount point id - * (app id), and the client id (rank) */ - unifyfs_val_t* val = unifyfs_vals[count]; - val->addr = kv->val.addr; - val->len = newlen; - val->delegator_rank = kv->val.delegator_rank; - val->app_id = kv->val.app_id; - val->rank = kv->val.rank; - - /* MDHIM needs to know the byte size of each key and value */ - unifyfs_key_lens[count] = sizeof(unifyfs_key_t); - unifyfs_val_lens[count] = sizeof(unifyfs_val_t); - - /* increment the number of keys we're deleting */ - count++; - } - } - - /* batch set file extent key/values from MDHIM */ - if (count > 0) { - ret = unifyfs_set_file_extents(count, - unifyfs_keys, unifyfs_key_lens, - unifyfs_vals, unifyfs_val_lens); - if (ret != UNIFYFS_SUCCESS) { - /* TODO: need proper error handling */ - LOGERR("unifyfs_set_file_extents() failed"); - goto truncate_rewrite_exit; - } - } + int i, j, rc; + int ret = (int)UNIFYFS_SUCCESS; -truncate_rewrite_exit: - /* clean up memory */ + /* get pointer to send buffer */ + char* sendbuf = thrd_ctrl->del_req_msg_buf; - if (NULL != unifyfs_keys) { - free_key_array(unifyfs_keys); - } + /* iterate over each active read request */ + RM_REQ_LOCK(thrd_ctrl); + for (i = 0; i < RM_MAX_ACTIVE_REQUESTS; i++) { + server_read_req_t* req = thrd_ctrl->read_reqs + i; + if (req->num_server_reads > 0) { + LOGDBG("read req %d is active", i); + debug_print_read_req(req); + if (req->status == READREQ_READY) { + req->status = READREQ_STARTED; + /* iterate over each server we need to send requests to */ + server_chunk_reads_t* remote_reads; + size_t packed_sz; + for (j = 0; j < req->num_server_reads; j++) { + remote_reads = req->remote_reads + j; + remote_reads->status = READREQ_STARTED; - if (NULL != unifyfs_vals) { - free_value_array(unifyfs_vals); - } + /* pack requests into send buffer, get packed size */ + packed_sz = rm_pack_chunk_requests(sendbuf, remote_reads); - if (NULL != unifyfs_key_lens) { - free(unifyfs_key_lens); - } + /* get rank of target server */ + int del_rank = remote_reads->rank; - if (NULL != unifyfs_val_lens) { - free(unifyfs_val_lens); + /* send requests */ + LOGDBG("[%d of %d] sending %d chunk requests to server %d", + j, req->num_server_reads, + remote_reads->num_chunks, del_rank); + rc = invoke_chunk_read_request_rpc(del_rank, req, + remote_reads->num_chunks, + sendbuf, packed_sz); + if (rc != (int)UNIFYFS_SUCCESS) { + ret = rc; + LOGERR("server request rpc to %d failed - %s", + del_rank, + unifyfs_rc_enum_str((unifyfs_rc)rc)); + } + } + } else { + /* already started */ + LOGDBG("read req %d already processed", i); + } + } else if (req->num_server_reads == 0) { + if (req->status == READREQ_READY) { + req->status = READREQ_STARTED; + } + } } + RM_REQ_UNLOCK(thrd_ctrl); return ret; } -/* given an app_id, client_id, global file id, - * and file size, truncate file to specified size +/* process chunk read responses from remote servers + * + * @param thrd_ctrl : reqmgr thread control structure + * @return success/error code */ -int rm_cmd_truncate( - int app_id, /* app_id for requesting client */ - int client_id, /* client_id for requesting client */ - int gfid, /* global file id */ - size_t newsize) /* desired file size */ +static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) { - /* set offset and length to request *all* key/value pairs - * for this file */ - size_t offset = 0; - - /* want to pick the highest integer offset value a file - * could have here */ - size_t length = (SIZE_MAX >> 1) - 1; - - /* get the locations of all the read requests from the - * key-value store*/ - unifyfs_key_t key1, key2; - - /* create key to describe first byte we'll read */ - key1.gfid = gfid; - key1.offset = offset; - - /* create key to describe last byte we'll read */ - key2.gfid = gfid; - key2.offset = offset + length - 1; - - /* set up input params to specify range lookup */ - unifyfs_key_t* unifyfs_keys[2] = {&key1, &key2}; - int key_lens[2] = {sizeof(unifyfs_key_t), sizeof(unifyfs_key_t)}; - - /* look up all entries in this range */ - int num_vals = 0; - unifyfs_keyval_t* keyvals = NULL; - int rc = unifyfs_get_file_extents(2, unifyfs_keys, key_lens, - &num_vals, &keyvals); - if (UNIFYFS_SUCCESS != rc) { - /* failed to look up extents, bail with error */ - return UNIFYFS_FAILURE; - } + // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked - /* compute our file size by iterating over each file - * segment and taking the max logical offset */ - int i; - size_t filesize = 0; - for (i = 0; i < num_vals; i++) { - /* get pointer to next key value pair */ - unifyfs_keyval_t* kv = &keyvals[i]; + int i, j, rc; + int ret = (int)UNIFYFS_SUCCESS; + shm_data_header* shm_hdr; - /* get last byte offset for this segment of the file */ - size_t last_offset = kv->key.offset + kv->val.len; + /* iterate over each active read request */ + for (i = 0; i < RM_MAX_ACTIVE_REQUESTS; i++) { + server_read_req_t* req = thrd_ctrl->read_reqs + i; + if (req->status == READREQ_STARTED) { + if (req->num_server_reads > 0) { + /* iterate over each server we sent requests to */ + server_chunk_reads_t* scr; + for (j = 0; j < req->num_server_reads; j++) { + scr = req->remote_reads + j; + if (NULL == scr->resp) { + continue; + } + LOGDBG("found read req %d responses from server %d", + i, scr->rank); + rc = rm_handle_chunk_read_responses(thrd_ctrl, req, scr); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to handle chunk read responses"); + ret = rc; + } + } + } else if (req->num_server_reads == 0) { + /* look up client shared memory region */ + int app_id = req->app_id; + int client_id = req->client_id; + app_client* client = get_app_client(app_id, client_id); + if (NULL != client) { + shm_context* client_shm = client->shmem_data; + assert(NULL != client_shm); + shm_hdr = (shm_data_header*) client_shm->addr; + + /* mark request as complete */ + req->status = READREQ_COMPLETE; + + /* signal client that we're now done writing data */ + client_signal(shm_hdr, SHMEM_REGION_DATA_COMPLETE); + + /* wait for client to read data */ + client_wait(shm_hdr); + } - /* update our filesize if this offset is bigger than the current max */ - if (last_offset > filesize) { - filesize = last_offset; + rc = release_read_req(thrd_ctrl, req); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to release server_read_req_t"); + ret = rc; + } + } } } - /* get filesize as recorded in metadata, which may be bigger if - * user issued an ftruncate on the file to extend it past the - * last write */ - size_t filesize_meta = filesize; - - /* given the global file id, look up file attributes - * from key/value store */ - unifyfs_file_attr_t fattr; - rc = unifyfs_get_file_attribute(gfid, &fattr); - if (rc == UNIFYFS_SUCCESS) { - /* found file attribute for this file, now get its size */ - filesize_meta = fattr.size; - } else { - /* failed to find file attributes for this file */ - goto truncate_exit; - } + return ret; +} - /* take maximum of last write and file size from metadata */ - if (filesize_meta > filesize) { - filesize = filesize_meta; - } +static shm_data_meta* reserve_shmem_meta(shm_context* shmem_data, + size_t data_sz) +{ + shm_data_meta* meta = NULL; + shm_data_header* hdr = (shm_data_header*) shmem_data->addr; - /* may need to throw away and rewrite keys if shrinking file */ - if (newsize < filesize) { - /* delete any key that extends beyond new file size */ - rc = truncate_delete_keys(newsize, num_vals, keyvals); - if (rc != UNIFYFS_SUCCESS) { - goto truncate_exit; - } + if (NULL == hdr) { + LOGERR("invalid header"); + } else { + pthread_mutex_lock(&(hdr->sync)); + LOGDBG("shm_data_header(cnt=%zu, bytes=%zu)", + hdr->meta_cnt, hdr->bytes); + size_t remain_size = shmem_data->size - + (sizeof(shm_data_header) + hdr->bytes); + size_t meta_size = sizeof(shm_data_meta) + data_sz; + if (meta_size > remain_size) { + /* client-side receive buffer is full, + * inform client to start reading */ + LOGDBG("need more space in client recv buffer"); + client_signal(hdr, SHMEM_REGION_DATA_READY); - /* rewrite any key that overlaps new file size */ - rc = truncate_rewrite_keys(newsize, num_vals, keyvals); - if (rc != UNIFYFS_SUCCESS) { - goto truncate_exit; + /* wait for client to read data */ + int rc = client_wait(hdr); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("wait for client recv buffer space failed"); + pthread_mutex_unlock(&(hdr->sync)); + return NULL; + } } + size_t shm_offset = hdr->bytes; + char* shm_buf = ((char*)hdr) + sizeof(shm_data_header); + meta = (shm_data_meta*)(shm_buf + shm_offset); + LOGDBG("reserved shm_data_meta[%zu] and %zu payload bytes", + hdr->meta_cnt, data_sz); + hdr->meta_cnt++; + hdr->bytes += meta_size; + pthread_mutex_unlock(&(hdr->sync)); } - - /* update file size field with latest size */ - fattr.size = newsize; - rc = unifyfs_set_file_attribute(1, 0, &fattr); - if (rc != UNIFYFS_SUCCESS) { - /* failed to update file attributes with new file size */ - goto truncate_exit; - } - -truncate_exit: - - /* free off key/value buffer returned from get_file_extents */ - if (NULL != keyvals) { - free(keyvals); - keyvals = NULL; - } - - return rc; + return meta; } -/* given an app_id, client_id, and global file id, - * remove file */ -int rm_cmd_unlink( - int app_id, /* app_id for requesting client */ - int client_id, /* client_id for requesting client */ - int gfid) /* global file id */ +int rm_post_chunk_read_responses(int app_id, + int client_id, + int src_rank, + int req_id, + int num_chks, + size_t bulk_sz, + char* resp_buf) { - int rc = UNIFYFS_SUCCESS; - - /* given the global file id, look up file attributes - * from key/value store */ - unifyfs_file_attr_t attr; - int ret = unifyfs_get_file_attribute(gfid, &attr); - if (ret != UNIFYFS_SUCCESS) { - /* failed to find attributes for the file */ - return ret; - } - - /* if item is a file, call truncate to free space */ - mode_t mode = (mode_t) attr.mode; - if ((mode & S_IFMT) == S_IFREG) { - /* item is regular file, truncate to 0 */ - ret = rm_cmd_truncate(app_id, client_id, gfid, 0); - if (ret != UNIFYFS_SUCCESS) { - /* failed to delete write extents for file, - * let's leave the file attributes in place */ - return ret; - } - } + int rc; - /* delete metadata */ - ret = unifyfs_delete_file_attribute(gfid); - if (ret != UNIFYFS_SUCCESS) { - rc = ret; + /* get application client */ + app_client* client = get_app_client(app_id, client_id); + if (NULL == client) { + return (int)UNIFYFS_FAILURE; } - return rc; -} + /* get thread control structure */ + reqmgr_thrd_t* thrd_ctrl = client->reqmgr; + assert(NULL != thrd_ctrl); -/* given an app_id, client_id, and global file id, - * laminate file */ -int rm_cmd_laminate( - int app_id, /* app_id for requesting client */ - int client_id, /* client_id for requesting client */ - int gfid) /* global file id */ -{ - int rc = UNIFYFS_SUCCESS; + server_chunk_reads_t* del_reads = NULL; - /* given the global file id, look up file attributes - * from key/value store */ - unifyfs_file_attr_t attr; - int ret = unifyfs_get_file_attribute(gfid, &attr); - if (ret != UNIFYFS_SUCCESS) { - /* failed to find attributes for the file */ - return ret; + /* find read req associated with req_id */ + if (src_rank != glb_pmi_rank) { + /* only need to lock for posting responses from remote servers. + * when response is local, we already have the lock */ + RM_REQ_LOCK(thrd_ctrl); } - - /* if item is not a file, bail with error */ - mode_t mode = (mode_t) attr.mode; - if ((mode & S_IFMT) != S_IFREG) { - /* item is not a regular file */ - LOGERR("ERROR: only regular files can be laminated (gfid=%d)", gfid); - return EINVAL; + server_read_req_t* rdreq = thrd_ctrl->read_reqs + req_id; + for (int i = 0; i < rdreq->num_server_reads; i++) { + if (rdreq->remote_reads[i].rank == src_rank) { + del_reads = rdreq->remote_reads + i; + break; + } } - /* lookup current file size */ - size_t filesize; - ret = rm_cmd_filesize(app_id, client_id, gfid, &filesize); - if (ret != UNIFYFS_SUCCESS) { - /* failed to get file size for file */ - LOGERR("lamination file size calculation failed (gfid=%d)", gfid); - return ret; + if (NULL != del_reads) { + LOGDBG("posting chunk responses for req %d from server %d", + req_id, src_rank); + del_reads->resp = (chunk_read_resp_t*)resp_buf; + if (del_reads->num_chunks != num_chks) { + LOGERR("mismatch on request vs. response chunks"); + del_reads->num_chunks = num_chks; + } + del_reads->total_sz = bulk_sz; + rc = (int)UNIFYFS_SUCCESS; + } else { + LOGERR("failed to find matching chunk-reads request"); + rc = (int)UNIFYFS_FAILURE; } - - /* update fields in metadata */ - attr.size = filesize; - attr.is_laminated = 1; - - /* update metadata, set size and laminate */ - rc = unifyfs_set_file_attribute(1, 1, &attr); - if (rc != UNIFYFS_SUCCESS) { - LOGERR("lamination metadata update failed (gfid=%d)", gfid); + if (src_rank != glb_pmi_rank) { + RM_REQ_UNLOCK(thrd_ctrl); } + /* inform the request manager thread we added responses */ + signal_new_responses(thrd_ctrl); + return rc; } -static int submit_read_request(reqmgr_thrd_t* thrd_ctrl, int num_keys, - unifyfs_key_t** keys, int* keylens) +static int send_data_to_client(shm_context* shm, chunk_read_resp_t* resp, + char* data, size_t* bytes_processed) { int ret = UNIFYFS_SUCCESS; - int app_id = -1; - int client_id = -1; - int num_vals = 0; - unifyfs_keyval_t* keyvals = NULL; + int errcode = 0; + size_t offset = 0; + size_t data_size = 0; + size_t bytes_left = 0; + size_t tx_size = MAX_DATA_TX_SIZE; + char* bufpos = data; + shm_data_meta* meta = NULL; - if (!thrd_ctrl || num_keys < 0 || !keys || !keylens) { - return EINVAL; + if (resp->read_rc < 0) { + errcode = (int) -(resp->read_rc); + data_size = 0; + } else { + data_size = resp->nbytes; } - app_id = thrd_ctrl->app_id; - client_id = thrd_ctrl->client_id; - - /* lookup all key/value pairs for given range */ - ret = unifyfs_get_file_extents(num_keys, keys, keylens, - &num_vals, &keyvals); - if (ret != UNIFYFS_SUCCESS) { - LOGERR("failed to get file extents (ret=%d)", ret); - return UNIFYFS_ERROR_MDHIM; - } + /* data can be larger than the shmem buffer size. split the data into + * pieces and send them */ + bytes_left = data_size; + offset = resp->offset; - /* this is to maintain limits imposed in previous code - * that would throw fatal errors */ - if (num_vals >= UNIFYFS_MAX_SPLIT_CNT || - num_vals >= MAX_META_PER_SEND) { - LOGERR("too many key/values returned in range lookup"); - ret = ENOMEM; - goto out_free; - } + for (bytes_left = data_size; bytes_left > 0; bytes_left -= tx_size) { + if (bytes_left < tx_size) { + tx_size = bytes_left; + } - /* if we get more than one write index entry - * sort them by file id and then by server rank */ - if (num_vals > 1) { - qsort(keyvals, (size_t)num_vals, sizeof(unifyfs_keyval_t), - compare_kv_gfid_rank); - } + meta = reserve_shmem_meta(shm, tx_size); + if (meta) { + meta->gfid = resp->gfid; + meta->errcode = errcode; + meta->offset = offset; + meta->length = tx_size; - server_read_req_t* rdreq = reserve_read_req(thrd_ctrl); - if (NULL == rdreq) { - LOGERR("failed to allocate server_read_req_t"); - ret = UNIFYFS_FAILURE; - } else { - rdreq->app_id = app_id; - rdreq->client_id = client_id; + LOGDBG("sending data to client (gfid=%d, offset=%zu, length=%zu) " + "%zu bytes left", + resp->gfid, offset, tx_size, bytes_left); - ret = create_chunk_requests(thrd_ctrl, rdreq, num_vals, keyvals); - if (ret != (int)UNIFYFS_SUCCESS) { - LOGERR("failed to submit read requests"); - release_read_req(thrd_ctrl, rdreq); + if (tx_size) { + void* shm_buf = (void*) ((char*) meta + sizeof(shm_data_meta)); + memcpy(shm_buf, bufpos, tx_size); + } + } else { + /* do we need to stop processing and exit loop here? */ + LOGERR("failed to reserve shmem space for read reply"); + ret = UNIFYFS_ERROR_SHMEM; } - } -out_free: - if (NULL != keyvals) { - free(keyvals); - keyvals = NULL; + bufpos += tx_size; + offset += tx_size; } - return ret; -} + *bytes_processed = data_size - bytes_left; -/* return number of slice ranges needed to cover range */ -static size_t num_slices(size_t offset, size_t length) -{ - size_t start = offset / meta_slice_sz; - size_t end = (offset + length - 1) / meta_slice_sz; - size_t count = end - start + 1; - return count; + return ret; } -/* given a global file id, an offset, and a length to read from that - * file, create keys needed to query MDHIM for location of data - * corresponding to that extent, returns the number of keys inserted - * into key array provided by caller */ -static int split_request( - unifyfs_key_t** keys, /* list to add newly created keys into */ - int* keylens, /* list to add byte size of each key */ - int gfid, /* target global file id to read from */ - size_t offset, /* starting offset of read */ - size_t length) /* number of bytes to read */ +/* process the requested chunk data returned from service managers + * + * @param thrd_ctrl : request manager thread state + * @param rdreq : server read request + * @param del_reads : remote server chunk reads + * @return success/error code + */ +int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq, + server_chunk_reads_t* del_reads) { - /* offset of first byte in request */ - size_t pos = offset; - - /* offset of last byte in request */ - size_t last_offset = offset + length - 1; - - /* iterate over slice ranges and generate a start/end - * pair of keys for each */ - int count = 0; - while (pos <= last_offset) { - /* compute offset for first byte in this segment */ - size_t start = pos; - - /* offset for last byte in this segment, - * assume that's the last byte of the same segment - * containing start, unless that happens to be - * beyond the last byte of the actual request */ - size_t start_slice = start / meta_slice_sz; - size_t end = (start_slice + 1) * meta_slice_sz - 1; - if (end > last_offset) { - end = last_offset; - } + // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked - /* create key to describe first byte we'll read - * in this slice */ - keys[count]->gfid = gfid; - keys[count]->offset = start; - keylens[count] = sizeof(unifyfs_key_t); - count++; + int i, num_chks, rc; + int ret = (int)UNIFYFS_SUCCESS; + chunk_read_resp_t* responses = NULL; + shm_context* client_shm = NULL; + shm_data_header* shm_hdr = NULL; + char* data_buf = NULL; - /* create key to describe last byte we'll read - * in this slice */ - keys[count]->gfid = gfid; - keys[count]->offset = end; - keylens[count] = sizeof(unifyfs_key_t); - count++; + assert((NULL != thrd_ctrl) && + (NULL != rdreq) && + (NULL != del_reads) && + (NULL != del_reads->resp)); - /* advance to first byte offset of next slice */ - pos = end + 1; + /* look up client shared memory region */ + app_client* clnt = get_app_client(rdreq->app_id, rdreq->client_id); + if (NULL == clnt) { + return (int)UNIFYFS_FAILURE; } + client_shm = clnt->shmem_data; + shm_hdr = (shm_data_header*) client_shm->addr; - /* return number of keys we generated */ - return count; -} - -/* given an extent corresponding to a write index, create new key/value - * pairs for that extent, splitting into multiple keys at the slice - * range boundaries (meta_slice_sz), it returns the number of - * newly created key/values inserted into the given key and value - * arrays */ -static int split_index( - unifyfs_key_t** keys, /* list to add newly created keys into */ - unifyfs_val_t** vals, /* list to add newly created values into */ - int* keylens, /* list for size of each key */ - int* vallens, /* list for size of each value */ - int gfid, /* global file id of write */ - size_t offset, /* starting byte offset of extent */ - size_t length, /* number of bytes in extent */ - size_t log_offset, /* offset within data log */ - int server_rank, /* rank of server hosting data */ - int app_id, /* app_id holding data */ - int client_rank) /* client rank holding data */ -{ - /* offset of first byte in request */ - size_t pos = offset; - - /* offset of last byte in request */ - size_t last_offset = offset + length - 1; - - /* this will track the current offset within the log - * where the data starts, we advance it with each key - * we generate depending on the data associated with - * each key */ - size_t logpos = log_offset; - - /* iterate over slice ranges and generate a start/end - * pair of keys for each */ - int count = 0; - while (pos <= last_offset) { - /* compute offset for first byte in this slice */ - size_t start = pos; - - /* offset for last byte in this slice, - * assume that's the last byte of the same slice - * containing start, unless that happens to be - * beyond the last byte of the actual request */ - size_t start_slice = start / meta_slice_sz; - size_t end = (start_slice + 1) * meta_slice_sz - 1; - if (end > last_offset) { - end = last_offset; - } - - /* length of extent in this slice */ - size_t len = end - start + 1; - - /* create key to describe this log entry */ - unifyfs_key_t* k = keys[count]; - k->gfid = gfid; - k->offset = start; - keylens[count] = sizeof(unifyfs_key_t); + num_chks = del_reads->num_chunks; + if (del_reads->status != READREQ_STARTED) { + LOGERR("chunk read response for non-started req @ index=%d", + rdreq->req_ndx); + ret = (int32_t)EINVAL; + } else if (0 == del_reads->total_sz) { + LOGERR("empty chunk read response from server %d", del_reads->rank); + ret = (int32_t)EINVAL; + } else { + LOGDBG("handling chunk read responses from server %d: " + "num_chunks=%d buf_size=%zu", + del_reads->rank, num_chks, del_reads->total_sz); + responses = del_reads->resp; + data_buf = (char*)(responses + num_chks); - /* create value to store address of data */ - unifyfs_val_t* v = vals[count]; - v->addr = logpos; - v->len = len; - v->app_id = app_id; - v->rank = client_rank; - v->delegator_rank = server_rank; - vallens[count] = sizeof(unifyfs_val_t); + for (i = 0; i < num_chks; i++) { + chunk_read_resp_t* resp = responses + i; + size_t processed = 0; - /* advance to next slot in key/value arrays */ - count++; + ret = send_data_to_client(client_shm, resp, data_buf, &processed); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("failed to send data to client (ret=%d)", ret); + } - /* advance offset into log */ - logpos += len; + data_buf += processed; + } - /* advance to first byte offset of next slice */ - pos = end + 1; - } + /* cleanup */ + free((void*)responses); + del_reads->resp = NULL; - /* return number of keys we generated */ - return count; -} + /* update request status */ + del_reads->status = READREQ_COMPLETE; -/* read function for one requested extent, - * called from rpc handler to fill shared data structures - * with read requests to be handled by the reqmgr thread. - * returns before requests are handled - */ -int rm_cmd_read( - int app_id, /* app_id for requesting client */ - int client_id, /* client_id for requesting client */ - int gfid, /* global file id of read request */ - size_t offset, /* logical file offset of read request */ - size_t length) /* number of bytes to read */ -{ - /* get application client */ - app_client* client = get_app_client(app_id, client_id); - if (NULL == client) { - return (int)UNIFYFS_FAILURE; - } + /* if all remote reads are complete, mark the request as complete */ + int completed_remote_reads = 0; + for (i = 0; i < rdreq->num_server_reads; i++) { + if (rdreq->remote_reads[i].status != READREQ_COMPLETE) { + break; + } + completed_remote_reads++; + } + if (completed_remote_reads == rdreq->num_server_reads) { + rdreq->status = READREQ_COMPLETE; - /* get thread control structure */ - reqmgr_thrd_t* thrd_ctrl = client->reqmgr; + /* signal client that we're now done writing data */ + client_signal(shm_hdr, SHMEM_REGION_DATA_COMPLETE); - /* get chunks corresponding to requested client read extent - * - * Generate a pair of keys for the read request, representing the start - * and end offset. MDHIM returns all key-value pairs that fall within - * the offset range. - * - * TODO: this is specific to the MDHIM in the source tree and not portable - * to other KV-stores. This needs to be revisited to utilize some - * other mechanism to retrieve all relevant key-value pairs from the - * KV-store. - */ - - /* count number of slices this range covers */ - size_t slices = num_slices(offset, length); - if (slices >= UNIFYFS_MAX_SPLIT_CNT) { - LOGERR("Error allocating buffers"); - return ENOMEM; - } + /* wait for client to read data */ + client_wait(shm_hdr); - /* allocate key storage */ - size_t key_cnt = slices * 2; - unifyfs_key_t** keys = alloc_key_array(key_cnt); - int* key_lens = (int*) calloc(key_cnt, sizeof(int)); - if ((NULL == keys) || - (NULL == key_lens)) { - // this is a fatal error - // TODO: we need better error handling - LOGERR("Error allocating buffers"); - return ENOMEM; + rc = release_read_req(thrd_ctrl, rdreq); + if (rc != (int)UNIFYFS_SUCCESS) { + LOGERR("failed to release server_read_req_t"); + } + } } - /* split range of read request at boundaries used for - * MDHIM range query */ - split_request(keys, key_lens, gfid, offset, length); - - /* queue up the read operations */ - int rc = submit_read_request(thrd_ctrl, key_cnt, keys, key_lens); - - /* free memory allocated for key storage */ - free_key_array(keys); - free(key_lens); - - return rc; + return ret; } -/* send the read requests to the remote delegators - * - * @param app_id: application id - * @param client_id: client id for requesting process - * @param req_num: number of read requests - * @param reqbuf: read requests buffer - * @return success/error code */ -int rm_cmd_mread( - int app_id, - int client_id, - size_t req_num, - void* reqbuf) +/* submit a client rpc request to the request manager thread */ +int rm_submit_client_rpc_request(unifyfs_fops_ctx_t* ctx, + client_rpc_req_t* req) { - int rc = UNIFYFS_SUCCESS; + assert((ctx != NULL) && (req != NULL)); /* get application client */ - app_client* client = get_app_client(app_id, client_id); + app_client* client = get_app_client(ctx->app_id, ctx->client_id); if (NULL == client) { - return (int)UNIFYFS_FAILURE; + LOGERR("app client [%d:%d] lookup failed", + ctx->app_id, ctx->client_id); + return EINVAL; } /* get thread control structure */ - reqmgr_thrd_t* thrd_ctrl = client->reqmgr; + reqmgr_thrd_t* reqmgr = client->reqmgr; + assert(NULL != reqmgr); + RM_REQ_LOCK(reqmgr); + arraylist_add(reqmgr->client_reqs, req); + RM_REQ_UNLOCK(reqmgr); - /* get the locations of all the read requests from the key-value store */ - unifyfs_extent_t* read_reqs = (unifyfs_extent_t*)reqbuf; + signal_new_requests(reqmgr); - /* count up number of slices these request cover */ - int i; - size_t slices = 0; - unifyfs_extent_t* req; - for (i = 0; i < req_num; i++) { - /* get offset and length of next request */ - req = read_reqs + i; - size_t off = req->offset; - size_t len = req->length; - - /* add in number of slices this request needs */ - slices += num_slices(off, len); - } - if (slices >= UNIFYFS_MAX_SPLIT_CNT) { - LOGERR("Error allocating buffers"); - return ENOMEM; - } - - /* allocate key storage */ - size_t key_cnt = slices * 2; - unifyfs_key_t** keys = alloc_key_array(key_cnt); - int* key_lens = (int*) calloc(key_cnt, sizeof(int)); - if ((NULL == keys) || - (NULL == key_lens)) { - // this is a fatal error - // TODO: we need better error handling - LOGERR("Error allocating buffers"); - return ENOMEM; - } - - /* we need to create a single server_read_req_t structure even with - * multiple gfids. */ - int num_keys = 0; - for (i = 0; i < req_num; i++) { - /* get the file id for this request */ - req = read_reqs + i; - int gfid = req->gfid; - size_t off = req->offset; - size_t len = req->length; - LOGDBG("gfid:%d, offset:%zu, length:%zu", gfid, off, len); - - /* Generate a pair of keys for each read request, representing - * the start and end offsets. MDHIM returns all key-value pairs that - * fall within the offset range. - * - * TODO: this is specific to the MDHIM in the source tree and not - * portable to other KV-stores. This needs to be revisited to - * utilize some other mechanism to retrieve all relevant KV - * pairs from the KV-store. - */ - - /* split range of read request at boundaries used for - * MDHIM range query */ - num_keys += split_request(&keys[num_keys], &key_lens[num_keys], - gfid, off, len); - } - - /* queue the read operations */ - rc = submit_read_request(thrd_ctrl, num_keys, keys, key_lens); - - /* free memory allocated for key storage */ - free_key_array(keys); - free(key_lens); - - return rc; + return UNIFYFS_SUCCESS; } -/* function called by main thread to instruct - * resource manager thread to exit, - * returns UNIFYFS_SUCCESS on success */ -int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl) +static int process_filesize_rpc(reqmgr_thrd_t* reqmgr, + client_rpc_req_t* req) { - if (thrd_ctrl->exited) { - /* already done */ - return UNIFYFS_SUCCESS; - } + int ret = UNIFYFS_SUCCESS; + size_t filesize = 0; - /* grab the lock */ - RM_LOCK(thrd_ctrl); + unifyfs_filesize_in_t* in = req->input; + assert(in != NULL); + int gfid = in->gfid; + margo_free_input(req->handle, in); + free(in); - /* if reqmgr thread is not waiting in critical - * section, let's wait on it to come back */ - if (!thrd_ctrl->has_waiting_delegator) { - /* reqmgr thread is not in critical section, - * tell it we've got something and signal it */ - thrd_ctrl->has_waiting_dispatcher = 1; - pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); + LOGDBG("getting filesize for gfid=%d", gfid); - /* we're no longer waiting */ - thrd_ctrl->has_waiting_dispatcher = 0; + unifyfs_fops_ctx_t ctx = { + .app_id = reqmgr->app_id, + .client_id = reqmgr->client_id, + }; + ret = unifyfs_fops_filesize(&ctx, gfid, &filesize); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("unifyfs_fops_filesize() failed"); } - /* inform reqmgr thread that it's time to exit */ - thrd_ctrl->exit_flag = 1; - - /* signal reqmgr thread */ - pthread_cond_signal(&thrd_ctrl->thrd_cond); + /* send rpc response */ + unifyfs_filesize_out_t out; + out.ret = (int32_t) ret; + out.filesize = filesize; + hg_return_t hret = margo_respond(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - /* release the lock */ - RM_UNLOCK(thrd_ctrl); + /* cleanup req */ + margo_destroy(req->handle); - /* wait for reqmgr thread to exit */ - int rc = pthread_join(thrd_ctrl->thrd, NULL); - if (0 == rc) { - pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); - pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); - thrd_ctrl->exited = 1; - } - return UNIFYFS_SUCCESS; + return ret; } -/* - * store all writes from app-client's index in the global metadata - * - * @param app_id: the application id - * @param client_id: client rank in app - * @return success/error code - */ -int rm_cmd_sync(int app_id, int client_id) +static int process_fsync_rpc(reqmgr_thrd_t* reqmgr, + client_rpc_req_t* req) { - size_t i; + int ret = UNIFYFS_SUCCESS; - /* assume we'll succeed */ - int ret = (int)UNIFYFS_SUCCESS; + unifyfs_fsync_in_t* in = req->input; + assert(in != NULL); + int gfid = in->gfid; + margo_free_input(req->handle, in); + free(in); - /* get memory page size on this machine */ - int page_sz = getpagesize(); + LOGDBG("syncing gfid=%d", gfid); - /* get application client */ - app_client* client = get_app_client(app_id, client_id); - if (NULL == client) { - return EINVAL; + unifyfs_fops_ctx_t ctx = { + .app_id = reqmgr->app_id, + .client_id = reqmgr->client_id, + }; + ret = unifyfs_fops_fsync(&ctx, gfid); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("unifyfs_fops_fsync() failed"); } - /* get pointer to superblock for this client and app */ - shm_context* super_ctx = client->shmem_super; - if (NULL == super_ctx) { - LOGERR("missing client superblock"); - return EIO; + /* send rpc response */ + unifyfs_fsync_out_t out; + out.ret = (int32_t) ret; + hg_return_t hret = margo_respond(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); } - char* superblk = (char*)(super_ctx->addr); - /* get pointer to start of key/value region in superblock */ - char* meta = superblk + client->super_meta_offset; + /* cleanup req */ + margo_destroy(req->handle); - /* get number of file extent index values client has for us, - * stored as a size_t value in meta region of shared memory */ - size_t extent_num_entries = *(size_t*)(meta); - - /* indices are stored in the superblock shared memory - * created by the client, these are stored as index_t - * structs starting one page size offset into meta region */ - char* ptr_extents = meta + page_sz; + return ret; +} - if (extent_num_entries == 0) { - /* Nothing to do */ - return UNIFYFS_SUCCESS; - } +static int process_laminate_rpc(reqmgr_thrd_t* reqmgr, + client_rpc_req_t* req) +{ + int ret = UNIFYFS_SUCCESS; - unifyfs_index_t* meta_payload = (unifyfs_index_t*)(ptr_extents); + unifyfs_laminate_in_t* in = req->input; + assert(in != NULL); + int gfid = in->gfid; + margo_free_input(req->handle, in); + free(in); - /* total up number of key/value pairs we'll need for this - * set of index values */ - size_t slices = 0; - for (i = 0; i < extent_num_entries; i++) { - size_t offset = meta_payload[i].file_pos; - size_t length = meta_payload[i].length; - slices += num_slices(offset, length); - } - if (slices >= UNIFYFS_MAX_SPLIT_CNT) { - LOGERR("Error allocating buffers"); - return ENOMEM; - } + LOGDBG("laminating gfid=%d", gfid); - /* pointers to memory we'll dynamically allocate for file extents */ - unifyfs_key_t** keys = NULL; - unifyfs_val_t** vals = NULL; - int* key_lens = NULL; - int* val_lens = NULL; - - /* allocate storage for file extent key/values */ - /* TODO: possibly get this from memory pool */ - keys = alloc_key_array(slices); - vals = alloc_value_array(slices); - key_lens = calloc(slices, sizeof(int)); - val_lens = calloc(slices, sizeof(int)); - if ((NULL == keys) || - (NULL == vals) || - (NULL == key_lens) || - (NULL == val_lens)) { - LOGERR("failed to allocate memory for file extents"); - ret = ENOMEM; - goto rm_cmd_sync_exit; - } - - /* create file extent key/values for insertion into MDHIM */ - int count = 0; - for (i = 0; i < extent_num_entries; i++) { - /* get file offset, length, and log offset for this entry */ - unifyfs_index_t* meta = &meta_payload[i]; - int gfid = meta->gfid; - size_t offset = meta->file_pos; - size_t length = meta->length; - size_t logpos = meta->log_pos; - - /* split this entry at the offset boundaries */ - int used = split_index( - &keys[count], &vals[count], &key_lens[count], &val_lens[count], - gfid, offset, length, logpos, - glb_pmi_rank, app_id, client_id); - - /* count up the number of keys we used for this index */ - count += used; - } - - /* batch insert file extent key/values into MDHIM */ - ret = unifyfs_set_file_extents((int)count, - keys, key_lens, vals, val_lens); + unifyfs_fops_ctx_t ctx = { + .app_id = reqmgr->app_id, + .client_id = reqmgr->client_id, + }; + ret = unifyfs_fops_laminate(&ctx, gfid); if (ret != UNIFYFS_SUCCESS) { - /* TODO: need proper error handling */ - LOGERR("unifyfs_set_file_extents() failed"); - goto rm_cmd_sync_exit; - } - -rm_cmd_sync_exit: - /* clean up memory */ - - if (NULL != keys) { - free_key_array(keys); + LOGERR("unifyfs_fops_laminate() failed"); } - if (NULL != vals) { - free_value_array(vals); - } - - if (NULL != key_lens) { - free(key_lens); + /* send rpc response */ + unifyfs_laminate_out_t out; + out.ret = (int32_t) ret; + hg_return_t hret = margo_respond(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); } - if (NULL != val_lens) { - free(val_lens); - } + /* cleanup req */ + margo_destroy(req->handle); return ret; } -/************************ - * These functions define the logic of the request manager thread - ***********************/ - -/* pack the chunk read requests for a single remote server. - * - * @param req_msg_buf: request buffer used for packing - * @param req_num: number of read requests - * @return size of packed buffer (or error code) - */ -static size_t rm_pack_chunk_requests(char* req_msg_buf, - remote_chunk_reads_t* remote_reads) +static int process_metaget_rpc(reqmgr_thrd_t* reqmgr, + client_rpc_req_t* req) { - /* send format: - * (int) cmd - specifies type of message (SVC_CMD_RDREQ_CHK) - * (int) req_cnt - number of requests in message - * {sequence of chunk_read_req_t} */ - int req_cnt = remote_reads->num_chunks; - size_t reqs_sz = req_cnt * sizeof(chunk_read_req_t); - size_t packed_size = (2 * sizeof(int)) + sizeof(size_t) + reqs_sz; + int ret = UNIFYFS_SUCCESS; - assert(req_cnt < MAX_META_PER_SEND); + unifyfs_metaget_in_t* in = req->input; + assert(in != NULL); + int gfid = in->gfid; + margo_free_input(req->handle, in); + free(in); - /* get pointer to start of send buffer */ - char* ptr = req_msg_buf; - memset(ptr, 0, packed_size); - - /* pack command */ - int cmd = (int)SVC_CMD_RDREQ_CHK; - *((int*)ptr) = cmd; - ptr += sizeof(int); + LOGDBG("getting metadata for gfid=%d", gfid); - /* pack request count */ - *((int*)ptr) = req_cnt; - ptr += sizeof(int); + unifyfs_fops_ctx_t ctx = { + .app_id = reqmgr->app_id, + .client_id = reqmgr->client_id, + }; + unifyfs_file_attr_t fattr; + memset(&fattr, 0, sizeof(fattr)); + ret = unifyfs_fops_metaget(&ctx, gfid, &fattr); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("unifyfs_fops_metaget() failed"); + } - /* pack total requested data size */ - *((size_t*)ptr) = remote_reads->total_sz; - ptr += sizeof(size_t); + /* send rpc response */ + unifyfs_metaget_out_t out; + out.ret = (int32_t) ret; + out.attr = fattr; + hg_return_t hret = margo_respond(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - /* copy requests into buffer */ - memcpy(ptr, remote_reads->reqs, reqs_sz); - ptr += reqs_sz; + /* cleanup req */ + margo_destroy(req->handle); - /* return number of bytes used to pack requests */ - return packed_size; + return ret; } -/* send the chunk read requests to remote delegators - * - * @param thrd_ctrl : reqmgr thread control structure - * @return success/error code - */ -static int rm_request_remote_chunks(reqmgr_thrd_t* thrd_ctrl) +static int process_metaset_rpc(reqmgr_thrd_t* reqmgr, + client_rpc_req_t* req) { - // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked - - int i, j, rc; - int ret = (int)UNIFYFS_SUCCESS; - - /* get pointer to send buffer */ - char* sendbuf = thrd_ctrl->del_req_msg_buf; - - /* iterate over each active read request */ - for (i = 0; i < RM_MAX_ACTIVE_REQUESTS; i++) { - server_read_req_t* req = thrd_ctrl->read_reqs + i; - if (req->num_remote_reads > 0) { - LOGDBG("read req %d is active", i); - debug_print_read_req(req); - if (req->status == READREQ_READY) { - req->status = READREQ_STARTED; - /* iterate over each server we need to send requests to */ - remote_chunk_reads_t* remote_reads; - size_t packed_sz; - for (j = 0; j < req->num_remote_reads; j++) { - remote_reads = req->remote_reads + j; - remote_reads->status = READREQ_STARTED; - - /* pack requests into send buffer, get packed size */ - packed_sz = rm_pack_chunk_requests(sendbuf, remote_reads); - - /* get rank of target server */ - int del_rank = remote_reads->rank; + int ret = UNIFYFS_SUCCESS; - /* send requests */ - LOGDBG("[%d of %d] sending %d chunk requests to server %d", - j, req->num_remote_reads, - remote_reads->num_chunks, del_rank); - rc = invoke_chunk_read_request_rpc(del_rank, req, - remote_reads->num_chunks, - sendbuf, packed_sz); - if (rc != (int)UNIFYFS_SUCCESS) { - ret = rc; - LOGERR("server request rpc to %d failed - %s", - del_rank, - unifyfs_rc_enum_str((unifyfs_rc)rc)); - } - } - } else { - /* already started */ - LOGDBG("read req %d already processed", i); - } - } else if (req->num_remote_reads == 0) { - if (req->status == READREQ_READY) { - req->status = READREQ_STARTED; - } - } + unifyfs_metaset_in_t* in = req->input; + assert(in != NULL); + int gfid = in->attr.gfid; + int attr_op = (int) in->attr_op; + unifyfs_file_attr_t fattr = in->attr; + if (NULL != in->attr.filename) { + fattr.filename = strdup(in->attr.filename); } + margo_free_input(req->handle, in); + free(in); - return ret; -} - -/* send the chunk read requests to remote delegators - * - * @param thrd_ctrl : reqmgr thread control structure - * @return success/error code - */ -static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) -{ - // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked + LOGDBG("setting metadata for gfid=%d", gfid); - int i, j, rc; - int ret = (int)UNIFYFS_SUCCESS; + unifyfs_fops_ctx_t ctx = { + .app_id = reqmgr->app_id, + .client_id = reqmgr->client_id, + }; + ret = unifyfs_fops_metaset(&ctx, gfid, attr_op, &fattr); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("unifyfs_fops_metaset() failed"); + } - /* iterate over each active read request */ - for (i = 0; i < RM_MAX_ACTIVE_REQUESTS; i++) { - server_read_req_t* req = thrd_ctrl->read_reqs + i; - if ((req->num_remote_reads > 0) && - (req->status == READREQ_STARTED)) { - /* iterate over each server we need to send requests to */ - remote_chunk_reads_t* rcr; - for (j = 0; j < req->num_remote_reads; j++) { - rcr = req->remote_reads + j; - if (NULL == rcr->resp) { - continue; - } - LOGDBG("found read req %d responses from server %d", - i, rcr->rank); - rc = rm_handle_chunk_read_responses(thrd_ctrl, req, rcr); - if (rc != (int)UNIFYFS_SUCCESS) { - LOGERR("failed to handle chunk read responses"); - ret = rc; - } - } - } else if ((req->num_remote_reads == 0) && - (req->status == READREQ_STARTED)) { - /* look up client shared memory region */ - int app_id = req->app_id; - int client_id = req->client_id; - app_client* client = get_app_client(app_id, client_id); - if (NULL != client) { - shm_context* client_shm = client->shmem_data; - assert(NULL != client_shm); - shm_data_header* shm_hdr = (shm_data_header*) client_shm->addr; - - /* mark request as complete */ - req->status = READREQ_COMPLETE; - - /* signal client that we're now done writing data */ - client_signal(shm_hdr, SHMEM_REGION_DATA_COMPLETE); - - /* wait for client to read data */ - client_wait(shm_hdr); - } + if (NULL != fattr.filename) { + free(fattr.filename); + } - rc = __release_read_req(thrd_ctrl, req); - if (rc != (int)UNIFYFS_SUCCESS) { - LOGERR("failed to release server_read_req_t"); - ret = rc; - } - } + /* send rpc response */ + unifyfs_metaset_out_t out; + out.ret = (int32_t) ret; + hg_return_t hret = margo_respond(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); } + /* cleanup req */ + margo_destroy(req->handle); + return ret; } -static shm_data_meta* reserve_shmem_meta(shm_context* shmem_data, - shm_data_header* hdr, - size_t data_sz) +static int process_read_rpc(reqmgr_thrd_t* reqmgr, + client_rpc_req_t* req) { - shm_data_meta* meta = NULL; - if (NULL == hdr) { - LOGERR("invalid header"); - } else { - pthread_mutex_lock(&(hdr->sync)); - LOGDBG("shm_data_header(cnt=%zu, bytes=%zu)", - hdr->meta_cnt, hdr->bytes); - size_t remain_size = shmem_data->size - - (sizeof(shm_data_header) + hdr->bytes); - size_t meta_size = sizeof(shm_data_meta) + data_sz; - if (meta_size > remain_size) { - /* client-side receive buffer is full, - * inform client to start reading */ - LOGDBG("need more space in client recv buffer"); - client_signal(hdr, SHMEM_REGION_DATA_READY); + int ret = UNIFYFS_SUCCESS; - /* wait for client to read data */ - int rc = client_wait(hdr); - if (rc != (int)UNIFYFS_SUCCESS) { - LOGERR("wait for client recv buffer space failed"); - pthread_mutex_unlock(&(hdr->sync)); - return NULL; - } - } - size_t shm_offset = hdr->bytes; - char* shm_buf = ((char*)hdr) + sizeof(shm_data_header); - meta = (shm_data_meta*)(shm_buf + shm_offset); - LOGDBG("reserved shm_data_meta[%zu] and %zu payload bytes", - hdr->meta_cnt, data_sz); - hdr->meta_cnt++; - hdr->bytes += meta_size; - pthread_mutex_unlock(&(hdr->sync)); + unifyfs_read_in_t* in = req->input; + assert(in != NULL); + int gfid = in->gfid; + off_t offset = in->offset; + size_t len = in->length; + margo_free_input(req->handle, in); + free(in); + + LOGDBG("reading gfid=%d (offset=%zu, length=%zu)", + gfid, (size_t)offset, len); + + unifyfs_fops_ctx_t ctx = { + .app_id = reqmgr->app_id, + .client_id = reqmgr->client_id, + }; + ret = unifyfs_fops_read(&ctx, gfid, offset, len); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("unifyfs_fops_read() failed"); } - return meta; -} - -int rm_post_chunk_read_responses(int app_id, - int client_id, - int src_rank, - int req_id, - int num_chks, - size_t bulk_sz, - char* resp_buf) -{ - int rc; - /* get application client */ - app_client* client = get_app_client(app_id, client_id); - if (NULL == client) { - return (int)UNIFYFS_FAILURE; + /* send rpc response */ + unifyfs_read_out_t out; + out.ret = (int32_t) ret; + hg_return_t hret = margo_respond(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); } - /* get thread control structure */ - reqmgr_thrd_t* thrd_ctrl = client->reqmgr; - assert(NULL != thrd_ctrl); + /* cleanup req */ + margo_destroy(req->handle); - RM_LOCK(thrd_ctrl); + return ret; +} - remote_chunk_reads_t* del_reads = NULL; +static int process_truncate_rpc(reqmgr_thrd_t* reqmgr, + client_rpc_req_t* req) +{ + int ret = UNIFYFS_SUCCESS; - /* find read req associated with req_id */ - server_read_req_t* rdreq = thrd_ctrl->read_reqs + req_id; - for (int i = 0; i < rdreq->num_remote_reads; i++) { - if (rdreq->remote_reads[i].rank == src_rank) { - del_reads = rdreq->remote_reads + i; - break; - } - } + unifyfs_truncate_in_t* in = req->input; + assert(in != NULL); + int gfid = in->gfid; + size_t filesize = in->filesize; + margo_free_input(req->handle, in); + free(in); - if (NULL != del_reads) { - LOGDBG("posting chunk responses for req %d from server %d", - req_id, src_rank); - del_reads->resp = (chunk_read_resp_t*)resp_buf; - if (del_reads->num_chunks != num_chks) { - LOGERR("mismatch on request vs. response chunks"); - del_reads->num_chunks = num_chks; - } - del_reads->total_sz = bulk_sz; - rc = (int)UNIFYFS_SUCCESS; - } else { - LOGERR("failed to find matching chunk-reads request"); - rc = (int)UNIFYFS_FAILURE; + LOGDBG("truncating gfid=%d, sz=%zu", gfid, filesize); + + unifyfs_fops_ctx_t ctx = { + .app_id = reqmgr->app_id, + .client_id = reqmgr->client_id, + }; + ret = unifyfs_fops_truncate(&ctx, gfid, filesize); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("unifyfs_fops_truncate() failed"); } - /* inform the request manager thread we added responses */ - signal_new_responses(thrd_ctrl); + /* send rpc response */ + unifyfs_truncate_out_t out; + out.ret = (int32_t) ret; + hg_return_t hret = margo_respond(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - RM_UNLOCK(thrd_ctrl); + /* cleanup req */ + margo_destroy(req->handle); - return rc; + return ret; } -/* process the requested chunk data returned from service managers - * - * @param thrd_ctrl : request manager thread state - * @param rdreq : server read request - * @param del_reads : remote server chunk reads - * @return success/error code - */ -int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, - server_read_req_t* rdreq, - remote_chunk_reads_t* del_reads) +static int process_unlink_rpc(reqmgr_thrd_t* reqmgr, + client_rpc_req_t* req) { - // NOTE: this fn assumes thrd_ctrl->thrd_lock is locked + int ret = UNIFYFS_SUCCESS; - int errcode, gfid, i, num_chks, rc; - int ret = (int)UNIFYFS_SUCCESS; - chunk_read_resp_t* responses = NULL; - shm_context* client_shm = NULL; - shm_data_header* shm_hdr = NULL; - shm_data_meta* meta = NULL; - void* shm_buf = NULL; - char* data_buf = NULL; - size_t data_sz, offset; + unifyfs_unlink_in_t* in = req->input; + assert(in != NULL); + int gfid = in->gfid; + margo_free_input(req->handle, in); + free(in); - assert((NULL != thrd_ctrl) && - (NULL != rdreq) && - (NULL != del_reads) && - (NULL != del_reads->resp)); + LOGDBG("unlinking gfid=%d", gfid); - /* look up client shared memory region */ - app_client* clnt = get_app_client(rdreq->app_id, rdreq->client_id); - if (NULL == clnt) { - return (int)UNIFYFS_FAILURE; + unifyfs_fops_ctx_t ctx = { + .app_id = reqmgr->app_id, + .client_id = reqmgr->client_id, + }; + ret = unifyfs_fops_unlink(&ctx, gfid); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("unifyfs_fops_unlink() failed"); } - client_shm = clnt->shmem_data; - shm_hdr = (shm_data_header*) client_shm->addr; - - num_chks = del_reads->num_chunks; - if (del_reads->status != READREQ_STARTED) { - LOGERR("chunk read response for non-started req @ index=%d", - rdreq->req_ndx); - ret = (int32_t)EINVAL; - } else if (0 == del_reads->total_sz) { - LOGERR("empty chunk read response from server %d", del_reads->rank); - ret = (int32_t)EINVAL; - } else { - LOGDBG("handling chunk read responses from server %d: " - "num_chunks=%d buf_size=%zu", - del_reads->rank, num_chks, del_reads->total_sz); - responses = del_reads->resp; - data_buf = (char*)(responses + num_chks); - for (i = 0; i < num_chks; i++) { - chunk_read_resp_t* resp = responses + i; - gfid = resp->gfid; - if (resp->read_rc < 0) { - errcode = (int)-(resp->read_rc); - data_sz = 0; - } else { - errcode = 0; - data_sz = resp->nbytes; - } - offset = resp->offset; - LOGDBG("chunk response for gfid=%d (offset=%zu, sz=%zu)", - gfid, offset, data_sz); - - /* allocate and register local target buffer for bulk access */ - meta = reserve_shmem_meta(client_shm, shm_hdr, data_sz); - if (NULL != meta) { - meta->offset = offset; - meta->length = data_sz; - meta->gfid = gfid; - meta->errcode = errcode; - shm_buf = (void*)((char*)meta + sizeof(shm_data_meta)); - if (data_sz) { - memcpy(shm_buf, data_buf, data_sz); - } - } else { - LOGERR("failed to reserve shmem space for read reply") - ret = (int32_t)UNIFYFS_ERROR_SHMEM; - } - data_buf += data_sz; - } - /* cleanup */ - free((void*)responses); - del_reads->resp = NULL; + /* send rpc response */ + unifyfs_unlink_out_t out; + out.ret = (int32_t) ret; + hg_return_t hret = margo_respond(req->handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } - /* update request status */ - del_reads->status = READREQ_COMPLETE; + /* cleanup req */ + margo_destroy(req->handle); - /* if all remote reads are complete, mark the request as complete */ - int completed_remote_reads = 0; - for (i = 0; i < rdreq->num_remote_reads; i++) { - if (rdreq->remote_reads[i].status != READREQ_COMPLETE) { - break; - } - completed_remote_reads++; - } - if (completed_remote_reads == rdreq->num_remote_reads) { - rdreq->status = READREQ_COMPLETE; + return ret; +} - /* signal client that we're now done writing data */ - client_signal(shm_hdr, SHMEM_REGION_DATA_COMPLETE); - /* wait for client to read data */ - client_wait(shm_hdr); +/* iterate over list of chunk reads and send responses */ +static int rm_process_client_requests(reqmgr_thrd_t* reqmgr) +{ + /* assume we'll succeed */ + int ret = UNIFYFS_SUCCESS; - rc = __release_read_req(thrd_ctrl, rdreq); - if (rc != (int)UNIFYFS_SUCCESS) { - LOGERR("failed to release server_read_req_t"); - } + /* this will hold a list of client requests if we find any */ + arraylist_t* client_reqs = NULL; + + /* lock to access requests */ + RM_REQ_LOCK(reqmgr); + + /* if we have any requests, take pointer to the list + * of requests and replace it with a newly allocated + * list on the request manager structure */ + int num_client_reqs = arraylist_size(reqmgr->client_reqs); + if (num_client_reqs) { + /* got some chunk read requets, take the list and replace + * it with an empty list */ + LOGDBG("processing %d client requests", num_client_reqs); + client_reqs = reqmgr->client_reqs; + reqmgr->client_reqs = arraylist_create(); + } + + /* release lock on reqmgr requests */ + RM_REQ_UNLOCK(reqmgr); + + /* iterate over each chunk read request */ + for (int i = 0; i < num_client_reqs; i++) { + /* process next request */ + int rret; + client_rpc_req_t* req = (client_rpc_req_t*) + arraylist_get(client_reqs, i); + switch (req->req_type) { + case UNIFYFS_CLIENT_RPC_FILESIZE: + rret = process_filesize_rpc(reqmgr, req); + break; + case UNIFYFS_CLIENT_RPC_LAMINATE: + rret = process_laminate_rpc(reqmgr, req); + break; + case UNIFYFS_CLIENT_RPC_METAGET: + rret = process_metaget_rpc(reqmgr, req); + break; + case UNIFYFS_CLIENT_RPC_METASET: + rret = process_metaset_rpc(reqmgr, req); + break; + case UNIFYFS_CLIENT_RPC_READ: + rret = process_read_rpc(reqmgr, req); + break; + case UNIFYFS_CLIENT_RPC_SYNC: + rret = process_fsync_rpc(reqmgr, req); + break; + case UNIFYFS_CLIENT_RPC_TRUNCATE: + rret = process_truncate_rpc(reqmgr, req); + break; + case UNIFYFS_CLIENT_RPC_UNLINK: + rret = process_unlink_rpc(reqmgr, req); + break; + default: + LOGERR("unsupported client rpc request type %d", req->req_type); + rret = UNIFYFS_ERROR_NYI; + break; + } + if (rret != UNIFYFS_SUCCESS) { + LOGERR("client rpc request %d failed (%s)", + i, unifyfs_rc_enum_description(rret)); + ret = rret; } } + /* free the list if we have one */ + if (NULL != client_reqs) { + /* NOTE: this will call free() on each req in the arraylist */ + arraylist_free(client_reqs); + } + return ret; } @@ -1984,11 +1391,12 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, * * @param arg: pointer to RM thread control structure * @return NULL */ -void* rm_delegate_request_thread(void* arg) +void* request_manager_thread(void* arg) { /* get pointer to our thread control structure */ reqmgr_thrd_t* thrd_ctrl = (reqmgr_thrd_t*) arg; + thrd_ctrl->tid = unifyfs_gettid(); LOGDBG("I am request manager thread!"); /* loop forever to handle read requests from the client, @@ -1999,6 +1407,12 @@ void* rm_delegate_request_thread(void* arg) /* grab lock */ RM_LOCK(thrd_ctrl); + /* process any client requests */ + rc = rm_process_client_requests(thrd_ctrl); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to process client rpc requests"); + } + /* process any chunk read responses */ rc = rm_process_remote_chunk_responses(thrd_ctrl); if (rc != UNIFYFS_SUCCESS) { @@ -2007,7 +1421,7 @@ void* rm_delegate_request_thread(void* arg) /* inform dispatcher that we're waiting for work * inside the critical section */ - thrd_ctrl->has_waiting_delegator = 1; + thrd_ctrl->waiting_for_work = 1; /* if dispatcher is waiting on us, signal it to go ahead, * this coordination ensures that we'll be the next thread @@ -2015,6 +1429,8 @@ void* rm_delegate_request_thread(void* arg) * some work (rather than the dispatcher grabbing the lock * and assigning yet more work) */ if (thrd_ctrl->has_waiting_dispatcher == 1) { + /* TODO - should this be pthread_cond_broadcast() since we + * might have multiple requestors waiting? */ pthread_cond_signal(&thrd_ctrl->thrd_cond); } @@ -2022,16 +1438,14 @@ void* rm_delegate_request_thread(void* arg) LOGDBG("RM[%d:%d] waiting for work", thrd_ctrl->app_id, thrd_ctrl->client_id); pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); + LOGDBG("RM[%d:%d] got work", thrd_ctrl->app_id, thrd_ctrl->client_id); /* set flag to indicate we're no longer waiting */ - thrd_ctrl->has_waiting_delegator = 0; - - /* go do work ... */ - LOGDBG("RM[%d:%d] got work", thrd_ctrl->app_id, thrd_ctrl->client_id); + thrd_ctrl->waiting_for_work = 0; + RM_UNLOCK(thrd_ctrl); - /* release lock and bail out if we've been told to exit */ + /* bail out if we've been told to exit */ if (thrd_ctrl->exit_flag == 1) { - RM_UNLOCK(thrd_ctrl); break; } @@ -2040,9 +1454,6 @@ void* rm_delegate_request_thread(void* arg) if (rc != UNIFYFS_SUCCESS) { LOGERR("failed to request remote chunks"); } - - /* release lock */ - RM_UNLOCK(thrd_ctrl); } LOGDBG("request manager thread exiting"); @@ -2052,127 +1463,12 @@ void* rm_delegate_request_thread(void* arg) /* BEGIN MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ -#if 0 // DISABLE UNUSED RPCS -/* invokes the server_hello rpc */ -int invoke_server_hello_rpc(int dst_srvr_rank) -{ - int rc = (int)UNIFYFS_SUCCESS; - hg_handle_t handle; - server_hello_in_t in; - server_hello_out_t out; - hg_return_t hret; - hg_addr_t dst_srvr_addr; - char hello_msg[UNIFYFS_MAX_HOSTNAME]; - - assert(dst_srvr_rank < (int)glb_num_servers); - dst_srvr_addr = glb_servers[dst_srvr_rank].margo_svr_addr; - - hret = margo_create(unifyfsd_rpc_context->svr_mid, dst_srvr_addr, - unifyfsd_rpc_context->rpcs.hello_id, &handle); - assert(hret == HG_SUCCESS); - - /* fill in input struct */ - snprintf(hello_msg, sizeof(hello_msg), "hello from %s", glb_host); - in.src_rank = (int32_t)glb_pmi_rank; - in.message_str = strdup(hello_msg); - - LOGDBG("invoking the server-hello rpc function"); - hret = margo_forward(handle, &in); - if (hret != HG_SUCCESS) { - rc = (int)UNIFYFS_FAILURE; - } else { - /* decode response */ - hret = margo_get_output(handle, &out); - if (hret == HG_SUCCESS) { - int32_t ret = out.ret; - LOGDBG("Got hello rpc response from %d - ret=%" PRIi32, - dst_srvr_rank, ret); - margo_free_output(handle, &out); - } else { - rc = (int)UNIFYFS_FAILURE; - } - } - - free((void*)in.message_str); - margo_destroy(handle); - - return rc; -} - -/* invokes the server_request rpc */ -int invoke_server_request_rpc(int dst_srvr_rank, int req_id, int tag, - void* data_buf, size_t buf_sz) -{ - int rc = (int)UNIFYFS_SUCCESS; - hg_handle_t handle; - server_request_in_t in; - server_request_out_t out; - hg_return_t hret; - hg_addr_t dst_srvr_addr; - hg_size_t bulk_sz = buf_sz; - - if (dst_srvr_rank == glb_pmi_rank) { - // short-circuit for local requests - return rc; - } - - assert(dst_srvr_rank < (int)glb_num_servers); - dst_srvr_addr = glb_servers[dst_srvr_rank].margo_svr_addr; - - hret = margo_create(unifyfsd_rpc_context->svr_mid, dst_srvr_addr, - unifyfsd_rpc_context->rpcs.request_id, &handle); - assert(hret == HG_SUCCESS); - - /* fill in input struct */ - in.src_rank = (int32_t)glb_pmi_rank; - in.req_id = (int32_t)req_id; - in.req_tag = (int32_t)tag; - in.bulk_size = bulk_sz; - - /* register request buffer for bulk remote access */ - hret = margo_bulk_create(unifyfsd_rpc_context->svr_mid, 1, - &data_buf, &bulk_sz, - HG_BULK_READ_ONLY, &in.bulk_handle); - assert(hret == HG_SUCCESS); - - LOGDBG("invoking the server-request rpc function"); - hret = margo_forward(handle, &in); - if (hret != HG_SUCCESS) { - rc = (int)UNIFYFS_FAILURE; - } else { - /* decode response */ - hret = margo_get_output(handle, &out); - if (hret == HG_SUCCESS) { - rc = (int)out.ret; - LOGDBG("Got request rpc response from %d - ret=%d", - dst_srvr_rank, rc); - margo_free_output(handle, &out); - } else { - rc = (int)UNIFYFS_FAILURE; - } - } - - margo_bulk_free(in.bulk_handle); - margo_destroy(handle); - - return rc; -} -#endif // DISABLE UNUSED RPCS - /* invokes the server_request rpc */ int invoke_chunk_read_request_rpc(int dst_srvr_rank, server_read_req_t* rdreq, int num_chunks, void* data_buf, size_t buf_sz) { - int rc = (int)UNIFYFS_SUCCESS; - hg_handle_t handle; - chunk_read_request_in_t in; - chunk_read_request_out_t out; - hg_return_t hret; - hg_addr_t dst_srvr_addr; - hg_size_t bulk_sz = buf_sz; - if (dst_srvr_rank == glb_pmi_rank) { // short-circuit for local requests return sm_issue_chunk_reads(glb_pmi_rank, @@ -2183,13 +1479,24 @@ int invoke_chunk_read_request_rpc(int dst_srvr_rank, (char*)data_buf); } + int ret = UNIFYFS_SUCCESS; + hg_handle_t handle; + chunk_read_request_in_t in; + chunk_read_request_out_t out; + hg_return_t hret; + hg_addr_t dst_srvr_addr; + hg_size_t bulk_sz = buf_sz; + assert(dst_srvr_rank < (int)glb_num_servers); dst_srvr_addr = glb_servers[dst_srvr_rank].margo_svr_addr; hret = margo_create(unifyfsd_rpc_context->svr_mid, dst_srvr_addr, unifyfsd_rpc_context->rpcs.chunk_read_request_id, &handle); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_create() failed"); + return UNIFYFS_ERROR_MARGO; + } /* fill in input struct */ in.src_rank = (int32_t)glb_pmi_rank; @@ -2203,153 +1510,165 @@ int invoke_chunk_read_request_rpc(int dst_srvr_rank, hret = margo_bulk_create(unifyfsd_rpc_context->svr_mid, 1, &data_buf, &bulk_sz, HG_BULK_READ_ONLY, &in.bulk_handle); - assert(hret == HG_SUCCESS); - - LOGDBG("invoking the chunk-read-request rpc function"); - hret = margo_forward(handle, &in); if (hret != HG_SUCCESS) { - rc = (int)UNIFYFS_FAILURE; + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; } else { - /* decode response */ - hret = margo_get_output(handle, &out); - if (hret == HG_SUCCESS) { - rc = (int)out.ret; - LOGDBG("Got request rpc response from %d - ret=%d", - dst_srvr_rank, rc); - margo_free_output(handle, &out); + LOGDBG("invoking the chunk-read-request rpc function"); + hret = margo_forward(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_forward() failed"); + ret = UNIFYFS_ERROR_MARGO; } else { - rc = (int)UNIFYFS_FAILURE; + /* decode response */ + hret = margo_get_output(handle, &out); + if (hret == HG_SUCCESS) { + ret = (int)out.ret; + LOGDBG("Got request rpc response from %d - ret=%d", + dst_srvr_rank, ret); + margo_free_output(handle, &out); + } else { + LOGERR("margo_get_output() failed"); + ret = UNIFYFS_ERROR_MARGO; + } } - } - margo_bulk_free(in.bulk_handle); + margo_bulk_free(in.bulk_handle); + } margo_destroy(handle); - return rc; + return ret; } + /* BEGIN MARGO SERVER-SERVER RPC HANDLER FUNCTIONS */ /* handler for remote read request response */ static void chunk_read_response_rpc(hg_handle_t handle) { int32_t ret; - hg_return_t hret; chunk_read_response_out_t out; /* get input params */ chunk_read_response_in_t in; - int rc = margo_get_input(handle, &in); - assert(rc == HG_SUCCESS); - - /* extract params from input struct */ - int src_rank = (int)in.src_rank; - int app_id = (int)in.app_id; - int client_id = (int)in.client_id; - int req_id = (int)in.req_id; - int num_chks = (int)in.num_chks; - size_t bulk_sz = (size_t)in.bulk_size; - - LOGDBG("received chunk read response from server %d (%d chunks)", - src_rank, num_chks); - - /* The input parameters specify the info for a bulk transfer - * buffer on the sending process. We use that info to pull data - * from the sender into a local buffer. This buffer contains - * the read reply headers and associated read data for requests - * we had sent earlier. */ - - /* pull the remote data via bulk transfer */ - if (0 == bulk_sz) { - /* sender is trying to send an empty buffer, - * don't think that should happen unless maybe - * we had sent a read request list that was empty? */ - LOGERR("empty response buffer"); - ret = (int32_t)EINVAL; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = (int32_t) UNIFYFS_ERROR_MARGO; } else { - /* allocate a buffer to hold the incoming data */ - char* resp_buf = (char*) malloc(bulk_sz); - if (NULL == resp_buf) { - /* allocation failed, that's bad */ - LOGERR("failed to allocate chunk read responses buffer"); - ret = (int32_t)ENOMEM; + /* extract params from input struct */ + int src_rank = (int)in.src_rank; + int app_id = (int)in.app_id; + int client_id = (int)in.client_id; + int req_id = (int)in.req_id; + int num_chks = (int)in.num_chks; + size_t bulk_sz = (size_t)in.bulk_size; + + LOGDBG("received chunk read response from server %d (%d chunks)", + src_rank, num_chks); + + /* The input parameters specify the info for a bulk transfer + * buffer on the sending process. We use that info to pull data + * from the sender into a local buffer. This buffer contains + * the read reply headers and associated read data for requests + * we had sent earlier. */ + + /* pull the remote data via bulk transfer */ + if (0 == bulk_sz) { + /* sender is trying to send an empty buffer, + * don't think that should happen unless maybe + * we had sent a read request list that was empty? */ + LOGERR("empty response buffer"); + ret = (int32_t)EINVAL; } else { - /* got a buffer, now pull response data */ - ret = (int32_t)UNIFYFS_SUCCESS; - - /* get margo info */ - const struct hg_info* hgi = margo_get_info(handle); - assert(NULL != hgi); - - margo_instance_id mid = margo_hg_info_get_instance(hgi); - assert(mid != MARGO_INSTANCE_NULL); - - /* pass along address of buffer we want to transfer - * data into to prepare it for a bulk write, - * get resulting margo handle */ - hg_bulk_t bulk_handle; - hret = margo_bulk_create(mid, 1, (void**)&resp_buf, &in.bulk_size, - HG_BULK_WRITE_ONLY, &bulk_handle); - if (hret != HG_SUCCESS) { - LOGERR("failed to prepare bulk transfer"); - ret = UNIFYFS_ERROR_MARGO; - goto out_respond; - } - - /* execute the transfer to pull data from remote side - * into our local bulk transfer buffer. - * NOTE: mercury/margo bulk transfer does not check the maximum - * transfer size that the underlying transport supports, and a - * large bulk transfer may result in failure. */ - int i = 0; - hg_size_t remain = in.bulk_size; - - do { - hg_size_t offset = i * MAX_BULK_TX_SIZE; - hg_size_t len = remain < MAX_BULK_TX_SIZE - ? remain : MAX_BULK_TX_SIZE; - - hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, - in.bulk_handle, offset, - bulk_handle, offset, len); + /* allocate a buffer to hold the incoming data */ + char* resp_buf = (char*) malloc(bulk_sz); + if (NULL == resp_buf) { + /* allocation failed, that's bad */ + LOGERR("failed to allocate chunk read responses buffer"); + ret = (int32_t)ENOMEM; + } else { + /* got a buffer, now pull response data */ + ret = (int32_t)UNIFYFS_SUCCESS; + + /* get margo info */ + const struct hg_info* hgi = margo_get_info(handle); + assert(NULL != hgi); + + margo_instance_id mid = margo_hg_info_get_instance(hgi); + assert(mid != MARGO_INSTANCE_NULL); + + /* pass along address of buffer we want to transfer + * data into to prepare it for a bulk write, + * get resulting margo handle */ + hg_bulk_t bulk_handle; + hret = margo_bulk_create(mid, 1, + (void**)&resp_buf, &in.bulk_size, + HG_BULK_WRITE_ONLY, &bulk_handle); if (hret != HG_SUCCESS) { - break; + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + goto out_respond; } - remain -= len; - i++; - } while (remain > 0); + /* execute the transfer to pull data from remote side + * into our local bulk transfer buffer. + * NOTE: mercury/margo bulk transfer does not check the maximum + * transfer size that the underlying transport supports, and a + * large bulk transfer may result in failure. */ + int i = 0; + hg_size_t remain = in.bulk_size; + do { + hg_size_t offset = i * MAX_BULK_TX_SIZE; + hg_size_t len = remain < MAX_BULK_TX_SIZE + ? remain : MAX_BULK_TX_SIZE; + + hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, + in.bulk_handle, offset, + bulk_handle, offset, len); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_transfer(off=%zu, sz=%zu) failed", + (size_t)offset, (size_t)len); + ret = UNIFYFS_ERROR_MARGO; + break; + } - if (hret == HG_SUCCESS) { - LOGDBG("transferred bulk data (%lu bytes)", in.bulk_size); + remain -= len; + i++; + } while (remain > 0); - /* process read replies (headers and data) we just received */ - rc = rm_post_chunk_read_responses(app_id, client_id, - src_rank, req_id, num_chks, bulk_sz, resp_buf); - if (rc != (int)UNIFYFS_SUCCESS) { - LOGERR("failed to handle chunk read responses") + if (hret == HG_SUCCESS) { + LOGDBG("successful bulk transfer (%zu bytes)", bulk_sz); + + /* process read replies we just received */ + int rc = rm_post_chunk_read_responses(app_id, client_id, + src_rank, req_id, + num_chks, bulk_sz, + resp_buf); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to handle chunk read responses"); ret = rc; + } + } else { + LOGERR("failed to perform bulk transfer"); } - } else { - LOGERR("failed to perform bulk transfer"); - ret = UNIFYFS_ERROR_MARGO; - } - /* deregister our bulk transfer buffer */ - margo_bulk_free(bulk_handle); + /* deregister our bulk transfer buffer */ + margo_bulk_free(bulk_handle); + } } + margo_free_input(handle, &in); } out_respond: - /* fill output structure */ - out.ret = ret; - /* return to caller */ + out.ret = ret; hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } /* free margo resources */ - margo_free_input(handle, &in); margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(chunk_read_response_rpc) diff --git a/server/src/unifyfs_request_manager.h b/server/src/unifyfs_request_manager.h index bb0924338..215efa181 100644 --- a/server/src/unifyfs_request_manager.h +++ b/server/src/unifyfs_request_manager.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017-2019, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -31,45 +31,60 @@ #define UNIFYFS_REQUEST_MANAGER_H #include "unifyfs_global.h" +#include "unifyfs_metadata_mdhim.h" + +typedef struct { + client_rpc_e req_type; + hg_handle_t handle; + void* input; + void* bulk_buf; + size_t bulk_sz; +} client_rpc_req_t; typedef struct { readreq_status_e status; /* aggregate request status */ + int in_use; /* currently using this req? */ int req_ndx; /* index in reqmgr read_reqs array */ int app_id; /* app id of requesting client process */ int client_id; /* client id of requesting client process */ - int num_remote_reads; /* size of remote_reads array */ - int in_use; /* occupied by a thread */ + int num_server_reads; /* size of remote_reads array */ chunk_read_req_t* chunks; /* array of chunk-reads */ - remote_chunk_reads_t* remote_reads; /* per-delegator remote reads array */ + server_chunk_reads_t* remote_reads; /* per-server remote reads array */ } server_read_req_t; -/* this structure is created by the main thread for each request - * manager thread, contains shared data structures where main thread - * issues read requests and request manager processes them, contains - * condition variable and lock for coordination between threads */ +/* Request manager state structure - created by main thread for each request + * manager thread. Contains shared data structures for client-server and + * server-server requests and associated synchronization constructs */ typedef struct reqmgr_thrd { - /* request manager thread */ + /* request manager (RM) thread */ pthread_t thrd; + pid_t tid; /* condition variable to synchronize request manager thread - * and main thread delivering work */ + * and margo rpc handler ULT delivering work */ pthread_cond_t thrd_cond; /* lock for shared data structures (variables below) */ pthread_mutex_t thrd_lock; - /* flag indicating that request manager thread is waiting - * for work inside of critical region */ - int has_waiting_delegator; + /* flag indicating request manager thread is waiting on thrd_cond CV */ + int waiting_for_work; - /* flag indicating main thread is in critical section waiting - * for request manager thread */ + /* flag indicating a margo rpc handler ULT is waiting on thrd_cond CV */ int has_waiting_dispatcher; + /* argobots mutex for synchronizing access to request state between + * margo rpc handler ULTs and request manager thread */ + ABT_mutex reqs_sync; + + /* array of server read requests */ int num_read_reqs; int next_rdreq_ndx; server_read_req_t read_reqs[RM_MAX_ACTIVE_REQUESTS]; + /* list of client rpc requests */ + arraylist_t* client_reqs; + /* buffer to build read request messages */ char del_req_msg_buf[REQ_BUF_LEN]; @@ -86,41 +101,29 @@ typedef struct reqmgr_thrd { int client_id; } reqmgr_thrd_t; +/* reserve/release read requests */ +server_read_req_t* rm_reserve_read_req(reqmgr_thrd_t* thrd_ctrl); +int rm_release_read_req(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq); + +/* issue remote chunk read requests for extent chunks + * listed within keyvals */ +int rm_create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, + server_read_req_t* rdreq, + int num_vals, + unifyfs_keyval_t* keyvals); /* create Request Manager thread for application client */ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id); /* Request Manager pthread main */ -void* rm_delegate_request_thread(void* arg); - -/* functions called by rpc handlers to assign work - * to request manager threads */ -int rm_cmd_mread(int app_id, int client_id, - size_t req_num, void* reqbuf); - -int rm_cmd_read(int app_id, int client_id, int gfid, - size_t offset, size_t length); - -int rm_cmd_filesize(int app_id, int client_id, int gfid, size_t* outsize); - -/* truncate file to specified size */ -int rm_cmd_truncate(int app_id, int client_id, int gfid, size_t size); - -/* delete file */ -int rm_cmd_unlink(int app_id, int client_id, int gfid); - -/* laminate file */ -int rm_cmd_laminate(int app_id, int client_id, int gfid); +void* request_manager_thread(void* arg); /* function called by main thread to instruct * resource manager thread to exit, * returns UNIFYFS_SUCCESS on success */ -int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl); - -/* retrieve all write index entries for app-client and - * store them in global metadata */ -int rm_cmd_sync(int app_id, int client_side_id); +int rm_request_exit(reqmgr_thrd_t* thrd_ctrl); /* update state for remote chunk reads with received response data */ int rm_post_chunk_read_responses(int app_id, @@ -134,18 +137,31 @@ int rm_post_chunk_read_responses(int app_id, /* process the requested chunk data returned from service managers */ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, server_read_req_t* rdreq, - remote_chunk_reads_t* del_reads); + server_chunk_reads_t* del_reads); -/* MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ +/** + * @brief hand over a read request to the request manager thread. + * + * @param req all members except for status and req_ndx need to be filled by + * the caller. @req->chunks and @req->remote_reads should be allocated from + * heap, and should not be freed by the caller. + * + * @return 0 on success, errno otherwise + */ +int rm_submit_read_request(server_read_req_t* req); -#if 0 // DISABLE UNUSED RPCS -int invoke_server_hello_rpc(int dst_srvr_rank); +/** + * @brief submit a client rpc request to the request manager thread. + * + * @param client application client context + * @param req pointer to client rpc request struct + * + * @return UNIFYFS_SUCCESS, or error code + */ +int rm_submit_client_rpc_request(unifyfs_fops_ctx_t* ctx, + client_rpc_req_t* req); -int invoke_server_request_rpc(int dst_srvr_rank, - int req_id, - int tag, - void* data_buf, size_t buf_sz); -#endif // DISABLE UNUSED RPCS +/* MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ int invoke_chunk_read_request_rpc(int dst_srvr_rank, server_read_req_t* rdreq, diff --git a/server/src/unifyfs_server.c b/server/src/unifyfs_server.c index cdd5b2d29..d77bbd141 100644 --- a/server/src/unifyfs_server.c +++ b/server/src/unifyfs_server.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -37,9 +37,10 @@ // server components #include "unifyfs_global.h" -#include "unifyfs_metadata.h" +#include "unifyfs_metadata_mdhim.h" #include "unifyfs_request_manager.h" #include "unifyfs_service_manager.h" +#include "unifyfs_inode_tree.h" // margo rpcs #include "margo_server.h" @@ -79,6 +80,8 @@ static int CountTasksPerNode(int rank, int numTasks); static int find_rank_idx(int my_rank); #endif +struct unifyfs_fops* global_fops_tab; + /* * Perform steps to create a daemon process: * @@ -381,15 +384,17 @@ int main(int argc, char* argv[]) exit(1); } - LOGDBG("initializing metadata store"); - rc = meta_init_store(&server_cfg); + LOGDBG("initializing file operations"); + rc = unifyfs_fops_init(&server_cfg); if (rc != 0) { LOGERR("%s", unifyfs_rc_enum_description(rc)); exit(1); } - LOGDBG("finished service initialization"); + /* initialize our tree that maps a gfid to its extent tree */ + unifyfs_inode_tree_init(global_inode_tree); + LOGDBG("publishing server pid"); rc = unifyfs_publish_server_pids(); if (rc != 0) { LOGERR("failed to publish server pid file: %s", @@ -397,6 +402,8 @@ int main(int argc, char* argv[]) exit(1); } + LOGINFO("server[%d] - finished initialization", glb_pmi_rank); + while (1) { sleep(1); if (time_to_exit) { @@ -405,6 +412,9 @@ int main(int argc, char* argv[]) } } + /* tear down gfid-to-extents tree */ + unifyfs_inode_tree_destroy(global_inode_tree); + LOGDBG("stopping service manager thread"); rc = svcmgr_fini(); @@ -601,9 +611,11 @@ static int unifyfs_exit(void) LOGDBG("stopping rpc service"); margo_server_rpc_finalize(); +#if defined(USE_MDHIM) /* shutdown the metadata service*/ LOGDBG("stopping metadata service"); meta_sanitize(); +#endif #if defined(UNIFYFSD_USE_MPI) LOGDBG("finalizing MPI"); @@ -909,7 +921,7 @@ unifyfs_rc disconnect_app_client(app_client* client) /* stop client request manager thread */ if (NULL != client->reqmgr) { - rm_cmd_exit(client->reqmgr); + rm_request_exit(client->reqmgr); } /* free margo client address */ @@ -958,7 +970,7 @@ unifyfs_rc cleanup_app_client(app_config* app, app_client* client) /* close client logio context */ if (NULL != client->logio) { - unifyfs_logio_close(client->logio); + unifyfs_logio_close(client->logio, 1); client->logio = NULL; } @@ -971,6 +983,7 @@ unifyfs_rc cleanup_app_client(app_config* app, app_client* client) /* free client structure */ if (NULL != client->reqmgr) { free(client->reqmgr); + client->reqmgr = NULL; } free(client); diff --git a/server/src/unifyfs_server_pid.c b/server/src/unifyfs_server_pid.c index 5a0cfa60c..eee856578 100644 --- a/server/src/unifyfs_server_pid.c +++ b/server/src/unifyfs_server_pid.c @@ -11,6 +11,7 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + #include #include #include @@ -24,29 +25,23 @@ extern unifyfs_cfg_t server_cfg; +static int* server_pids; // = NULL static pthread_cond_t server_pid_cond = PTHREAD_COND_INITIALIZER; static pthread_mutex_t server_pid_mutex = PTHREAD_MUTEX_INITIALIZER; static struct timespec server_pid_timeout; -static int* server_pids; -static pthread_mutex_t server_pid_alloc_lock = PTHREAD_MUTEX_INITIALIZER; - -static int alloc_server_pid(void) +static int alloc_server_pids(void) { int ret = 0; - - pthread_mutex_lock(&server_pid_alloc_lock); - { - if (!server_pids) { - server_pids = calloc(glb_pmi_size, sizeof(*server_pids)); - if (!server_pids) { - LOGERR("failed to allocate memory (%s)", strerror(errno)); - ret = ENOMEM; - } + pthread_mutex_lock(&server_pid_mutex); + if (NULL == server_pids) { + server_pids = (int*) calloc(glb_pmi_size, sizeof(int)); + if (NULL == server_pids) { + LOGERR("failed to allocate memory (%s)", strerror(errno)); + ret = ENOMEM; } } - pthread_mutex_unlock(&server_pid_alloc_lock); - + pthread_mutex_unlock(&server_pid_mutex); return ret; } @@ -84,14 +79,15 @@ static int server_pid_invoke_rpc(void) ret = out.ret; + margo_free_output(handle, &out); + margo_destroy(handle); + return ret; } -static void server_pid_handle_rpc(hg_handle_t handle) +static void server_pid_rpc(hg_handle_t handle) { int ret = 0; - int i = 0; - int count = 0; hg_return_t hret = 0; server_pid_in_t in; server_pid_out_t out; @@ -102,15 +98,15 @@ static void server_pid_handle_rpc(hg_handle_t handle) return; } - if (!server_pids) { - ret = alloc_server_pid(); - if (ret) { - LOGERR("failed to allocate pid array"); - return; - } + ret = alloc_server_pids(); + if (ret) { + LOGERR("failed to allocate pid array"); + return; } - - server_pids[in.rank] = in.pid; + assert((int)in.rank < glb_pmi_size); + pthread_mutex_lock(&server_pid_mutex); + server_pids[in.rank] = (int) in.pid; + pthread_mutex_unlock(&server_pid_mutex); out.ret = 0; hret = margo_respond(handle, &out); @@ -122,22 +118,14 @@ static void server_pid_handle_rpc(hg_handle_t handle) margo_free_input(handle, &in); margo_destroy(handle); - for (i = 0; i < glb_pmi_size; i++) { - if (server_pids[i] > 0) { - count++; - } - } - - if (count == glb_pmi_size) { - ret = pthread_cond_signal(&server_pid_cond); - if (ret) { - LOGERR("failed to signal condition (%s)", strerror(ret)); - } + ret = pthread_cond_signal(&server_pid_cond); + if (ret) { + LOGERR("failed to signal condition (%s)", strerror(ret)); } } -DEFINE_MARGO_RPC_HANDLER(server_pid_handle_rpc); +DEFINE_MARGO_RPC_HANDLER(server_pid_rpc); -static inline int set_timeout(void) +static inline int set_pidfile_timeout(void) { int ret = 0; long timeout_sec = 0; @@ -191,48 +179,58 @@ int unifyfs_publish_server_pids(void) int ret = UNIFYFS_SUCCESS; if (glb_pmi_rank > 0) { + /* publish my pid to server 0 */ ret = server_pid_invoke_rpc(); if (ret) { LOGERR("failed to invoke pid rpc (%s)", strerror(ret)); } } else { - ret = set_timeout(); + ret = alloc_server_pids(); if (ret) { return ret; } - if (!server_pids) { - ret = alloc_server_pid(); - if (ret) { - return ret; - } + ret = set_pidfile_timeout(); + if (ret) { + return ret; } + pthread_mutex_lock(&server_pid_mutex); server_pids[0] = server_pid; - if (glb_pmi_size > 1) { + /* keep checking count of reported servers until all have reported + * or we hit the timeout */ + do { + int count = 0; + for (int i = 0; i < glb_pmi_size; i++) { + if (server_pids[i] > 0) { + count++; + } + } + if (count == glb_pmi_size) { + ret = create_server_pid_file(); + if (UNIFYFS_SUCCESS == ret) { + LOGDBG("servers ready to accept client connections"); + } + break; + } ret = pthread_cond_timedwait(&server_pid_cond, &server_pid_mutex, &server_pid_timeout); if (ETIMEDOUT == ret) { LOGERR("some servers failed to initialize within timeout"); - goto out; + break; } else if (ret) { LOGERR("failed to wait on condition (err=%d, %s)", errno, strerror(errno)); - goto out; + break; } - } + } while (1); - ret = create_server_pid_file(); - if (UNIFYFS_SUCCESS == ret) { - LOGDBG("all servers are ready to accept client connection"); - } - } -out: - if (server_pids) { free(server_pids); server_pids = NULL; + + pthread_mutex_unlock(&server_pid_mutex); } return ret; diff --git a/server/src/unifyfs_service_manager.c b/server/src/unifyfs_service_manager.c index 3368c66bc..47955d0bd 100644 --- a/server/src/unifyfs_service_manager.c +++ b/server/src/unifyfs_service_manager.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -27,9 +27,6 @@ * Please read https://github.com/llnl/burstfs/LICENSE for full license text. */ -#include -#include - #include "unifyfs_global.h" #include "unifyfs_request_manager.h" #include "unifyfs_service_manager.h" @@ -41,8 +38,9 @@ typedef struct { /* the SM thread */ pthread_t thrd; - /* state synchronization mutex */ - pthread_mutex_t sync; + /* argobots mutex for synchronizing access to request state between + * margo rpc handler ULTs and SM thread */ + ABT_mutex sync; /* thread status */ int initialized; @@ -51,11 +49,9 @@ typedef struct { /* thread return status code */ int sm_exit_rc; - /* list of chunk read requests from remote delegators */ + /* list of chunk read requests from remote servers */ arraylist_t* chunk_reads; - /* tracks running total of bytes in current read burst */ - size_t burst_data_sz; } svcmgr_state_t; svcmgr_state_t* sm; // = NULL @@ -63,14 +59,14 @@ svcmgr_state_t* sm; // = NULL #define SM_LOCK() \ do { \ LOGDBG("locking service manager state"); \ - pthread_mutex_lock(&(sm->sync)); \ + ABT_mutex_lock(sm->sync); \ } while (0) /* unlock macro for debugging SM locking */ #define SM_UNLOCK() \ do { \ LOGDBG("unlocking service manager state"); \ - pthread_mutex_unlock(&(sm->sync)); \ + ABT_mutex_unlock(sm->sync); \ } while (0) /* Decode and issue chunk-reads received from request manager. @@ -78,10 +74,10 @@ do { \ * data for each request and construct a set of read replies * that will be sent back to the request manager. * - * @param src_rank : source delegator rank - * @param src_app_id : app id at source delegator - * @param src_client_id : client id at source delegator - * @param src_req_id : request id at source delegator + * @param src_rank : source server rank + * @param src_app_id : app id at source server + * @param src_client_id : client id at source server + * @param src_req_id : request id at source server * @param num_chks : number of chunk requests * @param msg_buf : message buffer containing request(s) * @return success/error code @@ -100,9 +96,8 @@ int sm_issue_chunk_reads(int src_rank, ptr += sizeof(int); /* extract number of chunk read requests */ - int num = *((int*)ptr); + assert(num_chks == *((int*)ptr)); ptr += sizeof(int); - assert(num == num_chks); /* total data size we'll be reading */ size_t total_data_sz = *((size_t*)ptr); @@ -134,22 +129,22 @@ int sm_issue_chunk_reads(int src_rank, char* databuf = crbuf + resp_sz; /* allocate a struct for the chunk read request */ - remote_chunk_reads_t* rcr = (remote_chunk_reads_t*) - calloc(1, sizeof(remote_chunk_reads_t)); - if (NULL == rcr) { + server_chunk_reads_t* scr = (server_chunk_reads_t*) + calloc(1, sizeof(server_chunk_reads_t)); + if (NULL == scr) { LOGERR("failed to allocate remote_chunk_reads"); return ENOMEM; } /* fill in chunk read request */ - rcr->rank = src_rank; - rcr->app_id = src_app_id; - rcr->client_id = src_client_id; - rcr->rdreq_id = src_req_id; - rcr->num_chunks = num_chks; - rcr->reqs = NULL; - rcr->total_sz = buf_sz; - rcr->resp = resp; + scr->rank = src_rank; + scr->app_id = src_app_id; + scr->client_id = src_client_id; + scr->rdreq_id = src_req_id; + scr->num_chunks = num_chks; + scr->reqs = NULL; + scr->total_sz = buf_sz; + scr->resp = resp; LOGDBG("issuing %d requests, total data size = %zu", num_chks, total_data_sz); @@ -206,9 +201,6 @@ int sm_issue_chunk_reads(int src_rank, /* update to point to next slot in read reply buffer */ buf_cursor += nbytes; - - /* update accounting for burst size */ - sm->burst_data_sz += nbytes; } if (src_rank != glb_pmi_rank) { @@ -219,10 +211,10 @@ int sm_issue_chunk_reads(int src_rank, assert(NULL != sm); SM_LOCK(); - arraylist_add(sm->chunk_reads, rcr); + arraylist_add(sm->chunk_reads, scr); SM_UNLOCK(); - /* rcr will be freed later by the sending thread */ + /* scr will be freed later by the sending thread */ LOGDBG("done adding to svcmgr chunk_reads"); return UNIFYFS_SUCCESS; @@ -237,7 +229,7 @@ int sm_issue_chunk_reads(int src_rank, } /* clean up allocated buffers */ - free(rcr); + free(scr); return rc; } @@ -254,9 +246,6 @@ int svcmgr_init(void) return ENOMEM; } - /* tracks how much data we process in each burst */ - sm->burst_data_sz = 0; - /* allocate a list to track chunk read requests */ sm->chunk_reads = arraylist_create(); if (sm->chunk_reads == NULL) { @@ -265,17 +254,12 @@ int svcmgr_init(void) return ENOMEM; } - int rc = pthread_mutex_init(&(sm->sync), NULL); - if (0 != rc) { - LOGERR("failed to initialize service manager mutex!"); - svcmgr_fini(); - return (int)UNIFYFS_ERROR_THRDINIT; - } + ABT_mutex_create(&(sm->sync)); sm->initialized = 1; - rc = pthread_create(&(sm->thrd), NULL, - sm_service_reads, (void*)sm); + int rc = pthread_create(&(sm->thrd), NULL, + service_manager_thread, (void*)sm); if (rc != 0) { LOGERR("failed to create service manager thread"); svcmgr_fini(); @@ -302,7 +286,6 @@ int svcmgr_fini(void) if (sm->initialized) { SM_UNLOCK(); - pthread_mutex_destroy(&(sm->sync)); } /* free the service manager struct allocated during init */ @@ -322,7 +305,7 @@ static int send_chunk_read_responses(void) arraylist_t* chunk_reads = NULL; /* lock to access global service manager object */ - pthread_mutex_lock(&(sm->sync)); + ABT_mutex_lock(sm->sync); /* if we have any chunk reads, take pointer to the list * of chunk read requests and replace it with a newly allocated @@ -337,15 +320,15 @@ static int send_chunk_read_responses(void) } /* release lock on service manager object */ - pthread_mutex_unlock(&(sm->sync)); + ABT_mutex_unlock(sm->sync); /* iterate over each chunk read request */ for (int i = 0; i < num_chunk_reads; i++) { /* get next chunk read request */ - remote_chunk_reads_t* rcr = (remote_chunk_reads_t*) + server_chunk_reads_t* scr = (server_chunk_reads_t*) arraylist_get(chunk_reads, i); - rc = invoke_chunk_read_response_rpc(rcr); + rc = invoke_chunk_read_response_rpc(scr); } /* free the list if we have one */ @@ -363,11 +346,11 @@ static int send_chunk_read_responses(void) * * @param arg: pointer to SM thread control structure * @return NULL */ -void* sm_service_reads(void* arg) +void* service_manager_thread(void* arg) { int rc; - LOGDBG("I am service manager thread!"); + LOGDBG("I am the service manager thread!"); assert(sm == (svcmgr_state_t*)arg); /* handle chunk reads until signaled to exit */ @@ -377,39 +360,12 @@ void* sm_service_reads(void* arg) LOGERR("failed to send chunk read responses"); } - pthread_mutex_lock(&(sm->sync)); - if (sm->time_to_exit) { - pthread_mutex_unlock(&(sm->sync)); break; } -#ifdef BURSTY_WAIT // REVISIT WHETHER BURSTY WAIT STILL DESIRABLE - /* determine how long to wait next time based on - * how much data we just processed in this burst */ - size_t bursty_interval; - if (sm->burst_data_sz >= LARGE_BURSTY_DATA) { - /* for large bursts above a threshold, - * wait for a fixed amount of time */ - bursty_interval = MAX_BURSTY_INTERVAL; - } else { - /* for smaller bursts, set delay proportionally - * to burst size we just processed */ - bursty_interval = - (SLEEP_SLICE_PER_UNIT * sm->burst_data_sz) / MIB; - } - if (bursty_interval > MIN_SLEEP_INTERVAL) { - usleep(SLEEP_INTERVAL); /* wait an interval */ - } -#else /* wait an interval */ usleep(MIN_SLEEP_INTERVAL); -#endif // REVISIT WHETHER BURSTY WAIT STILL DESIRABLE - - /* reset our burst size counter */ - sm->burst_data_sz = 0; - - pthread_mutex_unlock(&(sm->sync)); } LOGDBG("service manager thread exiting"); @@ -424,13 +380,13 @@ void* sm_service_reads(void* arg) * reply headers and corresponding data back to a server that * had requested we read data on its behalf, the headers and * data are posted as a bulk transfer buffer */ -int invoke_chunk_read_response_rpc(remote_chunk_reads_t* rcr) +int invoke_chunk_read_response_rpc(server_chunk_reads_t* scr) { /* assume we'll succeed */ - int rc = (int)UNIFYFS_SUCCESS; + int rc = UNIFYFS_SUCCESS; /* rank of destination server */ - int dst_rank = rcr->rank; + int dst_rank = scr->rank; assert(dst_rank < (int)glb_num_servers); /* get address of destinaton server */ @@ -442,34 +398,41 @@ int invoke_chunk_read_response_rpc(remote_chunk_reads_t* rcr) /* get handle to read response rpc on destination server */ hg_handle_t handle; + hg_id_t resp_id = ctx->rpcs.chunk_read_response_id; hg_return_t hret = margo_create(ctx->svr_mid, dst_addr, - ctx->rpcs.chunk_read_response_id, &handle); - assert(hret == HG_SUCCESS); + resp_id, &handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_create() failed"); + return UNIFYFS_ERROR_MARGO; + } /* get address and size of our response buffer */ - void* data_buf = (void*)rcr->resp; - hg_size_t bulk_sz = rcr->total_sz; + void* data_buf = (void*)scr->resp; + hg_size_t bulk_sz = scr->total_sz; - /* fill in input struct */ + /* register our response buffer for bulk remote read access */ chunk_read_response_in_t in; + hret = margo_bulk_create(ctx->svr_mid, 1, &data_buf, &bulk_sz, + HG_BULK_READ_ONLY, &in.bulk_handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + return UNIFYFS_ERROR_MARGO; + } + + /* fill in input struct */ in.src_rank = (int32_t)glb_pmi_rank; - in.app_id = (int32_t)rcr->app_id; - in.client_id = (int32_t)rcr->client_id; - in.req_id = (int32_t)rcr->rdreq_id; - in.num_chks = (int32_t)rcr->num_chunks; + in.app_id = (int32_t)scr->app_id; + in.client_id = (int32_t)scr->client_id; + in.req_id = (int32_t)scr->rdreq_id; + in.num_chks = (int32_t)scr->num_chunks; in.bulk_size = bulk_sz; - /* register our response buffer for bulk remote read access */ - hret = margo_bulk_create(ctx->svr_mid, 1, - &data_buf, &bulk_sz, HG_BULK_READ_ONLY, &in.bulk_handle); - assert(hret == HG_SUCCESS); - /* call the read response rpc */ LOGDBG("invoking the chunk-read-response rpc function"); hret = margo_forward(handle, &in); if (hret != HG_SUCCESS) { - /* failed to invoke the rpc */ - rc = (int)UNIFYFS_FAILURE; + LOGERR("margo_forward() failed"); + rc = UNIFYFS_ERROR_MARGO; } else { /* rpc executed, now decode response */ chunk_read_response_out_t out; @@ -480,7 +443,8 @@ int invoke_chunk_read_response_rpc(remote_chunk_reads_t* rcr) dst_rank, rc); margo_free_output(handle, &out); } else { - rc = (int)UNIFYFS_FAILURE; + LOGERR("margo_get_output() failed"); + rc = UNIFYFS_ERROR_MARGO; } } @@ -490,203 +454,101 @@ int invoke_chunk_read_response_rpc(remote_chunk_reads_t* rcr) /* free response data buffer */ free(data_buf); - rcr->resp = NULL; + scr->resp = NULL; return rc; } /* BEGIN MARGO SERVER-SERVER RPC HANDLERS */ -/* handler for server-server hello - * - * print the message, and return my rank */ -static void server_hello_rpc(hg_handle_t handle) -{ - /* get input params */ - server_hello_in_t in; - int rc = margo_get_input(handle, &in); - assert(rc == HG_SUCCESS); - - /* extract params from input struct */ - int src_rank = (int)in.src_rank; - char* msg = strdup(in.message_str); - if (NULL != msg) { - LOGDBG("got message '%s' from server %d", msg, src_rank); - free(msg); - } - - /* fill output structure to return to caller */ - server_hello_out_t out; - out.ret = (int32_t)glb_pmi_rank; - - /* send output back to caller */ - hg_return_t hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); - - /* free margo resources */ - margo_free_input(handle, &in); - margo_destroy(handle); -} -DEFINE_MARGO_RPC_HANDLER(server_hello_rpc) - -/* handler for server-server request - * - * decode payload based on tag, and call appropriate svcmgr routine */ -static void server_request_rpc(hg_handle_t handle) -{ - int32_t ret; - hg_return_t hret; - - /* get input params */ - server_request_in_t in; - int rc = margo_get_input(handle, &in); - assert(rc == HG_SUCCESS); - - /* extract params from input struct */ - int src_rank = (int)in.src_rank; - int req_id = (int)in.req_id; - int tag = (int)in.req_tag; - size_t bulk_sz = (size_t)in.bulk_size; - - LOGDBG("handling request from server %d: tag=%d req=%d sz=%zu", - src_rank, tag, req_id, bulk_sz); - - /* get margo info */ - const struct hg_info* hgi = margo_get_info(handle); - assert(NULL != hgi); - - margo_instance_id mid = margo_hg_info_get_instance(hgi); - assert(mid != MARGO_INSTANCE_NULL); - - hg_bulk_t bulk_handle; - void* reqbuf = NULL; - if (bulk_sz) { - /* allocate and register local target buffer for bulk access */ - reqbuf = malloc(bulk_sz); - if (NULL == reqbuf) { - ret = (int32_t)ENOMEM; - goto request_out; - } - hret = margo_bulk_create(mid, 1, &reqbuf, &in.bulk_size, - HG_BULK_WRITE_ONLY, &bulk_handle); - assert(hret == HG_SUCCESS); - - /* pull request data */ - hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, - in.bulk_handle, 0, - bulk_handle, 0, in.bulk_size); - assert(hret == HG_SUCCESS); - } - - switch (tag) { - default: { - LOGERR("invalid request tag %d", tag); - ret = (int32_t)EINVAL; - break; - } - } - - server_request_out_t out; -request_out: - - /* fill output structure */ - out.ret = ret; - - /* return to caller */ - hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); - - /* free margo resources */ - margo_free_input(handle, &in); - if (NULL != reqbuf) { - margo_bulk_free(bulk_handle); - free(reqbuf); - } - margo_destroy(handle); -} -DEFINE_MARGO_RPC_HANDLER(server_request_rpc) - -/* handler for server-server request - * - * decode payload based on tag, and call appropriate svcmgr routine */ +/* handler for server-server chunk read request */ static void chunk_read_request_rpc(hg_handle_t handle) { - hg_return_t hret; + int32_t ret = UNIFYFS_SUCCESS; /* get input params */ chunk_read_request_in_t in; - int rc = margo_get_input(handle, &in); - assert(rc == HG_SUCCESS); - - /* extract params from input struct */ - int src_rank = (int)in.src_rank; - int app_id = (int)in.app_id; - int client_id = (int)in.client_id; - int req_id = (int)in.req_id; - int num_chks = (int)in.num_chks; - size_t bulk_sz = (size_t)in.bulk_size; - - LOGDBG("handling chunk read request from server %d: " - "req=%d num_chunks=%d bulk_sz=%zu", - src_rank, req_id, num_chks, bulk_sz); - - /* get margo info */ - const struct hg_info* hgi = margo_get_info(handle); - assert(NULL != hgi); - - margo_instance_id mid = margo_hg_info_get_instance(hgi); - assert(mid != MARGO_INSTANCE_NULL); - - hg_bulk_t bulk_handle; - int reqcmd = (int)SVC_CMD_INVALID; - void* reqbuf = NULL; - if (bulk_sz) { - /* allocate and register local target buffer for bulk access */ - reqbuf = malloc(bulk_sz); - if (NULL != reqbuf) { - hret = margo_bulk_create(mid, 1, &reqbuf, &in.bulk_size, - HG_BULK_WRITE_ONLY, &bulk_handle); - assert(hret == HG_SUCCESS); - - /* pull request data */ - hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, - in.bulk_handle, 0, - bulk_handle, 0, in.bulk_size); - assert(hret == HG_SUCCESS); - - /* first int in request buffer is the command */ - reqcmd = *(int*)reqbuf; - } - } - - /* verify this is a request for data */ - int32_t ret; - if (reqcmd == (int)SVC_CMD_RDREQ_CHK) { - /* chunk read request command */ - LOGDBG("request command: SVC_CMD_RDREQ_CHK"); - sm_issue_chunk_reads(src_rank, app_id, client_id, req_id, - num_chks, (char*)reqbuf); - ret = (int32_t)UNIFYFS_SUCCESS; + hg_return_t hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("margo_get_input() failed"); + ret = UNIFYFS_ERROR_MARGO; } else { - LOGERR("invalid chunk read request command %d from server %d", - reqcmd, src_rank); - ret = (int32_t)EINVAL; + /* extract params from input struct */ + int src_rank = (int)in.src_rank; + int app_id = (int)in.app_id; + int client_id = (int)in.client_id; + int req_id = (int)in.req_id; + int num_chks = (int)in.num_chks; + size_t bulk_sz = (size_t)in.bulk_size; + + LOGDBG("handling chunk read request from server %d: " + "req=%d num_chunks=%d bulk_sz=%zu", + src_rank, req_id, num_chks, bulk_sz); + + /* get margo info */ + const struct hg_info* hgi = margo_get_info(handle); + assert(NULL != hgi); + + margo_instance_id mid = margo_hg_info_get_instance(hgi); + assert(mid != MARGO_INSTANCE_NULL); + + hg_bulk_t bulk_handle; + int reqcmd = (int)SVC_CMD_INVALID; + void* reqbuf = NULL; + if (bulk_sz) { + /* allocate and register local target buffer for bulk access */ + reqbuf = malloc(bulk_sz); + if (NULL != reqbuf) { + hret = margo_bulk_create(mid, 1, &reqbuf, &in.bulk_size, + HG_BULK_WRITE_ONLY, &bulk_handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* pull request data */ + hret = margo_bulk_transfer(mid, HG_BULK_PULL, hgi->addr, + in.bulk_handle, 0, + bulk_handle, 0, in.bulk_size); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_transfer() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* first int in request buffer is the command */ + reqcmd = *(int*)reqbuf; + + /* verify this is a request for data */ + if (reqcmd == (int)SVC_CMD_RDREQ_CHK) { + /* chunk read request command */ + LOGDBG("request command: SVC_CMD_RDREQ_CHK"); + ret = sm_issue_chunk_reads(src_rank, + app_id, client_id, + req_id, num_chks, + (char*)reqbuf); + } else { + LOGERR("invalid command %d from server %d", + reqcmd, src_rank); + ret = EINVAL; + } + } + margo_bulk_free(bulk_handle); + } + free(reqbuf); + } else { + ret = ENOMEM; + } + } + margo_free_input(handle, &in); } - /* fill output structure */ + /* return output to caller */ chunk_read_request_out_t out; out.ret = ret; - - /* return output to caller */ hret = margo_respond(handle, &out); - assert(hret == HG_SUCCESS); + if (hret != HG_SUCCESS) { + LOGERR("margo_respond() failed"); + } /* free margo resources */ - margo_free_input(handle, &in); - if (NULL != reqbuf) { - margo_bulk_free(bulk_handle); - free(reqbuf); - } margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(chunk_read_request_rpc) diff --git a/server/src/unifyfs_service_manager.h b/server/src/unifyfs_service_manager.h index 9f6dd15af..a2a9c9f2a 100644 --- a/server/src/unifyfs_service_manager.h +++ b/server/src/unifyfs_service_manager.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. * Produced at the Lawrence Livermore National Laboratory. * - * Copyright 2017, UT-Battelle, LLC. + * Copyright 2020, UT-Battelle, LLC. * * LLNL-CODE-741539 * All rights reserved. @@ -33,7 +33,7 @@ #include "unifyfs_global.h" /* service manager pthread routine */ -void* sm_service_reads(void* ctx); +void* service_manager_thread(void* ctx); /* initialize and launch service manager */ int svcmgr_init(void); @@ -50,6 +50,6 @@ int sm_issue_chunk_reads(int src_rank, char* msg_buf); /* MARGO SERVER-SERVER RPC INVOCATION FUNCTIONS */ -int invoke_chunk_read_response_rpc(remote_chunk_reads_t* rcr); +int invoke_chunk_read_response_rpc(server_chunk_reads_t* scr); #endif // UNIFYFS_SERVICE_MANAGER_H diff --git a/server/src/unifyfs_tree.c b/server/src/unifyfs_tree.c new file mode 100644 index 000000000..5fa3b3c4f --- /dev/null +++ b/server/src/unifyfs_tree.c @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include +#include +#include + +#include "unifyfs_tree.h" + +/** + * @brief given the process's rank and the number of ranks, this computes a + * k-ary tree rooted at rank 0, the structure records the number of children of + * the local rank and the list of their ranks + * + * @param rank rank of calling process + * @param ranks number of ranks in tree + * @param root rank of root of tree + * @param k degree of k-ary tree + * @param t output tree structure + */ +int unifyfs_tree_init( + int rank, /* rank of calling process */ + int ranks, /* number of ranks in tree */ + int root, /* rank of root of tree */ + int k, /* degree of k-ary tree */ + unifyfs_tree_t* t) /* output tree structure */ +{ + int i; + + /* compute distance from our rank to root, + * rotate ranks to put root as rank 0 */ + rank -= root; + if (rank < 0) { + rank += ranks; + } + + /* compute parent and child ranks with root as rank 0 */ + + /* initialize fields */ + t->rank = rank; + t->ranks = ranks; + t->parent_rank = -1; + t->child_count = 0; + t->child_ranks = NULL; + + /* compute the maximum number of children this task may have */ + int max_children = k; + + /* allocate memory to hold list of children ranks */ + if (max_children > 0) { + size_t bytes = (size_t)max_children * sizeof(int); + t->child_ranks = (int*) malloc(bytes); + + if (t->child_ranks == NULL) { + //LOGERR("Failed to allocate memory for child rank array"); + return ENOMEM; + } + } + + /* initialize all ranks to NULL */ + for (i = 0; i < max_children; i++) { + t->child_ranks[i] = -1; + } + + /* compute rank of our parent if we have one */ + if (rank > 0) { + t->parent_rank = (rank - 1) / k; + } + + /* identify ranks of what would be leftmost + * and rightmost children */ + int left = rank * k + 1; + int right = rank * k + k; + + /* if we have at least one child, + * compute number of children and list of child ranks */ + if (left < ranks) { + /* adjust right child in case we don't have a full set of k */ + if (right >= ranks) { + right = ranks - 1; + } + + /* compute number of children */ + t->child_count = right - left + 1; + + /* fill in rank for each child */ + for (i = 0; i < t->child_count; i++) { + t->child_ranks[i] = left + i; + } + } + + /* rotate tree neighbor ranks to use global ranks */ + + /* rotate our rank in tree */ + t->rank += root; + if (t->rank >= ranks) { + t->rank -= ranks; + } + + /* rotate rank of our parent in tree if we have one */ + if (t->parent_rank != -1) { + t->parent_rank += root; + if (t->parent_rank >= ranks) { + t->parent_rank -= ranks; + } + } + + /* rotate rank of each child in tree */ + for (i = 0; i < t->child_count; i++) { + t->child_ranks[i] += root; + if (t->child_ranks[i] >= ranks) { + t->child_ranks[i] -= ranks; + } + } + + return 0; +} + +void unifyfs_tree_free(unifyfs_tree_t* t) +{ + /* free child rank list */ + free(t->child_ranks); + t->child_ranks = NULL; + + return; +} + diff --git a/server/src/unifyfs_tree.h b/server/src/unifyfs_tree.h new file mode 100644 index 000000000..27c474735 --- /dev/null +++ b/server/src/unifyfs_tree.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef UNIFYFS_TREE_H +#define UNIFYFS_TREE_H + +#include +#include "unifyfs_meta.h" + +/* define tree structure */ +typedef struct { + int rank; /* global rank of calling process */ + int ranks; /* number of ranks in tree */ + int parent_rank; /* parent rank, -1 if root */ + int child_count; /* number of children */ + int* child_ranks; /* list of child ranks */ +} unifyfs_tree_t; + +/* given the process's rank and the number of ranks, this computes a k-ary + * tree rooted at rank 0, the structure records the number of children + * of the local rank and the list of their ranks */ +int unifyfs_tree_init( + int rank, /* rank of calling process */ + int ranks, /* number of ranks in tree */ + int root, /* rank of root process */ + int k, /* degree of k-ary tree */ + unifyfs_tree_t* t /* output tree structure */ +); + +/* free resources allocated in unifyfs_tree_init */ +void unifyfs_tree_free(unifyfs_tree_t* t); + +#endif /* UNIFYFS_TREE_H */ diff --git a/t/9100-metadata-api.t b/t/9100-metadata-api.t deleted file mode 100755 index 4396a4238..000000000 --- a/t/9100-metadata-api.t +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -test_description="Test Metadata API" - -# -# Source sharness environment scripts to pick up test environment -# and UnifyFS runtime settings. -# -. $(dirname $0)/sharness.d/00-test-env.sh -. $(dirname $0)/sharness.d/01-unifyfs-settings.sh - -# create a new directory for MDHIM data -export UNIFYFS_META_DB_PATH=$UNIFYFS_TEST_TMPDIR/meta2 -mkdir -p $UNIFYFS_META_DB_PATH - -$JOB_RUN_COMMAND $UNIFYFS_BUILD_DIR/t/server/metadata.t diff --git a/t/Makefile.am b/t/Makefile.am index 8002659bc..abfe4491e 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -13,7 +13,6 @@ TESTS = \ 9005-unifyfs-unmount.t \ 9010-stop-unifyfsd.t \ 9020-mountpoint-empty.t \ - 9100-metadata-api.t \ 9200-seg-tree-test.t \ 9201-slotmap-test.t \ 9300-unifyfs-stage-isolated.t \ @@ -29,7 +28,6 @@ check_SCRIPTS = \ 9005-unifyfs-unmount.t \ 9010-stop-unifyfsd.t \ 9020-mountpoint-empty.t \ - 9100-metadata-api.t \ 9200-seg-tree-test.t \ 9201-slotmap-test.t \ 9300-unifyfs-stage-isolated.t \ @@ -49,7 +47,6 @@ clean-local: libexec_PROGRAMS = \ common/seg_tree_test.t \ common/slotmap_test.t \ - server/metadata.t \ std/stdio-gotcha.t \ std/stdio-static.t \ sys/sysio-gotcha.t \ @@ -81,34 +78,12 @@ test_static_ldflags = \ $(CP_WRAPPERS) \ $(MPI_CLDFLAGS) -test_metadata_ldadd = \ - $(top_builddir)/t/lib/libtap.la \ - $(top_builddir)/t/lib/libtestutil.la \ - $(top_builddir)/common/src/libunifyfs_common.la \ - $(top_builddir)/server/src/libunifyfsd.a \ - $(top_builddir)/meta/src/libmdhim.a \ - $(LEVELDB_LDFLAGS) $(LEVELDB_LIBS) \ - $(MPI_CLDFLAGS) \ - $(MERCURY_LDFLAGS) $(MERCURY_LIBS) \ - $(ARGOBOTS_LDFLAGS) $(ARGOBOTS_LIBS) \ - $(MARGO_LDFLAGS) $(MARGO_LIBS) \ - -lpthread -lm -lstdc++ -lrt - test_common_cppflags = \ -I$(top_srcdir) \ -I$(top_srcdir)/common/src \ -D_GNU_SOURCE \ $(AM_CPPFLAGS) -test_meta_cppflags = \ - -I$(top_srcdir) \ - -I$(top_srcdir)/server/src \ - -I$(top_srcdir)/common/src \ - -D_GNU_SOURCE \ - $(AM_CPPFLAGS) \ - $(MARGO_CFLAGS) \ - $(MPI_CFLAGS) - test_cppflags = \ -I$(top_srcdir) \ -I$(top_srcdir)/client/src \ @@ -178,15 +153,6 @@ std_stdio_static_t_CPPFLAGS = $(test_cppflags) std_stdio_static_t_LDADD = $(test_static_ldadd) std_stdio_static_t_LDFLAGS = $(test_static_ldflags) -server_metadata_t_SOURCES = \ - server/metadata_suite.h \ - server/metadata_suite.c \ - server/unifyfs_meta_get_test.c - -server_metadata_t_CPPFLAGS = $(test_meta_cppflags) -server_metadata_t_LDADD = $(test_metadata_ldadd) -server_metadata_t_LDFLAGS = $(AM_LDFLAGS) - unifyfs_unmount_t_SOURCES = unifyfs_unmount.c unifyfs_unmount_t_CPPFLAGS = $(test_cppflags) unifyfs_unmount_t_LDADD = $(test_static_ldadd) diff --git a/t/common/seg_tree_test.c b/t/common/seg_tree_test.c index 7176eda19..a2b19655b 100644 --- a/t/common/seg_tree_test.c +++ b/t/common/seg_tree_test.c @@ -197,6 +197,24 @@ int main(int argc, char** argv) ok(max == 150, "max is 150 (got %lu)", max); ok(count == 1, "count is 1 (got %lu)", count); + seg_tree_clear(&seg_tree); + seg_tree_add(&seg_tree, 0, 0, 0); + seg_tree_add(&seg_tree, 1, 10, 101); + seg_tree_add(&seg_tree, 20, 30, 20); + seg_tree_add(&seg_tree, 31, 40, 131); + + /* Remove a single entry */ + seg_tree_remove(&seg_tree, 0, 0); + ok(1 == 1, "removed a single entry, got %s", print_tree(tmp, &seg_tree)); + is("[1-10:101][20-30:20][31-40:131]", print_tree(tmp, &seg_tree), + "removed a single range, got %s", print_tree(tmp, &seg_tree)); + + /* Remove a range spanning the two bordering ranges [20-30] & [31-40]. */ + seg_tree_remove(&seg_tree, 25, 31); + is("[1-10:101][20-24:20][32-40:132]", print_tree(tmp, &seg_tree), + "removed a range that truncated two entries, got %s", + print_tree(tmp, &seg_tree)); + seg_tree_clear(&seg_tree); seg_tree_destroy(&seg_tree); diff --git a/t/lib/testutil.c b/t/lib/testutil.c index 54af04cc1..26373ace6 100644 --- a/t/lib/testutil.c +++ b/t/lib/testutil.c @@ -30,7 +30,7 @@ static unsigned long seed; * be run individually if need be. If they run too fast, seeding srand() with * time(NULL) can happen more than once in a second, causing the pseudo random * sequence to repeat which causes each suite to create the same random files. - * Using gettimeofday() allows us to increase the granularty to microseconds. + * Using gettimeofday() allows us to increase the granularity to microseconds. */ static void test_util_srand(void) { @@ -63,7 +63,6 @@ void testutil_rand_string(char* buf, size_t len) idx = rand() % (sizeof(charset) - 1); buf[i] = charset[idx]; } - buf[i] = '\0'; } @@ -76,6 +75,7 @@ void testutil_rand_path(char* buf, size_t len, const char* pfx) { int rc; + memset(buf, 0, len); rc = snprintf(buf, len, "%s/", pfx); testutil_rand_string(buf + rc, len - rc); } diff --git a/t/server/metadata_suite.c b/t/server/metadata_suite.c deleted file mode 100644 index 75ca51e77..000000000 --- a/t/server/metadata_suite.c +++ /dev/null @@ -1,54 +0,0 @@ -#include -#include "unifyfs_configurator.h" -#include "unifyfs_metadata.h" - -#include "t/lib/tap.h" -#include "metadata_suite.h" - -int main(int argc, char* argv[]) -{ - /* need to initialize enough of the server to use the metadata API */ - unifyfs_cfg_t server_cfg; - int rc; - - MPI_Init(&argc, &argv); - - /* get the configuration */ - rc = unifyfs_config_init(&server_cfg, argc, argv); - if (rc != 0) { - exit(1); - } - - rc = meta_init_store(&server_cfg); - if (rc != 0) { - LOG(LOG_ERR, "%s", - unifyfs_rc_enum_description(UNIFYFS_ERROR_META)); - exit(1); - } - - /* - * necessary infrastructure is initialized - * running tests - */ - - plan(NO_PLAN); - - // keep the following two calls in order - unifyfs_set_file_attribute_test(); - unifyfs_get_file_attribute_test(); - - - /* - * shut down infrastructure - */ - - // shutdown the metadata service - meta_sanitize(); - MPI_Finalize(); - - // finish the testing - // needs to be last call - done_testing(); - - return EXIT_SUCCESS; -} diff --git a/t/server/metadata_suite.h b/t/server/metadata_suite.h deleted file mode 100644 index 56bbf0ffa..000000000 --- a/t/server/metadata_suite.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2019, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyFS. - * For details, see https://github.com/LLNL/UnifyFS. - * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. - */ - - -/* This is the collection of metadata tests to be run inside of - * metadata_suite.c. These tests are testing the wrapper functions found in - * server/src/unifyfs_metadata.c. - */ - - -#ifndef METADATA_SUITE_H -#define METADATA_SUITE_H - -int unifyfs_set_file_attribute_test(void); -int unifyfs_get_file_attribute_test(void); - -#endif /* METADATA_SUITE_H */ diff --git a/t/server/unifyfs_meta_get_test.c b/t/server/unifyfs_meta_get_test.c deleted file mode 100644 index 17718245e..000000000 --- a/t/server/unifyfs_meta_get_test.c +++ /dev/null @@ -1,42 +0,0 @@ -#include - -#include "metadata_suite.h" -#include "unifyfs_meta.h" -#include "unifyfs_metadata.h" -#include "t/lib/tap.h" - -#define TEST_META_GFID_VALUE 0xbeef -#define TEST_META_FID_VALUE 0xfeed -#define TEST_META_FILE "/unifyfs/filename/to/nowhere" - -int unifyfs_set_file_attribute_test(void) -{ - int rc; - - /* create dummy file attribute */ - unifyfs_file_attr_t fattr = {0}; - - fattr.gfid = TEST_META_GFID_VALUE; - snprintf(fattr.filename, sizeof(fattr.filename), TEST_META_FILE); - fflush(NULL); - - rc = unifyfs_set_file_attribute(1, 1, &fattr); - ok(UNIFYFS_SUCCESS == rc, "Stored file attribute"); - fflush(NULL); - return 0; -} - -int unifyfs_get_file_attribute_test(void) -{ - int rc; - unifyfs_file_attr_t fattr; - - rc = unifyfs_get_file_attribute(TEST_META_GFID_VALUE, &fattr); - ok(UNIFYFS_SUCCESS == rc && - TEST_META_GFID_VALUE == fattr.gfid && - (0 == strcmp(fattr.filename, TEST_META_FILE)), - "Retrieve file attributes (rc = %d, gfid = 0x%02X)", - rc, fattr.gfid - ); - return 0; -} diff --git a/t/std/fflush.c b/t/std/fflush.c index 27d1ba6cd..1daa146b7 100644 --- a/t/std/fflush.c +++ b/t/std/fflush.c @@ -33,7 +33,6 @@ int fflush_test(char* unifyfs_root) /* Generate a random file name in the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); - skip(1, 9, "remove when fflush() sync extents (issue #374)"); /* Write "hello world" to a file */ fp = fopen(path, "w"); @@ -65,7 +64,7 @@ int fflush_test(char* unifyfs_root) rc = fclose(fp); ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); - end_skip; + //end_skip; return 0; } diff --git a/t/std/fopen-fclose.c b/t/std/fopen-fclose.c index 6d3202250..9f9d9d850 100644 --- a/t/std/fopen-fclose.c +++ b/t/std/fopen-fclose.c @@ -34,52 +34,76 @@ int fopen_fclose_test(char* unifyfs_root) char path[64]; char path2[64]; FILE* fd = NULL; + int err, rc; - errno = 0; /* Generate a random file name in the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); testutil_rand_path(path2, sizeof(path2), unifyfs_root); /* Verify fopen a non-existent file as read-only fails with errno=ENOENT. */ + errno = 0; fd = fopen(path, "r"); - ok(fd == NULL && errno == ENOENT, + err = errno; + ok(fd == NULL && err == ENOENT, "%s:%d fopen non-existent file %s w/ mode r: %s", - __FILE__, __LINE__, path, strerror(errno)); - errno = 0; /* Reset errno after test for failure */ + __FILE__, __LINE__, path, strerror(err)); /* Verify we can create a new file. */ + errno = 0; fd = fopen(path, "w"); - ok(fd != NULL, "%s:%d fopen non-existing file %s w/ mode w: %s", - __FILE__, __LINE__, path, strerror(errno)); + err = errno; + ok(fd != NULL && err == 0, + "%s:%d fopen non-existing file %s w/ mode w: %s", + __FILE__, __LINE__, path, strerror(err)); /* Verify close succeeds. */ - ok(fclose(fd) == 0, "%s:%d fclose new file: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = fclose(fd); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fclose new file: %s", + __FILE__, __LINE__, strerror(err)); /* Verify we can create a new file with mode "a". */ + errno = 0; fd = fopen(path2, "a"); - ok(fd != NULL, "%s:%d fopen non-existing file %s mode a: %s", - __FILE__, __LINE__, path2, strerror(errno)); + err = errno; + ok(fd != NULL && err == 0, + "%s:%d fopen non-existing file %s mode a: %s", + __FILE__, __LINE__, path2, strerror(err)); /* Verify close succeeds. */ - ok(fclose(fd) == 0, "%s:%d fclose new file: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = fclose(fd); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fclose new file: %s", + __FILE__, __LINE__, strerror(err)); /* Verify opening an existing file with mode "r" succeeds. */ + errno = 0; fd = fopen(path, "r"); - ok(fd != NULL, "%s:%d fopen existing file %s mode r: %s", - __FILE__, __LINE__, path, strerror(errno)); + err = errno; + ok(fd != NULL && err == 0, + "%s:%d fopen existing file %s mode r: %s", + __FILE__, __LINE__, path, strerror(err)); /* Verify close succeeds. */ - ok(fclose(fd) == 0, "%s:%d fclose worked: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = fclose(fd); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fclose worked: %s", + __FILE__, __LINE__, strerror(err)); /* Verify closing already closed file fails with errno=EBADF */ - ok(fclose(fd) == -1 && errno == EBADF, - "%s:%d fclose already closed file %s should fail (errno=%d): %s", - __FILE__, __LINE__, path, errno, strerror(errno)); errno = 0; + rc = fclose(fd); + err = errno; + ok(rc == -1 && err == EBADF, + "%s:%d fclose already closed file %s should fail (errno=%d): %s", + __FILE__, __LINE__, path, err, strerror(err)); diag("Finished UNIFYFS_WRAP(fopen/fclose) tests"); diff --git a/t/std/fseek-ftell.c b/t/std/fseek-ftell.c index a27adb914..537dd46c3 100644 --- a/t/std/fseek-ftell.c +++ b/t/std/fseek-ftell.c @@ -34,8 +34,7 @@ int fseek_ftell_test(char* unifyfs_root) char path[64]; FILE* fp = NULL; - - errno = 0; + int err, rc; /* Create a random file at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); @@ -43,173 +42,282 @@ int fseek_ftell_test(char* unifyfs_root) skip(1, 3, "causing a hang on some architectures. Try after future update"); /* fseek on bad file stream should fail with errno=EBADF */ dies_ok({ fseek(fp, 0, SEEK_SET); }, - "%s:%d fseek on bad file stream segfaults: %s", - __FILE__, __LINE__, strerror(errno)); + "%s:%d fseek on bad file stream segfaults", + __FILE__, __LINE__); /* ftell on non-open file stream should fail with errno=EBADF * variable declaration and `ok` test are to avoid a compiler warning */ dies_ok({ int rc = ftell(fp); ok(rc > 0); }, - "%s:%d ftell on bad file stream segfaults: %s", - __FILE__, __LINE__, strerror(errno)); + "%s:%d ftell on bad file stream segfaults", + __FILE__, __LINE__); /* rewind on non-open file stream should fail with errno=EBADF */ - dies_ok({ rewind(fp); }, "%s:%d rewind on bad file stream segfaults: %s", - __FILE__, __LINE__, strerror(errno)); + dies_ok({ rewind(fp); }, "%s:%d rewind on bad file stream segfaults", + __FILE__, __LINE__); end_skip; /* Open a file and write to it to test fseek() */ + errno = 0; fp = fopen(path, "w"); - ok(fp != NULL, "%s:%d fopen(%s): %s", - __FILE__, __LINE__, path, strerror(errno)); - ok(fwrite("hello world", 12, 1, fp) == 1, "%s:%d fwrite() to file %s: %s", - __FILE__, __LINE__, path, strerror(errno)); + err = errno; + ok(fp != NULL && err == 0, "%s:%d fopen(%s): %s", + __FILE__, __LINE__, path, strerror(err)); + + errno = 0; + rc = (int) fwrite("hello world", 12, 1, fp); + err = errno; + ok(rc == 1 && err == 0, + "%s:%d fwrite() to file %s: %s", + __FILE__, __LINE__, path, strerror(err)); /* fseek with invalid whence fails with errno=EINVAL. */ - ok(fseek(fp, 0, -1) == -1 && errno == EINVAL, + errno = 0; + rc = (int) fseek(fp, 0, -1); + err = errno; + ok(rc == -1 && err == EINVAL, "%s:%d fseek with invalid whence should fail (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - errno = 0; /* Reset errno after test for failure */ + __FILE__, __LINE__, err, strerror(err)); + + /*--- fseek() with SEEK_SET tests ---*/ - /* fseek() with SEEK_SET tests */ /* fseek to negative offset with SEEK_SET should fail with errno=EINVAL */ - ok(fseek(fp, -1, SEEK_SET) == -1 && errno == EINVAL, - "%s:%d fseek(-1) to invalid offset w/ SEEK_SET fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) fseek(fp, -1, SEEK_SET); + err = errno; + ok(rc == -1 && err == EINVAL, + "%s:%d fseek(-1) to invalid offset w/ SEEK_SET fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* ftell after invalid fseek should return last offset */ - ok(ftell(fp) == 12, + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 12 && err == 0, "%s:%d ftell after fseek(-1) to invalid offset w/ SEEK_SET: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* fseek to valid offset with SEEK_SET succeeds */ - ok(fseek(fp, 7, SEEK_SET) == 0, + errno = 0; + rc = (int) fseek(fp, 7, SEEK_SET); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fseek(7) to valid offset w/ SEEK_SET: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* ftell after valid fseek with SEEK_SET */ - ok(ftell(fp) == 7, "%s:%d ftell after fseek(7) w/ SEEK_SET: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 7 && err == 0, + "%s:%d ftell after fseek(7) w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(err)); /* fseek beyond end of file with SEEK_SET succeeds */ - ok(fseek(fp, 25, SEEK_SET) == 0, "%s:%d fseek(25) past EOF w/ SEEK_SET: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) fseek(fp, 25, SEEK_SET); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fseek(25) past EOF w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(err)); /* ftell after fseek beyond end of file with SEEK_SET */ - ok(ftell(fp) == 25, "%s:%d ftell after fseek(25) w/ SEEK_SET: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 25 && err == 0, + "%s:%d ftell after fseek(25) w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(err)); /* fseek to beginning of file with SEEK_SET succeeds */ - ok(fseek(fp, 0, SEEK_SET) == 0, "%s:%d fseek(0) w/ SEEK_SET: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) fseek(fp, 0, SEEK_SET); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fseek(0) w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(err)); /* ftell after fseek to beginning of file with SEEK_SET */ - ok(ftell(fp) == 0, "%s:%d ftell after fseek(0) w/ SEEK_SET: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d ftell after fseek(0) w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(err)); + + /*--- fseek() with SEEK_CUR tests ---*/ - /* fseek() with SEEK_CUR tests */ /* fseek to end of file with SEEK_CUR succeeds */ - ok(fseek(fp, 12, SEEK_CUR) == 0, "%s:%d fseek(12) to EOF w/ SEEK_CUR: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) fseek(fp, 12, SEEK_CUR); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fseek(12) to EOF w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(err)); /* ftell after fseek to end of file with SEEK_CUR */ - ok(ftell(fp) == 12, "%s:%d ftell after fseek(12) w/ SEEK_CUR: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 12 && err == 0, + "%s:%d ftell after fseek(12) w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(err)); /* fseek to negative offset with SEEK_CUR should fail with errno=EINVAL */ - ok(fseek(fp, -15, SEEK_CUR) == -1 && errno == EINVAL, - "%s:%d fseek(-15) to invalid offset w/ SEEK_CUR fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) fseek(fp, -15, SEEK_CUR); + err = errno; + ok(rc == -1 && err == EINVAL, + "%s:%d fseek(-15) to invalid offset w/ SEEK_CUR fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* ftell after fseek to negative offset with SEEK_CUR */ - ok(ftell(fp) == 12, + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 12 && err == 0, "%s:%d ftell after fseek(-15) to invalid offset w/ SEEK_CUR: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* fseek to beginning of file with SEEK_CUR succeeds */ - ok(fseek(fp, -12, SEEK_CUR) == 0, + errno = 0; + rc = (int) fseek(fp, -12, SEEK_CUR); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fseek(-12) to beginning of file w/ SEEK_CUR: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* ftell after fseek to beginning of file with SEEK_CUR */ - ok(ftell(fp) == 0, + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 0 && err == 0, "%s:%d ftell after fseek(-12) to beginning of file w/ SEEK_CUR: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* fseek beyond end of file with SEEK_CUR succeeds */ - ok(fseek(fp, 25, SEEK_CUR) == 0, "%s:%d fseek(25) past EOF w/ SEEK_CUR: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) fseek(fp, 25, SEEK_CUR); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fseek(25) past EOF w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(err)); /* ftell after fseek beyond end of file with SEEK_CUR */ - ok(ftell(fp) == 25, "%s:%d ftell after fseek(25) past EOF w/ SEEK_CUR: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 25 && err == 0, + "%s:%d ftell after fseek(25) past EOF w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(err)); + + /*--- rewind test ---*/ - /* rewind test */ /* ftell after rewind reports beginning of file */ rewind(fp); - ok(ftell(fp) == 0, "%s:%d ftell after rewind reports beginning of file: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d ftell after rewind reports beginning of file: %s", + __FILE__, __LINE__, strerror(err)); + + /*--- fseek() with SEEK_END tests ---*/ - /* fseek() with SEEK_END tests */ /* fseek to negative offset with SEEK_END should fail with errno=EINVAL */ - ok(fseek(fp, -15, SEEK_END) == -1 && errno == EINVAL, - "%s:%d fseek(-15) to invalid offset w/ SEEK_END fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) fseek(fp, -15, SEEK_END); + err = errno; + ok(rc == -1 && err == EINVAL, + "%s:%d fseek(-15) to invalid offset w/ SEEK_END fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* ftell after fseek to negative offset with SEEK_END */ - ok(ftell(fp) == 0, + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 0 && err == 0, "%s:%d ftell after fseek(-15) to negative offset w/ SEEK_END: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* fseek back one from end of file with SEEK_END succeeds */ - ok(fseek(fp, -1, SEEK_END) == 0, "%s:%d fseek(-1) from EOF w/ SEEK_END: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) fseek(fp, -1, SEEK_END); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fseek(-1) from EOF w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(err)); /* ftell after fseek back one from end of file with SEEK_END */ - ok(ftell(fp) == 11, "%s:%d ftell after fseek(-1) from end w/ SEEK_END: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 11 && err == 0, + "%s:%d ftell after fseek(-1) from end w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(err)); /* fseek to beginning of file with SEEK_END succeeds */ - ok(fseek(fp, -12, SEEK_END) == 0, + errno = 0; + rc = (int) fseek(fp, -12, SEEK_END); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fseek(-12) to beginning of file w/ SEEK_END: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* ftell after fseek to beginning of file with SEEK_END */ - ok(ftell(fp) == 0, + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 0 && err == 0, "%s:%d ftell after fseek(-12) to beginning of file w/ SEEK_END: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* fseek beyond end of file with SEEK_END succeeds */ - ok(fseek(fp, 25, SEEK_END) == 0, "%s:%d fseek(25) past EOF w/ SEEK_END: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) fseek(fp, 25, SEEK_END); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fseek(25) past EOF w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(err)); /* ftell after fseek beyond end of file with SEEK_END */ - ok(ftell(fp) == 37, "%s:%d ftell after fseek(25) past EOF w/ SEEK_END: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == 37 && err == 0, + "%s:%d ftell after fseek(25) past EOF w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(err)); - ok(fclose(fp) == 0, "%s:%d fclose(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = fclose(fp); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fclose(): %s", + __FILE__, __LINE__, strerror(err)); + + /*--- non-open file stream tests ---*/ /* fseek in non-open file stream should fail with errno=EBADF */ - ok(fseek(fp, 0, SEEK_SET) == -1 && errno == EBADF, - "%s:%d fseek in non-open file stream fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) fseek(fp, 0, SEEK_SET); + err = errno; + ok(rc == -1 && err == EBADF, + "%s:%d fseek in non-open file stream fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* ftell on non-open file stream should fail with errno=EBADF */ - ok(ftell(fp) == -1 && errno == EBADF, - "%s:%d ftell on non-open file stream fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) ftell(fp); + err = errno; + ok(rc == -1 && err == EBADF, + "%s:%d ftell on non-open file stream fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* rewind on non-open file stream should fail with errno=EBADF */ + errno = 0; rewind(fp); - ok(errno == EBADF, + err = errno; + ok(err == EBADF, "%s:%d rewind on non-open file stream fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - errno = 0; + __FILE__, __LINE__, err, strerror(err)); diag("Finished UNIFYFS_WRAP(fseek/ftell/rewind) tests"); diff --git a/t/std/fwrite-fread.c b/t/std/fwrite-fread.c index 125e41811..4da434e73 100644 --- a/t/std/fwrite-fread.c +++ b/t/std/fwrite-fread.c @@ -16,11 +16,11 @@ * Test fwrite/fread/fgets/rewind/feof/chmod */ -#include -#include #include +#include #include #include +#include #include #include "t/lib/tap.h" #include "t/lib/testutil.h" @@ -34,169 +34,255 @@ int fwrite_fread_test(char* unifyfs_root) char* tmp; FILE* fp = NULL; int fd = 0; - - errno = 0; + int err, rc; testutil_rand_path(path, sizeof(path), unifyfs_root); - skip(1, 4, "causing a hang on some architectures. Try after future update"); + skip(1, 4, "causing hang on some architectures. Try after future update"); /* fwrite to bad FILE stream should segfault */ dies_ok({ FILE* p = fopen("/tmp/fakefile", "r"); fwrite("hello world", 12, 1, p); }, - "%s:%d fwrite() to bad FILE stream segfaults: %s", - __FILE__, __LINE__, strerror(errno)); + "%s:%d fwrite() to bad FILE stream segfaults", + __FILE__, __LINE__); /* fread from bad FILE stream should segfault */ dies_ok({ size_t rc = fread(buf, 15, 1, fp); ok(rc > 0); }, - "%s:%d fread() from bad FILE stream segfaults: %s", - __FILE__, __LINE__, strerror(errno)); + "%s:%d fread() from bad FILE stream segfaults", + __FILE__, __LINE__); /* fgets from bad FILE stream segfaults */ memset(buf, 0, sizeof(buf)); dies_ok({ char* tmp = fgets(buf, 15, fp); ok(tmp != NULL); }, - "%s:%d fgets() from bad FILE stream segfaults: %s", - __FILE__, __LINE__, strerror(errno)); + "%s:%d fgets() from bad FILE stream segfaults", + __FILE__, __LINE__); dies_ok({ int rc = feof(fp); ok(rc > 0); }, - "%s:%d feof() on bad FILE stream segfaults: %s", - __FILE__, __LINE__, strerror(errno)); + "%s:%d feof() on bad FILE stream segfaults", + __FILE__, __LINE__); end_skip; /* Write "hello world" to a file */ + errno = 0; fp = fopen(path, "w"); - ok(fp != NULL, "%s:%d fopen(%s): %s", - __FILE__, __LINE__, path, strerror(errno)); - ok(fwrite("hello world", 12, 1, fp) == 1, + err = errno; + ok(fp != NULL && err == 0, "%s:%d fopen(%s): %s", + __FILE__, __LINE__, path, strerror(err)); + + errno = 0; + rc = (int) fwrite("hello world", 12, 1, fp); + err = errno; + ok(rc == 1 && err == 0, "%s:%d fwrite(\"hello world\") to file %s: %s", - __FILE__, __LINE__, path, strerror(errno)); + __FILE__, __LINE__, path, strerror(err)); /* Seek to offset and overwrite "world" with "universe" */ - ok(fseek(fp, 6, SEEK_SET) == 0, "%s:%d fseek(6) to \"world\": %s", - __FILE__, __LINE__, strerror(errno)); - ok(fwrite("universe", 9, 1, fp) == 1, + errno = 0; + rc = fseek(fp, 6, SEEK_SET); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fseek(6) to \"world\": %s", + __FILE__, __LINE__, strerror(err)); + + errno = 0; + rc = (int) fwrite("universe", 9, 1, fp); + err = errno; + ok(rc == 1 && err == 0, "%s:%d overwrite \"world\" at offset 6 with \"universe\" : %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* fread from file open as write-only should fail with errno=EBADF */ - ok(fseek(fp, 0, SEEK_SET) == 0, "%s:%d fseek(0): %s", - __FILE__, __LINE__, strerror(errno)); - ok(fread(buf, 15, 1, fp) == 0 && errno == EBADF, + errno = 0; + rc = fseek(fp, 0, SEEK_SET); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fseek(0): %s", + __FILE__, __LINE__, strerror(err)); + + errno = 0; + rc = (int) fread(buf, 15, 1, fp); + err = errno; + ok(rc == 0 && err == EBADF, "%s:%d fread() from file open as write-only should fail (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - errno = 0; /* reset errno after test for failure */ + __FILE__, __LINE__, err, strerror(err)); - ok(fclose(fp) == 0, "%s:%d fclose(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = fclose(fp); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fclose(): %s", + __FILE__, __LINE__, strerror(err)); /* fsync() tests */ /* fsync on bad file descriptor should fail with errno=EINVAL */ - ok(fsync(fd) == -1 && errno == EINVAL, - "%s:%d fsync() on bad file descriptor should fail (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = fsync(fd); + err = errno; + ok(rc == -1 && err == EINVAL, + "%s:%d fsync() on bad file descriptor should fail (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* Sync extents */ + errno = 0; fd = open(path, O_RDWR); - ok(fd != -1, "%s:%d open() (fd=%d): %s", - __FILE__, __LINE__, fd, strerror(errno)); - ok(fsync(fd) == 0, "%s:%d fsync(): %s", - __FILE__, __LINE__, strerror(errno)); - ok(close(fd) == 0, "%s:%d close(): %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(fd != -1 && err == 0, "%s:%d open() (fd=%d): %s", + __FILE__, __LINE__, fd, strerror(err)); + + errno = 0; + rc = fsync(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fsync(): %s", + __FILE__, __LINE__, strerror(err)); + + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close(): %s", + __FILE__, __LINE__, strerror(err)); /* fsync on non-open file should fail with errno=EBADF */ - ok(fsync(fd) == -1 && errno == EBADF, - "%s:%d fsync() on non-open file should fail (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = fsync(fd); + err = errno; + ok(rc == -1 && err == EBADF, + "%s:%d fsync() on non-open file should fail (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* Laminate */ - ok(chmod(path, 0444) == 0, "%s:%d chmod(0444): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = chmod(path, 0444); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chmod(0444): %s", + __FILE__, __LINE__, strerror(err)); /* fopen a laminated file for write should fail with errno=EROFS */ + errno = 0; fp = fopen(path, "w"); - ok(fp == NULL && errno == EROFS, + err = errno; + ok(fp == NULL && err == EROFS, "%s:%d fopen laminated file for write fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - errno = 0; + __FILE__, __LINE__, err, strerror(err)); /* fread() tests */ + errno = 0; fp = fopen(path, "r"); - ok(fp != NULL, "%s:%d fopen(%s): %s", - __FILE__, __LINE__, path, strerror(errno)); + err = errno; + ok(fp != NULL && err == 0, "%s:%d fopen(%s): %s", + __FILE__, __LINE__, path, strerror(err)); /* fwrite to file open as read-only should fail with errno=EBADF */ - ok(fwrite("hello world", 12, 1, fp) == 0 && errno == EBADF, - "%s:%d fwrite() to file open for read-only fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) fwrite("hello world", 12, 1, fp); + err = errno; + ok(rc == 0 && err == EBADF, + "%s:%d fwrite() to file open for read-only fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); - ok(fread(buf, 15, 1, fp) == 1, "%s:%d fread() buf[]=\"%s\": %s", - __FILE__, __LINE__, buf, strerror(errno)); + errno = 0; + rc = (int) fread(buf, 15, 1, fp); + err = errno; + ok(rc == 1 && err == 0, + "%s:%d fread() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(err)); is(buf, "hello universe", "%s:%d fread() saw \"hello universe\"", __FILE__, __LINE__); - ok(fseek(fp, 6, SEEK_SET) == 0, "%s:%d fseek(6): %s", - __FILE__, __LINE__, strerror(errno)); - ok(fread(buf, 9, 1, fp) == 1, "%s:%d fread() at offset 6 buf[]=\"%s\": %s", - __FILE__, __LINE__, buf, strerror(errno)); + errno = 0; + rc = fseek(fp, 6, SEEK_SET); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d fseek(6): %s", + __FILE__, __LINE__, strerror(err)); + + errno = 0; + rc = (int) fread(buf, 9, 1, fp); + err = errno; + ok(rc == 1 && err == 0, + "%s:%d fread() at offset 6 buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(err)); is(buf, "universe", "%s:%d fread() saw \"universe\"", __FILE__, __LINE__); rewind(fp); - ok(fread(buf, 15, 1, fp) == 1, + errno = 0; + rc = (int) fread(buf, 15, 1, fp); + err = errno; + ok(rc == 1 && err == 0, "%s:%d fread() after rewind() buf[]=\"%s\": %s", - __FILE__, __LINE__, buf, strerror(errno)); + __FILE__, __LINE__, buf, strerror(err)); is(buf, "hello universe", "%s:%d fread() saw \"hello universe\"", __FILE__, __LINE__); /* fgets() tests */ rewind(fp); memset(buf, 0, sizeof(buf)); + errno = 0; tmp = fgets(buf, 15, fp); - ok(tmp == buf, "%s:%d fgets() after rewind() buf[]=\"%s\": %s", - __FILE__, __LINE__, buf, strerror(errno)); + err = errno; + ok(tmp == buf && err == 0, + "%s:%d fgets() after rewind() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(err)); is(buf, "hello universe", "%s:%d fgets() saw \"hello universe\"", __FILE__, __LINE__); rewind(fp); memset(buf, 0, sizeof(buf)); + errno = 0; tmp = fgets(buf, 6, fp); - ok(tmp == buf, "%s:%d fgets() w/ size = 6 after rewind() buf[]=\"%s\": %s", - __FILE__, __LINE__, buf, strerror(errno)); + err = errno; + ok(tmp == buf && err == 0, + "%s:%d fgets() w/ size = 6 after rewind() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(err)); is(buf, "hello", "%s:%d fgets() saw \"hello\"", __FILE__, __LINE__); rewind(fp); - ok(fread(buf, sizeof(buf), 1, fp) != 1, "%s:%d fread() EOF: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) fread(buf, sizeof(buf), 1, fp); + err = errno; + ok(rc != 1 && err == 0, + "%s:%d fread() EOF: %s", + __FILE__, __LINE__, strerror(err)); - ok(feof(fp) != 0, "%s:%d feof() past EOF: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = feof(fp); + err = errno; + ok(rc != 0 && err == 0, "%s:%d feof() past EOF: %s", + __FILE__, __LINE__, strerror(err)); - ok(fclose(fp) == 0, "%s:%d fclose(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = fclose(fp); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fclose(): %s", + __FILE__, __LINE__, strerror(err)); /* fwrite to closed stream fails with errno=EBADF */ - ok(fwrite("hello world", 12, 1, fp) == 0 && errno == EBADF, - "%s:%d fwrite() to closed stream fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) fwrite("hello world", 12, 1, fp); + err = errno; + ok(rc == 0 && err == EBADF, + "%s:%d fwrite() to closed stream fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* fread from closed stream fails with errno=EBADF */ - ok(fread(buf, 15, 1, fp) == 0 && errno == EBADF, - "%s:%d fread() from closed stream fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) fread(buf, 15, 1, fp); + err = errno; + ok(rc == 0 && err == EBADF, + "%s:%d fread() from closed stream fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* fgets from closed stream fails with errno=EBADF */ memset(buf, 0, sizeof(buf)); - tmp = fgets(buf, 15, fp); - ok(errno == EBADF, "%s:%d fgets() from closed stream fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + tmp = fgets(buf, 15, fp); + err = errno; + ok(err == EBADF, + "%s:%d fgets() from closed stream fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); - ok(feof(fp) != 0, "%s:%d feof() on closed stream: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = feof(fp); + err = errno; + ok(rc != 0 && err == 0, + "%s:%d feof() on closed stream: %s", + __FILE__, __LINE__, strerror(err)); diag("Finished UNIFYFS_WRAP(fwrite/fread/fgets/feof) tests"); diff --git a/t/sys/chdir.c b/t/sys/chdir.c index d7685e890..bf3f4add1 100644 --- a/t/sys/chdir.c +++ b/t/sys/chdir.c @@ -34,51 +34,59 @@ int chdir_test(char* unifyfs_root) "get_current_dir_name) tests"); char path[64]; - int rc; + int err, rc; char* str; - errno = 0; - testutil_rand_path(path, sizeof(path), unifyfs_root); /* define a dir1 subdirectory within unifyfs space */ char buf[64] = {0}; snprintf(buf, sizeof(buf), "%s/dir1", unifyfs_root); - rc = mkdir(buf, 0700); - ok(rc == 0 || errno == EEXIST, "%s:%d mkdir(%s): %s", - __FILE__, __LINE__, buf, strerror(errno)); errno = 0; + rc = mkdir(buf, 0700); + err = errno; + ok(rc == 0 || err == EEXIST, "%s:%d mkdir(%s): %s", + __FILE__, __LINE__, buf, strerror(err)); /* define a dir2 subdirectory within unifyfs space */ char buf2[64] = {0}; snprintf(buf2, sizeof(buf2), "%s/dir2", unifyfs_root); - rc = mkdir(buf2, 0700); - ok(rc == 0 || errno == EEXIST, "%s:%d mkdir(%s): %s", - __FILE__, __LINE__, buf2, strerror(errno)); errno = 0; + rc = mkdir(buf2, 0700); + err = errno; + ok(rc == 0 || err == EEXIST, "%s:%d mkdir(%s): %s", + __FILE__, __LINE__, buf2, strerror(err)); /* change to root directory */ + errno = 0; rc = chdir("/"); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, "/", strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "/", strerror(err)); /* check that we're in root directory */ + errno = 0; str = getcwd(path, sizeof(path)); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(err)); is(str, "/", "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, "/"); /* change to unifyfs root directory */ + errno = 0; rc = chdir(unifyfs_root); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, unifyfs_root, strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, unifyfs_root, strerror(err)); /* check that we're in unifyfs root directory */ + errno = 0; str = getcwd(path, sizeof(path)); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(err)); is(str, unifyfs_root, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); @@ -88,96 +96,120 @@ int chdir_test(char* unifyfs_root) /* try getcwd with a buffer short by one byte, * should fail with ERANGE */ - str = getcwd(path, len); // pass - ok(str == NULL && errno == ERANGE, - "%s:%d getcwd(buf, %d): expected NULL got %s errno %s", - __FILE__, __LINE__, len, str, strerror(errno)); errno = 0; + str = getcwd(path, len); // pass + err = errno; + ok(str == NULL && err == ERANGE, + "%s:%d getcwd(buf, short_len): (errno=%d) %s", + __FILE__, __LINE__, err, strerror(err)); - /* try getcwd with a NULL buffer */ + /* try getcwd with a NULL buffer (Linux glibc extension to POSIX) */ + errno = 0; str = getcwd(NULL, sizeof(path)); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd(NULL, good_len): (errno=%d) %s", + __FILE__, __LINE__, err, strerror(err)); if (str != NULL) { free(str); } /* try getcwd with a NULL buffer but short size, * should fail with ERANGE */ + errno = 0; str = getcwd(NULL, len); - ok(str == NULL && errno == ERANGE, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str == NULL && err == ERANGE, + "%s:%d getcwd(NULL, short_len): (errno=%d) %s", + __FILE__, __LINE__, err, strerror(err)); if (str != NULL) { free(str); } - errno = 0; /* try getcwd with a buffer and 0 size, should fail with EINVAL */ + errno = 0; str = getcwd(path, 0); - ok(str == NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str == NULL && err == EINVAL, "%s:%d getcwd(buf, 0): (errno=%d) %s", + __FILE__, __LINE__, err, strerror(err)); if (str != NULL) { free(str); } - errno = 0; /* try getcwd with a NULL buffer and 0 size, - * getcwd should allocate buffer */ + * getcwd should allocate buffer (Linux glibc extension to POSIX) */ + errno = 0; str = getcwd(NULL, 0); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd(NULL, 0): (errno=%d) %s", + __FILE__, __LINE__, err, strerror(err)); if (str != NULL) { free(str); } /* change to unifyfs/dir1 */ + errno = 0; rc = chdir(buf); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, buf, strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, buf, strerror(err)); /* check that we're in unifyfs/dir1 */ + errno = 0; str = getcwd(path, sizeof(path)); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(err)); is(str, buf, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, buf); /* change back to root unifyfs directory */ + errno = 0; rc = chdir(".."); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, "..", strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "..", strerror(err)); /* check that we're in root unifyfs directory */ + errno = 0; str = getcwd(path, sizeof(path)); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(err)); is(str, unifyfs_root, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); /* change to unifyfs/dir1 directory using relative path */ + errno = 0; rc = chdir("dir1"); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, "dir1", strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "dir1", strerror(err)); /* check that we're in unifyfs/dir1 directory */ + errno = 0; str = getcwd(path, sizeof(path)); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(err)); is(str, buf, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, buf); /* change to unifyfs/dir2 directory in strange way */ + errno = 0; rc = chdir("././.././dir2"); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, "dir1", strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "dir1", strerror(err)); /* check that we're in unifyfs/dir2 directory */ + errno = 0; str = getcwd(path, sizeof(path)); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(err)); is(str, buf2, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, buf2); @@ -188,67 +220,87 @@ int chdir_test(char* unifyfs_root) * these tests. For now, I'll leave this here as a reminder. */ #if 0 /* change to root directory */ + errno = 0; rc = chdir("/"); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, "/", strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "/", strerror(err)); /* check that we're in root directory */ + errno = 0; str = getwd(pathmax); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(err)); is(str, "/", "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, "/"); /* change to unifyfs root directory */ + errno = 0; rc = chdir(unifyfs_root); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, unifyfs_root, strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, unifyfs_root, strerror(err)); /* check that we're in unifyfs root directory */ + errno = 0; str = getwd(pathmax); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(err)); is(str, unifyfs_root, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); /* change to directory within unifyfs */ + errno = 0; rc = chdir(buf); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, buf, strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, buf, strerror(err)); /* check that we're in directory within unifyfs */ + errno = 0; str = getwd(pathmax); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(err)); is(str, buf, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, buf); /* change back to root unifyfs directory */ + errno = 0; rc = chdir(".."); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, "..", strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "..", strerror(err)); /* check that we're in root unifyfs directory */ + errno = 0; str = getwd(pathmax); - ok(str != NULL, "%s:%d getcwd: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d getcwd: %s", + __FILE__, __LINE__, strerror(err)); is(str, unifyfs_root, "%s:%d getcwd returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); #endif /* change to root directory */ + errno = 0; rc = chdir("/"); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, "/", strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "/", strerror(err)); /* check that we're in root directory */ + errno = 0; str = get_current_dir_name(); - ok(str != NULL, "%s:%d get_current_dir_name: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d get_current_dir_name: %s", + __FILE__, __LINE__, strerror(err)); is(str, "/", "%s:%d get_current_dir_name returned %s expected %s", __FILE__, __LINE__, str, "/"); @@ -257,14 +309,18 @@ int chdir_test(char* unifyfs_root) } /* change to unifyfs root directory */ + errno = 0; rc = chdir(unifyfs_root); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, unifyfs_root, strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, unifyfs_root, strerror(err)); /* check that we're in unifyfs root directory */ + errno = 0; str = get_current_dir_name(); - ok(str != NULL, "%s:%d get_current_dir_name: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d get_current_dir_name: %s", + __FILE__, __LINE__, strerror(err)); is(str, unifyfs_root, "%s:%d get_current_dir_name returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); @@ -273,14 +329,18 @@ int chdir_test(char* unifyfs_root) } /* change to unifyfs/dir1 */ + errno = 0; rc = chdir(buf); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, buf, strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, buf, strerror(err)); /* check that we're in unifyfs/dir1 */ + errno = 0; str = get_current_dir_name(); - ok(str != NULL, "%s:%d get_current_dir_name: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d get_current_dir_name: %s", + __FILE__, __LINE__, strerror(err)); is(str, buf, "%s:%d get_current_dir_name returned %s expected %s", __FILE__, __LINE__, str, buf); @@ -289,14 +349,18 @@ int chdir_test(char* unifyfs_root) } /* change back to root unifyfs directory */ + errno = 0; rc = chdir(".."); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, "..", strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "..", strerror(err)); /* check that we're in root unifyfs directory */ + errno = 0; str = get_current_dir_name(); - ok(str != NULL, "%s:%d get_current_dir_name: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d get_current_dir_name: %s", + __FILE__, __LINE__, strerror(err)); is(str, unifyfs_root, "%s:%d get_current_dir_name returned %s expected %s", __FILE__, __LINE__, str, unifyfs_root); @@ -305,14 +369,18 @@ int chdir_test(char* unifyfs_root) } /* change to unifyfs/dir2 directory */ + errno = 0; rc = chdir("dir2"); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, "dir2", strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "dir2", strerror(err)); /* check that we're in unifyfs/dir2 */ + errno = 0; str = get_current_dir_name(); - ok(str != NULL, "%s:%d get_current_dir_name: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(str != NULL && err == 0, "%s:%d get_current_dir_name: %s", + __FILE__, __LINE__, strerror(err)); is(str, buf2, "%s:%d get_current_dir_name returned %s expected %s", __FILE__, __LINE__, str, buf2); @@ -325,39 +393,53 @@ int chdir_test(char* unifyfs_root) * but when they do, we should check that fchdir works. */ #if 0 /* change to root directory */ + errno = 0; rc = chdir("/"); - ok(rc == 0, "%s:%d chdir(%s): %s", - __FILE__, __LINE__, "/", strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chdir(%s): %s", + __FILE__, __LINE__, "/", strerror(err)); /* open a directory in unifyfs */ + errno = 0; DIR* dirp = opendir(buf); - ok(dirp != NULL, "%s:%d opendir(%s): %s", - __FILE__, __LINE__, buf, strerror(errno)); + err = errno; + ok(dirp != NULL && err == 0, "%s:%d opendir(%s): %s", + __FILE__, __LINE__, buf, strerror(err)); + errno = 0; int fd = dirfd(dirp); - ok(fd >= 0, "%s:%d dirfd(%p): %s", - __FILE__, __LINE__, dirp, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, "%s:%d dirfd(%p): %s", + __FILE__, __LINE__, dirp, strerror(err)); /* use fchdir to change into it */ + errno = 0; rc = fchdir(fd); - ok(rc == 0, "%s:%d fchdir(%d): %s", - __FILE__, __LINE__, fd, strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fchdir(%d): %s", + __FILE__, __LINE__, fd, strerror(err)); closedir(dirp); /* open root directory */ + errno = 0; dirp = opendir("/"); - ok(dirp != NULL, "%s:%d opendir(%s): %s", - __FILE__, __LINE__, "/", strerror(errno)); + err = errno; + ok(dirp != NULL && err == 0, "%s:%d opendir(%s): %s", + __FILE__, __LINE__, "/", strerror(err)); + errno = 0; fd = dirfd(dirp); - ok(fd >= 0, "%s:%d dirfd(%p): %s", - __FILE__, __LINE__, dirp, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, "%s:%d dirfd(%p): %s", + __FILE__, __LINE__, dirp, strerror(err)); /* use fchdir to change into it */ + errno = 0; rc = fchdir(fd); - ok(rc == 0, "%s:%d fchdir(%d): %s", - __FILE__, __LINE__, fd, strerror(errno)); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fchdir(%d): %s", + __FILE__, __LINE__, fd, strerror(err)); closedir(dirp); #endif diff --git a/t/sys/creat-close.c b/t/sys/creat-close.c index f0194ad49..f7c92d556 100644 --- a/t/sys/creat-close.c +++ b/t/sys/creat-close.c @@ -34,41 +34,56 @@ int creat_close_test(char* unifyfs_root) char path[64]; int mode = 0600; int fd = -1; - - errno = 0; + int err, rc; /* Create a random file name at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); /* Verify closing a non-existent file fails with errno=EBADF */ - ok(close(fd) == -1 && errno == EBADF, + errno = 0; + rc = close(fd); + err = errno; + ok(rc == -1 && err == EBADF, "%s:%d close non-existing file %s should fail (errno=%d): %s", - __FILE__, __LINE__, path, errno, strerror(errno)); - errno = 0; /* Reset errno after test for failure */ + __FILE__, __LINE__, path, err, strerror(err)); /* Verify we can create a non-existent file. */ + errno = 0; fd = creat(path, mode); - ok(fd >= 0, "%s:%d creat non-existing file %s (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, + "%s:%d creat non-existing file %s (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(err)); /* Verify close succeeds. */ - ok(close(fd) == 0, "%s:%d close new file: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close new file: %s", + __FILE__, __LINE__, strerror(err)); /* Verify creating an already created file succeeds. */ + errno = 0; fd = creat(path, mode); - ok(fd >= 0, "%s:%d creat existing file %s (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, + "%s:%d creat existing file %s (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(err)); /* Verify close succeeds. */ - ok(close(fd) == 0, "%s:%d close %s: %s", - __FILE__, __LINE__, path, strerror(errno)); + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close %s: %s", + __FILE__, __LINE__, path, strerror(err)); /* Verify closing already closed file fails with errno=EBADF */ - ok(close(fd) == -1 && errno == EBADF, - "%s:%d close already closed file %s should fail (errno=%d): %s", - __FILE__, __LINE__, path, errno, strerror(errno)); errno = 0; + rc = close(fd); + err = errno; + ok(rc == -1 && err == EBADF, + "%s:%d close already closed file %s should fail (errno=%d): %s", + __FILE__, __LINE__, path, err, strerror(err)); /* CLEANUP * diff --git a/t/sys/creat64.c b/t/sys/creat64.c index b90a7b48f..4b4495c6c 100644 --- a/t/sys/creat64.c +++ b/t/sys/creat64.c @@ -34,30 +34,31 @@ int creat64_test(char* unifyfs_root) char path[64]; int mode = 0600; - int fd; + int err, fd; /* Create a random file name at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); - skip(1, 2, "remove when UNIFYFS(create64) has been implemented"); /* Verify we can create a non-existent file. */ errno = 0; fd = creat64(path, mode); - ok(fd >= 0, "creat64 non-existing file %s (fd=%d): %s", - path, fd, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, + "creat64 non-existing file %s (fd=%d): %s", + path, fd, strerror(err)); ok(close(fd) != -1, "close() worked"); /* Verify creating an already created file succeeds. */ errno = 0; fd = creat64(path, mode); - ok(fd >= 0, "creat64 existing file %s (fd=%d): %s", - path, fd, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, + "creat64 existing file %s (fd=%d): %s", + path, fd, strerror(err)); ok(close(fd) != -1, "close() worked"); - end_skip; - diag("Finished UNIFYFS_WRAP(creat64) tests"); return 0; diff --git a/t/sys/lseek.c b/t/sys/lseek.c index 5d8639ccd..8499412c6 100644 --- a/t/sys/lseek.c +++ b/t/sys/lseek.c @@ -37,158 +37,238 @@ int lseek_test(char* unifyfs_root) char path[64]; int file_mode = 0600; int fd = -1; - - errno = 0; + int err, rc; /* Create a random file at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); /* lseek in bad file descriptor should fail with errno=EBADF */ - ok(lseek(fd, 0, SEEK_SET) == -1 && errno == EBADF, + errno = 0; + rc = (int) lseek(fd, 0, SEEK_SET); + err = errno; + ok(rc == -1 && err == EBADF, "%s:%d lseek in bad file descriptor fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - errno = 0; /* reset errno after test for failure */ + __FILE__, __LINE__, err, strerror(err)); /* Open a file and write to it to test lseek() */ + errno = 0; fd = open(path, O_RDWR | O_CREAT | O_TRUNC, file_mode); - ok(fd >= 0, "%s:%d open worked: %s", __FILE__, __LINE__, strerror(errno)); - ok(write(fd, "hello world", 12) == 12, "%s:%d write worked: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, "%s:%d open worked: %s", + __FILE__, __LINE__, strerror(err)); + + errno = 0; + rc = (int) write(fd, "hello world", 12); + err = errno; + ok(rc == 12 && err == 0, + "%s:%d write worked: %s", __FILE__, __LINE__, strerror(err)); /* lseek with invalid whence fails with errno=EINVAL. */ - ok(lseek(fd, 0, -1) == -1 && errno == EINVAL, - "%s:%d lseek with invalid whence should fail (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) lseek(fd, 0, -1); + err = errno; + ok(rc == -1 && err == EINVAL, + "%s:%d lseek with invalid whence should fail (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); + + /*--- lseek() with SEEK_SET tests ---*/ - /* lseek() with SEEK_SET tests */ /* lseek to negative offset with SEEK_SET should fail with errno=EINVAL */ - ok(lseek(fd, -1, SEEK_SET) == -1 && errno == EINVAL, - "%s:%d lseek(-1) to invalid offset w/ SEEK_SET fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) lseek(fd, -1, SEEK_SET); + err = errno; + ok(rc == -1 && err == EINVAL, + "%s:%d lseek(-1, SEEK_SET) to invalid offset fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* lseek to valid offset with SEEK_SET succeeds */ - ok(lseek(fd, 7, SEEK_SET) == 7, - "%s:%d lseek(7) to valid offset w/ SEEK_SET: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 7, SEEK_SET); + err = errno; + ok(rc == 7 && err == 0, + "%s:%d lseek(7, SEEK_SET) to valid offset: %s", + __FILE__, __LINE__, strerror(err)); /* lseek beyond end of file with SEEK_SET succeeds */ - ok(lseek(fd, 25, SEEK_SET) == 25, - "%s:%d lseek(25) beyond EOF w/ SEEK_SET: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 25, SEEK_SET); + err = errno; + ok(rc == 25 && err == 0, + "%s:%d lseek(25, SEEK_SET) beyond EOF: %s", + __FILE__, __LINE__, strerror(err)); /* lseek to beginning of file with SEEK_SET succeeds */ - ok(lseek(fd, 0, SEEK_SET) == 0, "%s:%d lseek(0) w/ SEEK_SET: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 0, SEEK_SET); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d lseek(0, SEEK_SET): %s", + __FILE__, __LINE__, strerror(err)); + + /*--- lseek() with SEEK_CUR tests ---*/ - /* lseek() with SEEK_CUR tests */ /* lseek to end of file with SEEK_CUR succeeds */ - ok(lseek(fd, 12, SEEK_CUR) == 12, "%s:%d lseek(12) to EOF w/ SEEK_CUR: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 12, SEEK_CUR); + err = errno; + ok(rc == 12 && err == 0, + "%s:%d lseek(12, SEEK_CUR) to EOF: %s", + __FILE__, __LINE__, strerror(err)); /* lseek to negative offset with SEEK_CUR should fail with errno=EINVAL */ - ok(lseek(fd, -15, SEEK_CUR) == -1 && errno == EINVAL, - "%s:%d lseek(-15) to invalid offset w/ SEEK_CUR fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) lseek(fd, -15, SEEK_CUR); + err = errno; + ok(rc == -1 && err == EINVAL, + "%s:%d lseek(-15, SEEK_CUR) to invalid offset fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* lseek to beginning of file with SEEK_CUR succeeds */ - ok(lseek(fd, -12, SEEK_CUR) == 0, - "%s:%d lseek(-12) to beginning of file w/ SEEK_CUR: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, -12, SEEK_CUR); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d lseek(-12, SEEK_CUR) to beginning of file: %s", + __FILE__, __LINE__, strerror(err)); /* lseek beyond end of file with SEEK_CUR succeeds */ - ok(lseek(fd, 25, SEEK_CUR) == 25, - "%s:%d lseek(25) beyond EOF w/ SEEK_CUR: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 25, SEEK_CUR); + err = errno; + ok(rc == 25 && err == 0, + "%s:%d lseek(25, SEEK_CUR) beyond EOF: %s", + __FILE__, __LINE__, strerror(err)); + + /*--- lseek() with SEEK_END tests ---*/ - /* lseek() with SEEK_END tests */ /* lseek to negative offset with SEEK_END should fail with errno=EINVAL */ - ok(lseek(fd, -15, SEEK_END) == -1 && errno == EINVAL, - "%s:%d lseek(-15) to invalid offset w/ SEEK_END fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) lseek(fd, -15, SEEK_END); + err = errno; + ok(rc == -1 && err == EINVAL, + "%s:%d lseek(-15, SEEK_END) to invalid offset fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* lseek back one from end of file with SEEK_END succeeds */ - ok(lseek(fd, -1, SEEK_END) == 11, - "%s:%d lseek(-1) from EOF w/ SEEK_END: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, -1, SEEK_END); + err = errno; + ok(rc == 11 && err == 0, + "%s:%d lseek(-1, SEEK_END) from EOF: %s", + __FILE__, __LINE__, strerror(err)); /* lseek to beginning of file with SEEK_END succeeds */ - ok(lseek(fd, -12, SEEK_END) == 0, - "%s:%d lseek(-12) to beginning of file w/ SEEK_END: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, -12, SEEK_END); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d lseek(-12, SEEK_END) to beginning of file: %s", + __FILE__, __LINE__, strerror(err)); /* lseek beyond end of file with SEEK_END succeeds */ - ok(lseek(fd, 25, SEEK_END) == 37, - "%s:%d lseek(25) beyond EOF w/ SEEK_END: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 25, SEEK_END); + err = errno; + ok(rc == 37 && err == 0, + "%s:%d lseek(25, SEEK_END) beyond EOF: %s", + __FILE__, __LINE__, strerror(err)); + + /*--- lseek() with SEEK_DATA tests ---*/ - /* lseek() with SEEK_DATA tests */ /* Write beyond end of file to create a hole */ - ok(write(fd, "hello universe", 15) == 15, "%s:%d write to create hole: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) write(fd, "hello universe", 15); + err = errno; + ok(rc == 15 && err == 0, + "%s:%d write to create hole: %s", + __FILE__, __LINE__, strerror(err)); /* lseek to negative offset with SEEK_DATA should fail with errno=ENXIO */ - ok(lseek(fd, -1, SEEK_DATA) == -1 && errno == ENXIO, - "%s:%d lseek(-1) to invalid offset w/ SEEK_DATA fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) lseek(fd, -1, SEEK_DATA); + err = errno; + ok(rc == -1 && err == ENXIO, + "%s:%d lseek(-1, SEEK_DATA) to invalid offset fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* lseek to beginning of file with SEEK_DATA succeeds */ - ok(lseek(fd, 0, SEEK_DATA) == 0, "%s:%d lseek(0) w/ SEEK_DATA: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 0, SEEK_DATA); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d lseek(0, SEEK_DATA) w/ SEEK_DATA: %s", + __FILE__, __LINE__, strerror(err)); /* Fallback implementation: lseek to data after hole with SEEK_DATA returns * current offset */ - ok(lseek(fd, 15, SEEK_DATA) == 15, - "%s:%d lseek(15) to data after hole w/ SEEK_DATA returns offset: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 15, SEEK_DATA); + err = errno; + ok(rc == 15 && err == 0, + "%s:%d lseek(15, SEEK_DATA) to data after hole returns offset: %s", + __FILE__, __LINE__, strerror(err)); /* lseek beyond end of file with SEEK_DATA should fail with errno=ENXIO */ - ok(lseek(fd, 75, SEEK_DATA) == -1 && errno == ENXIO, - "%s:%d lseek(75) beyond EOF w/ SEEK_DATA fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) lseek(fd, 75, SEEK_DATA); + err = errno; + ok(rc == -1 && err == ENXIO, + "%s:%d lseek(75, SEEK_DATA) beyond EOF fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); + + /*--- lseek() with SEEK_HOLE tests ---*/ - /* lseek() with SEEK_HOLE tests */ /* lseek to negative offset with SEEK_HOLE should fail with errno=ENXIO */ - ok(lseek(fd, -1, SEEK_HOLE) == -1 && errno == ENXIO, - "%s:%d lseek(-1) to invalid offset w/ SEEK_HOLE fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) lseek(fd, -1, SEEK_HOLE); + err = errno; + ok(rc == -1 && err == ENXIO, + "%s:%d lseek(-1, SEEK_HOLE) to invalid offset fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* Fallback implementation: lseek to first hole of file with SEEK_HOLE * returns or EOF */ - ok(lseek(fd, 0, SEEK_HOLE) == 52, - "%s:%d lseek(0) to first hole in file w/ SEEK_HOLE returns EOF: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 0, SEEK_HOLE); + err = errno; + ok(rc == 52 && err == 0, + "%s:%d lseek(0, SEEK_HOLE) to first hole in file returns EOF: %s", + __FILE__, __LINE__, strerror(err)); /* Fallback implementation: lseek to middle of hole with SEEK_HOLE returns * EOF */ - ok(lseek(fd, 18, SEEK_HOLE) == 52, - "%s:%d lseek(18) to middle of hole w/ SEEK_HOLE returns EOF: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 18, SEEK_HOLE); + err = errno; + ok(rc == 52 && err == 0, + "%s:%d lseek(18, SEEK_HOLE) to middle of hole returns EOF: %s", + __FILE__, __LINE__, strerror(err)); /* lseek to end of file with SEEK_HOLE succeeds */ - ok(lseek(fd, 42, SEEK_HOLE) == 52, - "%s:%d lseek(42) to EOF w/ SEEK_HOLE: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 42, SEEK_HOLE); + err = errno; + ok(rc == 52 && err == 0, + "%s:%d lseek(42, SEEK_HOLE) to EOF w/ SEEK_HOLE: %s", + __FILE__, __LINE__, strerror(err)); /* lseek beyond end of file with SEEK_HOLE should fail with errno= ENXIO */ - ok(lseek(fd, 75, SEEK_HOLE) == -1 && errno == ENXIO, - "%s:%d lseek beyond EOF w/ SEEK_HOLE fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) lseek(fd, 75, SEEK_HOLE); + err = errno; + ok(rc == -1 && err == ENXIO, + "%s:%d lseek(75, SEEK_HOLE) beyond EOF fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); close(fd); /* lseek in non-open file descriptor should fail with errno=EBADF */ - ok(lseek(fd, 0, SEEK_SET) == -1 && errno == EBADF, - "%s:%d lseek in non-open file descriptor fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) lseek(fd, 0, SEEK_SET); + err = errno; + ok(rc == -1 && err == EBADF, + "%s:%d lseek in non-open file descriptor fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); diag("Finished UNIFYFS_WRAP(lseek) tests"); diff --git a/t/sys/mkdir-rmdir.c b/t/sys/mkdir-rmdir.c index 817b3402c..b276b9510 100644 --- a/t/sys/mkdir-rmdir.c +++ b/t/sys/mkdir-rmdir.c @@ -38,9 +38,7 @@ int mkdir_rmdir_test(char* unifyfs_root) char file_subdir_path[80]; int file_mode = 0600; int dir_mode = 0700; - int fd; - - errno = 0; + int err, fd, rc; /* Create random dir and file path names at the mountpoint to test on */ testutil_rand_path(dir_path, sizeof(dir_path), unifyfs_root); @@ -67,101 +65,138 @@ int mkdir_rmdir_test(char* unifyfs_root) todo("mkdir_1: we currently don't support directory structure"); /* Verify creating a dir under non-existent parent dir fails with * errno=ENOENT */ - ok(mkdir(subdir_path, dir_mode) == -1 && errno == ENOENT, + errno = 0; + rc = mkdir(subdir_path, dir_mode); + err = errno; + ok(rc == -1 && err == ENOENT, "%s:%d mkdir dir %s without parent dir should fail (errno=%d): %s", - __FILE__, __LINE__, subdir_path, errno, strerror(errno)); + __FILE__, __LINE__, subdir_path, err, strerror(err)); end_todo; /* end todo_mkdir_1 */ - errno = 0; /* Reset errno after test for failure */ /* Verify rmdir a non-existing directory fails with errno=ENOENT */ - ok(rmdir(dir_path) == -1 && errno == ENOENT, - "%s:%d rmdir non-existing dir %s should fail (errno=%d): %s", - __FILE__, __LINE__, dir_path, errno, strerror(errno)); errno = 0; + rc = rmdir(dir_path); + err = errno; + ok(rc == -1 && err == ENOENT, + "%s:%d rmdir non-existing dir %s should fail (errno=%d): %s", + __FILE__, __LINE__, dir_path, err, strerror(err)); /* Verify we can create a non-existent directory. */ - ok(mkdir(dir_path, dir_mode) == 0, "%s:%d mkdir non-existing dir %s: %s", - __FILE__, __LINE__, dir_path, strerror(errno)); + errno = 0; + rc = mkdir(dir_path, dir_mode); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d mkdir non-existing dir %s: %s", + __FILE__, __LINE__, dir_path, strerror(err)); /* Verify recreating an already created directory fails with errno=EEXIST */ - ok(mkdir(dir_path, dir_mode) == -1 && errno == EEXIST, - "%s:%d mkdir existing dir %s should fail (errno=%d): %s", - __FILE__, __LINE__, dir_path, errno, strerror(errno)); errno = 0; + rc = mkdir(dir_path, dir_mode); + err = errno; + ok(rc == -1 && err == EEXIST, + "%s:%d mkdir existing dir %s should fail (errno=%d): %s", + __FILE__, __LINE__, dir_path, err, strerror(err)); - /* todo_mkdir_2: Remove when issue is resolved */ - todo("mkdir_2: should fail with errno=EISDIR=21"); /* Verify creating a file with same name as a dir fails with errno=EISDIR */ + errno = 0; fd = creat(dir_path, file_mode); - ok(fd == -1 && errno == EISDIR, + err = errno; + ok(fd == -1 && err == EISDIR, "%s:%d creat file with same name as dir %s fails (fd=%d, errno=%d): %s", - __FILE__, __LINE__, dir_path, fd, errno, strerror(errno)); - end_todo; /* end todo_mkdir_2 */ - errno = 0; + __FILE__, __LINE__, dir_path, fd, err, strerror(err)); /* todo_mkdir_3: Remove when issue is resolved */ todo("mkdir_3: this fails because \"TODO mkdir_1\" is failing"); /* Verify we can create a subdirectory under an existing directory */ - ok(mkdir(subdir_path, dir_mode) == 0, + errno = 0; + rc = mkdir(subdir_path, dir_mode); + err = errno; + ok(rc == 0 && err == 0, "%s:%d mkdir subdirectory %s in existing dir: %s", - __FILE__, __LINE__, subdir_path, strerror(errno)); + __FILE__, __LINE__, subdir_path, strerror(err)); end_todo; /* end todo_mkdir_3 */ - errno = 0; /* can remove when test is passing */ /* Verify we can create a subfile under an existing directory */ + errno = 0; fd = creat(subfile_path, file_mode); - ok(fd >= 0, "%s:%d creat subfile %s in existing dir (fd=%d): %s", - __FILE__, __LINE__, subfile_path, fd, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, + "%s:%d creat subfile %s in existing dir (fd=%d): %s", + __FILE__, __LINE__, subfile_path, fd, strerror(err)); - ok(close(fd) == 0, "%s:%d close() worked: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(err)); /* Verify creating a directory whose parent is a file fails with * errno=ENOTDIR */ + errno = 0; fd = creat(file_path, file_mode); - ok(fd >= 0, "%s:%d creat parent file %s (fd=%d): %s", - __FILE__, __LINE__, file_path, fd, strerror(errno)); - ok(close(fd) == 0, "%s:%d close() worked: %s", - __FILE__, __LINE__, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, + "%s:%d creat parent file %s (fd=%d): %s", + __FILE__, __LINE__, file_path, fd, strerror(err)); + + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(err)); /* todo_mkdir_4: Remove when issue is resolved */ todo("mkdir_4: unifyfs currently creates all paths as separate entities"); - ok(mkdir(file_subdir_path, dir_mode) == -1 && errno == ENOTDIR, + errno = 0; + rc = mkdir(file_subdir_path, dir_mode); + err = errno; + ok(rc == -1 && err == ENOTDIR, "%s:%d mkdir dir %s where parent is a file should fail (errno=%d): %s", - __FILE__, __LINE__, file_subdir_path, errno, strerror(errno)); + __FILE__, __LINE__, file_subdir_path, err, strerror(err)); end_todo; /* end todo_mkdir_4 */ - errno = 0; - /* Verify rmdir a non-directory fails with errno=ENOENT */ - ok(rmdir(file_path) == -1 && errno == ENOTDIR, - "%s:%d rmdir non-directory %s should fail (errno=%d): %s", - __FILE__, __LINE__, file_path, errno, strerror(errno)); + /* Verify rmdir on non-directory fails with errno=ENOTDIR */ errno = 0; + rc = rmdir(file_path); + err = errno; + ok(rc == -1 && err == ENOTDIR, + "%s:%d rmdir non-directory %s should fail (errno=%d): %s", + __FILE__, __LINE__, file_path, err, strerror(err)); /* todo_mkdir_5: Remove when issue is resolved */ todo("mkdir_5: unifyfs currently creates all paths as separate entities"); /* Verify rmdir a non-empty directory fails with errno=ENOTEMPTY */ - ok(rmdir(dir_path) == -1 && errno == ENOTEMPTY, + errno = 0; + rc = rmdir(dir_path); + err = errno; + ok(rc == -1 && err == ENOTEMPTY, "%s:%d rmdir non-empty directory %s should fail (errno=%d): %s", - __FILE__, __LINE__, dir_path, errno, strerror(errno)); + __FILE__, __LINE__, dir_path, err, strerror(err)); end_todo; /* end todo_mkdir_5 */ - errno = 0; /* Verify we can rmdir an empty directory */ - ok(rmdir(subdir_path) == 0, "%s:%d rmdir an empty directory %s: %s", - __FILE__, __LINE__, subdir_path, strerror(errno)); + errno = 0; + rc = rmdir(subdir_path); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d rmdir an empty directory %s: %s", + __FILE__, __LINE__, subdir_path, strerror(err)); /* Verify rmdir an already removed directory fails with errno=ENOENT */ - ok(rmdir(subdir_path) == -1 && errno == ENOENT, - "%s:%d rmdir already removed dir %s should fail (errno=%d): %s", - __FILE__, __LINE__, subdir_path, errno, strerror(errno)); errno = 0; + rc = rmdir(subdir_path); + err = errno; + ok(rc == -1 && err == ENOENT, + "%s:%d rmdir already removed dir %s should fail (errno=%d): %s", + __FILE__, __LINE__, subdir_path, err, strerror(err)); /* Verify trying to rmdir the mount point fails with errno=EBUSY */ - ok(rmdir(unifyfs_root) == -1 && errno == EBUSY, - "%s:%d rmdir mount point %s should fail (errno=%d): %s", - __FILE__, __LINE__, unifyfs_root, errno, strerror(errno)); errno = 0; + rc = rmdir(unifyfs_root); + err = errno; + ok(rc == -1 && err == EBUSY, + "%s:%d rmdir mount point %s should fail (errno=%d): %s", + __FILE__, __LINE__, unifyfs_root, err, strerror(err)); /* CLEANUP * diff --git a/t/sys/open.c b/t/sys/open.c index 787fe43c7..30489ffa6 100644 --- a/t/sys/open.c +++ b/t/sys/open.c @@ -36,8 +36,7 @@ int open_test(char* unifyfs_root) char dir_path[64]; int file_mode = 0600; int dir_mode = 0700; - int fd; - int rc; + int err, fd, rc; /* Create a random file and dir name at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); @@ -47,15 +46,18 @@ int open_test(char* unifyfs_root) * errno=ENOENT */ errno = 0; fd = open(path, O_RDWR, file_mode); - ok(fd < 0 && errno == ENOENT, + err = errno; + ok(fd < 0 && err == ENOENT, "open non-existing file %s w/out O_CREATE fails (fd=%d, errno=%d): %s", - path, fd, errno, strerror(errno)); + path, fd, err, strerror(err)); /* Verify we can create a new file. */ errno = 0; fd = open(path, O_CREAT|O_EXCL, file_mode); - ok(fd >= 0, "open non-existing file %s flags O_CREAT|O_EXCL (fd=%d): %s", - path, fd, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, + "open non-existing file %s flags O_CREAT|O_EXCL (fd=%d): %s", + path, fd, strerror(err)); ok(close(fd) != -1, "close() worked"); @@ -63,32 +65,35 @@ int open_test(char* unifyfs_root) * errno=EEXIST. */ errno = 0; fd = open(path, O_CREAT|O_EXCL, file_mode); - ok(fd < 0 && errno == EEXIST, + err = errno; + ok(fd < 0 && err == EEXIST, "open existing file %s O_CREAT|O_EXCL should fail (fd=%d, errno=%d): %s", - path, fd, errno, strerror(errno)); + path, fd, err, strerror(err)); /* Verify opening an existing file with O_RDWR succeeds. */ errno = 0; fd = open(path, O_RDWR, file_mode); - ok(fd >= 0, "open existing file %s O_RDWR (fd=%d): %s", - path, fd, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, + "open existing file %s O_RDWR (fd=%d): %s", + path, fd, strerror(err)); ok(close(fd) != -1, "close() worked"); + errno = 0; rc = mkdir(dir_path, dir_mode); - ok(rc == 0, "mkdir(%s, %o) worked, rc = %d", dir_path, dir_mode, rc); - - /* todo_open_1: Remove when issue is resolved */ - todo("open_1: should fail with errno=EISDIR=21"); + err = errno; + ok(rc == 0 && err == 0, "mkdir(%s, %o) worked, (errno=%d): %s", + dir_path, dir_mode, err, strerror(err)); errno = 0; fd = open(dir_path, O_RDWR, file_mode); - ok(fd < 0 && errno == EISDIR, + err = errno; + ok(fd < 0 && err == EISDIR, "open directory %s for write should fail (fd=%d, errno=%d): %s", - dir_path, fd, errno, strerror(errno)); - end_todo; /* end todo_open_1 */ + dir_path, fd, err, strerror(err)); - /* ClEANUP + /* CLEANUP * * Don't unlink `path` so that the final test (9020-mountpoint-empty) can * check if open left anything in the mountpoint and thus wasn't wrapped diff --git a/t/sys/open64.c b/t/sys/open64.c index ec6b0ff83..127a1f085 100644 --- a/t/sys/open64.c +++ b/t/sys/open64.c @@ -36,7 +36,7 @@ int open64_test(char* unifyfs_root) char path[64]; int mode = 0600; - int fd; + int err, fd; /* Create a random file name at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); @@ -45,15 +45,18 @@ int open64_test(char* unifyfs_root) * errno=ENOENT */ errno = 0; fd = open64(path, O_RDWR, mode); - ok(fd < 0 && errno == ENOENT, + err = errno; + ok(fd < 0 && err == ENOENT, "open64 non-existing file %s w/out O_CREATE fails (fd=%d, errno=%d): %s", - path, fd, errno, strerror(errno)); + path, fd, err, strerror(err)); /* Verify we can create a new file. */ errno = 0; fd = open64(path, O_CREAT|O_EXCL, mode); - ok(fd >= 0, "open64 non-existing file %s flags O_CREAT|O_EXCL (fd=%d): %s", - path, fd, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, + "open64 non-existing file %s flags O_CREAT|O_EXCL (fd=%d): %s", + path, fd, strerror(err)); ok(close(fd) != -1, "close() worked"); @@ -61,15 +64,18 @@ int open64_test(char* unifyfs_root) * errno=EEXIST. */ errno = 0; fd = open64(path, O_CREAT|O_EXCL, mode); - ok(fd < 0 && errno == EEXIST, + err = errno; + ok(fd < 0 && err == EEXIST, "open64 existing file %s O_CREAT|O_EXCL fails (fd=%d, errno=%d): %s", - path, fd, errno, strerror(errno)); + path, fd, err, strerror(err)); /* Verify opening an existing file with O_RDWR succeeds. */ errno = 0; fd = open64(path, O_RDWR, mode); - ok(fd >= 0, "open64 existing file %s O_RDWR (fd=%d): %s", - path, fd, strerror(errno)); + err = errno; + ok(fd >= 0 && err == 0, + "open64 existing file %s O_RDWR (fd=%d): %s", + path, fd, strerror(err)); ok(close(fd) != -1, "close() worked"); diff --git a/t/sys/unlink.c b/t/sys/unlink.c index dd41ab08b..4f68768ef 100644 --- a/t/sys/unlink.c +++ b/t/sys/unlink.c @@ -27,42 +27,61 @@ static int unlink_after_sync_test(char* unifyfs_root) { char path[64]; - int fd; - errno = 0; + int err, fd, rc; testutil_rand_path(path, sizeof(path), unifyfs_root); + errno = 0; fd = open(path, O_WRONLY | O_CREAT, 0222); - ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + err = errno; + ok(fd != -1 && err == 0, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(err)); - ok(write(fd, "hello world", 12) == 12, "%s:%d write(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) write(fd, "hello world", 12); + err = errno; + ok(rc == 12 && err == 0, + "%s:%d write(): %s", + __FILE__, __LINE__, strerror(err)); - ok(fsync(fd) == 0, "%s:%d fsync(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = fsync(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fsync(): %s", + __FILE__, __LINE__, strerror(err)); - ok(close(fd) == 0, "%s:%d close(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close(): %s", + __FILE__, __LINE__, strerror(err)); struct stat sb = {0}; - ok(stat(path, &sb) == 0, "%s:%d stat(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = stat(path, &sb); + err = errno; + ok(rc == 0 && err == 0, "%s:%d stat(): %s", + __FILE__, __LINE__, strerror(err)); - ok(unlink(path) == 0, "%s:%d unlink(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = unlink(path); + err = errno; + ok(rc == 0 && err == 0, "%s:%d unlink(): %s", + __FILE__, __LINE__, strerror(err)); - todo("Failing with wrong errno. Should fail with errno=ENOENT=2"); - ok(stat(path, &sb) == -1 && errno == ENOENT, + errno = 0; + rc = stat(path, &sb); + err = errno; + ok(rc == -1 && err == ENOENT, "%s:%d stat() after unlink fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - end_todo; - errno = 0; /* Reset errno after test for failure */ + __FILE__, __LINE__, err, strerror(err)); - ok(unlink(path) == -1 && errno == ENOENT, - "%s:%d unlink() already unlinked file fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = unlink(path); + err = errno; + ok(rc == -1 && err == ENOENT, + "%s:%d unlink() already unlinked file fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); return 0; } @@ -70,46 +89,67 @@ static int unlink_after_sync_test(char* unifyfs_root) static int unlink_after_sync_laminate_test(char* unifyfs_root) { char path[64]; - int fd; - errno = 0; + int err, fd, rc; testutil_rand_path(path, sizeof(path), unifyfs_root); + errno = 0; fd = open(path, O_WRONLY | O_CREAT, 0222); - ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + err = errno; + ok(fd != -1 && err == 0, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(err)); - ok(write(fd, "hello world", 12) == 12, "%s:%d write(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) write(fd, "hello world", 12); + err = errno; + ok(rc == 12 && err == 0, "%s:%d write(): %s", + __FILE__, __LINE__, strerror(err)); - ok(fsync(fd) == 0, "%s:%d fsync(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = fsync(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fsync(): %s", + __FILE__, __LINE__, strerror(err)); - ok(close(fd) == 0, "%s:%d close(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close(): %s", + __FILE__, __LINE__, strerror(err)); /* Laminate */ - ok(chmod(path, 0444) == 0, "%s:%d chmod(0444): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = chmod(path, 0444); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chmod(0444): %s", + __FILE__, __LINE__, strerror(err)); struct stat sb = {0}; - ok(stat(path, &sb) == 0, "%s:%d stat(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = stat(path, &sb); + err = errno; + ok(rc == 0 && err == 0, "%s:%d stat(): %s", + __FILE__, __LINE__, strerror(err)); - ok(unlink(path) == 0, "%s:%d unlink(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = unlink(path); + err = errno; + ok(rc == 0 && err == 0, "%s:%d unlink(): %s", + __FILE__, __LINE__, strerror(err)); - todo("Failing with wrong errno. Should fail with errno=ENOENT=2"); - ok(stat(path, &sb) == -1 && errno == ENOENT, - "%s:%d stat() after unlink fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - end_todo; errno = 0; + rc = stat(path, &sb); + err = errno; + ok(rc == -1 && err == ENOENT, + "%s:%d stat() after unlink fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); - ok(unlink(path) == -1 && errno == ENOENT, - "%s:%d unlink() already unlinked, laminated file fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = unlink(path); + err = errno; + ok(rc == -1 && err == ENOENT, + "%s:%d unlink() already unlinked, laminated file fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); return 0; } @@ -120,59 +160,84 @@ int unlink_test(char* unifyfs_root) char path[64]; char dir_path[64]; - int fd; - int rc = 0; - errno = 0; + int err, fd, rc; testutil_rand_path(path, sizeof(path), unifyfs_root); testutil_rand_path(dir_path, sizeof(dir_path), unifyfs_root); - ok(unlink(path) == -1 && errno == ENOENT, + errno = 0; + rc = unlink(path); + err = errno; + ok(rc == -1 && err == ENOENT, "%s:%d unlink() non-existent file fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - errno = 0; /* Reset errno after test for failure */ + __FILE__, __LINE__, err, strerror(err)); + errno = 0; fd = creat(path, 0222); - ok(fd != -1, "%s:%d creat(%s) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + err = errno; + ok(fd != -1 && err == 0, "%s:%d creat(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(err)); - ok(fsync(fd) == 0, "%s:%d fsync(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = fsync(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fsync(): %s", + __FILE__, __LINE__, strerror(err)); - ok(close(fd) == 0, "%s:%d close(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close(): %s", + __FILE__, __LINE__, strerror(err)); struct stat sb = {0}; - ok(stat(path, &sb) == 0, "%s:%d stat(): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = stat(path, &sb); + err = errno; + ok(rc == 0 && err == 0, "%s:%d stat(): %s", + __FILE__, __LINE__, strerror(err)); - ok(unlink(path) == 0, "%s:%d unlink() empty file: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = unlink(path); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d unlink() empty file: %s", + __FILE__, __LINE__, strerror(err)); - todo("Failing with wrong errno. Should fail with errno=ENOENT=2"); - ok(stat(path, &sb) == -1 && errno == ENOENT, - "%s:%d stat() after unlink fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - end_todo; errno = 0; + rc = stat(path, &sb); + err = errno; + ok(rc == -1 && err == ENOENT, + "%s:%d stat() after unlink fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); - ok(unlink(path) == -1 && errno == ENOENT, + errno = 0; + rc = unlink(path); + err = errno; + ok(rc == -1 && err == ENOENT, "%s:%d unlink() already unlinked, empty file fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - errno = 0; /* Reset errno after test for failure */ + __FILE__, __LINE__, err, strerror(err)); /* Calling unlink() on a directory should fail */ - ok(mkdir(dir_path, 0777) == 0, "%s:%d mkdir(%s): %s", - __FILE__, __LINE__, dir_path, strerror(errno)); + errno = 0; + rc = mkdir(dir_path, 0777); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d mkdir(%s): %s", + __FILE__, __LINE__, dir_path, strerror(err)); - ok(unlink(dir_path) == -1 && errno == EISDIR, + errno = 0; + rc = unlink(dir_path); + err = errno; + ok(rc == -1 && err == EISDIR, "%s:%d unlink() a directory fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - errno = 0; /* Reset errno after test for failure */ - - ok(rmdir(dir_path) == 0, "%s:%d rmdir(): %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, err, strerror(err)); + errno = 0; + rc = rmdir(dir_path); + err = errno; + ok(rc == 0 && err == 0, "%s:%d rmdir(): %s", + __FILE__, __LINE__, strerror(err)); /* Tests for unlink after writing to a file */ int ret = unlink_after_sync_test(unifyfs_root); diff --git a/t/sys/write-read.c b/t/sys/write-read.c index 768b3d2d3..dadb88f00 100644 --- a/t/sys/write-read.c +++ b/t/sys/write-read.c @@ -31,145 +31,228 @@ int write_read_test(char* unifyfs_root) char path[64]; char buf[64] = {0}; int fd = -1; + int err, rc; size_t global; - errno = 0; - testutil_rand_path(path, sizeof(path), unifyfs_root); /* write to bad file descriptor should fail with errno=EBADF */ - ok(write(fd, "hello world", 12) == -1 && errno == EBADF, + errno = 0; + rc = (int) write(fd, "hello world", 12); + err = errno; + ok(rc == -1 && err == EBADF, "%s:%d write() to bad file descriptor fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - errno = 0; /* Reset after test for failure */ + __FILE__, __LINE__, err, strerror(err)); /* read from bad file descriptor should fail with errno=EBADF */ - ok(read(fd, buf, 12) == -1 && errno == EBADF, - "%s:%d read() from bad file descriptor fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) read(fd, buf, 12); + err = errno; + ok(rc == -1 && err == EBADF, + "%s:%d read() from bad file descriptor fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); /* Write "hello world" to the file */ + errno = 0; fd = open(path, O_WRONLY | O_CREAT, 0222); - ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); - ok(write(fd, "hello world", 12) == 12, + err = errno; + ok(fd != -1 && err == 0, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(err)); + + errno = 0; + rc = (int) write(fd, "hello world", 12); + err = errno; + ok(rc == 12 && err == 0, "%s:%d write(\"hello world\") to file: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* Write to a different offset by overwriting "world" with "universe" */ - ok(lseek(fd, 6, SEEK_SET) == 6, "%s:%d lseek(6) to \"world\": %s", - __FILE__, __LINE__, strerror(errno)); - ok(write(fd, "universe", 9) == 9, + errno = 0; + rc = (int) lseek(fd, 6, SEEK_SET); + err = errno; + ok(rc == 6 && err == 0, + "%s:%d lseek(6) to \"world\": %s", + __FILE__, __LINE__, strerror(err)); + + errno = 0; + rc = (int) write(fd, "universe", 9); + err = errno; + ok(rc == 9 && err == 0, "%s:%d overwrite \"world\" at offset 6 with \"universe\": %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); /* Check global size on our un-laminated and un-synced file */ testutil_get_size(path, &global); ok(global == 15, "%s:%d global size before fsync is %d: %s", - __FILE__, __LINE__, global, strerror(errno)); + __FILE__, __LINE__, global, strerror(err)); - ok(fsync(fd) == 0, "%s:%d fsync() worked: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = fsync(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d fsync() worked: %s", + __FILE__, __LINE__, strerror(err)); /* Check global size on our un-laminated file */ testutil_get_size(path, &global); ok(global == 15, "%s:%d global size after fsync is %d: %s", - __FILE__, __LINE__, global, strerror(errno)); + __FILE__, __LINE__, global, strerror(err)); /* read from file open as write-only should fail with errno=EBADF */ - ok(lseek(fd, 0, SEEK_SET) == 0, "%s:%d lseek(0): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 0, SEEK_SET); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d lseek(0): %s", + __FILE__, __LINE__, strerror(err)); + todo("Successfully reads and gets 0 bytes back"); - ok(read(fd, buf, 15) == -1 && errno == EBADF, + errno = 0; + rc = (int) read(fd, buf, 15); + err = errno; + ok(rc == -1 && err == EBADF, "%s:%d read() from file open as write-only (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); + __FILE__, __LINE__, err, strerror(err)); end_todo; - errno = 0; - ok(close(fd) == 0, "%s:%d close() worked: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(err)); /* Test O_APPEND */ + errno = 0; fd = open(path, O_WRONLY | O_APPEND, 0222); - ok(fd != -1, "%s:%d open(%s, O_APPEND) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + err = errno; + ok(fd != -1 && err == 0, "%s:%d open(%s, O_APPEND) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(err)); /* * Seek to an offset in the file and write. Since it's O_APPEND, the * offset we seeked to doesn't matter - all writes go to the end. */ - ok(lseek(fd, 3, SEEK_SET) == 3, "%s:%d lseek(3) worked: %s", - __FILE__, __LINE__, strerror(errno)); - ok(write(fd, "", 6) == 6, "%s:%d append write(\"\"): %s", - __FILE__, __LINE__, strerror(errno)); - ok(close(fd) == 0, "%s:%d close() worked: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 3, SEEK_SET); + err = errno; + ok(rc == 3 && err == 0, + "%s:%d lseek(3) worked: %s", + __FILE__, __LINE__, strerror(err)); + + errno = 0; + rc = (int) write(fd, "", 6); + err = errno; + ok(rc == 6 && err == 0, + "%s:%d append write(\"\"): %s", + __FILE__, __LINE__, strerror(err)); + + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(err)); /* Check global size on our un-laminated file */ testutil_get_size(path, &global); ok(global == 21, "%s:%d global size before laminate is %d: %s", - __FILE__, __LINE__, global, strerror(errno)); + __FILE__, __LINE__, global, strerror(err)); /* Laminate */ - ok(chmod(path, 0444) == 0, "%s:%d chmod(0444): %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = chmod(path, 0444); + err = errno; + ok(rc == 0 && err == 0, "%s:%d chmod(0444): %s", + __FILE__, __LINE__, strerror(err)); /* Verify we're getting the correct file size */ testutil_get_size(path, &global); ok(global == 21, "%s:%d global size after laminate is %d: %s", - __FILE__, __LINE__, global, strerror(errno)); + __FILE__, __LINE__, global, strerror(err)); /* open laminated file for write should fail with errno=EROFS */ + errno = 0; fd = open(path, O_WRONLY | O_CREAT, 0222); - ok(fd == -1 && errno == EROFS, + err = errno; + ok(fd == -1 && err == EROFS, "%s:%d open() laminated file for write fails (fd=%d, errno=%d): %s", - __FILE__, __LINE__, fd, errno, strerror(errno)); - errno = 0; + __FILE__, __LINE__, fd, err, strerror(err)); /* read() tests */ + errno = 0; fd = open(path, O_RDONLY, 0444); - ok(fd != -1, "%s:%d open(%s, O_RDONLY) for read (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + err = errno; + ok(fd != -1 && err == 0, + "%s:%d open(%s, O_RDONLY) for read (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(err)); /* write to file open as read-only should fail with errno=EBADF */ - ok(write(fd, "hello world", 12) == -1 && errno == EBADF, - "%s:%d write() to file open as read-only fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) write(fd, "hello world", 12); + err = errno; + ok(rc == -1 && err == EBADF, + "%s:%d write() to file open as read-only fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); - ok(read(fd, buf, 21) == 21, "%s:%d read() buf[]=\"%s\": %s", - __FILE__, __LINE__, buf, strerror(errno)); + errno = 0; + rc = (int) read(fd, buf, 21); + err = errno; + ok(rc == 21 && err == 0, + "%s:%d read() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(err)); buf[14] = ' '; /* replace '\0' between initial write and append */ is(buf, "hello universe ", "%s:%d read() saw \"hello universe \"", __FILE__, __LINE__); /* Seek and read at a different position */ - ok(lseek(fd, 6, SEEK_SET) == 6, "%s:%d lseek(6) worked: %s", - __FILE__, __LINE__, strerror(errno)); - ok(read(fd, buf, 9) == 9, "%s:%d read() at offset 6 buf[]=\"%s\": %s", - __FILE__, __LINE__, buf, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 6, SEEK_SET); + err = errno; + ok(rc == 6 && err == 0, + "%s:%d lseek(6) worked: %s", + __FILE__, __LINE__, strerror(err)); + + errno = 0; + rc = (int) read(fd, buf, 9); + err = errno; + ok(rc == 9 && err == 0, + "%s:%d read() at offset 6 buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(err)); is(buf, "universe", "%s:%d read() saw \"universe\"", __FILE__, __LINE__); - ok(lseek(fd, 0, SEEK_SET) == 0, "%s:%d lseek(0) worked: %s", - __FILE__, __LINE__, strerror(errno)); - ok(read(fd, buf, sizeof(buf)) == 21, "%s:%d read() past end of file: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = (int) lseek(fd, 0, SEEK_SET); + err = errno; + ok(rc == 0 && err == 0, + "%s:%d lseek(0) worked: %s", + __FILE__, __LINE__, strerror(err)); + + errno = 0; + rc = (int) read(fd, buf, sizeof(buf)); + err = errno; + ok(rc == 21 && err == 0, + "%s:%d read() past end of file: %s", + __FILE__, __LINE__, strerror(err)); - ok(close(fd) == 0, "%s:%d close() worked: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(err)); /* write to closed file descriptor should fail with errno=EBADF */ - ok(write(fd, "hello world", 12) == -1 && errno == EBADF, + errno = 0; + rc = (int) write(fd, "hello world", 12); + err = errno; + ok(rc == -1 && err == EBADF, "%s:%d write() to bad file descriptor fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); - errno = 0; /* Reset after test for failure */ + __FILE__, __LINE__, err, strerror(err)); /* read from closed file descriptor should fail with errno=EBADF */ - ok(read(fd, buf, 12) == -1 && errno == EBADF, - "%s:%d read() from bad file descriptor fails (errno=%d): %s", - __FILE__, __LINE__, errno, strerror(errno)); errno = 0; + rc = (int) read(fd, buf, 12); + err = errno; + ok(rc == -1 && err == EBADF, + "%s:%d read() from bad file descriptor fails (errno=%d): %s", + __FILE__, __LINE__, err, strerror(err)); diag("Finished UNIFYFS_WRAP(write/read) tests"); @@ -184,48 +267,61 @@ int write_pre_existing_file_test(char* unifyfs_root) char path[64]; char buf[300] = {0}; int fd = -1; + int err, rc; size_t global; - errno = 0; - testutil_rand_path(path, sizeof(path), unifyfs_root); + errno = 0; fd = open(path, O_RDWR | O_CREAT, 0222); - ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + err = errno; + ok(fd != -1 && err == 0, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(err)); /* Write 300 bytes to a file */ - ok(write(fd, "a", 300) == 300, + errno = 0; + rc = (int) write(fd, "a", 300); + err = errno; + ok(rc == 300 && err == 0, "%s:%d write() a 300 byte file: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); - ok(close(fd) == 0, "%s:%d close() worked: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(err)); /* Check global size is 300 */ testutil_get_size(path, &global); ok(global == 300, "%s:%d global size of 300 byte file is %d: %s", - __FILE__, __LINE__, global, strerror(errno)); + __FILE__, __LINE__, global, strerror(err)); /* Reopen the same file */ + errno = 0; fd = open(path, O_RDWR, 0222); - ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", - __FILE__, __LINE__, path, fd, strerror(errno)); + err = errno; + ok(fd != -1 && err == 0, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(err)); /* Overwrite the first 100 bytes of same file */ - ok(write(fd, buf, 100) == 100, + errno = 0; + rc = (int) write(fd, buf, 100); + err = errno; + ok(rc == 100 && err == 0, "%s:%d overwrite first 100 bytes of same file: %s", - __FILE__, __LINE__, strerror(errno)); + __FILE__, __LINE__, strerror(err)); - ok(close(fd) == 0, "%s:%d close() worked: %s", - __FILE__, __LINE__, strerror(errno)); + errno = 0; + rc = close(fd); + err = errno; + ok(rc == 0 && err == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(err)); /* Check global size is 300 */ testutil_get_size(path, &global); - todo("File is now 100 bytes instead of 300. See issue #488 for details"); ok(global == 300, "%s:%d global size of 300 byte file is %d: %s", - __FILE__, __LINE__, global, strerror(errno)); - end_todo; + __FILE__, __LINE__, global, strerror(err)); diag("Finished write-to-pre-existing-file tests"); diff --git a/util/unifyfs-stage/src/unifyfs-stage-transfer.c b/util/unifyfs-stage/src/unifyfs-stage-transfer.c index 2c5d5b69d..f70e6d3dc 100644 --- a/util/unifyfs-stage/src/unifyfs-stage-transfer.c +++ b/util/unifyfs-stage/src/unifyfs-stage-transfer.c @@ -78,7 +78,7 @@ static int md5_checksum(const char* path, unsigned char* digest) out: /* MD5_xx returns 1 for success */ - ret = ret == 1 ? 0 : EIO; + ret = (ret == 1 ? 0 : EIO); close(fd); return ret; diff --git a/util/unifyfs/src/unifyfs-rm.c b/util/unifyfs/src/unifyfs-rm.c index 8518c14c9..f69b9bd2c 100644 --- a/util/unifyfs/src/unifyfs-rm.c +++ b/util/unifyfs/src/unifyfs-rm.c @@ -219,7 +219,10 @@ static int wait_server_initialization(unifyfs_resource_t* resource, } while (1) { + int err; + errno = 0; fp = fopen(filename, "r"); + err = errno; if (fp) { while (fgets(linebuf, 31, fp) != NULL) { count++; @@ -235,12 +238,10 @@ static int wait_server_initialization(unifyfs_resource_t* resource, fclose(fp); break; - } - - if (errno != ENOENT) { + } else if (err != ENOENT) { fprintf(stderr, "failed to open file %s (%s)\n", - filename, strerror(errno)); - ret = -errno; + filename, strerror(err)); + ret = -err; break; } From 3ccc82de18c8f52e736443533d00259ef772351c Mon Sep 17 00:00:00 2001 From: CamStan Date: Wed, 18 Nov 2020 17:43:47 -0800 Subject: [PATCH 160/168] Update Travis CI to use Mercury v1.0.1 Travis is currently set up to use the default version of several of out dependencies. The default version for Mercury has changed to v2.0.0. We are able to build with this version, but there appears to be an issue when starting the UnifyFS server. This changes our Travis CI to use the previous default version of Mercury, v1.0.1. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bdf1c7596..b1852b8a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,7 +67,7 @@ install: - . $HOME/spack/share/spack/setup-env.sh - spack install leveldb && spack load leveldb - spack install gotcha@1.0.3 && spack load gotcha@1.0.3 - - spack install margo^mercury+bmi~ofi~boostsys && spack load argobots && spack load mercury && spack load margo + - spack install margo^mercury@1.0.1+bmi~ofi~boostsys && spack load argobots && spack load mercury && spack load margo - spack install spath && spack load spath # prepare build environment - eval $(./scripts/git_log_test_env.sh) From bf11231a1b99cb8a61d7e52a1f4682578676774e Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 16 Nov 2020 13:08:56 -0800 Subject: [PATCH 161/168] client: add option to sync after every write TEST_CHECKPATCH_SKIP_FILES="common/src/unifyfs_configurator.h" --- client/src/unifyfs.c | 26 ++++++++++++++++++++++++++ common/src/unifyfs_configurator.h | 1 + docs/configuration.rst | 1 + 3 files changed, 28 insertions(+) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 0fc40c0f0..3c876bde5 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -79,6 +79,11 @@ int unifyfs_client_id; static int unifyfs_use_single_shm = 0; static int unifyfs_page_size = 0; +/* Determine whether we automatically sync every write to server. + * This slows write performance, but it can serve as a work + * around for apps that do not have all necessary syncs. */ +static bool unifyfs_write_sync; + static off_t unifyfs_max_offt; static off_t unifyfs_min_offt; static off_t unifyfs_max_long; @@ -1706,6 +1711,15 @@ int unifyfs_fid_write( /* write succeeded, remember that we have new data * that needs to be synced with the server */ meta->needs_sync = 1; + + /* optionally sync after every write */ + if (unifyfs_write_sync) { + int ret = unifyfs_sync(fid); + if (ret) { + LOGERR("client sync after write failed"); + rc = ret; + } + } } } else { /* unknown storage type */ @@ -2316,6 +2330,18 @@ static int unifyfs_init(void) } } + /* Determine whether we automatically sync every write to server. + * This slows write performance, but it can serve as a work + * around for apps that do not have all necessary syncs. */ + unifyfs_write_sync = false; + cfgval = client_cfg.client_write_sync; + if (cfgval != NULL) { + rc = configurator_bool_val(cfgval, &b); + if (rc == 0) { + unifyfs_write_sync = (bool)b; + } + } + /* define size of buffer used to cache key/value pairs for * data offsets before passing them to the server */ unifyfs_index_buf_size = UNIFYFS_INDEX_BUF_SIZE; diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index ab136e13f..54d0f5584 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -74,6 +74,7 @@ UNIFYFS_CFG(client, recv_data_size, INT, UNIFYFS_DATA_RECV_SIZE, "shared memory segment size in bytes for receiving data from server", NULL) \ UNIFYFS_CFG(client, write_index_size, INT, UNIFYFS_INDEX_BUF_SIZE, "write metadata index buffer size", NULL) \ UNIFYFS_CFG(client, cwd, STRING, NULLSTRING, "current working directory", NULL) \ + UNIFYFS_CFG(client, write_sync, BOOL, off, "sync every write to server", NULL) \ UNIFYFS_CFG_CLI(log, verbosity, INT, 0, "log verbosity level", NULL, 'v', "specify logging verbosity level") \ UNIFYFS_CFG_CLI(log, file, STRING, unifyfsd.log, "log file name", NULL, 'l', "specify log file name") \ UNIFYFS_CFG_CLI(log, dir, STRING, LOGDIR, "log file directory", configurator_directory_check, 'L', "specify full path to directory to contain log file") \ diff --git a/docs/configuration.rst b/docs/configuration.rst index 7468f7ce6..6545487f6 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -71,6 +71,7 @@ a given section and key. local_extents BOOL service reads from local data if possible (default: off) recv_data_size INT maximum size (B) of memory buffer for receiving data from server write_index_size INT maximum size (B) of memory buffer for storing write log metadata + write_sync BOOL sync data to server after every write (default: off) ================ ====== ================================================================= The ``cwd`` setting is used to emulate the behavior one From 4e0e46d06b9fcc69df90396e80f3a4a28ce758f8 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 16 Nov 2020 18:04:01 -0800 Subject: [PATCH 162/168] flush spillover file before invoking sync rpc with server --- client/src/unifyfs-fixed.c | 83 ++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 90c33f6be..7833dcb83 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -191,53 +191,56 @@ int truncate_write_meta(unifyfs_filemeta_t* meta, off_t trunc_sz) */ int unifyfs_sync(int target_fid) { + int tmp_rc; int ret = UNIFYFS_SUCCESS; - off_t max_log_offset = 0; - - /* For each open file descriptor .. */ - for (int i = 0; i < UNIFYFS_MAX_FILEDESCS; i++) { - /* get file id for each file descriptor */ - int fid = unifyfs_fds[i].fid; - if (-1 == fid) { - /* file descriptor is not currently in use */ - continue; - } - - /* is this the target file? */ - if ((target_fid != -1) && (fid != target_fid)) { - continue; - } + /* if caller gave us a file id, sync that specific fid */ + if (target_fid >= 0) { + /* user named a specific file id, lookup its metadata */ + int fid = target_fid; unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if ((NULL == meta) || (meta->fid != fid)) { + /* bail out with an error if we fail to find it */ LOGERR("missing filemeta for fid=%d", fid); - if (fid == target_fid) { - return UNIFYFS_FAILURE; - } - continue; + return UNIFYFS_FAILURE; } + /* sync with server if we need to */ if (meta->needs_sync) { /* write contents from segment tree to index buffer */ off_t max_log_off = unifyfs_rewrite_index_from_seg_tree(meta); - if (max_log_off > max_log_offset) { - max_log_offset = max_log_off; - } /* if there are no index entries, we've got nothing to sync */ if (*unifyfs_indices.ptr_num_entries == 0) { - if (fid == target_fid) { - return UNIFYFS_SUCCESS; + /* consider that we've sync'd successfully */ + meta->needs_sync = 0; + return UNIFYFS_SUCCESS; + } + + /* ensure any data written to the spillover file is flushed */ + off_t logio_shmem_size; + unifyfs_logio_get_sizes(logio_ctx, &logio_shmem_size, NULL); + if (max_log_off >= logio_shmem_size) { + /* some extents range into spill over area, + * so flush data to spill over file */ + tmp_rc = unifyfs_logio_sync(logio_ctx); + if (UNIFYFS_SUCCESS != tmp_rc) { + LOGERR("failed to sync logio data"); + ret = tmp_rc; } + LOGDBG("after logio spill sync"); } /* tell the server to grab our new extents */ - ret = invoke_client_sync_rpc(meta->gfid); - if (ret != UNIFYFS_SUCCESS) { + tmp_rc = invoke_client_sync_rpc(meta->gfid); + if (UNIFYFS_SUCCESS != tmp_rc) { /* something went wrong when trying to flush extents */ LOGERR("failed to flush write index to server for gfid=%d", meta->gfid); + ret = tmp_rc; } + + /* we've sync'd, so mark this file as being up-to-date */ meta->needs_sync = 0; /* flushed, clear buffer and refresh number of entries @@ -245,22 +248,24 @@ int unifyfs_sync(int target_fid) clear_index(); } - /* break out of loop when targeting a specific file */ - if (fid == target_fid) { - break; - } + return ret; } - /* ensure any data written to the spillover file is flushed */ - off_t logio_shmem_size; - unifyfs_logio_get_sizes(logio_ctx, &logio_shmem_size, NULL); - if (max_log_offset >= logio_shmem_size) { - LOGDBG("before logio spill sync") - ret = unifyfs_logio_sync(logio_ctx); - if (ret != UNIFYFS_SUCCESS) { - LOGERR("failed to sync logio data"); + /* to get here, caller specified target_fid = -1, + * so sync every file descriptor */ + for (int i = 0; i < UNIFYFS_MAX_FILEDESCS; i++) { + /* get file id for each file descriptor */ + int fid = unifyfs_fds[i].fid; + if (-1 == fid) { + /* file descriptor is not currently in use */ + continue; + } + + /* got an open file, sync this file id */ + tmp_rc = unifyfs_sync(fid); + if (UNIFYFS_SUCCESS != tmp_rc) { + ret = tmp_rc; } - LOGDBG("after logio spill sync"); } return ret; From 9762474664db991241b803c0847ee15a4e423fae Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Thu, 19 Nov 2020 09:32:46 -0500 Subject: [PATCH 163/168] add output file CLI option to testutil When provided via '-o|--outfile ', rank 0 will open the named output file and that rank's calls to test_print() and test_print_once() will write to the output file. All other ranks will still write to stdout, which is the default when the option is not provided. Also updates the writeread example to use test_print_once() to print results to the output file. Similar changes will be necessary for the other examples. --- examples/src/testutil.c | 13 +++ examples/src/testutil.h | 178 +++++++++++++++++++++++++-------------- examples/src/writeread.c | 70 +++++++-------- 3 files changed, 164 insertions(+), 97 deletions(-) diff --git a/examples/src/testutil.c b/examples/src/testutil.c index 048876505..ecde7e1db 100644 --- a/examples/src/testutil.c +++ b/examples/src/testutil.c @@ -280,6 +280,7 @@ int stat_cmd(test_cfg* cfg, char* filename) int rc; const char* typestr; char* tmp; + char* newline; char datestr[32]; rc = stat(filename, &sb); @@ -357,15 +358,27 @@ int stat_cmd(test_cfg* cfg, char* filename) memset(datestr, 0, sizeof(datestr)); timestamp = sb.st_atime; ctime_r(×tamp, datestr); + newline = strchr(datestr, '\n'); + if (NULL != newline) { + *newline = '\0'; + } test_print(cfg, "Last file access: %s", datestr); memset(datestr, 0, sizeof(datestr)); timestamp = sb.st_mtime; ctime_r(×tamp, datestr); + newline = strchr(datestr, '\n'); + if (NULL != newline) { + *newline = '\0'; + } test_print(cfg, "Last file modification: %s", datestr); memset(datestr, 0, sizeof(datestr)); timestamp = sb.st_ctime; ctime_r(×tamp, datestr); + newline = strchr(datestr, '\n'); + if (NULL != newline) { + *newline = '\0'; + } test_print(cfg, "Last status change: %s", datestr); } diff --git a/examples/src/testutil.h b/examples/src/testutil.h index daa981e1d..d64b0c124 100644 --- a/examples/src/testutil.h +++ b/examples/src/testutil.h @@ -121,6 +121,8 @@ typedef struct { int use_mpi; int use_unifyfs; int enable_mpi_mount; /* automount during MPI_Init() */ + char* output_file; /* print test messages to output file */ + FILE* output_fp; /* I/O behavior options */ int io_pattern; /* N1 or NN */ @@ -196,44 +198,51 @@ void test_config_print(test_cfg* cfg) { assert(NULL != cfg); - fprintf(stderr, " Test Configuration\n"); - fprintf(stderr, "==========================\n"); - - fprintf(stderr, "\n-- Program Behavior --\n"); - fprintf(stderr, "\t debug = %d\n", cfg->debug); - fprintf(stderr, "\t verbose = %d\n", cfg->verbose); - fprintf(stderr, "\t use_mpi = %d\n", cfg->use_mpi); - fprintf(stderr, "\t use_unifyfs = %d\n", cfg->use_unifyfs); - fprintf(stderr, "\t mpi_mount = %d\n", cfg->enable_mpi_mount); - - fprintf(stderr, "\n-- IO Behavior --\n"); - fprintf(stderr, "\t io_pattern = %s\n", io_pattern_str(cfg->io_pattern)); - fprintf(stderr, "\t io_check = %d\n", cfg->io_check); - fprintf(stderr, "\t io_shuffle = %d\n", cfg->io_shuffle); - fprintf(stderr, "\t pre_trunc = %d\n", cfg->pre_wr_trunc); - fprintf(stderr, "\t post_trunc = %d\n", cfg->post_wr_trunc); - fprintf(stderr, "\t use_aio = %d\n", cfg->use_aio); - fprintf(stderr, "\t use_lio = %d\n", cfg->use_lio); - fprintf(stderr, "\t use_mapio = %d\n", cfg->use_mapio); - fprintf(stderr, "\t use_mpiio = %d\n", cfg->use_mpiio); - fprintf(stderr, "\t use_prdwr = %d\n", cfg->use_prdwr); - fprintf(stderr, "\t use_stdio = %d\n", cfg->use_stdio); - fprintf(stderr, "\t use_vecio = %d\n", cfg->use_vecio); - - fprintf(stderr, "\n-- IO Size Config --\n"); - fprintf(stderr, "\t n_blocks = %" PRIu64 "\n", cfg->n_blocks); - fprintf(stderr, "\t block_sz = %" PRIu64 "\n", cfg->block_sz); - fprintf(stderr, "\t chunk_sz = %" PRIu64 "\n", cfg->chunk_sz); - fprintf(stderr, "\t truncate_sz = %lu\n", (unsigned long)cfg->trunc_size); - - fprintf(stderr, "\n-- Target File --\n"); - fprintf(stderr, "\t filename = %s\n", cfg->filename); - fprintf(stderr, "\t mountpt = %s\n", cfg->mountpt); - - fprintf(stderr, "\n-- MPI Info --\n"); - fprintf(stderr, "\t app_id = %d\n", cfg->app_id); - fprintf(stderr, "\t rank = %d\n", cfg->rank); - fprintf(stderr, "\t n_ranks = %d\n", cfg->n_ranks); + FILE* fp = cfg->output_fp; + if (NULL == fp) { + fp = stderr; + } + + fprintf(fp, " Test Configuration\n"); + fprintf(fp, "==========================\n"); + + fprintf(fp, "\n-- Program Behavior --\n"); + fprintf(fp, "\t debug = %d\n", cfg->debug); + fprintf(fp, "\t verbose = %d\n", cfg->verbose); + fprintf(fp, "\t use_mpi = %d\n", cfg->use_mpi); + fprintf(fp, "\t use_unifyfs = %d\n", cfg->use_unifyfs); + fprintf(fp, "\t mpi_mount = %d\n", cfg->enable_mpi_mount); + fprintf(fp, "\t outfile = %s\n", cfg->output_file); + + fprintf(fp, "\n-- IO Behavior --\n"); + fprintf(fp, "\t io_pattern = %s\n", io_pattern_str(cfg->io_pattern)); + fprintf(fp, "\t io_check = %d\n", cfg->io_check); + fprintf(fp, "\t io_shuffle = %d\n", cfg->io_shuffle); + fprintf(fp, "\t pre_trunc = %d\n", cfg->pre_wr_trunc); + fprintf(fp, "\t post_trunc = %d\n", cfg->post_wr_trunc); + fprintf(fp, "\t use_aio = %d\n", cfg->use_aio); + fprintf(fp, "\t use_lio = %d\n", cfg->use_lio); + fprintf(fp, "\t use_mapio = %d\n", cfg->use_mapio); + fprintf(fp, "\t use_mpiio = %d\n", cfg->use_mpiio); + fprintf(fp, "\t use_prdwr = %d\n", cfg->use_prdwr); + fprintf(fp, "\t use_stdio = %d\n", cfg->use_stdio); + fprintf(fp, "\t use_vecio = %d\n", cfg->use_vecio); + + fprintf(fp, "\n-- IO Size Config --\n"); + fprintf(fp, "\t n_blocks = %" PRIu64 "\n", cfg->n_blocks); + fprintf(fp, "\t block_sz = %" PRIu64 "\n", cfg->block_sz); + fprintf(fp, "\t chunk_sz = %" PRIu64 "\n", cfg->chunk_sz); + fprintf(fp, "\t truncate_sz = %lu\n", (unsigned long)cfg->trunc_size); + + fprintf(fp, "\n-- Target File --\n"); + fprintf(fp, "\t filename = %s\n", cfg->filename); + fprintf(fp, "\t mountpt = %s\n", cfg->mountpt); + + fprintf(fp, "\n-- MPI Info --\n"); + fprintf(fp, "\t app_id = %d\n", cfg->app_id); + fprintf(fp, "\t rank = %d\n", cfg->rank); + fprintf(fp, "\t n_ranks = %d\n", cfg->n_ranks); + fprintf(fp, "\n==========================\n\n"); } static inline @@ -259,28 +268,28 @@ static inline void test_print(test_cfg* cfg, const char* fmt, ...) { int err = errno; - char buf[1024]; assert(NULL != cfg); - printf("[%d] ", cfg->rank); + FILE* fp = cfg->output_fp; + if (NULL == fp) { + fp = stdout; + } + + fprintf(fp, "[%d] ", cfg->rank); va_list args; va_start(args, fmt); - vsprintf(buf, fmt, args); - printf("%s", buf); + vfprintf(fp, fmt, args); va_end(args); if (err) { - printf(" (errno=%d, %s)", err, strerror(err)); - } - - /* Add in a '\n' if the line didn't end with one */ - if (buf[strlen(buf) - 1] != '\n') { - printf("\n"); + fprintf(fp, " (errno=%d, %s)", err, strerror(err)); } - fflush(stdout); + /* End with a newline */ + fprintf(fp, "\n"); + fflush(fp); } static inline @@ -294,17 +303,23 @@ void test_print_once(test_cfg* cfg, const char* fmt, ...) return; } + FILE* fp = cfg->output_fp; + if (NULL == fp) { + fp = stdout; + } + va_list args; va_start(args, fmt); - vfprintf(stdout, fmt, args); + vfprintf(fp, fmt, args); va_end(args); if (err) { - printf(" (errno=%d, %s)\n", err, strerror(err)); + fprintf(fp, " (errno=%d, %s)\n", err, strerror(err)); } - printf("\n"); - fflush(stdout); + /* End with a newline */ + fprintf(fp, "\n"); + fflush(fp); } static inline @@ -316,13 +331,18 @@ void test_print_verbose(test_cfg* cfg, const char* fmt, ...) return; } + FILE* fp = cfg->output_fp; + if (NULL == fp) { + fp = stderr; + } + va_list args; va_start(args, fmt); - vfprintf(stderr, fmt, args); + vfprintf(fp, fmt, args); va_end(args); - fprintf(stderr, "\n"); - fflush(stderr); + fprintf(fp, "\n"); + fflush(fp); } static inline @@ -334,13 +354,18 @@ void test_print_verbose_once(test_cfg* cfg, const char* fmt, ...) return; } + FILE* fp = cfg->output_fp; + if (NULL == fp) { + fp = stderr; + } + va_list args; va_start(args, fmt); - vfprintf(stderr, fmt, args); + vfprintf(fp, fmt, args); va_end(args); - fprintf(stderr, "\n"); - fflush(stderr); + fprintf(fp, "\n"); + fflush(fp); } /* ---------- Timer Utilities ---------- */ @@ -466,7 +491,7 @@ int test_is_static(const char* program) // common options for all tests -static const char* test_short_opts = "a:Ab:c:df:hkLm:MNn:p:PSt:T:UvVx"; +static const char* test_short_opts = "a:Ab:c:df:hkLm:MNn:o:p:PSt:T:UvVx"; static const struct option test_long_opts[] = { { "appid", 1, 0, 'a' }, @@ -482,6 +507,7 @@ static const struct option test_long_opts[] = { { "mpiio", 0, 0, 'M' }, { "nblocks", 1, 0, 'n' }, { "mapio", 0, 0, 'N' }, + { "outfile", 1, 0, 'o' }, { "pattern", 1, 0, 'p' }, { "prdwr", 0, 0, 'P' }, { "pre-truncate", 1, 0, 't' }, @@ -523,6 +549,8 @@ static const char* test_usage_str = " (default: 32)\n" " -N, --mapio use mmap instead of read|write\n" " (default: off)\n" + " -o, --outfile= output file name (or path)\n" + " (default: 'stdout')\n" " -p, --pattern= 'n1' (N-to-1 shared file) or 'nn' (N-to-N file per process)\n" " (default: 'n1')\n" " -P, --prdwr use pread|pwrite instead of read|write\n" @@ -612,6 +640,10 @@ int test_process_argv(test_cfg* cfg, cfg->use_mapio = 1; break; + case 'o': + cfg->output_file = strdup(optarg); + break; + case 'p': cfg->io_pattern = check_io_pattern(optarg); break; @@ -817,7 +849,8 @@ int lipsum_check(const char* buf, uint64_t len, uint64_t offset, val = start + i; if (ibuf[i] != val) { *error_offset = offset + (i * sizeof(uint64_t)); - fprintf(stderr, "DEBUG: [%" PRIu64 "] @ offset %" PRIu64 + fprintf(stderr, + "LIPSUM CHECK ERROR: [%" PRIu64 "] @ offset %" PRIu64 ", expected=%" PRIu64 " found=%" PRIu64 "\n", i, *error_offset, val, ibuf[i]); return -1; @@ -1195,8 +1228,21 @@ int test_init(int argc, char** argv, cfg->n_ranks = 1; } + if (NULL != cfg->output_file) { + if (cfg->rank == 0) { + // only rank 0 writes to output file + cfg->output_fp = fopen(cfg->output_file, "a"); + if (NULL == cfg->output_fp) { + test_print_once(cfg, + "USAGE ERROR: failed to open output file %s", + cfg->output_file); + exit(-1); + } + } + } + if (cfg->verbose) { - // must come after test_mpi_init() to pick up MPI info + // must come after MPI_Init() to have valid MPI info test_config_print(cfg); } @@ -1256,6 +1302,14 @@ void test_fini(test_cfg* cfg) free(cfg->mountpt); } + if (NULL != cfg->output_file) { + free(cfg->output_file); + if (NULL != cfg->output_fp) { + fflush(cfg->output_fp); + fclose(cfg->output_fp); + } + } + memset(cfg, 0, sizeof(test_cfg)); } diff --git a/examples/src/writeread.c b/examples/src/writeread.c index e62cc576a..0522435ca 100644 --- a/examples/src/writeread.c +++ b/examples/src/writeread.c @@ -387,41 +387,41 @@ int main(int argc, char* argv[]) double global_read_bw = bandwidth_mib(total_bytes, max_global_read_time); if (test_config.rank == 0) { - printf("File Create Time is %.6lf s\n", - time_create.elapsed_sec_all); - printf("Minimum Local Write BW is %.3lf MiB/s\n", - min_local_write_bw); - printf("Maximum Local Write BW is %.3lf MiB/s\n", - max_local_write_bw); - printf("Aggregate Local Write BW is %.3lf MiB/s\n", - aggr_local_write_bw); - printf("Global Write BW is %.3lf MiB/s\n", - global_write_bw); - printf("Minimum Local Sync Time is %.6lf s\n", - min_local_sync_time); - printf("Maximum Local Sync Time is %.6lf s\n", - max_local_sync_time); - printf("Global Sync Time is %.6lf s\n", - max_global_sync_time); - printf("Global Write+Sync BW is %.3lf MiB/s\n", - global_write_sync_bw); - printf("Stat Time Pre-Laminate is %.6lf s\n", - time_stat_pre.elapsed_sec_all); - printf("Stat Time Pre-Laminate2 is %.6lf s\n", - time_stat_pre.elapsed_sec_all); - printf("File Laminate Time is %.6lf s\n", - time_laminate.elapsed_sec_all); - printf("Stat Time Post-Laminate is %.6lf s\n", - time_stat_post.elapsed_sec_all); - printf("Minimum Local Read BW is %.3lf MiB/s\n", - min_local_read_bw); - printf("Maximum Local Read BW is %.3lf MiB/s\n", - max_local_read_bw); - printf("Aggregate Local Read BW is %.3lf MiB/s\n", - aggr_local_read_bw); - printf("Global Read BW is %.3lf MiB/s\n", - global_read_bw); - fflush(stdout); + errno = 0; /* just in case there was an earlier error */ + test_print_once(cfg, "File Create Time is %.6lf s", + time_create.elapsed_sec_all); + test_print_once(cfg, "Minimum Local Write BW is %.3lf MiB/s", + min_local_write_bw); + test_print_once(cfg, "Maximum Local Write BW is %.3lf MiB/s", + max_local_write_bw); + test_print_once(cfg, "Aggregate Local Write BW is %.3lf MiB/s", + aggr_local_write_bw); + test_print_once(cfg, "Global Write BW is %.3lf MiB/s", + global_write_bw); + test_print_once(cfg, "Minimum Local Sync Time is %.6lf s", + min_local_sync_time); + test_print_once(cfg, "Maximum Local Sync Time is %.6lf s", + max_local_sync_time); + test_print_once(cfg, "Global Sync Time is %.6lf s", + max_global_sync_time); + test_print_once(cfg, "Global Write+Sync BW is %.3lf MiB/s", + global_write_sync_bw); + test_print_once(cfg, "Stat Time Pre-Laminate is %.6lf s", + time_stat_pre.elapsed_sec_all); + test_print_once(cfg, "Stat Time Pre-Laminate2 is %.6lf s", + time_stat_pre.elapsed_sec_all); + test_print_once(cfg, "File Laminate Time is %.6lf s", + time_laminate.elapsed_sec_all); + test_print_once(cfg, "Stat Time Post-Laminate is %.6lf s", + time_stat_post.elapsed_sec_all); + test_print_once(cfg, "Minimum Local Read BW is %.3lf MiB/s", + min_local_read_bw); + test_print_once(cfg, "Maximum Local Read BW is %.3lf MiB/s", + max_local_read_bw); + test_print_once(cfg, "Aggregate Local Read BW is %.3lf MiB/s", + aggr_local_read_bw); + test_print_once(cfg, "Global Read BW is %.3lf MiB/s", + global_read_bw); } // cleanup From d2a5d822219ed19854534ea080c20385a89cad21 Mon Sep 17 00:00:00 2001 From: CamStan Date: Thu, 19 Nov 2020 17:23:36 -0800 Subject: [PATCH 164/168] Update docs and ci for v0.9.1 release Remove leveldb as required dependency in gitlab and travis. Add option to boostrap.sh if wanting to download and build leveldb. Update docs. --- .gitlab-ci.yml | 1 - .travis.yml | 1 - bootstrap.sh | 27 ++++++++----- docs/build-intercept.rst | 47 +++++++++++------------ docs/conf.py | 8 ++-- docs/dependencies.rst | 15 +++++++- docs/examples.rst | 82 +++++++++++++++++++++------------------- t/sys/chdir.c | 4 +- 8 files changed, 103 insertions(+), 82 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 577f2b668..3bae54585 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,7 +124,6 @@ before_script: - SPACK_COMPILER=${COMPILER//\//@} - SPACK_ARCH="$(spack arch -p)-$(spack arch -o)-$(uname -m)" - spack load gotcha %$SPACK_COMPILER arch=$SPACK_ARCH - - spack load leveldb %$SPACK_COMPILER arch=$SPACK_ARCH - spack load argobots %$SPACK_COMPILER arch=$SPACK_ARCH - spack load mercury %$SPACK_COMPILER arch=$SPACK_ARCH - spack load margo %$SPACK_COMPILER arch=$SPACK_ARCH diff --git a/.travis.yml b/.travis.yml index b1852b8a1..5849866d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,7 +65,6 @@ before_install: install: - . $HOME/spack/share/spack/setup-env.sh - - spack install leveldb && spack load leveldb - spack install gotcha@1.0.3 && spack load gotcha@1.0.3 - spack install margo^mercury@1.0.1+bmi~ofi~boostsys && spack load argobots && spack load mercury && spack load margo - spack install spath && spack load spath diff --git a/bootstrap.sh b/bootstrap.sh index e29f69ae4..81f1884b7 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -12,13 +12,16 @@ INSTALL_DIR=$ROOT/install cd deps repos=( https://xgitlab.cels.anl.gov/sds/bmi.git - https://github.com/google/leveldb.git https://github.com/LLNL/GOTCHA.git https://github.com/pmodels/argobots.git https://github.com/mercury-hpc/mercury.git https://xgitlab.cels.anl.gov/sds/margo.git ) +if [ $1 = "--with-leveldb" ]; then + repos+=(https://github.com/google/leveldb.git) +fi + for i in "${repos[@]}" ; do # Get just the name of the project (like "mercury") name=$(basename $i | sed 's/\.git//g') @@ -40,15 +43,19 @@ cd bmi make -j $(nproc) && make install cd .. -echo "### building leveldb ###" -cd leveldb -git checkout 1.22 -mkdir -p build && cd build -cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ - -DBUILD_SHARED_LIBS=yes .. -make -j $(nproc) && make install -cd .. -cd .. +if [ $1 = "--with-leveldb" ]; then + echo "### building leveldb ###" + cd leveldb + git checkout 1.22 + mkdir -p build && cd build + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ + -DBUILD_SHARED_LIBS=yes .. + make -j $(nproc) && make install + cd .. + cd .. +else + echo "### skipping leveldb build ###" +fi echo "### building GOTCHA ###" cd GOTCHA diff --git a/docs/build-intercept.rst b/docs/build-intercept.rst index 566104975..4887bff4f 100644 --- a/docs/build-intercept.rst +++ b/docs/build-intercept.rst @@ -4,16 +4,6 @@ Build & I/O Interception In this section, we describe how to build UnifyFS with I/O interception. -.. note:: - - The current version of UnifyFS adopts the mdhim key-value store, which strictly - requires: - - "An MPI distribution that supports MPI_THREAD_MULTIPLE and per-object locking of - critical sections (this excludes OpenMPI up to version 3.0.1, the current version as of this writing)" - - as specified in the project `github `_. - --------------------------- --------------------------------------- @@ -40,6 +30,22 @@ you can omit it during UnifyFS configuration by using the ``--without-gotcha`` configure option. Without GOTCHA, static linker wrapping is required for I/O interception. +HDF5 +**** + +UnifyFS includes example programs that use HDF5. If HDF5 is not available on +your target system, it can be omitted during UnifyFS configuration by using +the ``--without-hdf5`` configure option. + +MDHIM +***** + +Previous MDHIM-based support for file operations can be selected at configure +time using the ``--enable-mdhim`` option. Using this option requires LevelDB as +a dependency. Provide the path to your LevelDB install at configure time with +the ``--with-leveldb=/path/to/leveldb`` option. Note that this may not +currently be in a usable state. + PMI2/PMIx Key-Value Store ************************* @@ -58,13 +64,6 @@ mounting capability. With transparent mounting, calls to ``unifyfs_mount()`` and the namespace mountpoint. To enable transparent mounting, use the ``--enable-mpi-mount`` configure option. -HDF5 -**** - -UnifyFS includes example programs that use HDF5. If HDF5 is not available on -your target system, it can be omitted during UnifyFS configuration by using -the ``--without-hdf5`` configure option. - --------------------------- --------------------------- @@ -122,8 +121,10 @@ build is desired. Type ``spack info unifyfs`` for more info. ``spack install unifyfs+hdf5 ^hdf5~mpi`` Build with serial HDF5 Fortran ``spack install unifyfs+fortran`` Enable Fortran support + MDHIM ``spack install unifyfs+mdhim`` Enable MDHIM build options PMI ``spack install unifyfs+pmi`` Enable PMI2 support PMIx ``spack install unifyfs+pmix`` Enable PMIx support + spath ``spack install unifyfs+spath`` Normalize relative paths ========== ======================================== =========================== .. attention:: @@ -160,7 +161,6 @@ UnifyFS dependencies can then be installed. .. code-block:: Bash $ spack install gotcha - $ spack install leveldb $ spack install margo ^mercury+bmi .. tip:: @@ -186,7 +186,6 @@ manually build UnifyFS from inside the source code directory. .. code-block:: Bash $ spack load gotcha - $ spack load leveldb $ spack load mercury $ spack load argobots $ spack load margo @@ -196,7 +195,7 @@ manually build UnifyFS from inside the source code directory. $ make $ make install -To see all available build configuration options, type ``./configure --help`` +To see all available build configuration options, run ``./configure --help`` after ``./autogen.sh`` has been run. --------------------------- @@ -212,8 +211,8 @@ from the `UnifyFS repository `_. Build the Dependencies ********************** -UnifyFS requires MPI, LevelDB, GOTCHA, Margo and OpenSSL. -References to these dependencies can be found on our :doc:`` page. +UnifyFS requires MPI, GOTCHA, Margo and OpenSSL. +References to these dependencies can be found on our :doc:`dependencies` page. A `bootstrap.sh `_ script has been provided in order to make manual build and installation of dependencies @@ -234,11 +233,11 @@ this: $ export PKG_CONFIG_PATH=path/to/mercury/lib/pkgconfig:path/to/argobots/lib/pkgconfig:path/to/margo/lib/pkgconfig $ ./autogen.sh - $ ./configure --prefix=/path/to/install --with-gotcha=/path/to/gotcha --with-leveldb=/path/to/leveldb + $ ./configure --prefix=/path/to/install --with-gotcha=/path/to/gotcha $ make $ make install -To see all available build configuration options, type ``./configure --help`` +To see all available build configuration options, run ``./configure --help`` after ``./autogen.sh`` has been run. --------------------------- diff --git a/docs/conf.py b/docs/conf.py index 87684ba88..1f0ac6a60 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,17 +46,17 @@ # General information about the project. project = u'UnifyFS' -copyright = u'2017, Lawrence Livermore National Security LLC LLNL-CODE-741539, UT-Batelle LLC' -author = u'Kathryn Mohror, Adam Moody, Oral Sarp, Feiyi Wang, Hyogi Sim, Danielle Sikich, Joseph Moore, Ned Bass' +copyright = u'2020, Lawrence Livermore National Security LLC, LLNL-CODE-741539, UT-Batelle LLC' +author = u'Kathryn Mohror, Adam Moody, Oral Sarp, Feiyi Wang, Hyogi Sim, Swen Boehm, Michael Brim, Danielle Sikich, Joseph Moore, Ned Bass, Tony Hutter, Celso Mendes, Craig Steffen, Cameron Stanavige' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = u'0.9.0' +version = u'0.9.1' # The full version, including alpha/beta/rc tags. -release = u'0.9.0' +release = u'0.9.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/dependencies.rst b/docs/dependencies.rst index 85b6cbc4b..fa6e7a664 100644 --- a/docs/dependencies.rst +++ b/docs/dependencies.rst @@ -2,9 +2,11 @@ UnifyFS Dependencies ==================== -- `GOTCHA `_ version 1.0.3 +-------- +Required +-------- -- `leveldb `_ version 1.22 +- `GOTCHA `_ version 1.0.3 - `Margo `_ version 0.4.3 and its dependencies: @@ -22,3 +24,12 @@ UnifyFS Dependencies ``PKG_CONFIG_PATH`` environment variable and include in that variable the paths for the ``.pc`` files for Mercury, Argobots, and Margo separated by colons. + +-------- +Optional +-------- + +- `leveldb `_ version 1.22 + needed when building with ``--enable-mdhim`` configure option + +- `spath `_ for normalizing relative paths diff --git a/docs/examples.rst b/docs/examples.rst index f7859f337..26b002736 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -95,44 +95,50 @@ to aid in this process. Usage: write-static [options...] Available options: - -a, --appid= use given application id - (default: 0) - -A, --aio use asynchronous I/O instead of read|write - (default: off) - -b, --blocksize= I/O block size - (default: 16 MiB) - -c, --chunksize= I/O chunk size for each operation - (default: 1 MiB) - -d, --debug for debugging, wait for input (at rank 0) at start - (default: off) - -f, --file= target file name (or path) under mountpoint - (default: 'testfile') - -k, --check check data contents upon read - (default: off) - -L, --listio use lio_listio instead of read|write - (default: off) - -m, --mount= use for unifyfs - (default: /unifyfs) - -M, --mpiio use MPI-IO instead of POSIX I/O - (default: off) - -n, --nblocks= count of blocks each process will read|write - (default: 32) - -N, --mapio use mmap instead of read|write - (default: off) - -p, --pattern= 'n1' (N-to-1 shared file) or 'nn' (N-to-N file per process) - (default: 'n1') - -P, --prdwr use pread|pwrite instead of read|write - (default: off) - -S, --stdio use fread|fwrite instead of read|write - (default: off) - -U, --disable-unifyfs do not use UnifyFS - (default: enable UnifyFS) - -v, --verbose print verbose information - (default: off) - -V, --vecio use readv|writev instead of read|write - (default: off) - -x, --shuffle read different data than written - (default: off) + -a, --appid= use given application id + (default: 0) + -A, --aio use asynchronous I/O instead of read|write + (default: off) + -b, --blocksize= I/O block size + (default: 16 MiB) + -c, --chunksize= I/O chunk size for each operation + (default: 1 MiB) + -d, --debug for debugging, wait for input (at rank 0) at start + (default: off) + -f, --file= target file name (or path) under mountpoint + (default: 'testfile') + -k, --check check data contents upon read + (default: off) + -L, --listio use lio_listio instead of read|write + (default: off) + -m, --mount= use for unifyfs + (default: /unifyfs) + -M, --mpiio use MPI-IO instead of POSIX I/O + (default: off) + -n, --nblocks= count of blocks each process will read|write + (default: 32) + -N, --mapio use mmap instead of read|write + (default: off) + -o, --outfile= output file name (or path) + (default: 'stdout') + -p, --pattern= 'n1' (N-to-1 shared file) or 'nn' (N-to-N file per process) + (default: 'n1') + -P, --prdwr use pread|pwrite instead of read|write + (default: off) + -S, --stdio use fread|fwrite instead of read|write + (default: off) + -t, --pre-truncate= truncate file to size (B) before writing + (default: off) + -T, --post-truncate= truncate file to size (B) after writing + (default: off) + -U, --disable-unifyfs do not use UnifyFS + (default: enable UnifyFS) + -v, --verbose print verbose information + (default: off) + -V, --vecio use readv|writev instead of read|write + (default: off) + -x, --shuffle read different data than written + (default: off) One form of running this example could be: diff --git a/t/sys/chdir.c b/t/sys/chdir.c index bf3f4add1..67600e9fa 100644 --- a/t/sys/chdir.c +++ b/t/sys/chdir.c @@ -391,7 +391,7 @@ int chdir_test(char* unifyfs_root) /* TODO: Our directory wrappers are not fully functioning yet, * but when they do, we should check that fchdir works. */ -#if 0 + skip(1, 7, "fchdir tests until directory wrappers are fully functional") /* change to root directory */ errno = 0; rc = chdir("/"); @@ -442,7 +442,7 @@ int chdir_test(char* unifyfs_root) __FILE__, __LINE__, fd, strerror(err)); closedir(dirp); -#endif + end_skip; return 0; } From 16ff82867cc97291244b73cb02ad1a87256f428c Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Mon, 23 Nov 2020 13:06:30 -0500 Subject: [PATCH 165/168] fix stat_cmd() when errno != 0 from prior call --- examples/src/testutil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/src/testutil.c b/examples/src/testutil.c index ecde7e1db..bb3b40256 100644 --- a/examples/src/testutil.c +++ b/examples/src/testutil.c @@ -283,9 +283,10 @@ int stat_cmd(test_cfg* cfg, char* filename) char* newline; char datestr[32]; + errno = 0; rc = stat(filename, &sb); if (rc) { - test_print(cfg, "ERROR: stat(%s) - %s", filename, strerror(rc)); + test_print(cfg, "ERROR: stat(%s) failed", filename); return rc; } From c42a7973fb761012fc265be09da28e2c2f27e3f3 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 24 Nov 2020 21:24:05 -0800 Subject: [PATCH 166/168] fix: invoke correct rpc for truncate bcast tree --- server/src/unifyfs_group_rpc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/unifyfs_group_rpc.c b/server/src/unifyfs_group_rpc.c index 5adc64d13..9c308441a 100644 --- a/server/src/unifyfs_group_rpc.c +++ b/server/src/unifyfs_group_rpc.c @@ -706,6 +706,7 @@ int truncate_bcast_forward(const unifyfs_tree_t* broadcast_tree, /* forward request down the tree */ coll_request* req; + hg_id_t hgid = unifyfsd_rpc_context->rpcs.truncate_bcast_id; for (i = 0; i < child_count; i++) { req = requests + i; @@ -715,8 +716,7 @@ int truncate_bcast_forward(const unifyfs_tree_t* broadcast_tree, i, child, glb_servers[child].margo_svr_addr_str); /* allocate handle */ - rc = get_request_handle(unifyfsd_rpc_context->rpcs.truncate_id, - child, req); + rc = get_request_handle(hgid, child, req); if (rc == UNIFYFS_SUCCESS) { /* invoke truncate request rpc on child */ rc = forward_request((void*)in, req); From c4c2faa5d5e954d59fe1279c1f3f32c319b80c6c Mon Sep 17 00:00:00 2001 From: CamStan Date: Wed, 25 Nov 2020 11:20:14 -0800 Subject: [PATCH 167/168] Switch CI tests to use outfile and rename envars Main changes - Switch CI tests to use --outfile option in the examples - Update relevant CI envars to be prefixed with UNIFYFS_CI... - Change number of processes-per-node used by JOB_RUN_COMMAND to be 1 by default - Update CI testing docs 001-setup.sh - Change auto search preference to manual build first, then Spack 100-writeread-tests.sh - Update expected line count of output 990-stop-server.sh - Fix exit that kills interactive job allocation when running tests individually ci-functions.sh - Change unify_run_test() to get resulting output from outfile rather than stdout - Turn -k (--check) on for all tests except posix. read-posix tests fail with this on. Issue coming. - Update cleanup_hosts() to skip if PDSH not available - Update optional args and remove -v from default args testing.rst - Move CI config variables to their own sub-section - Put current high-impact variables in key variables section - Add additional variables section for others that are set and might be relevant to developers adding/writing tests - Updates to add clarity for running tests - Reword get_filename() docs for clarity and outfile addition - List additional testing helper functions available in ci-functions.sh that may be relevant to developers writing/adding future tests --- .gitlab-ci.yml | 8 +- docs/testing.rst | 395 ++++++++++++++++++++++-------------- t/ci/001-setup.sh | 103 +++++----- t/ci/002-start-server.sh | 6 +- t/ci/100-writeread-tests.sh | 8 +- t/ci/110-write-tests.sh | 4 +- t/ci/990-stop-server.sh | 13 +- t/ci/README.md | 12 +- t/ci/RUN_CI_TESTS.sh | 32 +-- t/ci/ci-functions.sh | 77 ++++--- t/ci/setup-lsf.sh | 22 +- t/ci/setup-slurm.sh | 2 +- 12 files changed, 399 insertions(+), 283 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3bae54585..3b2f27765 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,21 +27,21 @@ stages: .slurm-single-node-template: variables: JOB_LAUNCH_COMMAND: "srun -N1 -n1" - LLNL_SLURM_SCHEDULER_PARAMETERS: "-N 1 -p pbatch -t $UNIT_WALL_TIME -J unifyfs-unit-tests" + LLNL_SLURM_SCHEDULER_PARAMETERS: "-N 1 -p $QUEUE -t $UNIT_WALL_TIME -J unifyfs-unit-tests" .slurm-multi-node-template: variables: - LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -p pbatch -t $INTEG_WALL_TIME -J unifyfs-integ-tests" + LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -p $QUEUE -t $INTEG_WALL_TIME -J unifyfs-integ-tests" .lsf-single-node-template: variables: JOB_LAUNCH_COMMAND: "jsrun -r1 -n1" - LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -q pbatch -W $UNIT_WALL_TIME -J unifyfs-unit-tests" + LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -q $QUEUE -W $UNIT_WALL_TIME -J unifyfs-unit-tests" SCHEDULER_PARAMETERS: "-nnodes 1 -P $PROJECT_ID -W $UNIT_WALL_TIME -J unifyfs-unit-tests" .lsf-multi-node-template: variables: - LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes $NNODES -stage storage=${STORAGE_SIZE} -q pbatch -W $INTEG_WALL_TIME -J unifyfs-integ-tests" + LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes $NNODES $STAGE_STORAGE -q $QUEUE -W $INTEG_WALL_TIME -J unifyfs-integ-tests" SCHEDULER_PARAMETERS: "-nnodes $NNODES -P $PROJECT_ID -W $INTEG_WALL_TIME -J unifyfs-integ-tests" ##### Job Templates ##### diff --git a/docs/testing.rst b/docs/testing.rst index 18e01e94e..c427e1ad2 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -426,208 +426,268 @@ documentation. ------------ -Running the Tests -***************** +Configuration Variables +*********************** -.. attention:: +Along with the already provided :doc:`configuration` options/environment +variables, there are environment variables used by the integration testing +suite that can also be set in order to change the default behavior. - UnifyFS's integration test suite requires MPI and currently only supports - ``srun`` and ``jsrun`` MPI launch commands. Changes are coming to support - ``mpirun``. +Key Variables +^^^^^^^^^^^^^ -UnifyFS's integration tests are primarly set up to be run all as one suite. -However, they can be run individually if desired. +These environment variables can be set prior to sourcing the *t/ci/001-setup.sh* +script and will affect how the overall integration suite operates. -The testing scripts in `t/ci`_ depend on sharness_, which is set up in the -containing *t/* directory. These tests will not function properly if moved or if -they cannot find the sharness files. +``UNIFYFS_INSTALL`` +""""""""""""""""""" -.. important:: +USAGE: ``UNIFYFS_INSTALL=/path/to/dir/containing/UnifyFS/bin/directory`` - Whether running all tests or individual tests, first make sure you have - either interactively allocated nodes or are submitting a batch job to run - them. +The full path to the directory containing the *bin/* and *libexec/* directories +for your UnifyFS installation. Set this envar to prevent the integration tests +from searching for a UnifyFS installation automatically. Where the automatic +search starts can be altered by setting the ``$BASE_SEARCH_DIR`` variable. - Also make sure all :ref:`dependencies ` are installed and - loaded. +``UNIFYFS_CI_NPROCS`` +""""""""""""""""""""" -By default, the integration tests will use the number of processes-per-node as -there are nodes allocated for the job (i.e., if 4 nodes were allocated, then 4 -processes will be run per node). This can be changed by setting the -:ref:`$CI_NPROCS ` environment variable. +USAGE: ``UNIFYFS_CI_NPROCS=`` -.. note:: +The number of processes to use per node inside a job allocation. This defaults +to 1 process per node. This can be adjusted if more processes are desired +on multiple nodes or multiple processes are desired on a single node. - In order to run the the integration tests from a Spack_ installation of - UnifyFS, you'll need to tell Spack to use a different location for staging - builds in order to have the source files available from inside an allocation. +``UNIFYFS_CI_TEMP_DIR`` +""""""""""""""""""""""" - Open your Spack config file +USAGE: ``UNIFYFS_CI_TEMP_DIR=/path/for/temporary/files/created/by/UnifyFS`` - ``spack config edit config`` +Can be used as a shortcut to set ``UNIFYFS_RUNSTATE_DIR`` and +``UNIFYFS_META_DB_PATH`` to the same path. This envar defaults to +``UNIFYFS_CI_TEMP_DIR=${TMPDIR}/unifyfs.${USER}.${JOB_ID}``. - and provide a path that is visible during job allocations: +``UNIFYFS_CI_LOG_CLEANUP`` +"""""""""""""""""""""""""" - .. code-block:: yaml +USAGE: ``UNIFYFS_CI_LOG_CLEANUP=yes|YES|no|NO`` - config: - build_stage: - - /visible/path/from/all/allocated/nodes - # or build directly inside Spack's install directory - - $spack/var/spack/stage +In the event ``$UNIFYFS_LOG_DIR`` has **not** been set, the logs will be put in +``$SHARNESS_TRASH_DIRECTORY``, as set up by sharness.sh_, and cleaned up +automatically after the tests have run. The logs will be in a +*_/* subdirectory. Should any tests fail, sharness does not +clean up the trash directory for debugging purposes. Setting +``UNIFYFS_CI_LOG_CLEANUP=no|NO`` will move the *_/* logs +directory to ``$UNIFYFS_CI_DIR`` (the directory containing the integration +testing scripts) to allow them to persist even when all tests pass. This envar +defauls to ``yes``. - Then make sure to include the ``--keep-stage`` option when installing: +.. note:: - ``spack install --keep-stage unifyfs`` + Setting ``$UNIFYFS_LOG_DIR`` will put all created logs in the designated path + and will not clean them up. -Running All Tests -^^^^^^^^^^^^^^^^^ +``UNIFYFS_CI_HOST_CLEANUP`` +""""""""""""""""""""""""""" -To run all of the tests, simply run ``./RUN_CI_TESTS.sh``. +USAGE: ``UNIFYFS_CI_HOST_CLEANUP=yes|YES|no|NO`` -.. code-block:: BASH +After all tests have run, the nodes on which the tests were ran will +automatically be cleaned up. This cleanup includes ensuring ``unifyfsd`` has +stopped and deleting any files created by UnifyFS or its dependencies. Set +``UNIFYFS_CI_HOST_CLEANUP=no|NO`` to skip cleaning up. This envar defaults to +``yes``. - $ ./RUN_CI_TESTS.sh +.. note:: -or + PDSH_ is required for cleanup and cleaning up is simply skipped if not + found. -.. code-block:: BASH +``UNIFYFS_CI_CLEANUP`` +"""""""""""""""""""""" - $ prove -v RUN_CI_TESTS.sh +USAGE: ``UNIFYFS_CI_CLEANUP=yes|YES|no|NO`` -Running Individual Tests -^^^^^^^^^^^^^^^^^^^^^^^^ +Setting this to ``no|NO`` sets both ``$CI_LOG_CLEANUP`` and +``$UNIFYFS_CI_HOST_CLEANUP`` to ``no|NO``. -In order to run individual tests, testing functions and variables need to be set -up first, and the UnifyFS server needs to be started. To do this, first source -the *t/ci/001-setup.sh* script followed by *002-start-server.sh*. Then source -each desired test script after that preceded by ``$CI_DIR/``. When finished, -source the *990-stop-server.sh* script last to stop the server and clean up. +``UNIFYFS_CI_TEST_POSIX`` +""""""""""""""""""""""""" -.. code-block:: BASH +USAGE: ``UNIFYFS_CI_TEST_POSIX=yes|YES|no|NO`` - $ . full/path/to/001-setup.sh - $ . $CI_DIR/002-start-server.sh - $ . $CI_DIR/100-writeread-tests.sh - $ . $CI_DIR/990-stop-server.sh +Determines whether any ``-posix`` tests should be run since they +require a real mountpoint to exist. -Configuration Variables -^^^^^^^^^^^^^^^^^^^^^^^ +This envar defaults to ``yes``. However, when ``$UNIFYFS_MOUNTPOINT`` is set to a +real directory, this envar is switched to ``no``. The idea behind this is that +the tests can be run a first time with a fake mountpoint (which will also run +the posix tests), and then the tests can be run again with a real mountpoint and +the posix tests won't be run twice. This behavior can be overridden by setting +``UNIFYFS_CI_TEST_POSIX=yes|YES`` before running the integration tests when +``$UNIFYFS_MOUNTPOINT`` is set to an existing directory. -Along with the already provided :doc:`configuration` options/environment -variables, there are available environment variables used by the integration -testing suite that can be set in order to change the default behavior. They are -listed below in the order they are set up. +An example of testing a posix example can be see :ref:`below `. -``CI_PROJDIR`` -"""""""""""""" +.. note:: -USAGE: ``CI_PROJDIR=/base/location/to/search/for/UnifyFS/source/files`` + The posix mountpoint envar, ``UNIFYFS_CI_POSIX_MP``, is set to be located + inside ``$SHARNESS_TRASH_DIRECTORY`` automatically and cleaned up + afterwards. However, this envar can be set before running the integration + tests as well. If setting this, ensure that it is a shared file system that + all allocated nodes can see. -During setup, the integration tests will search for the ``unifyfsd`` executable -and installed example scripts if the UnifyFS install directory is not provided by -the user with the ``UNIFYFS_INSTALL`` envar. ``CI_PROJDIR`` is the base location -where this search will start and defaults to ``CI_PROJDIR=$HOME``. +Additional Variables +^^^^^^^^^^^^^^^^^^^^ +After sourcing the *t/ci/001-setup.sh* script there will be additional variables +available that may be useful when writing/adding additional tests. -``UNIFYFS_INSTALL`` +Directory Structure """"""""""""""""""" -USAGE: ``UNIFYFS_INSTALL=/path/to/dir/containing/UnifyFS/bin/directory`` +File structure here is assuming UnifyFS was cloned to ``$HOME``. -The full path to the directory containing the *bin/* and *libexec/* directories -for a UnifyFS installation. Set this envar to prevent the integration tests from -searching for a UnifyFS install directory automatically. +``UNIFYFS_CI_DIR`` + Directory containing the CI testing scripts. *$HOME/UnifyFS/t/ci/* +``SHARNESS_DIR`` + Directory containing the base sharness scripts. *$HOME/UnifyFS/t/* +``UNIFYFS_SOURCE_DIR`` + Directory containing the UnifyFS source code. *$HOME/UnifyFS/* +``BASE_SEARCH_DIR`` + Parent directory containing the UnifyFS source code. Starting place to auto + search for UnifyFS install when ``$UNIFYFS_INSTALL`` isn't provided. *$HOME/* -.. _ci-nprocs-label: +Executable Locations +"""""""""""""""""""" -``CI_NPROCS`` -""""""""""""" +``UNIFYFS_BIN`` + Directory containing ``unifyfs`` and ``unifyfsd``. *$UNIFYFS_INSTALL/bin* +``UNIFYFS_EXAMPLES`` + Directory containing the compiled examples_. *$UNIFYFS_INSTALL/libexec* -USAGE: ``CI_NPROCS=`` +Resource Managers +""""""""""""""""" -The number of processes to use per node inside a job allocation. This defaults -to the number of processes per node as there are nodes in the allocation (i.e., -if 4 nodes were allocated, then 4 processes will be run per node). This should -be adjusted if fewer processes are desired on multiple nodes, multiple processes -are desired on a single node, or a large number of nodes have been allocated. +``JOB_RUN_COMMAND`` + The base MPI job launch command established according to the detected + resource manager, number of allocated nodes, and ``$UNIFYFS_CI_NPROCS``. + + The LSF variables below will also affect the default version of this command + when using that resource manager. +``JOB_RUN_ONCE_PER_NODE`` + MPI job launch command to only run a single process on each allocated node + established according to the detected resource manager. +``JOB_ID`` + The ID assigned to the current CI job as established by the detected + resource manager. + +LSF +""" + +Additional variables used by the LSF resource manager to determine how jobs are +launched with ``$JOB_RUN_COMMAND``. These can also be set prior to sourcing the +*t/ci/001-setup.sh* script and will affect how the integration tests run. + +``UNIFYFS_CI_NCORES`` + Number of cores-per-resource-set to use. Defaults to 20. +``UNIFYFS_CI_NRS_PER_NODE`` + Number of resource-sets-per-node to use. Defaults to 1. +``UNIFYFS_CI_NRES_SETS`` + Total number of resource sets to use. Defaults to (number_of_nodes) * + (``$UNIFYFS_CI_NRS_PER_NODE``). + +Misc +"""" + +``KB`` + :math:`2^10`. +``MB`` + :math:`2^20`. +``GB`` + :math:`2^30`. -``CI_LOG_CLEANUP`` -"""""""""""""""""" +------------ -USAGE: ``CI_LOG_CLEANUP=yes|YES|no|NO`` +Running the Tests +***************** -In the event ``$UNIFYFS_LOG_DIR`` has **not** been set, the logs will be put in -``$SHARNESS_TRASH_DIRECTORY``, as set up by sharness.sh_, and cleaned up -automatically after the tests have run. The logs will be in a -*_/* subdirectory. Should any tests fail, the trash -directory will not be cleaned up for debugging purposes. Setting -``CI_LOG_CLEANUP=no|NO`` will move the *_/* logs directory -to ``$CI_DIR`` (the directory containing the integration tests) to -allow them to persist even when all tests pass. This envar defauls to ``yes``. +.. attention:: -.. note:: + UnifyFS's integration test suite requires MPI and currently only supports + ``srun`` and ``jsrun`` MPI launch commands. Changes are coming to support + ``mpirun``. - Setting ``$UNIFYFS_LOG_DIR`` will put all created logs in the designated path - and will not clean them up. +UnifyFS's integration tests are primarly set up to be run all as one suite. +However, they can be run individually if desired. -``CI_HOST_CLEANUP`` -""""""""""""""""""" +The testing scripts in `t/ci`_ depend on sharness_, which is set up in the +containing *t/* directory. These tests will not function properly if moved or if +they cannot find the sharness files. -USAGE: ``CI_HOST_CLEANUP=yes|YES|no|NO`` +Whether running all tests or individual tests, first make sure you have +either interactively allocated nodes or are submitting a batch job to run +them. -After all tests have run, the nodes on which the tests were ran will -automatically be cleaned up. This cleanup includes ensuring ``unifyfsd`` has -stopped and deleting any files created by UnifyFS or its dependencies. Set -``CI_HOST_CLEANUP=no|NO`` to skip cleaning up. This envar defaults to ``yes``. +Make sure all :ref:`dependencies ` are installed and loaded. .. note:: - PDSH_ is required for cleanup and cleaning up is simply skipped if not - found. + In order to run the the integration tests from a Spack_ installation of + UnifyFS, you'll need to tell Spack to use a different location for staging + builds in order to have the source files available from inside an allocation. -``CI_CLEANUP`` -"""""""""""""" + Open your Spack config file -USAGE: ``CI_CLEANUP=yes|YES|no|NO`` + ``spack config edit config`` -Setting this to ``no|NO`` sets both ``$CI_LOG_CLEANUP`` and ``$CI_HOST_CLEANUP`` -to ``no|NO``. + and provide a path that is visible during job allocations: -``CI_TEMP_DIR`` -"""""""""""""""" + .. code-block:: yaml -USAGE: ``CI_TEMP_DIR=/path/for/temporary/files/created/by/UnifyFS`` + config: + build_stage: + - /visible/path/from/all/allocated/nodes + # or build directly inside Spack's install directory + - $spack/var/spack/stage -Can be used as a shortcut to set ``UNIFYFS_RUNSTATE_DIR`` and -``UNIFYFS_META_DB_PATH`` to the same path. This envar defaults to -``CI_TEMP_DIR=${TMPDIR}/unifyfs.${USER}.${JOB_ID}``. + Then make sure to include the ``--keep-stage`` option when installing: -``CI_TEST_POSIX`` -""""""""""""""""" + ``spack install --keep-stage unifyfs`` -USAGE: ``CI_TEST_POSIX=yes|YES|no|NO`` +Running All Tests +^^^^^^^^^^^^^^^^^ -Determines whether any ``-posix`` tests should be run since they -require a real mountpoint to exist. +To run all of the tests, simply run ``./RUN_CI_TESTS.sh``. -This envar defaults to ``yes``. However, when ``$UNIFYFS_MOUNTPOINT`` is set to a -real directory, this envar is switched to ``no``. The idea behind this is that -the tests can be run a first time with a fake mountpoint (which will also run -the posix tests), and then the tests can be run again with a real mountpoint and -the posix tests wont be run twice. This behavior can be overridden by setting -``CI_TEST_POSIX=yes|YES`` before running the integration tests when -``$UNIFYFS_MOUNTPOINT`` is set to an existing directory. +.. code-block:: BASH -An example of testing a posix example can be see :ref:`below `. + $ ./RUN_CI_TESTS.sh -.. note:: +or + +.. code-block:: BASH - The the posix mountpoint envar, ``CI_POSIX_MP``, is set up inside - ``$SHARNESS_TRASH_DIRECTORY`` automatically and cleaned up afterwards. - However, this envar can be set before running the integration tests as well. - If setting this, ensure that it is a shared file system that all allocated - nodes can see. + $ prove -v RUN_CI_TESTS.sh + +Running Individual Tests +^^^^^^^^^^^^^^^^^^^^^^^^ + +In order to run individual tests, the testing functions and variables need to be +set up first and then the UnifyFS server needs to be started. + +First source the *t/ci/001-setup.sh* script whereafter sharness will change +directories to the ``$SHARNESS_TRASH_DIRECTORY``. To account for this, source +*002-start-server.sh* and each desired test script after that prefixed with +``$UNIFYFS_CI_DIR/``. When finished, source the *990-stop-server.sh* script +last to stop the server and clean up. + +.. code-block:: BASH + + $ . ./001-setup.sh + $ . $UNIFYFS_CI_DIR/002-start-server.sh + $ . $UNIFYFS_CI_DIR/100-writeread-tests.sh + $ . $UNIFYFS_CI_DIR/990-stop-server.sh ------------ @@ -647,12 +707,12 @@ as simple as possible. One particularly useful function is ``unify_run_test()``. Currently, this function is set up to work for the *write*, *read*, *writeread*, and *checkpoint-restart* examples. This function sets up the MPI job run command and -default arguments as well as any default arguments wanted by all examples. See +default options as well as any default arguments wanted by all examples. See :ref:`below ` for details. .. _helper-label: -Example Helper Functions +Testing Helper Functions ^^^^^^^^^^^^^^^^^^^^^^^^ There are helper functions available in `t/ci/ci-functions.sh`_ that can make @@ -672,14 +732,15 @@ example with the appropriate MPI runner and args. This function is meant to make running the cr, write, read, and writeread examples as easy as possible. The ``build_test_command()`` function is called by this function which -automatically sets any options that are always wanted (-vkf as well as -U and +automatically sets any options that are always wanted (-vkfo as well as -U and the appropriate -m if posix test or not). The stderr output file is also created (based on the filename that is autogenerated) and the appropriate option is set for the MPI job run command. -Args that can be passed in are ([-pncbx][-A|-M|-P|-S|-V]). All other args (see -:ref:`Running the Examples `) are set automatically, including the -filename (which is generated based on the input ``$app_name`` and ``$app_args``). +Args that can be passed in are ([-pncbx][-A|-M|-N|-P|-S|-V]). All other args +(see :ref:`Running the Examples `) are set automatically, +including the outfile and filename (which are generated based on the input +``$app_name`` and ``$app_args``). The third parameter is an optional "pass-by-reference" parameter that can contain the variable name for the resulting output to be stored in, allowing @@ -731,18 +792,18 @@ The results can then be tested with sharness_: USAGE: ``get_filename app_name app_args [app_suffix]`` -Builds and returns the filename for an example so that if it shows up in the -``$UNIFYFS_MOUNTPOINT`` (when using an existing mountpoint), it can be tracked -to its originating test for debugging. Error files are created with this -filename and a ``.err`` suffix and placed in the logs directory for debugging. +Builds and returns the filename with the provided suffix based on the input +app_name and app_args. + +The filename in ``$UNIFYFS_MOUNTPOINT`` will be given a ``.app`` suffix. -Also allows testers to get what the filename will be in advance if called +This allows tests to get what the filename will be in advance if called from a test suite. This can be used for posix tests to ensure the file showed -up in the mount point, as well as for cp/stat tests that potentially need the -filename from a previous test. +up in the mount point, as well as for read, cp, stat tests that potentially need +the filename from a previous test prior to running. -Note that the filename created by ``unify_run_test()`` will have a ``.app`` -suffix. +Error logs and outfiles are also created with this filename, with a ``.err`` or +``.out`` suffix respectively, and placed in the logs directory. Returns a string with the spaces removed and hyphens replaced by underscores. @@ -754,7 +815,8 @@ Returns a string with the spaces removed and hyphens replaced by underscores. Some uses cases may be: - posix tests where the file existence is checked for after a test was run -- cp/stat tests where an already existing filename from a prior test is needed +- read, cp, or stat tests where an already existing filename from a prior test + might be needed For example: @@ -777,9 +839,35 @@ For example: test_expect_success POSIX "$app_name $app_args: (line_count=$line_count, rc=$rc)" ' test $rc = 0 && test $line_count = 8 && - test_path_has_file_per_process $CI_POSIX_MP $filename + test_path_has_file_per_process $UNIFYFS_CI_POSIX_MP $filename ' +Additional Functions +"""""""""""""""""""" + +There are other convenience functions used bythat my be helpful in writing/adding tests are also +found in `t/ci/ci-functions.sh`_: + +``find_executable()`` + USAGE: ``find_executable abs_path *file_name|*path/file_name [prune_path]`` + + Locate the desired executable file when provided an absolute path of where + to start searching, the name of the file with an optional preceding path, + and an optional prune_path, or path to omit from the search. + + Returns the path of the first executable found with the given name and + optional prefix. +``elapsed_time()`` + USAGE: ``elapsed_time start_time_in_seconds end_time_in_seconds`` + + Calculates the elapsed time between two given times. + + Returns the elapsed time formatted as HH:MM:SS. +``format_bytes()`` + USAGE: ``format_bytes int`` + + Returns the input bytes formatted as KB, MB, or GB (1024 becomes 1KB). + Sharness Helper Functions ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -804,7 +892,6 @@ Expects two arguments: - $2 - Number of seconds to wait before giving up .. code-block:: BASH - :emphasize-lines: test_expect_success "unifyfsd is running" ' process_is_running unifyfsd 5 diff --git a/t/ci/001-setup.sh b/t/ci/001-setup.sh index f31898842..d4553bbbe 100755 --- a/t/ci/001-setup.sh +++ b/t/ci/001-setup.sh @@ -7,13 +7,13 @@ # desired. To run all tests simply run the RUN_TESTS.sh script. If Individual # tests are desired to be run, source the 001-setup.sh script first, followed by # 002-start-server.sh. Then source each desired script after that preceded by -# `$CI_DIR`. When finished, source the 990-stop-server.sh script last. +# `$UNIFYFS_CI_DIR`. When finished, source the 990-stop-server.sh script last. # # E.g.: # $ . full/path/to/001-setup.sh -# $ . $CI_DIR/002-start-server.sh -# $ . $CI_DIR/100-writeread-tests.sh -# $ . $CI_DIR/990-stop-server.sh +# $ . $UNIFYFS_CI_DIR/002-start-server.sh +# $ . $UNIFYFS_CI_DIR/100-writeread-tests.sh +# $ . $UNIFYFS_CI_DIR/990-stop-server.sh # # To run all of the tests, simply run RUN_CI_TESTS.sh # @@ -36,9 +36,9 @@ Then source any desired test files. Lastly, source 990-stop-server.sh. E.g.: $ . full/path/to/001-setup.sh - $ . $CI_DIR/002-start-server.sh - $ . $CI_DIR/100-writeread-tests.sh - $ . $CI_DIR/990-stop-server.sh + $ . \$UNIFYFS_CI_DIR/002-start-server.sh + $ . \$UNIFYFS_CI_DIR/100-writeread-tests.sh + $ . \$UNIFYFS_CI_DIR/990-stop-server.sh To run all of the tests, simply run RUN_CI_TESTS.sh. @@ -80,19 +80,19 @@ export SYSTEM_NAME=$(echo $(hostname) | sed -r 's/(^[[:alpha:]]*)(.*)/\1/') # Set up sharness variables and functions for TAP testing. echo "$infomsg Setting up sharness" -CI_DIR=${CI_DIR:-$(dirname "$(readlink -fm $BASH_SOURCE)")} -SHARNESS_DIR="$(dirname "$CI_DIR")" +UNIFYFS_CI_DIR=${UNIFYFS_CI_DIR:-$(dirname "$(readlink -fm $BASH_SOURCE)")} +SHARNESS_DIR="$(dirname "$UNIFYFS_CI_DIR")" UNIFYFS_SOURCE_DIR="$(dirname "$SHARNESS_DIR")" -export CI_PROJDIR=${CI_PROJDIR:-"$(dirname "$UNIFYFS_SOURCE_DIR")"} -echo "$infomsg CI_DIR: $CI_DIR" +BASE_SEARCH_DIR=${BASE_SEARCH_DIR:-"$(dirname "$UNIFYFS_SOURCE_DIR")"} +echo "$infomsg UNIFYFS_CI_DIR: $UNIFYFS_CI_DIR" echo "$infomsg SHARNESS_DIR: $SHARNESS_DIR" echo "$infomsg UNIFYFS_SOURCE_DIR: $UNIFYFS_SOURCE_DIR" -echo "$infomsg CI_PROJDIR: $CI_PROJDIR" +echo "$infomsg BASE_SEARCH_DIR: $BASE_SEARCH_DIR" -SHARNESS_TEST_DIRECTORY=${SHARNESS_TEST_DIRECTORY:-$CI_DIR} +SHARNESS_TEST_DIRECTORY=${SHARNESS_TEST_DIRECTORY:-$UNIFYFS_CI_DIR} source ${SHARNESS_DIR}/sharness.sh source $SHARNESS_DIR/sharness.d/02-functions.sh -source $CI_DIR/ci-functions.sh +source $UNIFYFS_CI_DIR/ci-functions.sh ########## Locate UnifyFS install and examples ########## @@ -106,22 +106,21 @@ echo "$infomsg Looking for UnifyFS install directory..." # Look for UnifyFS install directory if the user didn't already set # $UNIFYFS_INSTALL to the directory containing bin/ and libexec/ if [[ -z $UNIFYFS_INSTALL ]]; then - # Check for $SPACK_ROOT and if unifyfs is installed - if [[ -n $SPACK_ROOT && -d $(spack location -i unifyfs 2>/dev/null) ]]; + # Search for unifyfsd starting in $BASE_SEARCH_DIR and omitting SPACK_ROOT + unifyfsd_exe="$(find_executable $BASE_SEARCH_DIR "*/bin/unifyfsd"\ + $SPACK_ROOT)" + if [[ -x $unifyfsd_exe ]]; then + # Set UNIFYFS_INSTALL to the dir containing bin/ and libexec/ + UNIFYFS_INSTALL="$(dirname "$(dirname "$unifyfsd_exe")")" + # Else check for $SPACK_ROOT and if unifyfs is installed + elif [[ -n $SPACK_ROOT && -d $(spack location -i unifyfs 2>/dev/null) ]]; then # Might have a problem with variants and arch UNIFYFS_INSTALL="$(spack location -i unifyfs)" - # Else search for unifyfsd starting in $CI_PROJDIR and omitting spack_root - elif [[ -x $(find_executable $CI_PROJDIR "*/bin/unifyfsd" $SPACK_ROOT) ]]; - then - # Set UNIFYFS_INSTALL to the dir containing bin/ and libexec/ - UNIFYFS_INSTALL="$(dirname "$(dirname \ - "$(find_executable $CI_PROJDIR "*/bin/unifyfsd" $SPACK_ROOT)")")" else echo >&2 "$errmsg Unable to find UnifyFS install directory" - echo >&2 "$errmsg \`spack install unifyfs\`, set the" \ - "\$UNIFYFS_INSTALL envar to the directory containing bin/" \ - "and libexec/, or manually install to \$CI_PROJDIR/*" + echo >&2 "$errmsg Set \$UNIFYFS_INSTALL to the directory containing" \ + "bin/ and libexec/ or \`spack install unifyfs\`" exit 1 fi fi @@ -138,6 +137,7 @@ if [[ -d $UNIFYFS_INSTALL && -d ${UNIFYFS_INSTALL}/bin && else echo >&2 "$errmsg Ensure \$UNIFYFS_INSTALL exists and is the directory" \ "containing bin/ and libexec/" + exit 1 fi # Check for necessary Spack modules if Spack is detected @@ -145,7 +145,7 @@ fi # don't fail out if [[ -n $(which spack 2>/dev/null) ]]; then loaded_modules=$(module list 2>&1) - modules="gotcha leveldb argobots mercury margo" + modules="gotcha argobots mercury margo spath" for mod in $modules; do if ! [[ $(echo "$loaded_modules" | fgrep "$mod") ]]; then echo "$errmsg $mod not detected. Please 'spack load $mod'" @@ -160,9 +160,9 @@ fi # TODO: mpirun compatibility echo "$infomsg Finding job launcher" if [[ -n $(which jsrun 2>/dev/null) ]]; then - source $CI_DIR/setup-lsf.sh + source $UNIFYFS_CI_DIR/setup-lsf.sh elif [[ -n $(which srun 2>/dev/null) ]]; then - source $CI_DIR/setup-slurm.sh + source $UNIFYFS_CI_DIR/setup-slurm.sh else echo >&2 "$errmsg Failed to find a suitable parallel job launcher" exit 1 @@ -179,19 +179,23 @@ export UNIFYFS_LOG_VERBOSITY=${UNIFYFS_LOG_VERBOSITY:-5} # an alternate location for the logs if [[ -z $UNIFYFS_LOG_DIR ]]; then # User can choose to not cleanup logs on success - export CI_LOG_CLEANUP=${CI_LOG_CLEANUP:-yes} - # If no log cleanup, move logs to $CI_DIR - if [[ $CI_LOG_CLEANUP =~ ^(no|NO)$ || $CI_CLEANUP =~ ^(no|NO)$ ]]; then - logdir=$CI_DIR/${SYSTEM_NAME}_${JOB_ID}_logs + export UNIFYFS_CI_LOG_CLEANUP=${UNIFYFS_CI_LOG_CLEANUP:-yes} + # If no log cleanup, move logs to $UNIFYFS_CI_DIR + if [[ $UNIFYFS_CI_LOG_CLEANUP =~ ^(no|NO)$ ]] || \ + [[ $UNIFYFS_CI_CLEANUP =~ ^(no|NO)$ ]] + then + logdir=$UNIFYFS_CI_DIR/${SYSTEM_NAME}_${JOB_ID}_logs else # else put logs in sharness trash dir that sharness deletes logdir=$SHARNESS_TRASH_DIRECTORY/${SYSTEM_NAME}_${JOB_ID}_logs - echo "$infomsg Set CI_LOG_CLEANUP=no to keep logs when all tests pass" + echo "$infomsg Set UNIFYFS_CI_LOG_CLEANUP=no to keep logs when all" \ + "tests pass" fi - mkdir -p $logdir fi export UNIFYFS_LOG_DIR=${UNIFYFS_LOG_DIR:-$logdir} +mkdir -p $UNIFYFS_LOG_DIR echo "$infomsg Logs are in UNIFYFS_LOG_DIR: $UNIFYFS_LOG_DIR" +# sharedfs export UNIFYFS_SHAREDFS_DIR=${UNIFYFS_SHAREDFS_DIR:-$UNIFYFS_LOG_DIR} echo "$infomsg UNIFYFS_SHAREDFS_DIR set as $UNIFYFS_SHAREDFS_DIR" @@ -200,13 +204,13 @@ export UNIFYFS_DAEMONIZE=${UNIFYFS_DAEMONIZE:-off} # temp nlt=${TMPDIR}/unifyfs.${USER}.${SYSTEM_NAME}.${JOB_ID} -export CI_TEMP_DIR=${CI_TEMP_DIR:-$nlt} -$JOB_RUN_ONCE_PER_NODE mkdir -p $CI_TEMP_DIR -export UNIFYFS_RUNSTATE_DIR=${UNIFYFS_RUNSTATE_DIR:-$CI_TEMP_DIR} -export UNIFYFS_META_DB_PATH=${UNIFYFS_META_DB_PATH:-$CI_TEMP_DIR} +export UNIFYFS_CI_TEMP_DIR=${UNIFYFS_CI_TEMP_DIR:-$nlt} +$JOB_RUN_ONCE_PER_NODE mkdir -p $UNIFYFS_CI_TEMP_DIR +export UNIFYFS_RUNSTATE_DIR=${UNIFYFS_RUNSTATE_DIR:-$UNIFYFS_CI_TEMP_DIR} +export UNIFYFS_META_DB_PATH=${UNIFYFS_META_DB_PATH:-$UNIFYFS_CI_TEMP_DIR} echo "$infomsg UNIFYFS_RUNSTATE_DIR set as $UNIFYFS_RUNSTATE_DIR" echo "$infomsg UNIFYFS_META_DB_PATH set as $UNIFYFS_META_DB_PATH" -echo "$infomsg Set CI_TEMP_DIR to change both of these to same path" +echo "$infomsg Set UNIFYFS_CI_TEMP_DIR to change both of these to same path" # storage nls=$nlt @@ -219,28 +223,28 @@ echo "$infomsg UNIFYFS_LOGIO_SPILL_DIR set as $UNIFYFS_LOGIO_SPILL_DIR" ########## Set up mountpoints and sharness testing prereqs ########## # Running tests with UNIFYFS_MOUNTPOINT set to a real dir will disable posix -# tests unless user sets CI_TEST_POSIX=yes +# tests unless user sets UNIFYFS_CI_TEST_POSIX=yes export UNIFYFS_MP=${UNIFYFS_MOUNTPOINT:-/unifyfs} # If UNIFYFS_MOUNTPOINT is real dir, disable posix tests (unless user wants it) # and set REAL_MP prereq to enable test that checks if UNIFYFS_MOUNTPOINT is # empty if [[ -d $UNIFYFS_MP ]]; then - export CI_TEST_POSIX=no + export UNIFYFS_CI_TEST_POSIX=no test_set_prereq REAL_MP fi echo "$infomsg UNIFYFS_MOUNTPOINT established: $UNIFYFS_MP" -export CI_TEST_POSIX=${CI_TEST_POSIX:-yes} +export UNIFYFS_CI_TEST_POSIX=${UNIFYFS_CI_TEST_POSIX:-yes} # Set up a real mountpoint for posix tests to write files to and allow tests to # check that those files exist -if [[ ! $CI_TEST_POSIX =~ ^(no|NO)$ ]]; then - if [[ -z $CI_POSIX_MP ]]; then +if [[ ! $UNIFYFS_CI_TEST_POSIX =~ ^(no|NO)$ ]]; then + if [[ -z $UNIFYFS_CI_POSIX_MP ]]; then # needs to be a shared file system pmp=${SHARNESS_TRASH_DIRECTORY}/unify_posix_mp.${SYSTEM_NAME}.${JOB_ID} - mkdir $pmp fi - export CI_POSIX_MP=${CI_POSIX_MP:-$pmp} - echo "$infomsg CI_POSIX_MP established: $CI_POSIX_MP" + export UNIFYFS_CI_POSIX_MP=${UNIFYFS_CI_POSIX_MP:-$pmp} + mkdir -p $UNIFYFS_CI_POSIX_MP + echo "$infomsg UNIFYFS_CI_POSIX_MP established: $UNIFYFS_CI_POSIX_MP" # Set test_posix prereq test_set_prereq TEST_POSIX @@ -250,8 +254,9 @@ fi [[ -n $(which pdsh 2>/dev/null) ]] && test_set_prereq PDSH # skip cleanup_hosts test in 990-stop_server.sh if cleanup is not desired -export CI_HOST_CLEANUP=${CI_HOST_CLEANUP:-yes} -if ! [[ $CI_HOST_CLEANUP =~ ^(no|NO)$ || $CI_CLEANUP =~ ^(no|NO)$ ]]; then +export UNIFYFS_CI_HOST_CLEANUP=${UNIFYFS_CI_HOST_CLEANUP:-yes} +if ! [[ $UNIFYFS_CI_HOST_CLEANUP =~ ^(no|NO)$ ]] || \ + [[ $UNIFYFS_CI_CLEANUP =~ ^(no|NO)$ ]]; then test_set_prereq CLEAN fi diff --git a/t/ci/002-start-server.sh b/t/ci/002-start-server.sh index 9d962fd19..88e834657 100755 --- a/t/ci/002-start-server.sh +++ b/t/ci/002-start-server.sh @@ -44,8 +44,8 @@ fi # If running posix tests, posix mountpoint needs to be a real, shared dir # If it's not, prereq will not be set and posix tests will be skipped -test_expect_success TEST_POSIX "CI_POSIX_MP ($CI_POSIX_MP) is shared dir" ' - test_path_is_shared_dir $CI_POSIX_MP && +test_expect_success TEST_POSIX "POSIX_MP ($UNIFYFS_CI_POSIX_MP) is shared dir" ' + test_path_is_shared_dir $UNIFYFS_CI_POSIX_MP && test_set_prereq POSIX ' @@ -95,5 +95,3 @@ clean_fail() { cleanup_hosts } trap 'clean_fail $BASH_SOURCE' EXIT - -#TODO: can't call cleanup_hosts if PDSH prereq isn't set... diff --git a/t/ci/100-writeread-tests.sh b/t/ci/100-writeread-tests.sh index 7bc8bd18c..d73b735a6 100755 --- a/t/ci/100-writeread-tests.sh +++ b/t/ci/100-writeread-tests.sh @@ -68,7 +68,7 @@ unify_test_writeread() { # Evaluate output test_expect_success "$app_name $app_args: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 17 + test $lcount = 29 ' } @@ -86,11 +86,11 @@ unify_test_writeread_posix() { # Evaluate output test_expect_success POSIX "$app_name $1: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 17 && + test $lcount = 29 && if [[ $io_pattern =~ (n1)$ ]]; then - test_path_is_file ${CI_POSIX_MP}/$filename + test_path_is_file ${UNIFYFS_CI_POSIX_MP}/$filename else - test_path_has_file_per_process $CI_POSIX_MP $filename + test_path_has_file_per_process $UNIFYFS_CI_POSIX_MP $filename fi ' } diff --git a/t/ci/110-write-tests.sh b/t/ci/110-write-tests.sh index 066296b27..07640f5e1 100755 --- a/t/ci/110-write-tests.sh +++ b/t/ci/110-write-tests.sh @@ -88,9 +88,9 @@ unify_test_write_posix() { test $rc = 0 && test $lcount = 18 && if [[ $io_pattern =~ (n1)$ ]]; then - test_path_is_file ${CI_POSIX_MP}/$filename + test_path_is_file ${UNIFYFS_CI_POSIX_MP}/$filename else - test_path_has_file_per_process $CI_POSIX_MP $filename + test_path_has_file_per_process $UNIFYFS_CI_POSIX_MP $filename fi ' } diff --git a/t/ci/990-stop-server.sh b/t/ci/990-stop-server.sh index def649c87..a27dd5672 100755 --- a/t/ci/990-stop-server.sh +++ b/t/ci/990-stop-server.sh @@ -40,8 +40,8 @@ test_expect_success REAL_MP "Verify UNIFYFS_MOUNTPOINT ($UNIFYFS_MP) is empty" ' ' # Cleanup posix mountpoint -test_expect_success POSIX "Cleanup CI_POSIX_MP: $CI_POSIX_MP" ' - rm -rf $CI_POSIX_MP/*posix* +test_expect_success POSIX "Cleanup UNIFYFS_CI_POSIX_MP: $UNIFYFS_CI_POSIX_MP" ' + rm -rf $UNIFYFS_CI_POSIX_MP/*posix* ' # cleanup_hosts @@ -55,4 +55,11 @@ test_expect_success PDSH,CLEAN "Cleanup hosts" ' trap - EXIT # end here if running tests individually -[[ -z $full_run ]] && test_done +if [[ -z $full_run ]]; then + ( test_done; ) + test_exit_code=$? + + cd "$(dirname "$SHARNESS_TRASH_DIRECTORY")" + + return $test_exit_code +fi diff --git a/t/ci/README.md b/t/ci/README.md index 5aa441012..6614a6aff 100644 --- a/t/ci/README.md +++ b/t/ci/README.md @@ -47,16 +47,16 @@ $ prove -v RUN_CI_TESTS.sh In order to run individual tests, source the `001-setup.sh` script first, followed by `002-start-server.sh`. Then source each desired script after that -preceded by `$CI_DIR`. When finished, source the `990-stop-server.sh` script -last. +preceded by `$UNIFYFS_CI_DIR`. When finished, source the `990-stop-server.sh` +script last. E.g.: ```shell -$ . full/path/to/001-setup.sh -$ . $CI_DIR/002-start-server.sh -$ . $CI_DIR/100-writeread-tests.sh -$ . $CI_DIR/990-stop-server.sh +$ . ./001-setup.sh +$ . $UNIFYFS_CI_DIR/002-start-server.sh +$ . $UNIFYFS_CI_DIR/100-writeread-tests.sh +$ . $UNIFYFS_CI_DIR/990-stop-server.sh ``` If additional tests are desired, create a script after the fashion of diff --git a/t/ci/RUN_CI_TESTS.sh b/t/ci/RUN_CI_TESTS.sh index 48a6910a9..dd8c4034f 100755 --- a/t/ci/RUN_CI_TESTS.sh +++ b/t/ci/RUN_CI_TESTS.sh @@ -13,14 +13,14 @@ # # If individual tests are desired to be run, source the 001-setup.sh script # first, followed by 002-start-server.sh. Then source each desired script after -# that preceded by `$CI_DIR`. When finished, source the 990-stop-server.sh -# script last. +# that preceded by `$UNIFYFS_CI_DIR`. When finished, source the +# 990-stop-server.sh script last. # # E.g.: # $ . full/path/to/001-setup.sh -# $ . $CI_DIR/002-start-server.sh -# $ . $CI_DIR/100-writeread-tests.sh -# $ . $CI_DIR/990-stop-server.sh +# $ . $UNIFYFS_CI_DIR/002-start-server.sh +# $ . $UNIFYFS_CI_DIR/100-writeread-tests.sh +# $ . $UNIFYFS_CI_DIR/990-stop-server.sh # # Before doing either of these, make sure you have interactively allocated nodes # or are submitting a batch job. @@ -51,9 +51,9 @@ Then source any desired test files. Lastly, source 990-stop-server.sh. E.g.: $ . full/path/to/001-setup.sh - $ . $CI_DIR/002-start-server.sh - $ . $CI_DIR/100-writeread-tests.sh - $ . $CI_DIR/990-stop-server.sh + $ . \$UNIFYFS_CI_DIR/002-start-server.sh + $ . \$UNIFYFS_CI_DIR/100-writeread-tests.sh + $ . \$UNIFYFS_CI_DIR/990-stop-server.sh Before doing either of these, make sure you have interactively allocated nodes or are submitting a batch job. @@ -78,18 +78,18 @@ SECONDS=0 start_time=$SECONDS echo "Started RUN_TESTS.sh @: $(date)" -# Set up CI_DIR if this script is called first -CI_DIR=${CI_DIR:-"$(dirname "$(readlink -fm $BASH_SOURCE)")"} +# Set up UNIFYFS_CI_DIR if this script is called first +UNIFYFS_CI_DIR=${UNIFYFS_CI_DIR:-"$(dirname "$(readlink -fm $BASH_SOURCE)")"} # test_done gets called in 990-stop-server.sh if this is not set. # If not set, tests can be run individually full_run=true # setup testing -source $CI_DIR/001-setup.sh +source $UNIFYFS_CI_DIR/001-setup.sh # start unifyfsd -source $CI_DIR/002-start-server.sh +source $UNIFYFS_CI_DIR/002-start-server.sh # determine time setup took setup_time=$SECONDS @@ -101,13 +101,13 @@ echo "Setup time -- $(elapsed_time start_time setup_time)" ############################################################################## # writeread example tests -source $CI_DIR/100-writeread-tests.sh +source $UNIFYFS_CI_DIR/100-writeread-tests.sh # write example tests -source $CI_DIR/110-write-tests.sh +source $UNIFYFS_CI_DIR/110-write-tests.sh # read example tests -source $CI_DIR/120-read-tests.sh +source $UNIFYFS_CI_DIR/120-read-tests.sh ############################################################################## # DO NOT add additional tests after this point @@ -117,7 +117,7 @@ testing_time=$SECONDS echo "Testing time -- $(elapsed_time setup_time testing_time)" # stop unifyfsd and cleanup -source $CI_DIR/990-stop-server.sh +source $UNIFYFS_CI_DIR/990-stop-server.sh end_time=$SECONDS echo "All done @ $(date)" diff --git a/t/ci/ci-functions.sh b/t/ci/ci-functions.sh index c42672ee8..708a08bca 100755 --- a/t/ci/ci-functions.sh +++ b/t/ci/ci-functions.sh @@ -150,8 +150,9 @@ find_executable() local l_target="-path $2 -print -quit" local l_ret="$(find $1 -executable $l_prune $l_target)" + local l_rc=$? echo $l_ret - return 0 + return $l_rc } # Calculate the elapsed time between the two given times. @@ -160,7 +161,7 @@ find_executable() # $1 - The initial of the two times (in seconds) # $2 - The latter of the two times (in seconds) # -# Returns the elapsed time formated as HH:MM:SS +# Returns the elapsed time formatted as HH:MM:SS elapsed_time() { # USAGE: elapsed_time start_time_in_seconds end_time_in_seconds @@ -223,9 +224,9 @@ format_bytes() # # Bear in mind, the filename created in unify_run_test will have a .app suffix. # -# $1 - The app_name that will be prepended to the formated app_args in the +# $1 - The app_name that will be prepended to the formatted app_args in the # resulting filename -# $2 - The app_args that will be formated and appended to the app_name +# $2 - The app_args that will be formatted and appended to the app_name # $3 - Optional suffix to append to the end of the file # # Returns a string with the spaces removed and hyphens replaced by underscores @@ -265,26 +266,29 @@ get_filename() } # Builds the test command that will be executed. Automatically sets any options -# that are always wanted (-vkf and the appropriate -m if posix test or not). +# that are always wanted (-vkfo and the appropriate -m if posix test or not). # # Automatically builds the filename for -f based on the input app_name and # app_args and has .app appended to the end. This filename then also has .err # appended and is used for the stderr output file with JOB_RUN_COMMAND. # -# Args that can be passed in are ([-pncbx][-A|-M|-P|-S|-V]). All other args are -# set automatically. +# Args that can be passed in are ([-pncbx][-A|-M|-N|-P|-S|-V]). All other args +# are set automatically. # # $1 - Name of the example application to be tested (basetest-runmode) -# $2 - Args for $1 consisting of ([-pncbx][-A|-M|-P|-S|-V]). Encase in quotes. +# $2 - Args for $1 consisting of ([-pncbx][-A|-M|-N|-P|-S|-V]). Encase in +# quotes. # $3 - The runmode of test, used to determine if posix and set correct args # # Returns the full test command ready to be executed. build_test_command() { # USAGE: build_test_command app_exe_name app_args([-pncbx][-A|-M|-P|-S|-V]) + # runmode([static|gotcha|posix]) if [[ $# -ne 3 ]]; then echo >&2 "$errmsg USAGE: $FUNCNAME app_name" \ - "app_args([-pncbx][-A|-M|-P|-S|-V]) runmode" + "app_args([-pncbx][-A|-M|-N|-P|-S|-V])" \ + "runmode([static|gotcha|posix])" return 1 fi @@ -297,27 +301,31 @@ build_test_command() # Build example_command with options that are always wanted. Might need to # adjust for other tests (i.e., app-mpiio), or write new functions - local l_verbose="-v" local l_app_id="-a $app_id" + #local l_verbose="-v" # Sends DEBUG and test configuration info to stdout # Filename needs to be the write file if testing the read example local l_app_name=$(echo $1 | sed -r 's/(\w)-.*/\1/') if [[ $l_app_name = "read" ]]; then local l_app_filename="-f $(get_filename write-$3 "$2").app" else - #local l_check="-k" # not reliable atm local l_app_filename="-f ${l_filename}.app" fi - # Set mountpoint to an existing one if running posix test + # Set mountpoint to an existing dir & disable UnifyFS if running posix test if [[ $3 = "posix" ]]; then - local l_mount="-U -m $CI_POSIX_MP" + local l_mount="-U -m $UNIFYFS_CI_POSIX_MP" else local l_mount="-m $UNIFYFS_MP" + local l_check="-k" # read.c tests fail on posix files fi + # Add outfile option for checking line count after test completes + local l_outfile="-o ${UNIFYFS_LOG_DIR}/${l_filename}.out" + # Assemble full example_command - local l_app_args="$2 $l_app_id $l_check $l_verbose $l_mount $l_app_filename" + local l_app_args="$2 $l_app_id $l_check $l_verbose $l_mount $l_outfile \ + $l_app_filename" local l_full_app_name="${UNIFYFS_EXAMPLES}/${1} $l_app_args" # Assemble full test_command @@ -331,13 +339,13 @@ build_test_command() # testing files. # # The build_test_command is called which automatically sets any options that -# are always wanted (-vkf and appropriate -m if posix test or not). The stderr +# are always wanted (-vkfo and appropriate -m if posix test or not). The stderr # output file is also created (based on the filename that is autogenerated) and # the appropriate option is set for the JOB_RUN_COMMAND. # -# Args that can be passed in are ([-pncbx][-A|-M|-P|-S|-V]). All other args are -# set automatically, including the filename (which is generated based on the -# input app_name and app_args). +# Args that can be passed in are ([-pncbx][-A|-M|-N|-P|-S|-V]). All other args +# are set automatically, including the filename (which is generated based on +# the input app_name and app_args). # # The third parameter is an optional "pass-by-reference" parameter that can # contain the variable name for the resulting output to be stored in. @@ -346,18 +354,19 @@ build_test_command() # 2. app_output=$(unify_run_test $app_name "app_args") # # $1 - Name and runmode of the example application to be tested -# $2 - Args for $1 consisting of ([-pncbx][-A|-M|-P|-S|-V]). Encase in quotes. +# $2 - Args for $1 consisting of ([-pncbx][-A|-M|-N|-P|-S|-V]). Encase in +# quotes. # $3 - Optional output variable that is "passed by reference". # # Returns the return code of the executed example as well as the output # produced by running the example. unify_run_test() { - # USAGE: unify_run_test app_name app_args([-pncbx][-A|-M|-P|-S|-V]) + # USAGE: unify_run_test app_name app_args([-pncbx][-A|-M|-N|-P|-S|-V]) # [output_variable_name] if [[ $# -lt 2 || $# -gt 3 ]]; then echo >&2 "$errmsg USAGE: $FUNCNAME app_name" \ - "app_args([-pncbx][-A|-M|-P|-S|-V]) [output_variable_name]" + "app_args([-pncbx][-A|-M|-N|-P|-S|-V]) [output_variable_name]" return 1 fi @@ -368,14 +377,15 @@ unify_run_test() return 1 fi - # Skip this test if posix test and CI_TEST_POSIX=no|NO + # Skip this test if posix test and UNIFYFS_CI_TEST_POSIX=no|NO if ! test_have_prereq POSIX && [[ $l_runmode = "posix" ]]; then return 42 fi - # Fail if user passed in filename, mountpoint, verbose or disable - # UnifyFS since these are auto added - local opt='(-f|--file|-m|--mount|-v|--verbose|-U|--disable-unifyfs)' + # Fail if user passed in filename, check, mountpoint, outfile, verbose or + # disable UnifyFS since these are added automatically + local opt="(-f|--file|-k|--check|-m|--mount|-o|--outfile|-v|--verbose|-U|\ + --disable-unifyfs)" for s in $2; do if [[ $s =~ $opt ]]; then echo >&2 "$errmsg Call $FUNCNAME without $opt. Found $s" @@ -394,13 +404,17 @@ unify_run_test() local l_app_output; l_app_output="$($l_test_command)" local l_rc=$? - # Put the resulting output in the optional reference parameter + # Capture the outfile contents for line count checks + local l_outfile_name=$(get_filename $1 "$2" ".out") + local l_outfile_contents="$(cat ${UNIFYFS_LOG_DIR}/${l_outfile_name})" + + # Put the resulting outfile contents in the optional reference parameter local l_input_var=$3 if [[ "$l_input_var" ]]; then - eval $l_input_var="'$l_app_output'" + eval $l_input_var="'$l_outfile_contents'" fi - echo "$l_app_output" + echo "$l_outfile_contents" return $l_rc } @@ -410,6 +424,11 @@ unify_run_test() # that were leftover on the hosts. cleanup_hosts() { + if ! test_have_prereq PDSH; then + echo >&2 "$errmsg PDSH prereq not set, cleanup_hosts() skipped." + echo >&2 "$errmsg PDSH is required to run cleanup_hosts()." + return 1 + fi # Capture all output from cleanup in a log exec 3>&1 4>&2 @@ -446,7 +465,7 @@ cleanup_hosts() /dev/shm/logio_mem* \ "'${UNIFYFS_LOGIO_SPILL_DIR}'"/spill*.log \ /dev/shm/*-recv-* /dev/shm/*-req-* /dev/shm/*-super-* \ - "'$CI_TEMP_DIR'"' + "'$UNIFYFS_CI_TEMP_DIR'"' # Reset capturing all output exec 1>&3 2>&4 diff --git a/t/ci/setup-lsf.sh b/t/ci/setup-lsf.sh index 028483fde..c636d5146 100755 --- a/t/ci/setup-lsf.sh +++ b/t/ci/setup-lsf.sh @@ -10,8 +10,8 @@ # Returns the unique hosts being used on LSF, exluding the launch host. get_lsf_hosts() { - # NOTE: There's potential that some versions of LSF may change to where - # they put the user on the first compute node rather than a launch node. + # NOTE: It's possible that some systems with LSF may place the user on the + # first compute node rather than a launch node. local l_hosts=$(uniq $LSB_DJOB_HOSTFILE | tail -n +2) echo $l_hosts } @@ -33,31 +33,31 @@ get_hostlist() nnodes=$(get_lsf_hosts | wc -w) # Define each resource set -nprocs=${CI_NPROCS:-$nnodes} -ncores=${CI_NCORES:-20} +nprocs=${UNIFYFS_CI_NPROCS:-1} +ncores=${UNIFYFS_CI_NCORES:-20} # Total resource sets and how many per host -nrs_per_node=${CI_NRS_PER_NODE:-1} -nres_sets=${CI_NRES_SETS:-$(($nnodes * $nrs_per_node))} +nrs_per_node=${UNIFYFS_CI_NRS_PER_NODE:-1} +nres_sets=${UNIFYFS_CI_NRES_SETS:-$(($nnodes * $nrs_per_node))} if [[ $ncores -gt 20 ]]; then - echo >&2 "$errmsg Number of cores-per-resource-set (\$CI_NCORES=$ncores)" \ - "needs to be <= 20." + echo >&2 "$errmsg Number of cores-per-resource-set" \ + "(\$UNIFYFS_CI_NCORES=$ncores) needs to be <= 20." exit 1 fi if (($nres_sets % $nrs_per_node)); then echo >&2 "$errmsg Total number of resource sets ($nres_sets) must be" \ "divisible by resource-sets-per-node ($nrs_per_node). Set" \ - "\$CI_NRES_SETS and/or \$CI_NRS_PER_NODE accordingly." + "\$UNIFYFS_CI_NRES_SETS and/or \$UNIFYFS_CI_NRS_PER_NODE accordingly." exit 1 fi if [ $(($nrs_per_node * $ncores)) -gt 40 ]; then echo >&2 "$errmsg Number of cores-per-resource-set ($ncores) *"\ "resource-sets-per-node ($nrs_per_node) = $(($nrs_per_node*$ncores))" \ - "needs to be <= 40. Set \$CI_NCORES and/or \$CI_NRS_PER_NODE" \ - "accordingly." + "needs to be <= 40. Set \$UNIFYFS_CI_NCORES and/or" \ + "\$UNIFYFS_CI_NRS_PER_NODE accordingly." exit 1 fi diff --git a/t/ci/setup-slurm.sh b/t/ci/setup-slurm.sh index 48749ef35..d827cf7f1 100755 --- a/t/ci/setup-slurm.sh +++ b/t/ci/setup-slurm.sh @@ -20,7 +20,7 @@ get_hostlist() # Variables specific to SLURM nnodes=$SLURM_NNODES nres_sets=$SLURM_NNODES -nprocs=${CI_NPROCS:-$nnodes} +nprocs=${UNIFYFS_CI_NPROCS:-1} app_out="-o" app_err="-e" From f950aa37966313fb6628d92070704bc6c410c65b Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 30 Nov 2020 15:21:55 -0800 Subject: [PATCH 168/168] fix: handle read extent with no registered data chunks --- server/src/unifyfs_p2p_rpc.c | 98 +++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/server/src/unifyfs_p2p_rpc.c b/server/src/unifyfs_p2p_rpc.c index 92a1e553e..c41e379ca 100644 --- a/server/src/unifyfs_p2p_rpc.c +++ b/server/src/unifyfs_p2p_rpc.c @@ -257,6 +257,7 @@ static void find_extents_rpc(hg_handle_t handle) const struct hg_info* hgi = margo_get_info(handle); assert(hgi); + margo_instance_id mid = margo_hg_info_get_instance(hgi); assert(mid != MARGO_INSTANCE_NULL); @@ -315,35 +316,37 @@ static void find_extents_rpc(hg_handle_t handle) margo_free_input(handle, &in); } - /* fill rpc response struct with output values */ - hg_bulk_t bulk_resp_handle; - find_extents_out_t out; - out.ret = ret; - out.num_locations = 0; + /* define a bulk handle to transfer chunk address info */ + hg_bulk_t bulk_resp_handle = HG_BULK_NULL; if (ret == UNIFYFS_SUCCESS) { - void* buf = (void*) chunk_locs; - size_t buf_sz = (size_t)num_chunks * sizeof(chunk_read_req_t); - hret = margo_bulk_create(mid, 1, &buf, &buf_sz, - HG_BULK_READ_ONLY, &bulk_resp_handle); - if (hret != HG_SUCCESS) { - LOGERR("margo_bulk_create() failed"); - ret = UNIFYFS_ERROR_MARGO; - } else { - out.num_locations = num_chunks; - out.locations = bulk_resp_handle; + if (num_chunks > 0) { + void* buf = (void*) chunk_locs; + size_t buf_sz = (size_t)num_chunks * sizeof(chunk_read_req_t); + hret = margo_bulk_create(mid, 1, &buf, &buf_sz, + HG_BULK_READ_ONLY, &bulk_resp_handle); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_create() failed"); + ret = UNIFYFS_ERROR_MARGO; + } } } + /* fill rpc response struct with output values */ + find_extents_out_t out; + out.ret = ret; + out.num_locations = num_chunks; + out.locations = bulk_resp_handle; + /* send output back to caller */ hret = margo_respond(handle, &out); if (hret != HG_SUCCESS) { LOGERR("margo_respond() failed"); } - if (out.num_locations) { - margo_bulk_free(bulk_resp_handle); - } /* free margo resources */ + if (bulk_resp_handle != HG_BULK_NULL) { + margo_bulk_free(bulk_resp_handle); + } margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(find_extents_rpc) @@ -428,39 +431,44 @@ int unifyfs_invoke_find_extents_rpc(int gfid, /* set return value */ ret = out.ret; if (ret == UNIFYFS_SUCCESS) { - /* allocate local buffer for chunk locations */ + /* get number of chunks */ unsigned int n_chks = (unsigned int) out.num_locations; - buf_sz = (size_t)n_chks * sizeof(chunk_read_req_t); - buf = malloc(buf_sz); - if (NULL == buf) { - LOGERR("allocation for bulk locations failed"); - ret = ENOMEM; - } else { - /* create a margo bulk transfer handle for locations array */ - hg_bulk_t bulk_resp_handle; - hret = margo_bulk_create(mid, 1, &buf, &buf_sz, - HG_BULK_WRITE_ONLY, - &bulk_resp_handle); - if (hret != HG_SUCCESS) { - LOGERR("margo_bulk_create() failed"); - ret = UNIFYFS_ERROR_MARGO; + if (n_chks > 0) { + /* got some chunks to read, allocate a buffer + * to hold chunk location data */ + buf_sz = (size_t)n_chks * sizeof(chunk_read_req_t); + buf = malloc(buf_sz); + if (NULL == buf) { + LOGERR("allocation for bulk locations failed"); + ret = ENOMEM; } else { - /* pull locations array */ - hret = margo_bulk_transfer(mid, HG_BULK_PULL, - preq.peer, out.locations, 0, - bulk_resp_handle, 0, - buf_sz); + /* create a margo bulk transfer handle for + * locations array */ + hg_bulk_t bulk_resp_handle; + hret = margo_bulk_create(mid, 1, &buf, &buf_sz, + HG_BULK_WRITE_ONLY, + &bulk_resp_handle); if (hret != HG_SUCCESS) { - LOGERR("margo_bulk_transfer() failed"); + LOGERR("margo_bulk_create() failed"); ret = UNIFYFS_ERROR_MARGO; } else { - /* lookup requested extents */ - LOGDBG("received %u chunk locations for gfid=%d", - n_chks, gfid); - *chunks = (chunk_read_req_t*) buf; - *num_chunks = (unsigned int) n_chks; + /* pull locations array */ + hret = margo_bulk_transfer(mid, HG_BULK_PULL, + preq.peer, out.locations, 0, + bulk_resp_handle, 0, + buf_sz); + if (hret != HG_SUCCESS) { + LOGERR("margo_bulk_transfer() failed"); + ret = UNIFYFS_ERROR_MARGO; + } else { + /* lookup requested extents */ + LOGDBG("received %u chunk locations for gfid=%d", + n_chks, gfid); + *chunks = (chunk_read_req_t*) buf; + *num_chunks = (unsigned int) n_chks; + } + margo_bulk_free(bulk_resp_handle); } - margo_bulk_free(bulk_resp_handle); } } }

    *5#BGvC&@ zI}vAVIeSGyMxWga;ontFJIUaItAVv@!h+7wH+`&^J>EKLn zox52O2OIcR+pI{e*=43vP7|+B=;e-ZcT38)cX}O8x|3TG(R|Y2Xus*+>iusi2dkwWq%2j=xq96S71g&W3Zm0Lc8==AFBJLGe~7lV`*ySG<664w zuCl%JmBp58yAmZag=DOJX%=^krREmj6`-;K8oF^TlKZEY#%E9Mde}?-((RESAHJ~( zt1b1kSWUjsbnsg#JiA%*dZ05%4_E8s&*tx~qbuYU3a&b}OO%Sw&Rz6}%xf!g^%##> zO}tZS&-Bc_))3K@$8={vdakzRxtwDM_0`_0j|F?%Fx`x6k=DhO9K2z@>{pc98Fi@Y zRfGz!ZazKS;}DeVja2(+)wUR?vllnPX=c5R%8A=6+ukK^yRmqV+U{_scjuc`KafcZ>EqVk>nLI`Ar0T3h$SLx%Zx%1$*+WvD=|$NLu++y=&am#X43Tt^() z6Mj90WFikF9~kS+5k0s&IN?J&Tpnxg+pO_f_02-9N4nPw;*FnHWOKuLM&aWE5$_Yl zwjykPIiZO~Xhy`x={qUPDJG|H3;uC4;HjkVTbxZzpK&EzSNe64Dm!|Ly?QJomM4L@ zu|%LoIeK!B^4;3=b3rq(Kas0ppECvlgh8CBgG`OsYY#${f7-m;|MB(GLR=E#j^X2C zy0W?(XLXpO=JDl!qQ$+^T2Mw--yTvXtc3C*bc^8|%OZRpQ;ND}6M0EPmLZyH*nVaY zI42H87T#>$7gOh@&K2MO{*ZlEr&b=_!N%(9KD z_rH_e@PgCaD6&X(HCT){a^J8nCPn@ZkHzkIQ%_M%q}6^}Q{Q1U>_)8#SCP9Ghb(!D zt^Y^YHLZoURnCKLaXEumehAYZxm~Mm=~q@SVwY&tLb&$!J*aCA!l#u#h3#JXtp>8M zCO)x4k5#zy=eox@^fe&JP*Y#K+Vnzx z<8-k0P-$;XiXDm+?Y#5Nn_^vS-}o4B8owU%+@y{ZHGdHhHfeb<;pM+YQ^@W(?onpR zJT>-Rf1P8F8t-ML6F1nV7lTjiTE@9=&se`%S|GSriHH3Z)+@$|sc*()iFt5}_jC%* z=Zf4p&@a`HK%++x1{ROe35mRI)bveJaaloVarS~nCd6f^SJ&TbyA$i7YX}T32$hFO>;z6<~L_2{qzrS>)=}_hlg|k3}{))$w3)Xrx zuH02pPB)aQ*LTnTB6q(d=(v{Ldv3L3&6ez|?4!KG9Q(xngTfSS3RKLh>KnI8M>3tU z26}EE(q9uiz&coc8PD>SUQDi&p)LZ`8BbX_%!V7^o?e{pqqKSM0q1e)E(rm)&{n~n z@|%JCd#^h(ii_T2lQ6jKJ_Dq9(w5tLfvN$H8b8dT) zDu%jOkB?J@H1yU3-3*;yMD#2=2Cj%SZ-3JF)Pp=E7c9%dQZBiu=8#}5ec?9xu*C5h zj^7|Szt!zCH1&dl+BlQ?9n@3w^Y+rh`)lOG06jryJpf73}ee3%#rW5F*EmDeL?*?O8$t!?)cxLXq4CZKN2pMTvjYBHsGf~OCJpD^4dEkX5pOn#9VmZ!z(@lC44oZzBzEMa z)`oG9ozegmO!(O4mGi2Rn@XsWmXT}myI#T83py3n~5{oMI6~GDT&<}ljRBvt0gLRvB1=KP%SAR?Ty`M$UI#P7!28Kd%4%B9o)vD&D z&1=l<9%rw=u##}mjcg%=%!3-;1iVgq9EskB^&|}C!XS#U3r;|K1 z4ln$E3q6*9+{C8|=tdr)oSVgkw8}JypE?^VP|p6*7%0B_6C>tkEC#*Q&)!}bkp~ou z5KzY#whyYH9YCj0WxA`;&0|;t0yW5O;a`Yn%vHH1VQ?n0V%UH}N^`@6)fI~13G3gO z77w3Pb*9M_IX>E`P&SQWyJcr* zXQa8LV2%xgWt4&zhz|V+GFisHZ7JrF-IZ*3f9kxU-E`QkxzVw)?038lDwN}eR8&P5 zpe17~pK^S@9(Ret8_1s6{Qx{;fAB`Nw6vz3YseA_u%Hab#n^~y9})~wi+sQXrj=;zlRTs z37IazohJbODaWF$=HWAz>G8H>kXdUAEcezscJ;U{3K#-GKo8;&xD4Pb^tD)dk4S+b z-ZIQ#Xf)dV=DZ6cIr_|CgmCIxfGoUCeed^?z@|o@@j=}ba0Sbt61IGvxAv;#UVA9$ z5&9Rrg+Byf#FjkQj;i3nt4QILmX)c? zMdO3=0)(CpE%c9CCHIz@O`3xdb{up^VCP5E3C{;K*yNo2O`b=0YlLD|<*k5fOBMy8 z#>Ot!m-IvVe3-? zK<{IAGk=dGH4mt1w8NG!O8$HzRXXDwEpqkWcLQudo%%hu!(q3h`KMwz^QeM3R9Jdr za}&rFz!Dq*7&kDm{ru#c3jbjwgA(TR7?5Yah~vl?{Ntr!9~6cTl+SJ3S_Htjz<7Ci z&00Q${^v({m>*qxL3fN(9ZdFQpa-a0p1Q+dc^HHy2T+;a4uEv-xg(eVvjkiiMC*>S zmUlqdflormlJLi^zVs4+W#c5e6TkZ7Z9`t;W4gov#V|7v^EYj!&V#!5n+DGe z>z^LkOC|tVotYW*>d0$SaKwHk>VyWh>gtk!&-4xGf&uLg#Nptu7V0Un+dm%AAIU9% z1Igutn?{|CG7g81*q;C`0eZwtoRAtCOvTceKE6{q1O?2-N5thH5d#7~yRi2_!*XE` zW&jd%3&8$}z4_vE9?H*OM&$PXn0EZ=9j9Z#+qonfhK3xy-6{bT25ZAe1YqusAV7Nn zg~PAl{JBdM@1%tbjLYUG7~`0o8Ry9ub8(!@sitPk``j4nH0Z}Ruwpu=cmKTsV20A8EJx#6+ zbYlEDqn^7(qS4Z}rymFHMq$9cg9BMlPAqHE;t^|ynUL%4jnB#E2K66A0ky3y7feN} z|J3-w=^)H#2$)Rgn_Yg#7eOJ*GJpjIy_RZ^0h}G?oZzW~L)dU*K-ZrXK+L-?B_#A; zeS8KCy7uC+{6Vnr9oHr>8_VD^ATCu7TL!JIY8c33xkJzd*CB;RkCMp%OT>z0ZWnO+ zEmSZ&gFVuP_;}J9u`sC5nC@5t^Z_8~1E5}XOaO%-3Z$BJ6_u3c!2Pn!$?-NnGXC9* zVeoVeO*1)S@WjTrCVfV#6c}Z0kH4k(&w}P&G=^O?(F<6fe0LjgMgUXbJrs1A<==h| zgIfXF?2mKQy?%UAh8-)W14b|E7zTQvhPJ!EZnlB0G5H=g#jnBRf9(#4_iI29nO*R_ zOyXG31F*4yi2^#RI=J+mPl7dk=PHK)9dg34XhiR>%5P| z@do$ShTU3OS48MW(^Uvsjc=&2kyo(eY6n^bh+oOjN69zXHzX(AeILn|^`HJoci?1?~P(r?*%?faMQ( zgmXOOjf~#y953<>bf6C}7ewH8TyLEQK&URJ-h^_ppuH=+-4axK=R}36A*<}4bB@@A z&X!N~I<>;pLloSNiqePa(dp6BLbRTvXH&!OQ1GF{NS_A_px9_c*pr!bUmQ%M2~{Ou z^#82QoQR_dP&cx<^DLj~1s8Ajt&WU)>qs3$1w*$#TI*Irc;a+qCkMmUY8><1nmNTf z26~inQfRMM($i;cWf&g}RuMm&lNZA&0&CT1eXV+qjIU#p1Rm~fLu0Rb6$CKL@|B4yWF?ABJ7n$3GF8y!R!rA3 zo3y1h-0F}16c|-Ec^EnvsDxi^)Awb>5*8Jg$Qmn3nN*~gi-dFVj<6-E>2W0e7T573 zT84h5lhEECdu-ev8cF{zpYDAdkYcb=nmzY8HtxJ3*peY~2g4u|+N0y8ZIy7LZS5Ko zty$KTx$Ym_dWpn8RUcwmAE_6uE;V?Iq~)`P$b9Q!wY864k{6W!56H~cw}vf@y4S@GjTEDeX@kt^CK5FwKW9-p(cZWEuOGjm;v@X zJ}f>u7uMnLFSlORuJtTIIez1IzOQJrUE)G@N2$fw)N2zNquA!x)Q)p4K~Imi=--@T z0p;E3c9Lee7yLv;bBGl-l96j2LC%%LlQZ%PCQ|TJIKlR(`EC;$PP0cvtG8Kwp18Gu zlBz5#*T96PAZDXsfTfQnQLXpkDY!VGXXSti(NVR|@lRX=W+KwKalO_!@8>u}rQ{#-z-3Z}G zrHCC?ZKEL3u*u^DRT_LC#K=6r5r>Sn4QnqA z`BnD*N5_=1;548;vm~z3V`Ln>=p!4X$G&TNIh}Dchyx43o?KF1(T#1zcth1h1jX>P zAYflQMu*$qeR^P6Wnz(V9T|mi(#mn+Q2epu@Q}DU21GPJFFc-B?j*S~BVMt0&_?|C zlx>s*Ln^u9PlxqK3&DLZJqg1OVrFJo2%lf!No?21DdgHdkKka^A}``kCH!0txVXrE zx32al{TiOPdp^FZ5d_W3X$d5leYD^cMrQY$@OuG8^y@D~>+IiAXIG6Yd`KLV_38=h z^R~XH5GMsn7_GbJHGa*}R!ZflfaW6a;>*7&;qYmGHj&ar{3x!9Uo=}dI;G*Ip3X=~ zzLa5i!w;|jl&tB?VvOjinWkfN19+zp^iSMN%#aK->Lq3r=}m(x9egcml%jGjJxN+~ z&*YM&O3E4HjjS4(CH5&1zr=jgznrC1jArw7k%}(%{hI$(!ok6JOy8HaUqk$p?INXdp-DtfR?u6W_=u@t1L64+>yvS_Q^ud(PqI zy9K+X`4D$jG{Tsh1m@q&@3@`sv0V%lCP@s|iU`z{%lG*ikzkge7;K(Fc{9Es`G<&8 zPF{sC;ia_w(MZBu|Cyr#Om}UbvFTq&V*+QN$HdbrDfu1z32D5rR&sc{V_g-FL|Z8_ zU$j(w!83@=E<}itnU?FZPeA~e2zz&EiFoPMZw~xtlqSKx)@JeJU-DmD8BRfqIi=Y(YbV5b$>wVVjoJD9Jdj1TZA?t+^n0zU}AP-t1SP+?@%I^PHU&67u zfi;tzF^i$_{iBM3gWZ9~2U%Hx+(M2aE^AoavjUmEu^5_qHsj>kt_HtlD`z6rS8;{9 z!}}vc`%>j+qb2cGZ3v{N_(}%Wsn5@X<}}qqXK8HKfs#&;!#yx7``T=&@-!{GO+`NInrdgY%!m zRurd7dYb4DFb>#<&lMpMN-6Uku1Pj6Q&)2{!P}SW-CI_>%eUj&B}KT^uGFYB^Wn=0 z1xN>kkOy!gIe*-4M^9qeJX~KAZ?FKleO(byuic(p>9}8^$YZA(#pYQ7<2CPnkW-u;`8s?Y$wr^y^3CN8$}nGX zuXx9rfNa@yhh`CuQ;{cnOd^d24sZCOA}}RBhiXN< zio#qrZeYdfbbNZ@!G6V4%y6~#2ZXvi@22zwGo8S(7y)4dbX&|5c*hD89L#8eWT#m2 z!dNT0SAC`d&4+F3uh<~uIiS{%ZB+?kMQbp83lg1S`Uu_skyt~w8j;x<&l)qnnIi77 zp(*N%?!HHy;C=j`vN=;L$*QtB< zb5?aPoF^I%`Dh;yg+vA>e>tW8!d_wwkNMh%{YNiMpaL%o&QU;?d%ipOlqWp$V`m(U zdmzcqBQ!rERBeK3=NIa zy^GJMD;n;6*R#~VoO8Rbz3a&-8~aiuCfM`Jh4`p_4pKq@7e;ignC1sVXtW~phfJQ5 zMi{g$EWVCr?3@*-fGhUpZxK_3FXv_q?_;A$!@nm*g>uFDQs7G;zS#=8Imle~Wp|$` zyhu+x%9Q1yfs<^By=>mPYvZc2(7kXm{yWe3v&gM)zZ$+yAb%7>2O$rTTL2Td&ka9w zG=a|*&oB|nqy!Ut2t2%fojDm9eK{~IRYCE3A^taRM4awNZfBjq#Q4+~5mpzOp*ee$ zjZlv-mo0+R$)QXGCAV@n70ZhEtcbIv=BMUcoJOBrMKv-e{lwOmDTZ#ia`bQIOV=)X z`n7zcw6ud}S$7zRiz!-&j8uxFvLN-z8VeTs!QkuPo(5>f45|=inpLs5`%t`GjrN+}P8vF!SF0mU2Tvi{lrP zdMV|TY!h#mQd1GqK5K4Kts+&U9-oQom)|{IrUBBdIKS{yoNRCdOk6CcuU+amED!io ztyM)X3bFz2_1lydcGkl)R4M18#TeGL^D06E`6MeQ)^o0!Ff6IP2vj7ou{1e@>y|Hd zw_z=Sl{^53V!*q1-X`=KDl&eQO3}$+isnM+O3~%%i!z*5u_BuLC9#XQoc zZ+&0GDmf=OB$`L<-IlG$@eO|0rz5^@MbCu!VEo(?>^3KLOx2SuwB+m9mVg>iyX*N# zWa(-_|KN*XKQ1$0+I@O=X~5I)qeBiqtG;*Xl1u$M&%dytw=Zzl*rqZQ&YmPZC`pQa z)1SO68wq2BY-DezmZImghhDOm0oiRJ7&N__y zsVZZ)k?g0%oBn}7fK03~z6{yN;ff8Wk62IHypPDEK!0Jm&lL*c%R!OjZx9VtL*pl# zItxvT9a(A&@c@n3v08*<6DR_!$wyV({|I9UFmp4KguTd@v6Ray@jgnQd}B4uetk1_qaWRYjhzDXcbw8szltHs#s^`GqwUWB#(T?$GBpIItbg_ z6SWK0L*3^%WGp6Yqlb}|wB}=XH`~A7=#1_yRRjnR>~E{*6xIqw(63^dROJ=iGh5%I zxc`m}N}>g}r0jrWG5_&3l6ai_3zl*a4Jmw!W&7Lp<%|rw1)(JHF?Qllt$x;~+eo5R z8y}0k0{8E6C5P9jM}%4=HuTTqHfATE8J&N8UKiqwzSB$?Q%_WA$&&7Dqs;uXaJPRk zXRw`M)rsM@gchQ7`|&*b(ymQ2l;XJz0IESK*(<#09O27Wdv{?qo)^>XekdUpAQr2X zJ6!v4#OmZc@b<&Ed-h=uP+0$Q+kns3_4Trhkp7SI$ur9H*C>o~1v~V#M&u-fs7vs9 zPyu4IfbK${zjQ>!c!m66VIskho5U?UwAJLfX#LI7;MlMQZkdaOn^w$Mv9*<>57Y0D z$rvv72jmQ9sLHZ;Y;m8(das4_d4_b-dV|!8?)Ye$0f;bk7o~S9QB0UXy1TGp8F))onN35Bb z4tLKIqNSxh6^d-ds1e=7{J$Z>SMM>nIZH_g0CJ=&Qg!?nwGspBVi{j-g)3h7lfWBI zv@wb!%@oaVFG#C9=p_=$<{ZAG^2Sdk@id1l2`wNw1@6i4@Eb4P*rfHn8Y4E2kDwMY zgsPeFYh~LB-EX3Kv$b9{n+7=XUd7Nu&$>C8KT|DHJXv7Suucn9$*3Q{{z;H90I-=| z(`2Z2PWZere0if|vaCOsRkQS=n3AIFYeHe$h`5Q{>o~rlt@?~G#6q*F+*QR-tEoQf zIy25@`2lw3T*+0u6_lEuF(Ne|0G!Axs9@St;0>eezIH`)tnJ=PeCJf_*wGd2Mn?xZ zv1;J%m|02MY*#;0u8ktz?<5@NM#Ms+F0v@c?7T~4>6_<{tl*S=md1t9TxS!3-Atgl ziLZSlp4-b3+dj2U_>P%5lv3ToV1h{&}}LR?&PW;DFrp@gn}uND0O9l{CPkX0`t zR6m77iCo``bUUTuE6_~S`sufyHIcX5C~eTvFe;a`3GKI0Icsq$<@>++-Y_^7m;=LK zJIltzFFI86eSfwue|5z}fk1AOA6&)bmS%g%nIjg+;sA`T#RhXw2c^{WJbqw;_$GG32y4wd=Rq{iEDUmCqgr8&P}>ZFoOHLKVe)~{+P z@UF)-r<+P1Mnk19#B)70yszL|)DO6jU`=%q;}{h&K1aOa2zG2+aN#AD>A>xt{mTId zwoNfN>J$-cYPM$?sQsoa$D*4I`vdsia5fs8?D>89w7ljjbocoRg2+$*a= ziYs2j!-3W?kD&C+Mh0Y6i~RFwi8rgXB?$u*oTj0>Nmhlm#<15l+I*LDz;_bIIz9zt+6-k+Sv~ZkTtO>l?Pp{ z1ZrZt0QgRtJY**RvFgPYUKysOKp9D1U#9&ZFD>&&Z8}jVvd@016el~F8{6X>A1*t7 zZ!9ZqwJ~(AeqBR0eCeMA2qy+% zPhM2sERbHLPUN?xfq4EJ-DT2Cco$$g- zE_owJ>aZn~uP%U`BtjXdaKaPi1JmU08vtnz;XR=0sg&LyQR!o}`(DK?#%Xcun5I@@!JJ<`q~uXJ_zzyC%eQ zMeSc;tNB0IMbbVMtJ)g1dTJ@?8X2giULW%6RN8kNm~hz#!)2x?GmdrBxGWIKIgZ0F z#C_?3G>@e^eZmn#`Jd>gf*nv>m-f$n`F=WSc^nGG_2iNi*Ux3u({}gy79!LQf zt@rDiI$g&h5DvT~7N2R06KjYUrxs6hLz+Tf^}f1?FjQv}?e z`+my_6EXm}Tz;eJ^>`&Br__9bYoWiO4w^Op^Q?NjvRUvUSwuv;>R+cL49qy!@o zOr4@FM|%ypT7Ey;VlJY+-tB~+I{NI%3i}bHXI0NeevoUk$0LqGcyWg(9U4IERP;GI zw7H|U@+34OW+VIiFHnT_Hs)ZX`=WBCH8#xF1D>2-ndvw%%~gp;#G%K=91ZIihf&iV z`d@KrrR~h;qiw=K+`l9LnH6d^wW8E<%axryJDMoUC}%q$P@czr%)`>Y-(_6<#1_OX zEW{avKmy0#m~7)}C_nnSX5ytQPf_v%1|=tU zGjMN7EPxa;sYGs_IAsZB*}2=zyZ9pi0Vx5?-MIJ1gvAUySr^CB{BA#&k$CL!T!5?p z!_EHp%O@Z7Fk-6h+mU;EQjet@d$RZqz$NgmMqO^Y8(CQtOfDvp6_O?pN(%H(lu`d4Y8 zQiMuK(JNOnn@=1>@rE27ff>z9Q6C*2l{mWE1cilCNN>#OalX%a%eQg8$aVi4jbi5}D z9!`f@bNPt2l#?}&C%NT4``M&Lp`wMe>+N11*LamSusRWX+Vb|*8d_rY8`#s~yvbLZ zQUcO1e7`77+PtuZRFu_CA*aj_4@6@FIKBAK_6y#<+&9WpGJQk-I>(@F0JGg`kh%b3 z{3sYJf)yXg5|dD#w=9(V0>iDgu^A~_>VD{8U)=|jV3z-!6AMy{IWV-Q6GaW_Z7xpe zZ?!Xl&esT1!zd~UF78XKtAyy7P(044&Xwlq2ujZ{S5OQjY+T6?X9VW|l-nAs{3^e) zhGv4j=Vd#WP{O z;zlp4Mn(2tu!m&@@QY}1hky2DL~uLif8eFZ#Q_j!Ld@K|9nlx{gj3<&tI5OThyH_` z-jgJaGzYiNGbL5W?;@ca+B%9(rofeo%DaW+n9FGTex1Ex^JUw4-3SU!_q5&XcUzgt!yS&JoRH=irV*8upn+stK`i=uXw1`$$I z=h!E{L~ST0L37L_6PATAI*5zdUul^*i!@@H*njwJ>Lo5W$hX_NqY%Db!Nf1YHq8jG z@3nV~bD5>5qMndw(LNw3i3w&yrul?PZa;2ZYEtKxd?7EM$gvKG=H^D`wi8kWax`Xe zW=5iF(rf5@>+Zk4u*ffSZf6AwoY$VoO(-?%X-eh<(l3Opoh*?bPQ03JmTq=lL(vd= z$lf<^4Q(SB=MY8PGzG2c?xITRJ)4yLev?li?o(F+AFmPDSXf6yWy#GzcN=G()kdUG zhu?e#tvT$!IRj<<_WDz11L@~R5ArSH`| zh=*K|-_{qslNcysYZs;cl1TiPIp*Qr^>-zNZVdRagN`73KLd23yq;xg`V_0DniIlV zE|B!8L#alFiE|t(9Y(vv7|%VNoAVY?dMXWOu8(n2qOYHnnu91}E;*>5Kn91GGkyi(7AKL}Y>nL=q}pZ^0o CB%?(D literal 57613 zcmb@ubyQVf*Drn!2M&#NcM3>|bVx~;G}0i5bax$6=>{p0?v&00g0x5pNGjb(iqzc) zzwh%r_rCYuaerg{{$Om_d+xR7n)z9C%}vB}RXJ=7atsg%gsmX|OalZ$0e(gLiiQLN zfdxo?(ty9ft{QSupsFvF+rS^lR+1``AW&@_=Cv7u6x~Vwg)0bz^9k_-?sqJ)0D)HW z6rM?Hc^U8BV761qWFN$Awp*zlwUdz^>k`U5c@q7JW-~e%4_S(ng1+gO#Vkf-JXqSn zR7z@Acn9{f{RfKH7pKQ)d$@FULh&E)_(+Y>%!Zi;NM)gezt-)zo>2D-ZgM*hAfv|~ zCf%?P{haCWIV)^!YrVP*ct7@Ms;p|-$Vj`(r|g@t?LK-JU1d1cg3D|-$D99l`QvtE z=GgUtV%9EWJ&l9(G}mDTj)O6`UhcTS%*-KQAtpCarX^XGS2 z+hSx3&)FCG?>NTC5!FABz4Y7c3}!_++3!$wVpL&{R_8#5 z?(g`&CcBH=FcOCk=RWoq&?{#N;>Ddu(qG)lXb55>hGo-irJ0c4P@t^ySA=&wv^X3X zN*;cDheJ(Ho+$9RhO3vCDV^AW`VB}?v6fY{h<4H*WshsINeK($2flokPw=|8k`9i2YWCbk+cA!W9^`DN?$z zz^4szxn;`evHv~HjjWP(eHuNVr_e+~SfR|G{Xrg{T^xN-o2v=E{dl{HrxdOO>nio`o~jzf{`u${NLWo)&;lB#2T)lgX}2I6mDW zO-VAYWl?4Tv{keOSmI9Tg?PQnx`d2_eUAURx)7@+psgtphB@KcwQrG?SO{zR57$9K zOdH0d;ea|kuV(7yrT2|ozC6rX$Ig-#G2(N;60`a(&LB}w&u*b`mB9=ega@^$js$gZ z4J;+3uqG#HQMIop&|nSOt*Cva7bq) z1#qoBr77#&IKQ8Dk%xWL3Y&|scQ+97ArmI8%gKq+wv6HN|E0UA4}>GtAfMdUXB<@5 zD~-ikm4=An3{Wq|x0T#f*Z)4%0bS%->}2D4I6bHhdD)A@sR?-N;MZ=7!$2wTaC~x^ zb;5BV)Qq;h@-gb^D9J)D&A3RhpHSv8N=&Gyswg8Z@1(L*%|IeF4w#R4LJ~VWuf%%t zhj2EnnXmp>L@&QiSB*>aFna21_p$C$sb+EOq46LM%>E0LoSM%{#ULNXMV<@3+rZw1 zLc>+?d2&(BVyf{5A=$5Q-JT0A=rKG_J4h*xmLg*w+|+<7`cFpW_V!GE@0aiG7bdrh zVxKde>sRr}aO)3dxn0XLt7Pz6eaWH{@fZ+kXy)?G#NIMpJJqOGLmIXUr*oYkoE@I~ zFaBn2TZH|-N3gL-Hp>ZZ9v#)Vam#+(Hzi3;44Q_y{v>M53w^)P8huMeSjd8QsC>dv?lGJn%LLN46_3B=CBb zX4un$$l;A;N=HXBPnvR&e}v(~@7>4FY=gC~cJ(1AgT`VVBTP?zas(5=67%Zl zdK#y}`9wNkKM7b=3uuVI-ofn$&=;toCKY|caS+;DNkv?4M1I#zUznB36sB1>b8k&{ zV?On3Q#h05IEQl1THx?fx|_rc;ewRDec>++@Oqc0%`UMa@kw0(B^~1J90(WGp;`1g z_w73VHn#N9ZS15=<4vC8T>sw6`;yGyks!m80|i`Rz4g;XA$NLeVXyDk^L7Ly#y#$qZ?&j}--Rl_6!!A+nk^U*WXbWXQ((e3M{NrCU{b;hPw=ljE{9Mbf;A>F?tl0Km7DrRshC=6-Vh8W<8G_Ti$u&)PwWR43Au71#U zdseMdCLow1WKF-fvQuNCb9LrA_S1PcPrN=mv^Qk49VvpQs^i(v>_}i>K%B+xAKkm~ z5xKOE+uA79f@ngea4JLA3HGg(oaH8Bs=ZfIB2D6<)M0iwv9~|F&$iT?-c75&q2vt+ zW}NL-VNy9n@^Bh9Lv5n`_r=-_m4^6||8umbiDRsLT%&u<)h~SAevYpG{ceMV4f>4e<(+@tpY?@V&`i~(&F{^s%QdA!rsIat( zTX~p-B{D@wX=71|t7>mBbdBA2r8o2hUVEbw{Cn>InG@iztfy3V%MhDTo*UT|ANv)a zlrMn5$bgG#IF63Ek70hS*p+^hxz44q*Jp2vsY~1B%9O-XH;)Mq?y8uLa-;J=yS_#g zjCX(q8rQ86m|(Ac!#}`8ykCZsikVrVui$`2$p@rDF7(ZBd;XF@c2&;ZOP=j!@4wTG zC@5A=&kGQ@JYE|V6z1VEe_sqSdL6hqnm;l(Nq4u3J-|WxSXelcf{cP9Q_wl@CgMXy z*Oi4YN!^W&C5B~j1&-|DPv^}q*i361g1QXNUOnCCBX+en0mOMZ>1S}cn`T4C(i-;g^ivZKT zECiqZufN6S;84HtWGUd8e0UMaL^O9rk}}?XY4fg4@)=^>!jG%PA>(z{$)!#e0CE6^ zduR$E(PUk#sG$zGy1G=|I$grQV*(@b7hWZlIz0j)699JVlK}*9Pe`aD`R$nalE?^w zf;8z_vmj$s?Y1+VqIna?jo~-grrjKN0=;dXht!mm^=h?!y|(+rlkcCu|IQw{cM~;q zVCqfq($4NtYi84W?P|Yza8x-DU3ALicg?D`)ZJ zM(BitK}nB9h6KJ>O1qf32|fBC;ok)mdK?W5yuFcYZ5BBTytVgs~fFxk`Eqm9EXD<^zmJiEWDqb*X{?Pk7gi2N(%HOqUa*6~93nq<}n!s^v zXf+?ZFiwgw0ubhk#QSiif~S}o70mY3vQ{7rx5@DZ=8ov!QKy!>K_^6yFg#;%8nel7 zJm)2{hnBh?*5jmx`f3JeECmV20dfMymUMIS0npM-RPfG*AK{z3ER|z`0$B$=tt*s`EIi z#hf8&X_$>MbO1c=udx^GtZZ)@MwRDXCN?aC13$R5+kRiL#n%TTSS8U$T6=|H?x5NJ8u#qKb23V(=80xnq0`Hyhyj}Geii` z9yLl!+fe&kx9*Z(S6Pa7CrsnJz@tZQr(yCXS&2yD0?3&wq7P~9RRDL6V^-)6= zdMzS)16hEmKQU#Q?G2ZOeDv?Y2cTObc&^6uu@leAHe~ z;#*yuX{pT6C=#zrY7Eil$XA@~-Y5Xz>-i3A3PZ5{qM;>u>4l$_*81u6NEk!o7gkF^ zTjauePY398Y!evNV+~9KecCBZMHXx_< z+7P2DuZEfsY{#2b1>LbF0Silgbc659-8|0S5syDxs(xy(xaNIidMz{4aktP zldyFV`~CKqbTWI+8$50PfCn`ru9a(mazN=@PQw|JNHfb$C#@?j$w4(q|Bgw{4T^k30p1Ekc)ny=OI41_;wMubzx6(E`F;1zv~?Zjt5sq||NeZ_>UJNA$@P?` z8r|kbrXw|%ITVaDsM%*-oZWcQ9s$S=rNdk!OkuJOc3&b%IX<0K%hb}Y2`PtkT)|)X z1_lmdvWlY*;Z&h*8;y5~cUb|@<@&7pc7OAtGKEq!r8g<(E-KJ}$1(6`;p2lGUtDUY zla}nR*^X5?ms0@5Flt(SS)%Z8a`)Rky;~OHMe25u{V8WCX{@-q4pg0}(o0SB!ZM(AU zuqB(M>r_R$Kz>&$eXE#1efK9&G8^CFZsEb=)75tDGRujQRM4xmALrtoHj^VL0_CX+ ztdm39Kg3~aem~30WFG34wcu;L0DT-zPwS-mwLvp&)C1^xZSGs zzp46Bf7il<`r2yVvaxtg{PU_hlGj9mg2!>+xR~$x?eor4LL3Deg}_|t_1E6b409<| z`{+EyB^Zw23Ww?Bhh*rcw4dlDe1y0yGON;tJWRD|T$?TI)D%E(B5cUm=iLy*k#$Id zP4|T)`xN_D<1#^5W*)iRTl-dIg!osGNHIB5CUxeDcA20~6*iP~jobJZ`Ymjv@PIMpc&?Qw~k&=ED>b3a2HH@{Q9*bAW%cZ5HO_KtSOqp_oCPiD0y`%GfF>D zYlQN}orc%5fgR{hZJkOQt!rKxnlS85c)0SAq(8N(lhfA%`rh|`z*@cP138yEd@q2C z!oC&y!dQ@LQGfo;*SBnscrb>@l6vGUH07;)|2}Ts&>PSeyqB=9RlE%Q}22?ln}x7T`-E^iTu_CjWl>!bC!C zR?S@A>XiO`jScqetx|Ich*oj`Ib0o9&-SRpfqmnZ{sP7`OF*RLqvW!9G-<7+K#dHv&Q`I8kD)ZN2LtCLw_BrNl zCVD#rh>?#M7KKXz*EX-FzE=xN86dW1t#oqTsIrhmKrI;JFr}1IhlUd2hjT?r@j~j_ z+_2F%S2a~4r(dNlCBssL@CUl{xs8vS<~bgvEH1imX;CEuLcxbLlN>)*cFWpxu37Ka zWQ>2ZWJIvwspey!Y_9J9{2$qyg6_^ITN2@&Xp3>TpH&s;$(s`%IM_YRA#+DkFW$2dvXR5W|DwbCwow#6!$ zua;};)rNnUzWD;J;cc!j_V=YpKZfS|pC}v$wSTf$CV#f95bMuD&2e73?<4B->>oCO zC}G{{*9^Yy91B?N(AMlBnAklQVxf4o^^(Hz_-O$lga;>{RGgFm?o#%vuFGGTpMS&u zVJIcM(1WaI)b^9Ik0kjE90er#=I?Re`k*zUNLiBak+yxhnn!;EMiD&1TGmr9^>hIg zzF@$|B&=2qdhdNZvse_RJqN~W(dK!h5C?jH6=bjo2+lRK?1&5=-@57NGrX@LO%SosuR@ZWUOOECB{G65 zjtyw1pAVKX!(^;mD=M=SD^{0Ih^BQ0c`6Nw3Xz>85ulHZovq8Zj0dI}k)0@2lu{lH zrmP}`t0Y^mmzjdzutS{)Ebrq(x_ghW=8aYUPjirfIzZRs3L5L&3OJ*dzzMr%{&qaC zYFK+);GifhWBF%7Y?(}3WncG!gRf%DbfSfMKAR|NjZK5!F+RK~(zx52rTr-TP0WRO z?!jVPnX_q2HZR)}YlmBZcIHs?i>%2GMHC}wx~;AZ3zFARGV5_Sm8fdA@N@vzQ~;NW z%i5&)v8|r8w6x2>dh4{UbNCnfZ1eYCmW!ob#Ym*-4(9F2g9%()^KmwFer{QR8aRmH z@#IyO1BPvEBcq%71)$G<(LwjMv>e&1gC-guLUheOXGX%pVkX!H&YNGwWo`WO@JL5@eB@S4odE4;!j9Jio-a?v5jpQRvK^JC1rpYhAAOJny$y?}NtY-H>ZIe8 z>zLhn2eDEq!YCW!)XXnLD*d4!KC-VZlMQ}!Qcq&Rt94zSGYu?Mp|$`at3w(8vkbQv zeHs_W_hdIX(PBjL^;cq7%Hpx(L*wy9XD%Q964u&LlaVDn%@DpVXCA0Ogectt2^D{- z23LR1GQOpwtVA5v@%-j>^Hn#?{REZ8Re1+dMmo71ayo76dX!)nnaC{>_o3^n z%7jQ&JZoSkw~Vb+KvBOaI#kPg@cZ0G1WPd5N+*wu0%^%oL<+0E5PmDlS@qa$R2D|u z-Sx=WQ{OQSg|5I)rH#sX-dHc8+=J0O*?Q@D?jQ1)-r&N$k~QZ*;qAyrfFCCN_|P>A z)S~@DlIkl6?snY5_`6&|igR6NSvk3c*Z^zQTTG#N>2uHEHPktY<3lanxGbSqZ+xth zi9wbeyZ3}e&q0oqiw+vuX6r*K$4N%VyC26*s_Ma5MJk!h=*Tt6s8}l)@n2O;=jtpI z-w8NUQ}37GZs*(%dVB?tbaYnndmlv%L!a%!15ME-zmrj~ueb41=; zcQ%L{`bLai0?;2@_ZbrSdf|%+ZLZrC9qQm^6dc4{Dbq6x=-eciog0R3&HXPs`~+ey z=>Csef`0aQHrW&24-%JS}mV`_B4#i zBHOgDVWK=D(G+XT>Jp{r0YJWy0v0h;Uk8NGhh2cz(lJ4gqQjc}{HBZCGpVm` zHO2RfsQs_y7?!g2S>pBWBc2wt29T|kBKk`1@PSs0 zlvv{p%SWzO6F#jvWp-Z~qn=13;_??!Go|5*sn+Wlzi8lP=nr)vm@{P?it+cXywzf2 z)um}YHUPFBh@pTS*0KL-DF6j!4q7XGa6PY;f0lSpDbZ+NwvnA;k6RvR=-OmcZ;W92 zDP*?8N;B`I+1|YHdu7@PYkMB&k8eW&&~eDr!+xa7x1XV@^|lH0zQw7x!|X4i`((J> z0s;VFR5x#0rS<)6j9_ot_8H8|lWIpfdO08XbE%S59qHxEmr2YjJZkg+>iJY)JOadj z4P z{=-!;%F5Pj04i_eV^TEz*&Ffc(Fs|O3QfG85Y;1*|eO6ILE?Su9App#Lh|x1y!H?do zF?m(G`Sf5JQ}QKNe!2O&e*lY15t0epvrk)S*4Fj3RZIr`!Kd9R;QIB}EGSe|5u{WN z+4e&Ih;1rTefYh>7T6fbuavAMcQciz(TaZQ+Ws9MMy~VK@~v=F@z>kq*TCjLDvzpr z4_ZSh6{t|W`jA?whw5|HH+!EEs-s$9R}>$PQsc!1he??H#Iwdi>t!jjDyW34;${)Q z`8qdoBR@db{=Dc21Bp~6h}6SC)=e++?w76V52B0fGR2~2*0el}@4vUW7Q5wOn-a!T ziYmUPr+afGC8YHdB<--f6~l!(i6OukVuE`Q+QkEsoQnaMPEiVbDA+I#0psRezlPqMYYi;#Z^w)?V`qM^1# zf`taqJ53?z3B3Eny2>B z*VQ|_*h1iyHUbbmeTmLwu33JPwRi(K)A`y2>$$<32@(&+*-pp=tII=uy&dMbHjjPb zBRC|%$+iddh0FCb34;mgGdp>tUysCsLhIZ7KGu_rC*^C72g0G;5+;hw6Z~|pLPgL^ zXq^Em!{;xj=i0|pH#W||Y@OV1J(7_1Wswvz25u?kxXjgJ(?Aq(Q^0Vy@Ptw*Wu#KY z&u7H7%-RGK6BxL_2YX|GXAF@|bEv7@J4ZHG=uB9S+8VSk{tin0*z&RYS;fdYk6>Qd z;xbV@vQ3F$VI~+0w{VMGq+cO(cq;t1j)PcfZ!iS{5lLwqNWbuyjmkG7Y9xw!9|X=a z*@(s~(6y5yU%V}VN_4t8C2tKbf@I~sC$(99%117vC>-x830Z^NF#hGrl8FG!A{RsQ{isDNSEcnC$g?4SZt~B zO-aRM@-I0FVnGs=Fl%nU4+X}~W7TEJ22|TT1Al`-JR8#Q z+c%9W@)zaoBprv{vux&0J)O_j1LQ})8}pAe#< z1IonDmg9>ef<__&PVo^UGB5!t?cb3O2uPe|FDgjk-FX>(Dfd(*@lfQN8~p04mWP)M zD8!D(T5!$5T6Ef+zHZxfD={`%AYAT3^q$)0b~3~m*L1z3lUJJ?16l%31&wXRUuiP^ z!+~oghP57O*vdq=%8)f(Ui|j=s{Ba$lYS~P5s$r^>O_!iY`ce(larepA-Z>ODI?vF zN~oc;)<->e78qXWYhzgP77AmLYPOBIq-8>!!rRo99u&k09*!G_y?ji8?HE$Pt2_a2?7i<5xH&?Lc*~Z(F#t`-D&a?q9W^|hNB24QZqC9o} z71b$Yq3A~4q+-|t6#X^`0Sg@CjOw!|(7i_+GGShrrUM=GpXdW=)T(cAeup$=bw0D? z*Fs<&!x{xXBOtpWP*-J2nYYz?V+0Yi$h6yiZh8v&-FzbEM{@U7+#8aoTY%Fbozr=1 zJnhh7p+CpIgP#u6F2oXL%NBhGfi(9U%XF*xQlLa+j4onHC*axb0cwsGM=DU30&kluQe z0e~a58ngjesQdN~L_DfjL+p^B!BF^}1g@#prO}{9p00Bq*&15D#<{Bx8oPm4mibQr z<>#}`}S%N7{$37snnNC zdZUNv=M$K=?|SM5sIhCF7F{$r1zW<0)|FY-qSe8sdZ-EYc zU;hYWXKV&h3%Um(b0aEOQ{$cFG!On1;T9(5Fe8fJYtMOMMt_$AnC5eVln=+LUymPH zs@!~*Ojcx31z4>VW6U*NRs9}(5Af?OS6=en`y)k%JH_dHnpX@uZ<6k zkMT6SNFWhB`vXY%PZXhWLuz%QrlYa9wC8@$*ExP59JN=J$-+RM*?I08;wWw7$6lFXx1ZOS?b`Vw8%VZ$|$JRIyMs}pI`#I=LJvosvvQiCXOsYn(XUN)1+6e~Ua z)7OFgcQjpjG4P{u&C3@aBo~XAwB?4h>;{*tM|xB4DdN5ft)ZY;X+RYNM&B}Ga;8~|{20yx6MM77unBa3Ed{U9^TBv5Z5o5XD z|8(D2S0&AO?LEzXg{ngv!R}(TaT+Mq%nU;6&jsoEvdn`s)uza~(@G@2=u5_Pu>r&l zcg=WK63JExVKgebNp>D4v*p&b_b6M>D9zJSRK&o0RPKuy_X6-e0m4Nq>p*f83^#0Y zh~{NJ7|TM4qYoy?jlYWe^7%|9P=Es@*o0xN2I6|*66CvGSBUVE`xkxX2m&|ndh%Iy zpB@yJC)^9GQc52m{?_}|-6N=RxPIUA8Y_6#7&`-k)INk9E=^EvO-a;OwJ zyTG|woymZ5(C}%o-B&*s48o?!;IoQUUMoQx3Xh#QdTaidN#7caX1IvOb5wb2U|md1 z|6R5YojbG!3&op(e0zLc{eny3xGNJ#v+1vTOLlecO?4sjzs%slUgpRamA5qJqUU;w zEoRen?M?D7Ayy*-{To(mmdo9?yIgEwu#k&1Y7 zAE*&e?ly__cxfiV742$s9LX;S^Y%9319XZ%4z z=6#lUYqX*CHw=>y-qD5pt!jATwT=8wWsd*j4ML0+p5If&Pc1bH=xT;-fDqI9>obK) zBrmeZix_U0g8Hl_Uk`{`#w=}G-U;09eex=a9y`cLhXKn}gYe%Zz^?8A4o=bKF)cc( z$BPcH*MCrGI*o2V@XL5lv2=RHk=Zax<$|6fikDj8FD<{%fdbO=oY#2RkJ>Lu)Wly7 zCB*9i&1(Hw4K_drA+Y?1Qa<13X6TobX;Yq`{IuG*G|`o$NMTj%Sur}KP^Tiyml1e3 zaocQn=FglGyk5R!rqRv)`$y^6-ff}x&$tYT-k+;2?_iVH7Exgf=%hb37eil|OyIFV(a zL8)6D53~AJqI0|)&4C`s#B2IviceUki81kzS*24cphdEgajENv)_uFdr`0|akx?a( zH1*4OVC6bI7Z6wHF(DD#Whf|>pcRaDiLp|fW1n_H^5*YT2-o9iknC2E%P^X0+&Cr> zbB>BU4ZW|!GN;jS0S0fveRH_i<+(@v#ssv~S~40=IW?=88N)}DOydP=uVuM_YB2gQ z9P`qnn{4rd!#GEIFL}H6GRWd#Ewg%3XCgE;g~2#YJr_ibn=g)wY%(SuJ)tTN9f*A znoAX+;@cth9G4cqPb8s-CA^Gl1xNRWJa}4y0eV(}VJf7Z0a|Lb&m`oOQb(mWJZDOx zFQEK<`zx?G&X`UpoN4wsRvdoeF1E?+M%a;J^;xufo zaRm8B7{i%(ty%CtZ76ULWa@S1kpNy}D~Xk<#Z3znljZuKN}AKoUre$rHQt+olD^e# zB;P2rk=CHT_xQa#LO=t&CL($I)alz=%KP85{ZC(M^htp7b?$9Q|LwEt=}()~p~?#4 zO(!J$;vGlFKy5fI%i=?O3J;} zIljbkY60iUYj=od)>EKoWOH!rE{vnb^-5X;{+SKw7-$QLhc_^ukUXgBXE1)KdwJ<^ z>kMUu$tCyGF($Y7BF2ufpKmOEp3p2stMGwsJ~=-g*fuaE79X=EcY8B7+B1j7-ww8_ zt(J_=0VQvy-|Dw-Y`GGYb&Fd2fE5$xnd?bu`Dsv+S{my1YR(poHi4syy^<68dGO+0 z&?bWz_wte1di0_fp*0yiw8EYDW`I+!sXRLAMa43m(7z6NJ&uWHnski(A4dLv zD{cSV6%bS_Ut3OFeS!Aabij$u;*N}f4bJ>Lr=hK}L@P2-@`k1^RwD7mj6^920LPKT zk53%6yXSNrg;(`Y%qOrBwQgBv;=wmL0qc0KjuJf>Y7cy`h~A#eu{X{pZuAvU0CnZQ z&jr)tCk~QdUBs0iA*$h@60gkgF%k9j=W7A)dV4@PBt-E#dW^2(%E(Zs=2^jLqKi++ zz?+sEEdnJlLg)MUpibNF7p=JiH{7+{u*@I~dG<$%Ei?x_SSzngSGk);BIMglK>{Jh zS3gd9Sm^hI!3*3|0fHSPy9$W=4~Wu%9x>(bD3E6Fk^yB_fO}3-_$WhvfY(Z}3c#rm zH;e-7{L|!@PLIC<_jjI?k;g@C)B&kp^G0+R<>mI6XU0SUkOTe~>4xG;*}$O6Xmp`d zAnbRzQfQ@Ma&Ed%Tnz}XsT3Mr0rWK-Ad$!Z3PYz0jVJ#ACIIg8NkEwYKfYKp3fnY1 zDnt~(W%$%dJ6=(u#+3=l@X5czZA$x&MH+LJn(L^kMKJxLRB}2ikDT$z+j&gvcX>Qf zP-S(Z0j!_rQjnsOkvy=b779;Da|iDT%!;%1raf-y7jS&%cUPLDkvuUVK%M>Tz&5VS z=_5J+(zFBQRkHe^{%tAt6XFW(PeoYik_*NeX7GNlcF7n}q~f=b`S;{rV>43{$AvoN z-1*T%xtd>0@}Y@x2tp|V(>-ggKus;?QDIL#?6RLW>EI_rCBfgl?g$V^0Wsx25sZY| z!fSV(Pt2u0cR$o%VAP4TzZ-<3WKMzD^S+S+V-`rp#2X)XQt+oaas`neeYNx~t0GD# zA>dI}gb0m&5MJuXFJ>>1%tQqR1$lWxDH$0j$!kPQns!6mX?w%(+r}#a3Yd)QO9w z+MeDVj#>58Y>96&&G_s~r~Xk((}OC}hUVK{lkB6V-7hwwTB9<>0b7LAJez)hIDpos z`i)HA zNk|RGl+NnPaFI~7t4NN1@;@uxy|xP;U7uHSf+9Y1D(Q5yG+v<8nxc!gWQwgx_M834 zh+B*12sFq%hgRVHSo4akXJn-L7`HgyTAiq@1vjojfj$yhQPSb+#+jvJLC^5{lVUTl zj$37GgDYCLk@q^;sNvQ{uexBAog$3<6F$j5?96jZ;!?Ce0cf|oI*j6`L|VJk)LzGY-Gho?{3u2^bJV(T?TBzjw(M8uT7 z-b&~!!Aq0{W=(4G^UVb(L8T!FQDrUi2ofebt`D;EUEmpq(M>aJmF9srQ^|S(E*Io@S5hOOSf%{AibU9g zO(V1pT)ZocbS}s#%ShS4=|l@3BhUa7g`g5-j&VG-*N!lHIR+?GM`s>P+*QWHV-YC{k8A--iixyz+R z*#xje{h|P~{GCTU{lXFnrMWZkVaBXpEhaC(@XJ)n(n<|ydvTt)RzL_Nc<_p|Sh~bab(L8qzFX$F9O>B5%<8{=@KUMg&g#+sj%_8d zY_d_w+&I(SmM-*x(;LnSmjNS&jjOwm+D{{tzK;)Tno`ukmZ&Wl?AjwgZyC&xgDG(Q z$0{6N+{!h_$jMaSjp;EC(IVVLtFWW@RF3a7167+#hfA}IaQ3c>>(h5mA(RMWbaYCW znOvTqc>J~VZM-GIQ6rC0iAwmNP7~6eZS&!Q&`lLqzcGRZ+&2Xk7;}VthP`sJ0|-A& zZVBZRrjz?%w=4H3C~F7HO&b<`FYrkpDwLK*`kaX+6-~dUHwF6Wz!5t=>^^}jGBQ$2 zvWnwtKt}ZZo1}Jh^K4m}Ksuw@lcX`q4$Foknc7)Q*2rgM_kLDZpOz*4QLtk}ayn{y zc#4aPbjT{(JvGJ~%a(#Jv8Fp*n#UjBSV_h(d61$#!gz0Ylv*FY56y=5+SqBWmTKt@ zy;W%~ckxs(o1ym~i&mZ)W{R0q#9@9e6xZ?mkxWszF@!70312g1MV*fX@DJl6m+kmp zFZ(I>kQwAANFBZEAtPd;p~yv7x0P)+drEuUPx_6csm|d#pQ+C4UdPuPkPdKwlJPR# z1lkS~kp>k0yeDh4d16mq?9QO?V)#<$KMcJnsjPTLw8W>@Cms0u7h%}!*vb~$EY|}REzW6!e)!_hzFhJu{Jujwrhe-(vBq?2|vO% zc0n?ESQ^rtIm+7KF|qXk0orM0%OE`Ve`jZynX_iEV%=f%>QF=kEMh~%78B3qi<%*H z%_2*Y|L)IHUnQ#pVh~SXh=8!0kzT`Z0MOwf{x;!yoF(k)S3K|m^lY3C`oZQicqqwJ zy!@v@i4Vw~eC~sR)bm~JS*Y%@Tn^zuF%_QXc-Qwdpr!Yjkt3Mt&8l{bS?o0e47px1cpiAEo zZXN}p%@8H*PA_>R=7rE2x ze?NW=&y|E!fykW#f#$;k7l89Q90`c(-_V6-o*V}KN4&|U-BHZ-H#u)65m|t)L{t46 z3PM{575^z>>EasrYskw%$L7U*w(3(b%GFT(C#2WgphhcnU$xUt zjt=8^wG25He)X{&sg%4=B7+6LXir3?UgbE|A%AaV`8<_-z6~6X4UW?1;xs7ylQjy> zpmxTF9H`V@9~9hSHy6vx1sNdO1adn6`{eL2T~}v#Wn~5EP#jHaJo)H6lEnP~npuI| zmpu^3#&Iito7d%lck?ZfJr{f-g9t0n2HWdo1`Ikk6pZg5;bk`F{b#=3Lq#s8h^!SS zFMfN2A_umL`a<%}(pjq`9-3v$YkHsJE#pl~=7X|vB{|W%-+pAe4i}1T@VNC#UlM%t^@2&-~5safi93}yQ)j)W0{7^VUxYw%sm6(B6^=HY9r2pI_%{7cfUAJpnJ|InAMj%U|~X2 zZV-pp$zUv_Hmio974lrFd46KMSc_H&g<0Ye#Rh+5SSR!m(Z_^cXtOOm5u!EqCK}Kr zqMZhTX8??uXLUIQ2%|si3}rg4Y9wl2{`LIm76z?^3+WNOu5CJgIJbzx$toWUnXl$k zV3NXfhJxzYoWPEM3DxxV>Wx~-K|K!_vqe?~)UTIJx3oJ|O+~=;t-9<)zN~y5q9w_u zW<3=^W$5EDRLm*z?T^;fvN@ORDW0;y@2N5iX$ze4@fhto7L(sE8gmx!k?F_pC_mp7P*vs z!~BGfDZ)1riP&4=mw=<7#0w?D%|0F51Nkh`tK>CnEAec~C?#z)elEZ#J~pq3v_!D< z*2IQPJNPy3yNC0&4BQv4r-3JF0FGjM4I##UWLWOg>I1m2#U|m~yOu({KYKA3-c82p znZT>;L%8IQBKz6E>wN2`Yko(GaV6$jnY^opci-6W_jGVr<*P4u8pks%Lj>5(k{Lil zGte+jCDd9N*$w-@*7QA0xA9k^Ouu9O+W zSj4q^6pfcpbO-?}3%j9)g*Z=Nb!KaR=ul&@EIB~SpNU=tvShUc;*gGCgSqSHgDLe$ zMXr0iE12>cp3F|}6bxLr{({}oXOJSpaToz`@f*BHx~fd%-0J|ur3EfZe_<}$IE+=7 zcLf=^nEXYXnh{XtmOL=aTd4A7B7zeIsNY`C*^@6u{(#l|XdsF*ksc~5EjAIlMai94 z^EzAvKK9lznsfUB9Tmi();{TEqqbDJCdUuoqFDY#mO|EPmt3l97lVMsxfDmv!-tnY z?H`mK!gRTAapiFQFRsMl?TzL)3R=Z(z>!^j!f{=QacOM$BP?AkeZFXl0_=-`>YrS_ zX&kG>2rVPp%%{=Hm4ZlXtlL&q2GK4%0xIMEmoqkB}#B3LK*m-Cs+IsR`WXU&bXNwsl ze`dkvs3e0u1rJyU>k!556}bX$p_RIt!DY#JJfO38{v5vw?C8z-;Q|nMx`eh_M;fJ= z!mT~wHaWk>`Qn`=GrJe!9B;{av&e&#yP#+?d}zVm7kvgc?Nz4xgW?F-MNAZ?J#a+022mD3^sFIOf3JP{#^O5<+hgFL#sL4kC5Gg3^u3rb zvuf5`-MK8iDuWJ>QPwLlr*9TzA+5mS31Hcm$B?P<_=bb2W#7@vFT5dwcCX?4Fld%y z47aJ?_R-N1uy_8Yh41uL4?+!`+4;3Arrt?afEj|Mz3L-?DAW*!o!?&Bo&w0BjP;7~ zgH2UI>*uJ9-^9Goox+?sTWfT&D6_jkwE&HuO2Gcu|L$s;2KzBOd*(sg=Y}v8(R~VY z-W3lZzgCMJEbs#V^FgFo!Yfi)yMht16GCR3J@0*I-6m+4^>Wqsay<F#p~+?Me|7nI8KGXec@nc0)y1j2PI$#wdIwd=qPdPffrsTG_C^F`gI zA5D^z#J#QV;PRzw{u54$-Ao0fbVa*n^4+kRgSO>>V|3<$TXEM(k8XyBeWvMg6+&!5 zNtXA=o8SKS!jm`uA?L!lisgj8gCfJD5*zl?SckmUm++9`nUzE0(jqVjC=C*lf`mwyASEp* zWgy*BN~a)F`*{aH-_JLG=j^lg*?X^j);j;pVwm@R;(q$Ruj_iW$bZ9=*eRrn##MkJ z3QF#lE9mtn>5a(g_df@uAx0~}=IPrN2v-;z-n?GF8E`ZO_{M|Z^j$`eYSXutC2tkP zI2ZIyy-6;k-WnYp&3-~Fv615UG`Gq&^hkxppl&~*bwBh?MW;z$L+|w79-_!g$Old)eni4Q~hyd&AVfzUzD$(N=VJh~TN1S=9Pm(0o%U3R1HL_53^3}dd z58YArRM#`wtWL(5@J=dm8252Q7D3uo)C9I*bAOD%pK_Mwz8$;Kd&zk7+W4$YLD zzSMpMdH}E#t!#SO+7=AnfH!InTY`2kP|Kk(n1j z8=GT47;w}w8TIjvGz=(KWz{5aAh)JlCJlb}KS=}`t%viEx|+0j83pMnifSWn9~Y)^ z7M4}E6a4yoBY>M2(eO1o8*Aox9n7kt7tu*u8U){_03#tXV{$u~o!P%eRj7V!}f5o$pPp!OU4$7G5;_TqO!v zzSrRB|=qU#|Zm-`6m>!>n`1@XlGQxB!R5>3Hi;EhUS@>>^SaLhh zDt#W)ys6l7sU%0>YE%`xd(Q}(bdOGDPocs44Rj7++ci+rB&79@BGv3^X7NHne;H=4 z*+Fqrl>mJ1ozUzaXnC#F>kho%kXy5%qJ{J?g`75o#gx-SBy|o(uRlNdA_ zFgR-$x+Dt!(9nH^{;`I!hYpw?tS^tRH?KGgR}?6$<|ASYc$^Dv-)$wzEwQ#&PrL~o z6<#1HC7ioQj+#{!LdzAeo_*O?6)E`WR5oq}Jw{`1bpF_9#Hc;k{0M#z#3?T_Etfnj z*C$BQ5&T{*?tFNyh|`#EGVOdj!KGkCGBnH<3@x(J{If{UE&g47f+07Hy)?Hgz_r&F zp{*cq4R-?`w*!h-aFuZQnn{!Z2M5?Ys5-$X=_Yx#iDY_E+6ofbh`Jl1(>}S%mt}f_ z{(R3sVzW~#P%YC5&7Qj2)8#o*TIU4=Zy3p73l~#f5+9)Vf^=;_IEzIdn;q-Fc(S7~ zD6YyP@hO1}Y$Nzsv^wi#&1OlQ{+(!)z<|2F$SjEOp1is-W0x7y=6!m4)nMv3kw=$V zOqfQ?4;JM+52l(vC}QX)63vBAHF~G(-8}?U;kUa@4*euDI$G%%Xtqbq{T{#JK8V$I zWMkxcd~D$^A)|EVYU6&UQKW+2Y~Ba};Wux-o)P+vo7rnHW23`d@o-2|En?4nt3x+y z>_l~!$x^jg{7UNP;bYuiiD@-fjHcHCBp*sH1jG+Iffz#ZXm`Y5%jT)Tj+WEQXCG^EbQ zSfv9#YEbY?NLPYszbT_0`4uw!U%AwurhlEJUp@V1;=_ScS0Ze#yE${LaQ~5j`Ikdc z8kZtUd?9n1g+nV1MeJ~C*vI}kPEBndo#b}_E7;||7Z|V}c)B~oxQ?ut!ian%08!q> z_FvOo-!J~rfbl8CI~aH8N}7093!|ozCSV>#VF|tMHOIUX0$_fXYdOcBw}X1ISY19zHy3ydigq=CG_7i`@SZ2yorKVTVZ{_4P~@Z)~>0oWm_$LugC6e7ywZ+)yDv3CJ}rTHiBKO#u)1i{3` zUl;|*0b_aC^z35CTOHsj^Y#!NK(wj$C8iZJpG(Njrccui1S>B@U4)@vw? z+$4?bhYNg8DR8y7{8CbH@4hG4T@Y=dqM@nL1X?o;;iGEw!*6rpYvjiD_g|{8xz@L} z-P{d{Te@3%>(^I>O1}f=grW#&qxTGV;IV_?teMnFl=4#`)-~1OS?nvIp$x}o`YOSh zBhiMM62_qKmp5`T0AcUL!%u;<-^??!e3x`JuXzsX->MgXxicF~Xc}<1Xp`kHve9W3 zE$A}+_Isw^M=7fE?^V{lOxg;e!ByYJ@qPA>SJfhfuKvonKU0+oc9hDOi7BVg*uOnh zq_vyOykl7<>C@+0LzA$=~f>|lsq=;X!3gr@7#}m(HSl~*X>RUV~P;= z9z1PLehuZ~OQu!+M_$UXI~SiUWE8*#c!p6V9CzZzUfZNc4yinMxe3PMX|K(`7mS|0 zjmP&muy(W4>M}9@r|Y#g9Zi}ktj*#Jak5c<3+)?R7O6A8hil&vn83Wa$-%kHdXBjV zRVvyD=Mt9|si~U-L@MU*hu^c3Ln8H*anre8Wpe~1`+)bZ(yH?f9WO9-fz%6*MkB^%y<-GmvDA8Rc7-v4~5y9sNm{4$0h>kbt-Ef7NC-(KoRwXEhzM;lt8rV55&&mx*0&wr3$ z9lY$P!p~Xpa_Lkc;xydL^y%bxy-nvuW~`6ZAd>!R{Ww&3*6eI$##O|sM>kI! z9;;)%nPghwnwY3ow+y20SU5i71oT&(pICmlJXS3%^H$ED;njVFA&-q2gB?m@URxlr zYU8UHm4%=i#XM%eZmqz6!-sdCFQp%~S6V=(xYQ5q+3flIv`0RN7^ZOX%^nEyPrR~h zt>UZ+yGu`BUc=)+=FmD^Kb0_lJAOG+J!X!%E>3tBQiFKU>l}A`Z1wh4`ItLX8?UAs zy6I~LELZGGMpg=B3F0LNLHv|R1}SYu{U#r~Tb0SlyY<786P+iQ6kNE*YFzd9Xr2b&_=pi4qD zVr)p77t^xoB071{;gf#4Ex%0O7<;WakS^-dyjs@CL_@kSYpA95XHu{*q!#(4O!y83 z`_tgjObb_y@Iz(-e-^88DE&_gF@p{=m$i9(a-*c-C;bl)^j?11()`<*bX2T?CbYj7 zo8IIRTuq5K)8F}1YyAifP`aqa_ix0fsXfjFG`YP>hfV$w#*k_P&@QRsYwNezTDUY} zy&EF8m_w@Zpnq1vrCi(c8NZhWd~Iy@0cY~}W_4o?_niB0S8b;p*3KP&U7EQk*sqU6 zzYgVe@BSw0Te+*jkFa{ty#Z*&!R-@sf#LQR_xO0Yyh?>&ZYz&h4)Mj{J3xY)Ew+xJhNpRY`^>~7ho+bp!0+QEQeO^Y4pAtyT z62b;vUswR^xVYssPNT{9GS7adexKeNxdmi(LWFRjKK?%)OYlDoLipeK;^4j3Cru}~5%vXrbOf2= z6c3dFsv(Ub%2Y50+VLHu(GId;N4RB9t}FbiARp~sItMuj1dI9Dw(7ad!+0(a4U-Ph zrQW^Gk$h0fa(Y4P~lQVFlQgqCbbj$H4J z8jG$+HTI*4^nzi*tWO}ogjHLV&D*2#dEUtqg&6Pix6IUUe3vNc9MpPIPylx78xWIn zd;ZovA}mb<2(-Ztak8)Y?bkp+3j}>Ey5g&s`fooNdFU-m~Y>0RT7HpnRI3 ze6BojozR7C9;P8xo8{1!YZ9vXWx!RVSFWpv^#9T0l2k1t3QVg5K|MHffSD@0r^ZFx=!F*92+O85sXbFV(z~S&hm90l5 zI?m94A+2%fGY1bS1(tfAec3y<+~2seGKhFX8t+pldk->F9;YrhKb%2StZ{(mUI(4Y z55jmHVVMWrM>E0wi@{j*x%M02Y%qYL&xR|P&`z8t1mTGk@*BvZ8?Mx&?IpF|rc4Gh z44^hHA-MMIr9_hnGp&G;o(km&ng5tCe%L&5UR@x{Wm^7Q|6O4a+aR{`XX|4B0H-f^ zuKR$=L0ds5ej1gXil;cJ)XR-@4KH}(6^7t+96w#3v-wOo^*61T4*Kb^}aIU0q$kA;yMAqTI^{ zo@}0zxwBljF*-3`2#tavHwNV9eB;wVI1F%3Vd32gtS&%)0}V$(FeG5e3X-3A3s3Ka zqg$giMtl{oNU}6-;wrW_^}=!`JOeo0_N%8oOsvdH4wn$ON?Rsl5`Z;!cSSke4`-LH zTfeYLuIDPM)WH!L%u@f4+o@-L6UA>ukutZE8!B!j*?Ip3u1|@*^Jh1EgeB@dobWVu+p&}fsau&%A#8pnK1Du8s*|x5=OwBU z_NKgDyWXR`*C@-G+z3Irv!j`SZvNXs{%v6^O1BW}gw##0maEAd&V~bFrrwJ&9)qY{ zmxtYE$w7VBE8%O+91rRhS{(dOx59g53LZ%u`rk7flLzrbsz`}d3#oKUYVf$eTIb_T0rN{S5!e}8AMeVB!!x-0 z7VMor(W5&(ISnAUfO@E}5t4a^SxGT)J&k*qF8xmBLgS_&C#)-Agtq;`Ro3L%w{aD1 zyvY~#_4g6{v!-P%Jm!CAfDWJK5_{<6~=(myTkvftz}{%e-z)-mQ97b z93p#(7{#(;^&znU{o`PEq)>ar(OvB}ZUoDO06&9^MCa`p@7EB1LZ9vMv@WSaPI|7u ze+ECH^W$^Lr+3YWh0Rfdb=F%Pd6!4ME`qqan;G*iL0sy^Y5?_GZ5iI}*i|{l@NCgp z>GfxivXLBwrcqjN%=lyCdrF}TkpstH(7%5h=V}NJkCnRK{Ww=M=ItL{CZY&2uUTt- zk|&l1rdEXdIpenzC%=|16N++i4_{V(0Ok%xG^FUdkLCBa7&)pisNtSk#bc?JSYnwP z)Y!4@KZ4%@_&~G2qjg*FJ_gf-3rNHEWp+^E`fIJdMIwk!)6Xd7U=XH?J`bbKhCjyw zI5(1R7>hwAjJLRfkJPR;ukBw`i4uqpU%((K9t}#?H=&FWG9NJimlEAnpz4MalZ$PMOjuw)J~AYD~#Tc^&EM9?y;A8Fglo{ixR)=~gNLNCqSIT)`( z;FBij7^c~54BZ+KYW_bmAx$V6_mV)^gmHD_oqBftR#>&fxbu0)982>Td~zc@EWJwS z$v9XRK)tVdeyopll|h9_3|{Om#YmcY@b{zm-C~zGS}^SouEf)!A6AZ^)lOZHQ$v`f zQ=suRz_NBB7`1+E8_#dvmUzN+Zj^XF_Uh3b2{B&j^XPfoS8Ukh+ft+x*PiSVg9Z|s zrLY6s@IiXl?D(Q_t(df{0!RKFuh?}CGXKQe+k)Td2tUCUF{jcdU((VWdW_`1@hiUJ zK&`!ZsGyfV<6YD|*g6F~MSfYVEFBDOv)BIfYN6tmxn4nn2W~2=Syg8cxKnwwespPHb}imDk?dI;!1#Wl9{Mnj#v0! zWs=R|&}{Q5Gn|Aqjn#T!Rc2*ckR2MEx?RebZ9zgkJXdLGo=bQZ`3ey5IO!1l2~hbG zV?DlZiVdN+jM?8T>E^4Afk0KBll7W>+wAO%E0}DK_gU3ii?ybueBJm_SCqk;Jt3qd z_PX>jcdn(WfFcySaErUE!>cgm$ASjC(6$9@XWJ79GxuC5s{v~i+R1`M>M+W=Vi|H7 zcCXWz&ci0GUP|jzN0ip%x*JpZY12JHQ5)<4uvlKy>@*iaz3B96pR=$dl{ZeLyGlM9 zho%qO0UGK9u?b7WH`?@3_72*w^Wh1|gS*z+k8p7?O|3N_dHlX+?a0|VJo!Au(R%W> z^43JP76d-Hkq??nAE3*L|m+uBRxkKS~g$6%!PtbmOp zaB#?G%j~Cy{2!q0@9a3yQ&)4*kX`jcakwsF7KmrjV|_al5yYM zW3oBypxMW=JE^&~T$qkh4RsVcp#SDU)8+DeW%uyZUqv@-Y6@=Z21~kH$4JLqk``ie zRl0wHcnhp!36Dn&K=VD1x>OhifJ4!Bq1&XLeIz#dvNh`=*hh)X*tkj>?Wz1`axP;Z zNyW$r>NkVyDI#(9bG<-_Hp@UktC`kEg%8_ChBJyNeG zdo-=_DG-FXlYZ$wG)$sQAKx3=pwOC6(N& z;`l0N$Hu!7#i8xQZ9hCnCrg#h6BLv!b^R@ZgK3S8!nmjp<%EW{y>QeKAbV4ka%qa@ zEtBKDom3Kh!?;wXfsN3de;dZx_vERf6r)tS9!D`mBF#5))E(JZ6qdEoJ>qdwG$WZ- z4@<6jpK)Yosq#gB;^?{sK~_pWX@05bOu+~+w?I=xl3j*SVrHqCNKvVLe;U&21tX=E zYa*N?~ukL}P^WyT7GZ=z^C76hlm(%qwWAFD zP3dc6O3kS@Ue5bQP$qd5AIA^+J|uo^u*)#GPn3S2)-T)euR&KbpBHXL8500?gULa2 z#TGgRJgX{K*>-0t=KGgO;HPsfHJcfX_no_Q}=>xfMAW#uMxGk_X=)67*>_ z81mX9tX!F)E0&2b<@Bk+ryexl@l=Z-e#zU*CBJ{8LjfYr1F;NTelG>w3rAS_BC41NXq*OGUh zWT~k6k^jxOQ{(s*)sT+)8ypNu%_mDQ={naAGJ}4LSh4)mAgbIoM`^=rCQ$3%t~c|a zmN%2QXz&y4Orb9Bt9Cf=S^r*n4OJOEj2mmHFCpq;8_ZjG^iP^wZY<6rEBY?Rzo~19 zzTo$){fim>hbMoV=Y0G4Vwk!D!9(q4=Zc`@e;JT0i%Hs`fZO;bnfk9;%iBQ|8z9xs zHO5nVt{JX-@cnLYQIIB(zI%~+Yll>cYS-%cN$cd*r?my_7nrw|U}Y(zzI?50!NU)I zpZDNs)#vsJn+wq~5Qsmaeeh$N2&o9BT&C=(<2%K4yor|NWLnr3KaG>*wf?N5_iMd^ z0WwsZ0^dif^?*Sm>WYyf<6Gm&ecI1M#A`Bf#{Ju!-u8Wf=gkDvQe`bJ-DM2Evpbkq>tBwzsBTb=@cD6?hihv zsptg2Q13uL^4S7O8dt0R{~1_ls`5>Wq);$%T@(E$>Q_>0(m1Z7ZRyA~#?DbGzsm@_VTG0nfd zn}bztk+#GRpYnwhw{uJ%_QuuX3->2Q+gg8d+P3PGND{&+tv~A*KL&lRhsK&6F&fPn z&UJL_+x8<~Tv<|S=~UMFx~xa}7@`-UMq01-F3R8=`q#}NIJRIYPQE8#UwgOE z?1g{YMV>z>2O~iIY5XF*Qgs+aT|oWQx^v$Db11 z{bP@LWV^gWpG1-`CY+ZD;ReA5^W46|44)~j*~!2%d7fu(KIK+*{m6@-5h7nClD4MS86^u7x6IL`9P!!*EufVYTzF%!8FXiMEyjjEQoEf@XE5j*J*gQ9JI z1l1cUP{5w$wZ|azqQ?#pJlO3%5%_>BhuSK-a(j4lM6Y`GwFU0omO32^yrxfMpOuHv zRBK@iO-7I>vjvV1pk9#h43QaygcwQ!nfl`Xa=>4!SnQsH|xFJ2~c-(Fc z1b5~?P8{3Lt=4i>OfW)PpjlcdK+c~H4t4P?R9mwlrB*2YYP#EspQ`pN&D87_zJ|Z3&jYV|0=ta3;>RQI}nx>s2(>+ zR($VEWy~h+?7FD6J;yN3rR005%9nQA(V>x!EEunbu3Y| zapDfR&gKJ+EBc7Cv;!4cM6?HbfS%G#1Z28%5sh#DOk4FC<%d2weUSY^A!L5z9g;6< z>$5Cj(vVPGE^*iBsR|1VDRmcwxRR%;ki0x9D{|S-WY?UR;yQQ<`IM3zf>LR3JS7(1@?2|+!<*pZew=#UNR8b=hSwND z9sv-%qZ7Qyg15Mb0nq1WPQD9Re_T=Lg_3N?1VBq?fnn58tHvnCynYzALzzq!hU$1U zGGy8(WW5tvL9P3tHkLjZa(bItL_uQ;?eFGJbK9^VdRiv4LD+{Wl2ZAG?31cX>xbj* z*g!`-#+0F)(dQ3{DmMn>99s*b^!mE$C7H)0>RzKVu=I6Ev`__?2X2Wv`N%;#KHw!c zggHAWhV!PeYR~~!xo|}%-C|tl<5$s^k=LvM^t`w5PGn7+_38_J%I|Ku*A0OmvhUDL z2TYgeK2Hx2;VWV^!JNJtCLOtuCu*LGx26jKZ{}TIG8p^)oNcsKI2qjgF0Jtw;(3A1 z%!e{twV3?i$@t)~2ICSYAR?HloAKpz83Q68mSzct>{OHh z(O@e9_ln=sA=C};`Fnr6%47Twega>7!{@6^B!Hi~`sICK-amZ%APBK%B|OXcu}&qr z_ThPSsu+r2Y*fz;N|S=;9YDZIKbX4~I80w9LxF_{RY*u)v0fC2KOv!(<4IBd&0&Vc=Dk@Kd-KZL+D<4>%_4@{L+l6cE=r9VIwQikl0iZE_e$7{jdoEe1G4krUDdt*)x$=NA;87tihel%1qzA@M@aR6Y?T4D z)=YZ|xdkzo6bnVVHP_Tey0*R_H*?{ zte~(ht}dVgB$#bui5O1r5!`6$)%5(+x*%GZ$a(Hf{u6+u`TzXCm}XMyRRZH0A-B`3JWCu9a_i zRo96qQ`Dy$XP+ID#<0iObWcVJAo7AUFM@A2^?RWI;VY4XKvko2Yx6!2T%0f{xnXx& zBlz7!@cBv@q(bIYk@0r!=ZY<={ntCTPDB0N+MB<(^k>q5q*$7GIuE+b#Oal0*zO&PJcE*nY6D#*%$aQD(4flY1U$uT10Ld?4Ju0$kEhmdvtQ z#ftFk%P{V~E8?FNdtUJi2nvdvhtp|>&@-4pG;SLzUEm7fCD?4wb{5C#y8PDWHkhA& z)b-2p$x*~To-5k!cCzP*Ya>j&ULg&I0sDh?1V!KH5PUV0W4YJB#h__YE7cf5yBAti zuS2E1U&M|zVQvHj=FoL>j6RYn4HrOH#{g;QRg2XB2_<3O`dK{ddRx9t^LOf^@^m)$ z;R`IRu<_W0xUKzA=NYZ(r-gCK95AOJ}3N)Zg|PA0G9Lu3laiaXc* z%shpx+JgJRQ9)Ro?F%aXB0?;wT@>&sfC@sVGe*XJAU^D>1z=K&Po6w+!wwgM1Arl6 zqZgj;OqI2&87yGA`OHY&NzZ%m4lH13VMZ84aRwJ&05Uo_YUAJmpRMcctGnBv{^@U?0pU~kmp&2e^h*u<7$mvB2_wKi)$$8mI0oYqFLmO0(F~w<*!N~?IJn&{rL-|Dkigzqlv`=+^ zKBtL4Dx=X2t{<9Jc3v0AS=+K>9ap;i+@~k2Sb)A)=*}f-E?0v~q5dn6Q!YiCBgE9h zECiiCv9P|vkVV%!6rN!)2dc{}f*6$;eFb(6OtfGcNcg8z<8%6c&fcr%wE`8X835X6 zb@QP9R-bTtqmZt{wJ)AZ9|FqDcsc856#Zl<;4=-F!xT~@YCfGcwe)OrP7Xrr@EVDR zm&;dNOtp#7=t(8sZv`u>EIoi3#Z>EcUz6;qkL&y-g#d!0t!ALfh2iHZXKY>haiLff z1|cC~V|mV>!U7(fqsAb^1EcZ+_748~&2fI=6k|l@!mtL3HCuuv@5za1VTN>`GZ3>G z2U1}!LjB)oZ!d%U6;CFsgl8X`$yVW*n<>G_|FSjlMpgC2K?H}ewHV8RiDW(;nt3M2 zgXD}l3#hA+wI6YSBddU&LZoO%1r8b2#=0#oW^4oIOD{R9jdnoe0#J9k=7GDJZz(gl zk;VH{On}Tk+fYa}A&g5EOY5i?)| z7_yP&XMaV}o;qP_EmN6Kmblh!O^rWZ(Wqz7R(i&OsSn@VV&hGd#}tYm?*h)fpYy`; zIJDr1cy!y>VlD9-vf}7T)?NX%<4ee#1bD!l`?S_{O&G?+1|4GQxI&_JuGJ%WBymS8 zKkXEC*?>UkKSoU`-0|5rHoAn+XjkvwC`4oUC_C(HTb&tz>RQW9nR5=s6t;O7f<}IG zzyT0?Df^dY?ndO{l7 z42r;jSgrsA!f8_*{(!0i!Au(W`SreCup7AEwTq&?A0ds`?mk(5V0rdaiDeM+MhD%v z%dhtb$YA7VLYiH-h6U8YDXY6W0gr|B!12*@Ek1DL`TBoYsZ{|3(1uSgV}%!Bz;HOJ z?|fl<{Vn_!f6s$L8tuo8e1Pq;_p1zlXjsKs4YA+PV?Oc`%#knhjs~w$OUFq5RMId& zXP<&pdx8n(kL1hNTk=OsaVFvUqUKhofcpC;^s>ha9;S#CMo!I6fFJYSdY>zMaxF))50EBt#UNLJv+h3K9KxGdQiuQ0b= zbY5mdNxmO6M|ZErTZvO{-&xW{Sc~GSVExm#BZp$xI!sakg+*uF5dB7OxoZ*Qo2om(G5c5^4C9_2RhMRMK{u47FLZGKwWrhRs_kjGqYa$ zm(Kd7_JzsOr7ex4Tq(;+G33cj6K~FIDxKcobJ8vLOrd!$Q zc~D25N_Q#s@|w%diPUx(m`3?OdQowB6*~|xr4C<6Dz33PjQ-KwJ2&~yWdroHsH7Su zK1QWu#){567Txh$+0EVrKJ<#u*5kXZ=*5L*6aQ|#K{ofBDdzp--isU-%(4E-o%QG< z|0z)96O8ACU_idqKGoqQ8TUYc`q9t^);(^gf9qm+-8f?$t(!olbK<&TgdR9vXBV#c9a?wE9mqw~0 zMf7B34X>TfEcFy^FA8JyC-a}V01K?cAodD;d77aXnRNP z{vkUGf1{r+?hw=$JOk2UsbN1)ry+T|VNcaQL|c6jV!CBIRYRwKAvivN&v^DTd2pTu zs6SR)0H_r$@l!$w7qQ>2U9L>s9Vxi()BrsS2(A1*$WjC7=nEb_7y`%MYY#R!Jd`^h zOmzpA-Y-a$Sn1;hmpT#*#jeLe6*tuq)+Em@akFConNhiI}jKHuUU2AjzxdorOUWkfi_;s-Fy97%WZ=gq|Mc zeL2S^FNkgcvRX(YMsAb`35drSlo(p*gv8x9PkibO0uLQPj8rXxNQVF-q=9hL?4Y#L z7$CNZg4I4q7e;U0!NxcRAXiu#h>%3YpESgTiF8E%ix(`QO9Qd#7hBKYc@h+ak-!j= z2>b86_WvXwWi_85h&;y}{xY&we9(jz(i3T3to0 z*-dTIAvp{9(Y|+}?L8qG9o<(LXFI7jw;uj>SW>1uo?EurnEE|fWPDX(o>x(>{J5jT z5qU_x;Xema@>ED<9=h&yv!?4M)B}Okj4Mhk_JZbyCUn6+-!z^QIE(^b8+D!eSL4Uc z&lP~{PvH<;dHQo)vc=(}4EFMSh|c%Se@7!^0^CX!MrS)5AFcA9=T3_#B}LiIO8KjY zxa9AaI@K?vM^pwL&n)54p8%=88(W7iX?p(Vn0)TrG(C4-+XL#C^VJE9k4o6#TWjuq z`TZusyVqv4Did^|YpeQFtK>nI9H;r|4}B-fcC+PS-0n z4=o&n_|q@VB!awcxE* zvDi%SUO=AM0WPy!Uov%k4z<5essSQIkZ99tWW8+Pin`4m2Mr3TR~a`Pbxw&6LTdms z@{xgk1lGvi)*!KGjj-!pUuXd94@2DN%Gv7R`fp9$$PMXc-6gN1+kl^dL#J@BeDwap z#IMa^4X&FV@V_YGlDd@RDFD;5-judNIdb+qeAaeQyZg{_Az1 z4xw$NWKsp^qBe7MN3$N0Qy9jf_gW*%r8=t}l|G|9sI{(+^?mygbXF; z*MS0_J%et4>-w^uZ0fy96)64)sStX6;J=>^jb%s}*;E!=8S7b5v9rJJ@3E1dGloPn zAAJQpi^iyjq|%kSAKxOkM-$J2t=jHb=$t}rx1p*#Nda+h-mxlw-~po(&5?X`VPb~(aN9WFMdOJf?#-`|U20_})%hXS5O-(zjd0MITE+Ya0MlgnIl1v%2y?6V5gO;dbe6EifDUMZ|VR0G_#$pq0a zS93ok9s}gRQVjDxx2<*S1_AXe8gOmj-n(!TW+1F#m5$@eU6^!GDo)?T<{9uBs*L~B z#}_RgKncob3|VGzz@CMqvjawkd1Qa8pdxdQ>tlu?-Ra|{2Hq42&y_5ZC*E)8X#$UG z106EKMVy6h7x&DOeA^vcTNRBbn9NiaHO9uLtuVqwIno*yy1a`jX)}pO_bjL1`igb9 zf4XLq8M_Nz-|YUaJ#@*pdu_7b9o$T(L-K8>m$&v%2d?-so>^Z^2q@{IY(BMo%u_P1 zaJgu#B4CbEJK$dY(KrU!RE3!W_;q1N7x~`+9_Mhw$e2ym3D6ksIOY&Eze5nq_a)t> z+30?VDg2I3XzjGth0PTMCFr^XaJ(iy9=0W2A+TrJFM75$@j73`nvV{)scu3 zNg?_qw|y)ZVRlVY4>P)e{rXS-&PV&}@wekRY#{t>$~u4$Ugr~Yym#c5um@uU*2#~uJD;T zeFQXzpL>nlbxGW}I8jr(9Ke30y)F2wpngEiaB7Z-*w1YF?J%)Jt5-4ZLJf}1g)O^o zIhqISsHb72Thlq1Vt~N9KW%dX(5uY5-?v!mgyh|A$9vZ-tgh6Qc<{dF5v}8i8W#y` z;=@%Nk@uGF?=qRWTTz_?epGxKN6iOKno(C;v7)u-+T_o=QDjW za>FtcDBTJg`F;1M0JoBZGh&2K#ZSQtgs+EfPw;8&cmT`w!%>$~6vbHN!(+o%-6O2K zhIdOadei%cJ)SW@ zKfA4_8cmXRbP*b%LhzAb)-CUQzv&T}7d^~cPZ>kvSS-L`n@oK|9dfBp?LSxIj#hTYd*i1U#XuHEeE9XX-*{qa2t3X zG{$(X2=9SJlsZsHK&sm6uYr6NOC*yPQpbxBLkwuOJ>Y}BBZEE7_-B)XwqwtJ#nw*V zkMt$J@TRmK8fc(Quo3^<-y^S1$*uKR?t5+idI~NJ)4>>An8}*CE0{HL7zvSaum3EA z$g~3)H>AP51=zeWB`-7V>j#<#C6GL|(?{r@b2F*BW&v2&qL*_ZHV)<1Ysmmdhj{Z= zw!;ORPkO0FNdu_t&r)D2!luvuI^s|NYcjXjG`3v?>*F4{i_r)cur`Mwo-P$?*3PJ{ zfIC}wEh2tjf0)rt>EqT{mpe}VKUr6=JAEC--X^YPDZ|Lc-|2i+DJFfpVhQoBkZG;k zjY4M)s9aZC=Lv|tZsN(DYtVU-{AdSV$xG_UgV1-VZMFdRVFv`I*U#1OneuaO?xsQR z4D>OaF`;NuEnmGUyyPQYYP@v5|AY}f=sL7kNVqq+%^Ggxe-)M~NXAel zu+nrkA(}91yw<7u&UR5|F4uB`+?dnPWLtZ5KUU#7gQB)tQYBe}EW=wO&BX}2z0$7- zGk{|}tuhE$nnW@(tkGDJ>ij2jJWno}8s*)%cElQs}MHF4rR7LP!hAk<}AuEWcL}hcl2;`FGO(ztnAY- zCRP6S4h)-8kr=Xmyj$rrb;cG4>?L}~qrxUi!x(fAg5$uMzOJlhw?M|-3=*u4>zaMw z+H~XHcAwyqKHk8kOlqh?@{R68VvTQ zIAi`8=;6@4+45VTMg9R17?#iw_&<^5`c5$L;Ku1lZOc)Ie(iM+AVHqFoy^!2a`7{C zD(C~0XqsTaponxjCXca!c09Qu(y5%M9IH`^CS%r5C=z??U7qECttHR zg=|xS5Yl^{Dvc1W28%IRqZ|yxyNJca#buDHMVSh=9l-0z2nZhK9L;!i)hsrhaY=Z6`VQ9&CXSw!wXtofCi)Q(g+|OyJ*Ilb_xp`5%i7VqD$uB--4^?09J9r2)jih-^>00tG zyY%W;pp?!DmF(^{;hTaVCoYKk&r6nF2Ld9;0etjKxL~*1$CmCC{; zT~_M|g9_6m-S}T8<}$Z|q&j#?vU(2seHs1xVyFQ+foo9OxmWl(0&b)b4L}x6tsG8K zK>r4@vs1NA4YiPt z#i)oGLY5FikcUsl?Yl_m9gi!ZFwA_VC-{Ogww+b8CmIT$1ZK>9PphjH~F zxEI>V179S|JHh3at9Ks%^`7;YgQGTUAZw=tqGBXE5#sq*s{+_p=Q6YZjbEV4 zeKPb}-Fsb5bM*eqsZ0*r?jlI9s4L*cFVO3PuC4w$lWd?dUI17Xc6ZOvi$u6rYqwY^ z57VdJ7R}83575Iy8HZjS43GaIX?*Y!(Ev^z1|*aJ|Ns5J+71%67}tSd!&~bg6QuIm zdP(h*4)-kPaK@>?m!CP$Yc4pW*duZZ20F=ty=`s4BSVp1!x!r?^4TJCR48uYPiH^Z zCY9Ic9XG<8g{S1OMR1th;rDssrJzl!-(+jUjiNS>$YCq;aO<3<`R^uLb`!Wngm;h2 zHNj;ReVC%3#Jp>M6bQDU8h{t*0gYRizQ~r%N3-BU^DbX7bBz_SL_EV0b%qKoRA|2t zA%L>5p2 zsFm7*k?jIoV*dOwf8%#6*-XYhMKK)cma=3ighB6Gq2E;@TaOwf5N)OPnDlEFuo+Vi zvmyHiUYmv}{(X%L`|}?gP1NOrBEc&@wdlg?gX~rwQO_Qy0r*>#`>nv=zy0*3bU_Qf z;GN!|EpdO6zGEa%M)`KW_u~(1Qu!gyajHT}XOz3W`F80Y9xYP+MuR}>)e=n6LL3fh zsL^=66nJylIEcP{RI+x3Wf;AF<|wtE!+UTdPxKgzo(V(kvi$&Cj&Jhh^{veE!fp;W zHD(0nhn(m0~v`4Kg7mg-7qN^*j)xUDN60H@fH=9m_o)gkX&80rVOAA;Ygslb*)-( zfGfvqNxNE52V6OH@Fi+jBm?vzeWCoz9l4f3JFwD#l8TB7iHr{s*CH5tYd!7IDSaE& zXxUnJw{3{Eukqe*{Z-!DRLq~Z&Yb3fkXSKJsREP@i=^s2IvdJ}4Uv>TIo*-ul&uz= zqj5$h2!U-y?Dbu|)Mg&*&aHxY5Q;A$HYmFj?y)n`$NJoZID7|H%~M(UL&VKD&EWouOWZB% z?|@@0Gsm73o*kvoh{#XY$6v?){4p(b)Tl5`qQ>jfVeh!~S%`9L-lSC-rjf}Gi*DvRJU1=Q`caejKzk&%NzzkB+<1>8PwyU2 zK&=g{UYI^~y z19dB}RaM?%%JI~N&mWObYF_Vz42cad*e97Li*ddI(|v*m(8^sry2$r#s{m!X!4p<` zn=!Gy;uG5!@8SL+ih1(YM#~FcUtYus%a!x0vwtV=t^MGTJ94q<)MJ*1vCa}3tH!c5 zU_MHdW_??+LN4{*Tb)k!FOtiWqA!yzepwKWmzIao+(J|fjvKjadl|&r+Pq_&yuezh zq$DUM!|3a4p==Zt_lPvrbb#|~)U6^h<_G_+y|<34@@pPP4F|kxMLHP5YJ00>(T;b%cF30p9L=AQt%ei!8*FyIn=#wx5AwkO6v8_dw@1rEn)3Otl$ zL#v$HHgL*nf1nYRB-Fi5Als*^At>2SFTv|yWhEnv@p&&kR0cPg{!(mFp?pyIp^$db zG#obYNlaEIZ94mCzX| zofmuCdOsCt3pMzze)>gwPw$Ua@!JpN*WkXN=sw~HR#`3U8rG}i#c4O$JOSbW8*oLv zu#i@dJt3{Fx=1~qkT;<3UjBob%;D$I$tIuiifc&RRCiP% z1nYD$`TA%3_W)nZo^GWF(H9U}Hgs$E+A&tr`*6{5Tug5Jl(!CScDTV39ij9Egm7&G zmSvt$&5);G@ra>Di^=1ou>HU$reGvI z_G;|AfbjoYB=hvrPbXpZ}r=HjhROWcy!JlV#7 zo_cpl^=!{SRDbqmpR`P>px$>i=nviLFKH_Gm!<(o)gyM*r%T*EI6T=re}>b#oOA;F z&!jbX4uy~N>N(8s+elKvvm*vqZeA<4>$Nk}_$K{?$@n&n2hLbUa6ec0bgz+KhB)!q z#fBKTwTMN4VS4XWSv#^Mpo|hnT}jf|#uIT2h>()(?^N-jMW2Jdd74g&8GV(oFk~+n zxbM4uaYh({a$VjZIs9Gd^E%W-H3CtxGm?wS>ff$M?M0~#ZQ=B^sXlG}4#C4NMF}sA z7+Ja5WP7p0*&81M{*HXcA2Pgu_nh6?4f!5c|DhPot^hK|=L9Toxufq#D{eF<(LD4k+@_jI!k{C;z(o3g0O(e$=eG`SlwGlS`&adwY z7E-MW*&X*;A*X+~M!7^OY3MGyqQGy6=ImJS-f>q#K7S5d zJZO8uO_YH^=z!TSZL!lS0mP#BNk%rzLcV3nEq8U$mnwd~v zg$mcl_fCIZ3hVVt3Nnp;ky^RNR?Pg|Cr16&v$xx}rwo-99C79M<;J;3 zuQe~18+@uvsQLq%t^_3sIHMrqeIfv45F>O!*(F-1caEfLKj;xc$`ZJX)Ht?Z(%pG8 z)snR~X`o#(R4hXJSumt%00yedOnjZLbX$Pb*-19p$&MZCBqZ@TLad*{QF1}n60H6<%o0AOVbn3xyba0TD!}+T;h|h-Ggg$sRayQ7Z@yTkOf$q%3RPSUeg32;BQ5 z0Gy}*V_$aE)WC6#)L|Y)oJt6G;dD_0N|zSus*D)^h-Rt+h_gu0M2Qn(Yi}0)_?U_b17d^$i-}SAq zK$&+`#U~JjF-xzwOn&l>AT2N?pbdegM+ZpUiuRgEtF1pr&o+DmXL9rEvZSDcyR+JZ zqn|FK{2{W%4)2_VG4JCZ%y{RS%Ovo`qj8n07y8BPQ!6jdX@dinRBbcK&~{uVk(vWqyf7?1+e$mPMDkoI*& z!RU*3t^05Z!@7)ce!`xkLVvtt@KxPRir>7|BNU)x#`MnU>w7kL%@kQh!g5wD0wuD( z{O0g=2d|-#Y(Z_yt{0&MsE6+>2f1m|j9F`K1_}YML`js1UJ4>CF4`2wV7{LQ`Z7u8 zyTfAy@g81k?ny&wV6XlWq`F;HiNHjclJ&8?@zN2H*IQu2drv0~$&?Y9xE#xbx?wZ* zkBoaY^IxQ6zb{8h6wPlW?{`xDn@l!xjHIfboqSR;{zsbCx77a^(G2F1;_;T9ef3T} zBRq@I@{!?%o(9MSH!#3iZn_Bx{omV~coHQyiKR?-75~0zQUd9t@;YCI z)E^Xj?&&~6|M!aBzhSbOxH(?-G3JxzAgk;Rs$vJawBHQOReYgbr?Yh4Wh5MZ@_*_O zdVbyeBhbtQ$SGF+t|?Z0?{8C}OnM(8Z#pJUZGuc^h`ppZ&@*uT-%g5%8C|S(dLP-* z+hP&^Vyaim6l|<@>n^tZU!dZ!@G%UnyEC+`4xKSU8P5`~ykc6fya~QY|N26hXE$|4DB4yL0i3(b`W5*Kr^A$!kP-*Z-S^AV^Y3Z0Z~aW4 z@nkR4h6NZ_4xUyYfoS?nM@^BwH!jD0O_TEtV+PQioQt9jNx_F2#T#~FR+go}MTh+S zzoYmh=|7@)0#J0^m1)X6+TjisomXtYF}7}a!7!zoQ+^%G{TfIrwO={T!?m)CXCyrd zDP_=;p7fp*b}J*GsVM1XdRlp33*#S9?N0kosHRE&L3Ka_LpjKbjtW2MP;JPsp{n>8 zPbjAjPwv0t^S<~$;`11^hx)Z-I!cIMdvwS}xSx76!YUAA)G@x~Ue9gwh4Hf?gRR5b zVV{&f^L7I1Quy}cGPxV*lPWkj_jS_7{^GK9vax!eUxFs7?I7~UThwjTQ7%Y`xtF?B=U#BZT%i8wB4scj0Wy%+Z49}N`xfn`oj)F)8% z!i4=R;3QQZ&}%%pn)bgE284wO$^Vx^C^3+YoTZ+o{Kp7O0^VBu^~~kJiheI-40_Xj zOa}ihjY1k>9^*U%eMy2}I=(`+!7UZLlv~X2TSh?{5wVX6!DJ_UxAmj4Skr95+yCR6 zZk-S9u#*i5K+dSZa&MRVc5&Zi8f)4@dK{e+t>Pb}j%HM)?Fo-@;BP-ch%n+@Hqko` zmT_=(Tu#^nQ*#Gf;j>YE1pIZ9cEfMe^i@x%D*dR6@o#g{hJo--cs^oG@<9CCwhtUF z8V@W=yVZXqpP6%sJjpNQ6YqI{ocW#y>U9UA-jbJ^9 zsaY!%m6C7g$U_di<4y+F;bF`7Utaw`@85Wx>vXsQZYD{93Tq%QjdB&~;L_6geMltf zn3$OC%(bk1NJvQ7(V?mX&3iiD;y#mXr)&DzLMTHsov9BnP>gOZ2@Rt2JIyWD>>ac-7$vwfz&( z{>Kp|=#VX+vuuK*`rEH#C(jl2)ku#CUT*5!N zsOH^mL!O(0P4w?`E-#g46honiw=@k~%>ur7?zF@VA$kokd6xYLgX}u~1RFLQ9#x%$ zS+5>Y9k{%0l|Fc_B2)Tgh^%LblJiE(H6`@KJCJWik&jinw^Q2+>>>9{&$#Gsk3Tl# zt|g{WmJz$wQK=2f)@t?0AK!m#Z@OmGc;E41i;TV;P?V=mkZ6?9UyE$CC9HqR9erj- z9N>l4B=PH{p?+aO@~_weMTfF|TQKKT&O7F$Tck%= z`d2neEZ2xCv&9{>SYV~yg@%1w!>h)Ej~&wcRaWpTiNCZD)1fXap~Ii|&z_W3jkE5o zOOAZ4$|<4%D2XA_Axmh5)%OFf8*)5WM)kBAy|o>r^ROLc{S{+KgtDrkep{$@LE84X zwPgudt3@Evkm;A+SJsS@q%D}=hbB8V?EMa$!^5RniLsMqQWnl73Q3g8SK$g`nkK24sKlFwmN&<05ueu0P&*y?AqHr2pqn@LLuWKO$^ogAcn0F7xon0uhD-? zk^AMWeaaf%H*jGdlwFsRG+_4vAraP*+m)Ge7L8JFKC{2$B13yfC7?XBBrV~I*M;Y^ zAMYEuVjY(EA}gYG1ux~v=BpwQZn~R@{yXjhZu2ig?P9_t6q~-yE~msin$^&s>EWe% zUlF#Wh)%TWIx}pil|z{~3PslbYZxYFpr)=!t72UMFC1mX#QsI!$;WRge`^v%jJ*(y z4=U8YzTzqj2U-r#(b{Ltr`NYZAm+ghGdDJag*tyI@nBn2#q@L|(BrIy=+){f>ci$H z8HMP0>gqDXZ{$$2fu#CdD7H3Nd=V&n3&MnAA3$sh>SfdQ<~)m5EKO5QOKBWgPCiX_ z$1{iYFdF3j(Gi0fSkyh+!6!^7%~Xs|>#LLC_S6j!j-&(uu@PXvi>UAm4r(Wj9GA^- zgvD2^F7S|%C}S=0%!3omJ_R1d-p~fXQAH`K#T^g>@34b+tqmVXqd$Mxq%-m_x+2f=9 zX}=@GzvNoqQnnyJC#0VN;aA!|z%3-4uKQ4D#YEgpxb~K>2oG9!TW&!Vyi0tVE zXb@0%Ya%0=IFlDikn%=a9WE%I4x&;7DEEApi?qe!O(nlc8qg~lv zoG0YE7=$W5PsU^$2ymxvfxvIaJE9k@k)WBOOC9np5d{aY_%d=3J?+Ehaq8pAj`-=t z|2+E2T^tYjmmo-<540m->dH;lV=xQLRMrDdha)TiOjB3>|6l%h(m=sm%6@ag{A5Y( z8b7bXyqo|OOSeF5$H--Rc~(#FRVqI(hl_`|o(a6KYjRDDWwSu8@Xt;Rf&NHZS~jbQ zpgun`A7EstK!15`Qk%z3jvB7?Rylb^zT#7=n~ESGXeoe>vs2T4KaI?#4Wv#eW%8q- z!TC?7@5|>#*ql}PJLM$tqDrO{j)Y3V4H{6|2x8!GdVY#eolIsh9&KTeeR$-$s-Lau z>aeeRmba_r(rp}+^0>MFu0mKVReMaBFiU7OsJl>ga!Qu!8M?zbuiMmBeFhF&J;pL; zs2}J{Q6y?JH^@UF%npLA;2usCXbf4x+iTMW?ogylD968yqg2!KidW6-W0OO-GXEtQYYWyb=sKx(2qMDn| z%q`8Ra4+31+O==nsOPDSJY55Z1w$RNcer9lrMxawNd(HHLL@MerIV(;`fHRl6*M!| z)8H(lx)nGap_ov;3Ypwz=YC;|(RZgRHtut;z>q21c-2B61z9Z-)$^Ehv~3%RGxRjl z9Zf6FJwfG&F3`)W=xWjmUdVktIRQd6&YJKK;RYa_4}oqJ9S=sGnOh)V~d6i{^Pwa8dLSWoZ0kMYL{#9D=l_z zrTZAqA?dJZl|4@t?|yU09&%N+bAw3wyg4McqW8~sdiB!k(DWFX`Pkp;aV*RD{vk{PA?aH}|!Ym5j zw{RzEwDmD|Sk2rrda6z~?3$vO2ycE?sea-^U%8|!;2GBgty%Kz#q5h>^(->?E&*=P z;C!)ql8&h9*wPaSn)tk6b29vO;NvyOGiX1zPx%zZ-VB6P_R}yKe$cJ6%KCiD%>8zO zi=_{J#=i^=ZoV9WyCSKco;NKKO@hr|kAq7JAe}xJ6#**2P-+RPZ;9oB7Ec?aMdGXz zq2-_^n;1gqzeO{cu=sF0Xp{#so6Bm=oXNP1S%qatdIky(o#)#yf6Kpg_!>u8#+&L- z{_~2p^K|nv=sr+Th6C?w^(X_tYe&-8HU^YmaAAF^L!;JuSbcsvOb^87cG8|vy^B<3Jr#ZHQXYUWmuHvMzJ>=IAw!*Jk18s_!|AqX9&U-vLTxiOk?J4E_n{^8Lm zBf;-WadTgZ6*|1?Cu}l?s;sBZD;9!I`vraCQ}MltroApYk%E#Jm3wSXt|cuJwe{DK z2)@{ts`|^?{9e0vyGrE^7`X~nEj<$oVveAg2B`4H?x96}ho@yJyVtpN4&H3r9}-$O z3_rMcimcolo*se6xv|D^E4VNmh4$7?5Ys#wYQSUvy!A%;=@~n>3~W95^8Mhi#4L0L zr*Myo<|nutp$74clt(iCB0h;0YYOomj`PjoR`TrEkleGMRmiynlRtQ70JI88q{Wt%qG7qAnbg_$+$Xafhb}ku%>KoP+ZHO%Z!oASp2(8~r}QKV zF_HUCz?$%VNmn8Ri_%p*8%c_5Pb9vkvg$Ml3*CV12TFx50>w_jGRq@0u z##HAXWG=%t!~`R#x3F9?&ZiCPi#)O?J4yp*|{kgdeqZ_Em8vgbvKWRk_ zzI20VdL(M6R#rcWo!RlJJLYXJo42Y_Pd>0xr=9f(CfJ+{jU9mWk8R0QbB8g*&34X* zPf1j(gwXmXO080cb9c^y#64vBuP{6WWDLG%sl+@-qt!})OY!!ePWdwl1$r~LRaxIK zn=Ao|yT^2&57gLXj6ZREdvm}1v)BT)v2-^#gS6(aW85Shhw01J9%VF$O(K_=ucEy? zbOIIJ4<^j{%db5Z++dlSyJft%Q(Q9b(#5gTl*eSpUxnvfN=yF)GpPk(Ipd!oNWhUk z+C4pUy+-_#N<`2@l1JD9^}WPHFKy~fO({Gj-!O&2JIT{7)*~IBszR9s%B?5&Fr)Rd z{+UWj)_Rx68dj`PZgY-E^l z9W~G(;IL&C+3AF5vR^dIRRu9oKf3}J&_66)c6fyJ(fSi(YB8?*EOUoe>;f5M3)lNH zXWd2^_lz(qzY0@tGSvhGrlmimzGW>u6Sb2;948-+cnW-vx?GZ+C5|w)Dbyfxy@YbH6ZX+I zrt0yDs&`A-tjDjtfAVoSTr>706$|`d%;|aS;dl2(WqWK&#Ct14?T^0C5j)x2=7DKQ zG&ADE(uL7cH*s%|32Ub_wm|djB)SZ4gWl8=yq!Xre6JeS`L#Kd0z4U#Mjt=53od>| zpBKZh14lqEzT%t_;r>oQgb7Uqcx)g!<)HF_`E^~&3y}ivQ`tnGE!E$mwt1NGrkvth z-&RY1Pagj&m-!^G6{GCucLGWjlQh*c{X-W*{M~M7YJan+eN3-QMboKkmf;tz=;us9 z=FQ65)dr*9OC<^kn{5$@Ojsw3<_@@P22#E*G`%GDSzg-d_$<^Fa<5DXJZb0-41XkU zE-iI&pFxE-$BIqasP28Bxm*i6DQD<_gOzjNXtxWj2W9&m-xZa!G)9<~N2{QTKt{M&gpR-R^V0vq#Ek1DJm;Jrir z$U85d)af*l*gSl|+7LXEqHE%oF!xA-e6q{O*82E=nrv>V|oYFV19NGYB{SG;ifHO6d;rS=>dM;HlYF951 zxwkEKq%a~!mIfkQ1ufp8K_n0J(U3N%;IDR3)W026At1kwGwe(Lo;M`Y?52wFEGo&Y zrwyUybEWoe`(*>#t9a-Rt)Z>ahuqJ=k)i8FS1zqn`S=s*%t>k9ZHL7}F@dYw;sP60-pv7F|2z_&hWpp$rlzOIlh z3XwJ!_0|n&mGK~iRQdO9fTai#3~W*nn+L^J9>mPcpM?&A32b?>hL)4NQvP{a$?&SW ze!GL1in%5J;i(Dal5**>Z5Oja-{f}+&@q%gA*UsBxt1v8RBuc~+6E8RQZ~W%s(PpA0Q_Z=Z~rGFQ^Ssi0R)#d@DDPbZtsXt@H}c=~|^OfTo&V;(ny zdxhXit|E|)(}z>9^dxT5YT`2hzJk9=;`M6#aVcB2Z$6d!5$jjX$HH-%Nco>}y+63l%o`)vQ@AU4VPqO{sb(Rw#A!gKyZ`0;epg z!Yh;6B3jvi4{4Y!b_4MNSL4@B(o%gFnux5=JZ%|B(94 z^aNd^pXE5`OlQ{>XXT+&H67d|`H3dpbO}UgA)}>4~hI)NDu&Ew_yJ z$mh~R`)h!G|Kh$D#3dokK!XTT5Ue8|4zUcsysVN40*=(LlO-c5Jz&0%&VI~--WImr z2r98~x_~!Sc3mp>HMt{D1(1ZIcal^WDs6!loz7wt*!wGS?t( z^!aOE9*c>ht`o{QQhj&#^F0!48h$P=YH3|RZo}S0kcj)t_fF&(xK9(YIIC~fr8PHK zG<~rhE{J0}X-o%*6H1ePiOXME@h`Ygd}+o;3&lurmgd7>9oJs5Z8MTFZ^_U`lGa4~ z3-IHGN)DL|fLGQbfRXeeH|9vKp5KOr4g=^XGZIh*y zdd8>1*iAPGe5)-AphNf4ueZ?9mfTT^Y!fSpfBbBpaNtQ#+q;|h+S=<9^MKe=33E@5 zv08`9$R<;H#lP$I^!B(0uIbN&ri(+aA`q5nqfW<;5B)%Sk^jKur>A9o{@Gz@3Nk578-ohLRHrlH_ZFK~f&Jsyb;367yDFh`h1{}yLXMwxc1gbwQn?T(fN;-aktB7*Z zgRnd&3RDu##;b#}J7`*)5cWrFAcbV5A4bWoOrAqdey zzAxbhVMJjW=gV{v7m$!aP3v=+29ll%H;AQ9&jCKk8|&>kQ1(F*_OCJV2rzJk_wg>$ zNsjO;KgXv&xXe9Ci=T;q8=qPQSs{gN)fd-bp8}1oyA)%K1zel0N#b3>%i%$h4F^Iv zoB1$8CCvVv?Sc}NdeQp{kPGCuP)@2m^mTx8i5_49r}ca5t7dMZG7BA%cC$X55mX?} z=ow-hNLP3U&V?}C58lY&QArQrV!lfP5;tUV8400RN&dbHH@E{*l&qI@!T;DGK}Yy| z{97NCo_NGh0U?ESffN>? zhZNMD_{4CG=6mLYT2Q1ELVwJNnczNLc*({TZ6RQxK@v{T6=&`I*! z=9|sYB0HBu9@lfsFApF0-d+8O>9Xa%@DR%5da$Ise-ABpY~#56KG}Ke+R@NUQN0_0 zSSZ%kh!xHz*ElN2H=YVag1)nHx-k{IS>i*)2M2V%4~;=kfE^LdJeuR2qX+nhgY~|a z2+ThQuj z+0I_q5AaOcdADL~fM)EnE})(BG&=w3k(AX_S;~B=ZwU@k*YBP!%^=&?dYCT`D%pda zIH>iyp_NjH>H}6#tHr*pyCkDOGszCbn&=LOH&50$OUXcbnk<%bi>ck|1XTKBcr0i5 zWODWA7=ZKT^gGE`XD?%r-U&4io#b>Nn0zx=p6xRB!HHkwBVTdawe5Oo*Q(t$%*oO5 zkIs6BOgYM~c@Bcl7@6NQe`&{=@$58(>LvK%->+D8Cf_pGU9mCTyrSEKk*o)?t5L2b z_dK7m9%GdbXn{*-m8J|5o%VkmefwFSBmy<*rCG$}G@JiO(zc_pshn?nFzt~NK`;3F z?cMip-+!++%@rvtLO81@*eqSou4a?w?s{onJtMh!JOh#-JaV<|`))h&lbafW9q>L8 z`2f5Bq!jEusI$5H&UQ{1Rjp3RMFNK$rqh&kYds>47{d~F**_RwAx%jC(#w6Gm9+xZ~*!lYZqEH(l6SzX8LpbHLr z=;mCL>kBH^5KURK)0-?fRmp4BUvRStcs5hO7+GApqzk zQlLuIl8wUDWRKt{#!^gIgo2RX6u0LIi#S3_Uek<>nQTm4I zEbn!?%f?^+!#G8B;e_++aZcBn_ng|hL>A3JO;%Nkh(inrJqZ-W|?P>PtY8c7^LR7Y2 z0r*Hj_`tOxxpoCY4pNmh~Nqvni4wTXI0pG=Kj@z#Bt;%+A^NtoD=QPlc6d5A;0r3+e6H{N{$*c zDR~G^wW)uHvxah>Lz%|bu59~+GxET3JI1OM;eguA*$SBSb`Gn(?2v1_SR)ji`nw>f zm7Se@?%a6uOeZUI6Vdl1RAr&Ky;@I}eT1L!Bmujp81;&h3L%k>V3Jq7d}Ss4&b^h^ zUbr$3PFXf>V+igO;biKou7!$fsZv;ZtgXm4aoC(6ZJ70SeY<7iVpkvSI3KTVKji0H z9JkuwTZ|r}iwSdz-u=4B+cj%dxw!t;2}Vfi)>M(rm&D2yqiPf3a#wLVR1j^t`177= zIBdf>r*iwC#-sj=R+j15r;@0$*FkEGZG3%Y~ zOn=YfIe0?Eq*CedJ%67KKT`~c3577+dolajs_W4PStEY@YaQd01G4tw&QG6Qu}73N za)N5FguQGizn$DeX%SXjy`nIshNjcBk*uPTo;f0%ZDV+;7Zg0ht2 ze$+J27H-w!NRNZ^&Bc4Om`wXt+$(EUI+){dOPw|4W!^*~v?GUR%f8g7UcgmMFReeJreSnk%=KxM@U;D@zIUl+dRWSA=BoLn5}|=JX<(96v;2{Z4LJ z@D9&& zcRP7Nj^)XB4>HJ#8;Q?L#jWu~H~RuZE)zdJ#*jp^a5u&;VsR6~T{cXKQXKbatxO+i z*;nA>cFRjy)T|y5o$3r{Hy#8D481o@hI8+#sdn%kjt44w$J_z#{*m6inE@p&A0el` z9+85vYK;K8Nt^YeAyO`j>1Ka&P{*eY|5R~aqw@7f-7QO$7qQ;NB~n7>NEXx^PX`9^ zYG5{d-K!-sqEi%@#ROV!SPl{ux}Kkw zZW={xK08=F@|O~A^eDdsI_3ygj=)a_BV=8rX5i^4n#smv?) z6?tt2_`B3s{T_P^$E3R!D8Ud*K+g;Nxps3GzR*@Cz%onrPE z(lG9aEd(29KXRgJ(>}LT_pVi;+BkCY*+f-45qp&rh*k5^r4^Jo-n!)!k~v8nII$#_ z5M3q8($zcEP=Ve+kHCx0bLZ9l*(F8)*@>_1NPD3;} zU1v{5e9VfRPY&2Rxpa>AmQ_qY2WUq(bjWsxY%t8LOol7WXRs?omnL6<7j8ed>DrAf zRW)x}=tbn`j;=E*D#oYXq-8K%LHEmsbIsoLL1QO*&={jEm7>Z zdz%AHkLB(;le-lVX_)u3!C8l8WuLl(+F2>`kx9HfX7!W`MC_*NRtUsib_y61+!$OfZ3>eLrHR{YysC-j3a7VfS_Vm+7|*?fjXBrb%@ASCqceD;nK_voJxT@%VYLcs zjHenSiMM=$9YR#ISB#DVRHMS9lUUDt5&MQ3t|e12@+G>|C#TpiM%m58?4PfgGcqt` zf35C`;ujwX43VEyu$gb{+PWTCWwU1vTeRJD-mD7Fd4e9}uw2=AOXo9--kknuMfJ(T zPH1Bz6CFt~diKEUWrjZ42ZOBzyzYV8u^*3pRDX^Y$L)P?U`Szj!p%Wi<4_)QPf$jKx|gWiR{#8UZP<{nk=Xf;T=xD?SO%$n6xz zi87m!^(ySp=e8_(&!&)3Ucg{WpVDs=?*~j^Fwh|7?wNX-^MmhVWs%Wde5ApAu{?HV z$^{1Nwsg4o0usxt&=<^t@yb$s`~AOVH7k(eu*X_hH82=#59z;sAb)v&c8b$@nl^Q% zgSp{8E`hAN)QD-s_c2TujGHcL)2r*W)=xj0-tBfW{QZoeSDPquNlghE3}!9H;@o!~ zfP!Q}czc}w0v4Cx<;bTnHi-t96uCRmA~eQmn&NSCso~Gf8!kI`5>ZF1ZC$7PaayNs z#blu&ewCskW#*faoKn0=)=E*H;;Gg$N&dAy5U|~(`+bM-UCbfpndZ07hH_-24}bdY zK2lM=+mWI5GvRz>U)r!pX!Gn%+ZsP(R(D^Zl%UIxeN2w5@iozBV?U=beXCp!gOkdi z+PgT8)p%69R9Bwg)U|Mi7UQFF*m5MLdG!!MNBV%ULq%nted>pU)w7Yq6}g579e#(4 z8CCd;iF@ZrS1q#qYm2Uf)iBUd>*bW(?&umK&>$h=&$Y5Lj9U7pe74Y9oj&Svem2MH zyWZb~J0u#(x7S&nR4(XZKKfdeS7uzHsE?K{f2Gmxjk7AbpJd>{IRR5vduSqi-$T)I znK$xiRB?kZ@ugj=(h6T2o+LfVIkt}bv6($+$P_i)ib%ZBJS_&ntZIFh(xu9~e<(iC zvlB#^nUCx(PCJ%^t7j-HH_s9xmKo&0lC~$ip(q8!Md^0oSi_ynW9uXhNjdasc(^aO ztBEmHoQq3n=YZ?^-iIIhXXbYq+RYWucxV~#GEJyYvDV3*KL}4x?$tUPu!H$D?pFAv-BRYBK zG2ys}ENoF+8_kkdl?Nr*DLNU>>r8@q)Pmu9Q!9~#di_<_dVP>>=3uxe)xnnQ`sqOQ zkj{tJFQg0gy0;1R)#IC_U}V^U0hrZITc%sDx8tnx$GF~!PEvlJRPU8`&S(~qem^8S zsZ&QTC3vNf^?c>3g^R%$!B@4>^#=-5r9PrvvW_cZ*16kV_)>yV79DcuTAS4@Q&Ot8 znt~^X7|eM2G~72~O3~pn=PJ*%lqNFppB@k1b3=f|@_h`i4MzCN^w}pgK9AK+N7hfF zH8M-la~?xIfS(ypLnd+b%yqK+1*|EtMjVr#T{DaS8L=Flo6e4g(E&w$(2bN8jRwS) z5)}vQqUdaUE}yFwssY7{-0{g^1}bQVs1^OKDT%Yn-lnpHE)$h<%FeEp^0Rz+uIr=y z5)GWt=gaY8+j2#r6!uszBxgFJx-C=%Z2HMc_D!GxcQ4+ znh^#mabn;LwOQzkr-H375Zb|Jrf^nUfWC06x&sD;Eulet^#Aq;H?utR^ID*(^6vFZ zn`!;5`;3(Fh~ITSJam$Xjrqq`O}7`?N78U+IAuqDTuSnIZt;h3u9M|AWj6Cf;PWHhH26vXyF-0nu(50u#WNVnbh_iQjx7Ahlb*z}0q@PJ>Hs zA=_JzH0<=pdlm6(*=qO>KU<4A3%zzR#Y?T&fN-2E2-dhId&cqxB}CVh9P!a>T5b~jwp@u zzufW!(Z9W~K>MGGo$1bZzX~ghh9Sq!#^02CaHHMnW!-c{fj+&DHLtnF{@g?UFvqqw z2U*))iPD#*Yex^29xo5;Ro}EdV)pGU1jRIcQd!9*s}$omogYh7{z$`Kw4dPank) zp~ET#A<}$9pQf=I5}t|X(^*ytW$;DaFRg_=HijS=5`>A&ke)R37{4)NhIKPux#i|- z%;DCgabOC)Ul;{ZE&^kEkJH2%dO-8yfxB*f1dljx!3@YR;3KY4Xj()@gk=H5tks$BM zTX%Wvs(Ey3(@_dSqzE1JU6MJ7YA&XH4;aQ8Fx_N?O-{$jaRK5G0}tReU&|Ot;R%`Z=FEw{EZTzPfmC~KAjx}?ELY%>`K;K(v7!U;9!ykN>PfcXi+BGU4gL6b From 225f9f9edf5300b18d593f8c987cca613b0e7611 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 30 Jan 2020 13:41:12 -0800 Subject: [PATCH 064/168] client: do not unlink superblock file on unmount The client was calling shm_unlink during unmount, which deleted the superblock shared memory file so that future clients could not reattach to it. This has been changed to not shm_unlink that file on the client. The server now deletes that file during its cleanup logic. Also, this modifies the client to immediately delete its read request and read reply shared memory files upon returning from the mount_rpc. The server attaches to those segments during the mount rpc, so it's safe to delete the files at that point. It also deletes those files in more failures cases for improved cleanup after failure. --- client/src/unifyfs.c | 53 ++++++++++++++++++++++++++++----------- common/src/unifyfs_shm.c | 52 ++++++++++++++++++++++---------------- common/src/unifyfs_shm.h | 18 +++++++------ server/src/unifyfs_init.c | 15 ++++++++--- 4 files changed, 91 insertions(+), 47 deletions(-) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index ff21f692d..05ac7b18a 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -2510,21 +2510,6 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, return ret; } - /* call client mount rpc function here - * to register our shared memory and files with server */ - LOGDBG("calling mount"); - invoke_client_mount_rpc(); - -#if defined(UNIFYFS_USE_DOMAIN_SOCKET) - /* open a socket to the server */ - rc = unifyfs_init_socket(local_rank_idx, local_rank_cnt, - local_del_cnt); - if (rc < 0) { - LOGERR("failed to initialize socket, rc == %d", rc); - return UNIFYFS_FAILURE; - } -#endif - /* create shared memory region for read requests */ rc = unifyfs_init_req_shm(local_rank_idx, app_id); if (rc < 0) { @@ -2536,9 +2521,47 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, rc = unifyfs_init_recv_shm(local_rank_idx, app_id); if (rc < 0) { LOGERR("failed to init shared receive memory"); + unifyfs_shm_unlink(shm_req_name); return UNIFYFS_FAILURE; } + /* Call client mount rpc function + * to register our shared memory and files with server */ + LOGDBG("calling mount"); + ret = invoke_client_mount_rpc(); + if (ret != UNIFYFS_SUCCESS) { + /* If we fail to connect to the server, bail with an error */ + LOGERR("Failed to mount to server"); + + /* TODO: need more clean up here, but this at least deletes + * some files we would otherwise leave behind */ + + /* Delete file for shared memory regions for + * read requests and read replies */ + unifyfs_shm_unlink(shm_req_name); + unifyfs_shm_unlink(shm_recv_name); + + return ret; + } + + /* Once we return from mount, we know the server has attached to our + * shared memory regions for read requests and read replies, so we + * can safely remove these files. The memory regions will stay active + * until both client and server unmap them. We keep the superblock file + * around so that a future client can reattach to it. */ + unifyfs_shm_unlink(shm_req_name); + unifyfs_shm_unlink(shm_recv_name); + +#if defined(UNIFYFS_USE_DOMAIN_SOCKET) + /* open a socket to the server */ + rc = unifyfs_init_socket(local_rank_idx, local_rank_cnt, + local_del_cnt); + if (rc < 0) { + LOGERR("failed to initialize socket, rc == %d", rc); + return UNIFYFS_FAILURE; + } +#endif + /* add mount point as a new directory in the file list */ if (unifyfs_get_fid_from_path(prefix) < 0) { /* no entry exists for mount point, so create one */ diff --git a/common/src/unifyfs_shm.c b/common/src/unifyfs_shm.c index 75d63a4c2..8ab912ac9 100644 --- a/common/src/unifyfs_shm.c +++ b/common/src/unifyfs_shm.c @@ -25,10 +25,9 @@ #include "unifyfs_log.h" #include "unifyfs_const.h" -/* TODO: same function exists in client code, move this to common */ -/* creates a shared memory of given size under specified name, - * returns address of new shared memory if successful, - * returns NULL on error */ +/* Creates a shared memory of given size under specified name. + * Returns address of new shared memory if successful. + * Returns NULL on error. */ void* unifyfs_shm_alloc(const char* name, size_t size) { int ret; @@ -93,10 +92,10 @@ void* unifyfs_shm_alloc(const char* name, size_t size) return addr; } -/* unmaps shared memory region from memory, and releases it, - * caller should provide the address of a pointer to the region - * in paddr, sets paddr to NULL on return, - * returns UNIFYFS_SUCCESS on success */ +/* Unmaps shared memory region from memory. + * Caller should provide the address of a pointer to the region + * in paddr. Sets paddr to NULL on return. + * Returns UNIFYFS_SUCCESS on success. */ int unifyfs_shm_free(const char* name, size_t size, void** paddr) { /* check that we got an address (to something) */ @@ -107,7 +106,8 @@ int unifyfs_shm_free(const char* name, size_t size, void** paddr) /* get address of shared memory region */ void* addr = *paddr; - /* if we have a pointer, try to munmap and unlink it */ + /* if we have a pointer, munmap the region, when all procs + * have unmapped the region, the OS will free the memory */ if (addr != NULL) { /* unmap shared memory from memory space */ errno = 0; @@ -119,19 +119,6 @@ int unifyfs_shm_free(const char* name, size_t size, void** paddr) /* not fatal, so keep going */ } - - /* release our reference to the shared memory region */ - errno = 0; - rc = shm_unlink(name); - if (rc == -1) { - int err = errno; - if (ENOENT != err) { - /* failed to remove shared memory */ - LOGERR("Failed to unlink shared memory %s errno=%d (%s)", - name, err, strerror(err)); - } - /* not fatal, so keep going */ - } } /* set caller's pointer to NULL */ @@ -139,3 +126,24 @@ int unifyfs_shm_free(const char* name, size_t size, void** paddr) return UNIFYFS_SUCCESS; } + +/* Unlinks file used to attach to a shared memory region. + * Once unlinked, no other processes may attach. + * Returns UNIFYFS_SUCCESS on success. */ +int unifyfs_shm_unlink(const char* name) +{ + /* delete associated file if told to unlink */ + errno = 0; + int rc = shm_unlink(name); + if (rc == -1) { + int err = errno; + if (ENOENT != err) { + /* failed to remove shared memory */ + LOGERR("Failed to unlink shared memory %s errno=%d (%s)", + name, err, strerror(err)); + } + /* not fatal, so keep going */ + } + + return UNIFYFS_SUCCESS; +} diff --git a/common/src/unifyfs_shm.h b/common/src/unifyfs_shm.h index 33afe6a19..afc2c3d56 100644 --- a/common/src/unifyfs_shm.h +++ b/common/src/unifyfs_shm.h @@ -19,17 +19,21 @@ extern "C" { #endif -/* allocate and attach a named shared memory region of a particular size - * and mmap into our memory, returns starting memory address on success, - * returns NULL on failure */ +/* Allocate and attach a named shared memory region of a particular size + * and mmap into our memory. Returns starting memory address on success. + * Returns NULL on failure. */ void* unifyfs_shm_alloc(const char* name, size_t size); -/* unmaps shared memory region from memory, and releases it, - * caller should povider the address of a pointer to the region - * in paddr, sets paddr to NULL on return, - * returns UNIFYFS_SUCCESS on success */ +/* Unmaps shared memory region from memory. + * Caller should povider the address of a pointer to the region + * in paddr. Sets paddr to NULL on return. + * Returns UNIFYFS_SUCCESS on success. */ int unifyfs_shm_free(const char* name, size_t size, void** paddr); +/* Delete file used to attach to shared memory segment. + * Returns UNIFYFS_SUCCESS on success. */ +int unifyfs_shm_unlink(const char* name); + #ifdef __cplusplus } // extern "C" #endif diff --git a/server/src/unifyfs_init.c b/server/src/unifyfs_init.c index a5f49873d..2d2df5e69 100644 --- a/server/src/unifyfs_init.c +++ b/server/src/unifyfs_init.c @@ -661,22 +661,31 @@ static int unifyfs_exit(void) /* free resources allocate for each client */ for (j = 0; j < MAX_NUM_CLIENTS; j++) { - /* release request buffer shared memory region */ + /* Release request buffer shared memory region. Client + * should have deleted file already, but will not hurt + * to do this again. */ if (app->shm_req_bufs[j] != NULL) { unifyfs_shm_free(app->req_buf_name[j], app->req_buf_sz, (void**)&(app->shm_req_bufs[j])); + unifyfs_shm_unlink(app->req_buf_name[j]); } - /* release receive buffer shared memory region */ + /* Release receive buffer shared memory region. Client + * should have deleted file already, but will not hurt + * to do this again. */ if (app->shm_recv_bufs[j] != NULL) { unifyfs_shm_free(app->recv_buf_name[j], app->recv_buf_sz, (void**)&(app->shm_recv_bufs[j])); + unifyfs_shm_unlink(app->recv_buf_name[j]); } - /* release super block shared memory region */ + /* Release super block shared memory region. + * Server is responsible for deleting superblock shared + * memory file that was created by the client. */ if (app->shm_superblocks[j] != NULL) { unifyfs_shm_free(app->super_buf_name[j], app->superblock_sz, (void**)&(app->shm_superblocks[j])); + unifyfs_shm_unlink(app->super_buf_name[j]); } /* close spill log file and delete it */ From f9cd6ae60f936aa9dfb4a0a42c355a74d4edb58d Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Fri, 31 Jan 2020 15:12:06 -0500 Subject: [PATCH 065/168] documentation updates 1. update unmount info to remove "only rank zero" 2. remove NUMA config option blurb --- docs/api-mount.rst | 14 ++++++-------- docs/build-intercept.rst | 11 ----------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/docs/api-mount.rst b/docs/api-mount.rst index 5ce67a556..06ea571f3 100644 --- a/docs/api-mount.rst +++ b/docs/api-mount.rst @@ -19,17 +19,19 @@ In this section, we describe how to use the UnifyFS API in an application. Mounting --------------------------- -In ``C`` applications, include *unifyfs.h*. See writeread.c_ for a full +In C or C++ applications, include ``unifyfs.h``. See writeread.c_ for a full example. .. code-block:: C + :caption: C #include -In ``Fortran`` applications, include *unifyfsf.h*. See writeread.f90_ for a +In Fortran applications, include ``unifyfsf.h``. See writeread.f90_ for a full example. .. code-block:: Fortran + :caption: Fortran include 'unifyfsf.h' @@ -48,7 +50,7 @@ filesystem. For instance, to use UnifyFS on all path prefixes that begin with call UNIFYFS_MOUNT('/tmp', rank, size, 0, ierr); -Where /tmp is the path prefix you want UnifyFS to intercept. The rank and rank +Where ``/tmp`` is the path prefix you want UnifyFS to intercept. The rank and rank number is the rank you are currently on, and the number of tasks you have running in your job. Lastly, the zero corresponds to the app id. @@ -61,17 +63,13 @@ When you are finished using UnifyFS in your application, you should unmount. .. code-block:: C :caption: C - if (rank == 0) { - unifyfs_unmount(); - } + unifyfs_unmount(); .. code-block:: Fortran :caption: Fortran call UNIFYFS_UNMOUNT(ierr); -It is only necessary to call unmount once on rank zero. - .. explicit external hyperlink targets .. _ifort_issue: https://github.com/LLNL/UnifyFS/issues/300 diff --git a/docs/build-intercept.rst b/docs/build-intercept.rst index 6e18b27ad..e8dd48f8f 100644 --- a/docs/build-intercept.rst +++ b/docs/build-intercept.rst @@ -226,17 +226,6 @@ this: $ make $ make install -.. note:: - - You may need to add the following to your configure line if it is not in - your default path on a linux machine: - - ``--with-numa=$PATH_TO_NUMA`` - - This is needed to enable NUMA-aware memory allocation on Linux machines. Set the - NUMA policy at runtime with ``UNIFYFS_NUMA_POLICY = local | interleaved``, or set - NUMA nodes explicitly with ``UNIFYFS_USE_NUMA_BANK = `` - --------------------------- --------------------------- From 350d4d7bc49cf84fb479d7b76b8a27ed1fdb8074 Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Fri, 31 Jan 2020 15:20:14 -0500 Subject: [PATCH 066/168] remove unused NUMA settings and configuration option --- client/src/unifyfs.c | 31 --------------- configure.ac | 2 - m4/check_numa.m4 | 91 -------------------------------------------- 3 files changed, 124 deletions(-) delete mode 100644 m4/check_numa.m4 diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 05ac7b18a..5aba007d6 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -48,10 +48,6 @@ #include #include -#ifdef HAVE_LIBNUMA -#include -#endif - #ifdef UNIFYFS_GOTCHA #include "gotcha/gotcha_types.h" #include "gotcha/gotcha.h" @@ -136,11 +132,6 @@ static size_t unifyfs_spillover_size; /* maximum number of chunks that fit in spillover storage */ long unifyfs_spillover_max_chunks; -#ifdef HAVE_LIBNUMA -static char unifyfs_numa_policy[10]; -static int unifyfs_numa_bank = -1; -#endif - extern pthread_mutex_t unifyfs_stack_mutex; /* keep track of what we've initialized */ @@ -1892,28 +1883,6 @@ static int unifyfs_init(int rank) unifyfs_max_index_entries = unifyfs_index_buf_size / sizeof(unifyfs_index_t); - /* if we're using NUMA, process some configuration settings */ -#ifdef HAVE_LIBNUMA - char* env = getenv("UNIFYFS_NUMA_POLICY"); - if (env) { - sprintf(unifyfs_numa_policy, env); - LOGDBG("NUMA policy used: %s", unifyfs_numa_policy); - } else { - sprintf(unifyfs_numa_policy, "default"); - } - - env = getenv("UNIFYFS_USE_NUMA_BANK"); - if (env) { - int val = atoi(env); - if (val >= 0) { - unifyfs_numa_bank = val; - } else { - LOGERR("Incorrect NUMA bank specified in UNIFYFS_USE_NUMA_BANK." - " Proceeding with default allocation policy."); - } - } -#endif - /* record the max fd for the system */ /* RLIMIT_NOFILE specifies a value one greater than the maximum * file descriptor number that can be opened by this process */ diff --git a/configure.ac b/configure.ac index efc8ca147..9bcb9b151 100755 --- a/configure.ac +++ b/configure.ac @@ -104,8 +104,6 @@ AS_IF([test "x$enable_pmi" = "xyes"],[ AM_CONDITIONAL([USE_PMI2], [false]) ]) -CHECK_NUMA - AC_ARG_WITH(pkgconfigdir, [AS_HELP_STRING([--with-pkgconfigdir=DIR],[pkgconfig file in DIR @<:@LIBDIR/pkgconfig@:>@])], [pkgconfigdir=$withval], diff --git a/m4/check_numa.m4 b/m4/check_numa.m4 deleted file mode 100644 index a7cb5dee1..000000000 --- a/m4/check_numa.m4 +++ /dev/null @@ -1,91 +0,0 @@ -dnl @synopsis CHECK_NUMA() -dnl -dnl This macro searches for an installed numa library. If nothing was -dnl specified when calling configure, it searches first in /usr/local -dnl and then in /usr. If the --with-numa=DIR is specified, it will try -dnl to find it in DIR/include/numa.h and DIR/lib/libnuma.a. If -dnl --without-numa is specified, the library is not searched at all. -dnl -dnl If either the header file (numa.h) or the library (libnuma) is not -dnl found, the configuration exits on error, asking for a valid numa -dnl installation directory or --without-numa. -dnl -dnl The macro defines the symbol HAVE_LIBNUMA if the library is found. You -dnl should use autoheader to include a definition for this symbol in a -dnl config.h file. Sample usage in a C/C++ source is as follows: -dnl -dnl #ifdef HAVE_LIBNUMA -dnl #include -dnl #endif /* HAVE_LIBNUMA */ -dnl -dnl @category InstalledPackages -dnl @author Loic Dachary -dnl @version 2004-09-20 -dnl @license GPLWithACException - -AC_DEFUN([CHECK_NUMA], [ - -# -# Handle user hints -# -LOOK_FOR_NUMA="no" -AC_MSG_CHECKING(if numa is wanted ) -AC_ARG_WITH([numa], - [AS_HELP_STRING([--with-numa=DIR],[root directory path of libnuma installation (defaults to /usr/local or /usr if not found in /usr/local)])], - [if test "$withval" != "no" ; then - AC_MSG_RESULT(yes) - LOOK_FOR_NUMA="yes" - if test "$withval" != "yes" ; then - # - # given a path to look in - # - NUMA_HOME="$withval" - fi - else - AC_MSG_RESULT(no) - fi], - [AC_MSG_RESULT(no)] -) - -# -# Locate numa, if wanted -# -if test "$LOOK_FOR_NUMA" = "yes" ; then - # - # determine where to look for libnuma - # - if test -n "${NUMA_HOME}" - then - AC_MSG_NOTICE([include libnuma from ${NUMA_HOME}]) - - # - # Look for NUMA where user tells us it is first - # - CFLAGS="-I${NUMA_HOME}/include $CFLAGS" - LDFLAGS="-L${NUMA_HOME}/lib $LDFLAGS" - else - AC_MSG_NOTICE([checking for libnuma installation in default locations]) - fi - - # - # Locate numa - # - AC_LANG_SAVE - AC_LANG_C - AC_CHECK_HEADER(numa.h, [numa_cv_numa_h=yes], [numa_cv_numa_h=no]) - AC_CHECK_LIB(numa, numa_num_possible_nodes, [numa_cv_libnuma=yes], [numa_cv_libnuma=no]) - AC_LANG_RESTORE - - # - # Determine whether we found it - # - if test "$numa_cv_libnuma" != "yes" -o "$numa_cv_numa_h" != "yes" - then - # - # If either header or library was not found, bomb - # - AC_MSG_ERROR(either specify a valid numa installation with --with-numa=DIR or disable numa usage with --without-numa) - fi -fi - -]) From 3a381ad4e1add6372b28c5a3834b7ebca6c39f67 Mon Sep 17 00:00:00 2001 From: CamStan Date: Tue, 4 Feb 2020 08:48:55 -0600 Subject: [PATCH 067/168] Replace temp UnifyFS logo with official logo The current UnifyFS logo in the documentation was a temp logo while the official logo was being designed. This adds the new logo, a new smaller logo, removes the old image, and updates the name of the image in conf.py --- docs/conf.py | 4 ++-- docs/images/UnifyFS-logo.png | Bin 0 -> 24301 bytes docs/images/UnifyFS-logo_sml.png | Bin 0 -> 17422 bytes docs/images/unify-logo.png | Bin 184178 -> 0 bytes 4 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 docs/images/UnifyFS-logo.png create mode 100644 docs/images/UnifyFS-logo_sml.png delete mode 100644 docs/images/unify-logo.png diff --git a/docs/conf.py b/docs/conf.py index f1abca286..9066c3847 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -127,7 +127,7 @@ # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = 'images/unify-logo.png' +html_logo = 'images/UnifyFS-logo.png' # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 @@ -231,7 +231,7 @@ def setup(app): app.add_stylesheet("theme_overrides.css" ) # The name of an image file (relative to this directory) to place at the top of # the title page. -latex_logo = 'images/unify-logo.png' +latex_logo = 'images/UnifyFS-logo.png' # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. diff --git a/docs/images/UnifyFS-logo.png b/docs/images/UnifyFS-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..fa374b5a0cad4257d659be312413b6e94c4684d5 GIT binary patch literal 24301 zcmeEui9eKo^zS2y5S6_`>Kh@1kX=fPJ&b*qea$u)`;tniWZx_Mo_!c=sOt0DuY!0F;q5 z7r^i69j6?F|6KHVZsY|3!nes^6n=S%z5s9&c=6K9@0!%Qo= zgcc1)juM6;4Sc+A{h7J3-<}~g+|@Puo(?lv8TNg3vJtC!B$&=p`;x}p(y`gvvAH;k zb^hgG=IZ+FWC|SO;|__9b#?h5dXp>QyVxC#6Wz0)=OP~IWEo@)^4$`m@XSg$^AhtJ zAe;=~xIyOr=g^zBoI17`Tr2N|HJD4)e;D85w}b@2LReCk7KLk zc_P~{ca+y0E4~N`X4mD3B#cZ}^F;13E#Ag#(Ex1$7m*YIa2Ju=ZAd?1^XpWZb{c9| zIe-LCGcR|fzT5dEj;9AKYa_8rju}lJfrIaTroAQu%JQAo+>iqKiEi0Sq{HcK+0=8u z-Bgf<9DX)g86ULqkkzz8mp@W5?(~b8}0;6A{SPjkeq%k z)qjcA{P#1D$3;S3PfQe-d$Gmg)W@GANyXQ{;`$0z9fbx8dJNnoPIsymq2W}($A~ep zmF4gVJntgI|4gG-GLlm;&Z}5I6b78W7wsEYuV+iqnPzb(f@N_Pi8W^GkMMmO@G{C-G(tWy< zvR%2&2{f?f!ZbGJykqOOJ%}mHI}Ukni;y`OY;_IpuUHbSp+e{H{yw7r_6OtHeU^2bHn6ob6)aL z0sDz$EKSHEF_;JTAdM`M_5d(DWP&FTp&s|ZAFT=s#HlU1NL}8F=GrbQCMDKnzf?f3 z6U6H-0opzi7w2SoyPsv)kl7V8dO0pkBUIrg`%zWDH516brfsoy=W+cpB(LczM*sTq2;6767+%rH{e_p+ku09r%_jjKz#+ z>&dNmk?jW{rNx$mHA`KtGB`0;a!RMhGmyU3WsfK5*f#k3T45=X$UoNS$MGNurw}e{tEo{vYX2$}bmS z-EXeh>=^D_TD1OxF~P~QkRv0|o>rMKj)6>$EL`pwzQt=Q4_!F4lzCLPcu|ux^Z7#4 z>Y#o7n0#def|1X)N9HO>#VUg5R}ZL!zhRS2=Lp7ca|ul_M$TRz zlLCD)9POO*9hB$UqqLiBfPhfXDf+dvd!A(qm)h*krA1m`EOkiU27G$9mM~NW6^F|w z?Y@UfG?V{#_{+PgXB`h(LwrGk9%6Q*fawP4a8Am+3n&4-@e*mRhNGvnZ`Xtp2$VoA zpZ6dc2n-V*b-`F>!HN=^@4g;II)YdW1ciX@z+ASZrSK-Hni*kHxGVI@^J%& zh@0NHvZ+>ijaO@JnZ)VrV$|e~q=GgxmMe^&+jCKicqTwqBnqX%di8G7TC4$aK?MNz z!-o*hbmoJNmqi(^CXFMYVX`GP7TXL;RLo|aB;`9#i++woF+s4s`F|>}0!mepSoxbC z_LuZIo9X6^Znw(U4jqDr+3~mhA z?>YgYsvMP#6ZW7l4``@1e-?7kYw-X}xIwmJ>QnB9%EydyXN&eB<;S$4Ivh?P%mmwC z50FDm@U!xEZJSL?n%sL#4cW9j%^Z*MAgU#5{f&kNFnJO+eW_{e-2R?Iy;Q()j4Hur z3iL4zZJAsT8^}lr2G7ZW59%WBcoaNC`K8DT0yph#9}c`Ot@Ra2w&eVa42%Z7#Dtk` zmT=3@o?*SNR*~94|CGI!ijqQV!0Cw=L1G5BYFHZZ(C~L+!3R`>K?s2yX~g+ipO?Y>_}Cyaj>h`Kc@sr4PHWDWofWro!4huxT?MeTL&)H zjx~|&`9l@8$H2mP`FjCzrrEeFG5@N$2K82Z@Y6SV(7By#$7hoonH~eQ5@YP6Pb^9)8LjS0Pu;nEhq@~E)1WwDISz8wG{TwYAArof z0w_UfkOl>B--Q=)6FK~^z$vva*rr_8?fpFdo9HKjz5UF~oEKbHiJZna>pum)w8N+>9uMb+uq^x2i%N(pcov7eVE_s#b7jBu zB^E{97a#Q|-b*z$Hl|loQ}fpfy#Nq&yz`D^sYF_JM*b=GzuG_vB;NM6$XnTuRV@=4 zuO4!X+L$ajki4TCqo#I&Jya)!om<*u`&F?ra{M>!mM@zM5c&t?M`tf-DAPr#&Sc?l z!Q=J4EBGxK`3(itsJQ~ga$IYt4{mR2S};64Y{WNxmO%(1a0b>h{M^_mi5qw-X? zY=OIp%%xoY{+ZT^O#>btxg_euB>rI^#sN;UX=C!)OnSG{$n47#es0uB!1ziew>?3D4$vM3(&3FlSv$H=; zr&uUAhh0E%_&T z9~{JooID65%4)sYnysI5DmBWGtetk_Pyz(*Xm|%SA=Qg(;@WWo`wb@ajy5dBFCXQZ z8sA{C&jFxVb+v_IQjv6XlRXfwyTjl--YmhwI=jb1Pd~TEVO-{WhzmI@*p^Zxad)ir zUyl`c9&zc9?#jy}ZrK6a7sK~EmZ#01I&%OyKlZ&X3e-a6*wRL&1W4EJYP#@?D|&5& z^~Sr}1)nv=yYf%`qu-D&#{v|x=gI=wM#==A_lO3?qn@*CGyT)5tqi;M_tErTQsoN! zo{C3HF&@|qFGl@>+Mx-3au4q2Gi3tT+je3X`iwAO;Kxq4rJs^|Y&~vs@SCb#mQmya zwz|y!_61=751n+}_Xu;4BQPpmHR%fs#J4&~bW(OT-O8^B<^o|c`M^-r?<{sclR70n zjM&`92^5wxy2=1K#o!xbP6KbWf{D)UvV`%pO!ION3!8b=!EZi;j;6EivnXGaY_}FC z-rf@bY9fZ0VL$~Fz@yFgO_}FfcdT}LodK(%-AZ{OF zbT7bHX6Wb^R_0tVg5)>5uE6bAyD{03@n~Yxeb}NCy5(t$xiem$&%@%Ehgh$#Gfzek zfi&>ADK#K1J)5z9#jf-3-{Top3P7$c>jw3&f1UzLwil7&tbF|^`}M611_NihhAz%W z+H6(upp`#)_ho(V+*@Xj>~zA-GNA!tVkux)73qBK>Tgdl7ME zRjmq2>@yYb>%l`PDS7C}f}W7C_6HQoH^a5yBds%bx1du;zg?mt?JQSFv1A2;Va47p zs?kK($}u-nZ0PUjA!h^ks57yn+3h}Yt>}=WiWW9W*BQg?OT#_ZQ(Pv$`v&U&a?AM3 zlIbqr@)l9r+)*GlIa?7IwW9d4V;6`CwbvtpN3yc%yre3ke7(wbeN6Lj?2l?yMD`|E zM01~Pr!No{5A(MXU#+JUWV3) zf~I~fR&5_bgD-;;6!+o&Jn}x_t8nzzq-;W-aLH4eYn_C6#nZ2QhgfJfEt-T)u-g9p zOs&#x&eiI*sQRE}XJE~_nr?zD6DA~Svt>%VZ(ZgI( zU|&6tlfBT&6FR8v!abUXU|F#*khPGNVd)`phX?!a7&q)s*H-%Pb#HsqqqBpX_YzSt zkBFNfkR(j3UhnkE@;&)iJ>z_hWgk6r>8GSS`^@y* z^F6K+QA{od`~E=a=`HVy!qg%WnfRjkv#B*=OVIr{+Ywe9HW9`1+&QC?c5b{i>D8H) z@~4fnQcr<^0!<)f4dnkw&pwjN-`m9R@4IJ75r=%fUmPK71=48Re!G%Ai3`|FRbL_- zSuE&;0eqORm+)MJH_zP3i87|kWqQYuFA)8M^l4xcZqQMW!6cPTWAY!DNY7rJc9+`y z7*_sgf8R7E%k51}w)vKxlj82Ly6r=zc<*zEhop(JbisY60Q=Xk`5&b3S+$ItGtAoj z3Wk-Nx!Qo|0qajzQPC^1^1pQzckf@8beUnUa|zk^z0Cdiuz!|&t#`8JM&)2s)WAQ5 z;#Rp5Y)PzaE$WkyYXm*M$ZW?qF>J|?52u1xK5>e^Jrj#WSJ^moBmQ1q!OUjL0@iH` zXs0Os63!XA7@p+~PJtORR@c>5%_%{@cOU93@W9>7BDjj5b1viDp^z-srR!4em(uH8 zwoc5ZXGibVU^=Z!YENpB#tbtidzqDpy}$b3vwsW*cHKYjj!hj{^i3ov(a3U5&s<#M z4heiV)$sc9RF#d%vGIVCLQCAkX~(A8p(6$T@_IH_W#<->Z6#Oh^biuc=|H~tJtp|1 zp|dZ1FQw_V#Jw{1H->plccO5V*QI<~;;rwuzV1!tS(|vqgYF6B-8y#(1O>uDdp{wbsj=pt6{h;a-Z>*j|Z{CZ>#k*{rAd`gBJ3$r5 zoYmQG?0L=ByBm$hWTdC#YC_l6^$P0QWI~KhO*i+=<9kz_>pFA{;Xz2d zn`MifA>=3+TEXLu$+BigU#hXW+qyX*uv_jX7<=f`*&S;&TE}T|dcvv5uJcxbDT$SJ z^vnyIxZTF7DC;qKrgmmzXt)`<)0^&x7kotcL$b7-9|-yf>t%5|E01+E+<{U;%}N^= zr47uRr!5u9Ent0nZ%;oh8Y}yNlYb)br*JiK?Yn{*uscz1=6^_!#mlrP$*lIt;^O{eg*vi*3{=;#I!j{cy@&?926 zbN5X1^WR`XXEp^@S@?TYuea=arKer*X#2an=Wcjo^+bAR3UyhsuH|?SJX_oTTS8}(j-iah zGrt^mY!?5K%nI+8tVjW=fQ^UfwD}@F!ph%lAphp6M&kDJKi<)h8T^zoW@mM@tkQ8q zhyBE@dha~4!pLx@fq6!jwPm-DaNLx5LFUm(ccqnRq{iO$kBDp`H$&RJ*_q59RnbWF z9Oyd+Txm^=(>qoso;K0seNl$Hb+sG2Z8ZgLKNw?=;X!VI?DDy6MJI&gYq=s(ZnI5Zu#foTa#}TId9Hm<#(Kl zzhhziE+Gfyi+uvqsn}Kacqo75Wd`o)ARSRviv486C~lI-nWGjv`$NvN_*vu+oVd~I zmSQ&#*K-RT(Th^>bc%G;6oN&ag+S$#^c8x{29YES>Xwt0ILvi+_6DR)GZ7O8L)6U< z`b;`+YJNNpt!kKjQk>ZehXomwF1B%okd_VSHfmA9C>GGrFifR}(P7IUNw7Vu?Nog} zm-xi3KuGCZ^vv9xzf1P)RY@TidZGM)=Jw} zalK91ex}-g)(2Qx#RDXGUX`H)Rn&TFhye|k2BCbA{iZE7&_s;5nkM>VNDqtFSC_Ue zk4e6j=Svc5zwrkWzSi9x{Xe?^M_H7)L;)IP2^-v9^AQZQGZ&ac@a=QGB%5Z^5w|j- z327cOTg|EkTY}Eg8T;HrPAerkiEB+9A}c?C9zNgVbyDR+x!YaltZlzO@7Pf;rNgjcb@k1PHx?x_{j$zZuKtwv8nGk z$@gv4{}}wKCoC)?9@e}3GtZ6haP7eED6L045Id{$_UMt!?~1=L2YcESO}bRt#L3fy zUg9l#e^d6dY6IPQvzj333r`|~j?MXv&20kioBb*k@7WJjp-#dR#89d&9LJ@4EUXDGjJ7*3tkg}z8824K_{cu` z6YwKnU8}*yXA1-~@`HQApza(OF8_VdZn8;Aia;mvg&B_dPp{>RV`U8ISL~HJyymrD zIYpR@tRWA?H+uqeRyW{R=r`D%9~}5{zm>i8%)>zZ#4`CKV$-g^56Wc#RgZ3OeAcn( zHQU*JXY>V7s{K1+PH#sV)$_=r)G3I+e@FW;vPNRW{gI#W@%A}oBgDr5+erv78@uUY-;2q}DZAWkTSXPX<*N;@c%=W~KR4CD_RK zANwDNGF4aKY@OlT`nPXe!ScE)N$YVY3lSi(l3Dlhpoio(#uu5a&hs9&Bbk1q=M%q7 zGu3Gd{~0P?45a9tb1aWJP=mf*mXFhw7GYGiS5e~#BTmAK`UDMBl%cAiT3n(+VDtKi z!)S*)qgvL7!L~j99;d$tJIjQ;e7G4xN7Ta9Sso zXr3~RcPBu7Rnt)(-ygLN_C&-ND$Yoym5BIS$$OD8@#dGu+^LiKP)hV{HpW@Fl7!;2kN8c3XPI<2EaZYyj#0Ott zRj;AdZW!x1ftdUiJde}1u%*e0q1BBtaQ=d2?=`u?X20s5=xFeTSCdnuy;$2O-6inP zUlT>A=nxY3zN>xXI*HUH5{1)HWbwV{B7O^yR#fqo*G zNa`H!o%t)AtQPD^Ua0R~QABb*9S!m^ZF~@_O!*j@Mu@7C@-pWl zZMi^D1pb=fyik=M^d-K(Fn4+A`5V(gyC?65T5coq52uMgzNR>VXBW1fGKzXg83|b6 zj+52~T0`75w;4VrGSg=0i#G8tf-y}Tyg3BEbie4tEq?HYOZq;B{*HVbSy91*}o*OWMDS80;`}8%`@*@ck=Q%~egS(}NJ^hP`d; zZH4=nJH)HCI3z;3m>LK_O~6wyV2iWZQ-@YBn^k=LBh;CLsnBi{)RzeUfUdNVCCuBL z0TcA-7S(oD(10_DJ)#aILv@dt6qSp`Mm0V+v%?c9)8QLTnZ)~K17 z{6F4&P{k7|*DzUU)(FBOU`f(pzv+R?jCdR?8dS?4p0ITC#jeyQCN;eHs>NN)R7fI$ zinh$QFz-gmoXz!AOfR!$&pP3DlNJ{^Pn0!NeX^=qo^^a`eHg=rxg@i3wFHTgObcYQ zln2oo;W(lf26IOYYl*!1)8MANY+W2F^!+`JAwCbyBWH*r%~83HLCnL5VL#&BY0)99 z%6`LLhn|<=3F-YN$sO@$Hlv}VOxbRub{8zmbPTTw&OteTX3a{EvCB@`SG?OT8T}rt zBCz<-SyWeN4eE+&wXaReEjRyt=#wntVX+2^*xp(qm?pRCPg8E+_^^#K)U$dIUDJv0 zNv3}&FIsPjk{^f zuP2L&&tnC_Fu%JhKt#|XVW964u2ZaHK0R@-rOd4_cWL2Zy}%kP&?W%78Pw$ld*Q6Z zB40lD8P!+L#XoIIyVRwZgRD9@MaCdWxLWon2Erb=_e_XpgPrvmK_+)Z z&SPKQynq$;qn=+NJFMO!%yQ>@iuPe|;zkU?|0n88v~HqiqQIf~mFJ(TJByLxJ`Iti zxu%*F{(|P70&l3blID-V3eLCfKI2laoTT?mW%L`eqs!i6cwEJ_UkTwo&nB`4`<6`OTXLndis|8-@ zD8YrItRj}N={FlOap0Q94WW%cU4x@w0;WeXOQPcGM}Xt%U52>={UaF5aR->(Wzj1V ztxZx};1NrpQYydQ;}t)OB-T=zReI@8(~5S3o&W44&gdCnh~OOWm^X_vI_ZQgGI z2w=uDo5)Zeb5+X_1vc=FpbrxSG;At81gJg`N4g?2@64T6&5#`4mDsF_@5oDn&`4OI zDv}eK>l%z<^fE<}GK!yPZ|YLb{jRC_mCavQj8{`J3-ZNN5 z6?=43>?X*FR!B(u&4i-#bh}6_V1e^~lvrV}qxMFf100(ccZ=MT``z@lX@3N1 z>wk(GH+~~Cn=tGgf;8^}*L!}?;d^S6cuc&muf*P0^8fhk;Mz2vuE>-us=GM`65l8z$b-cMgaltF3wP^Nm3C!a_-G_<2jpqK2e zXtN(afSe!pNwr>H>Zn6doUv}kRu)i*e*fAqkHsT*K?#n?+tC}k-_Oll_fa7O*=LqFvdw7Si zlFKH2OPt-n!S^n8QBQXv-R=pukKfwqS?_ok*hgYtxvdndLV;B#WE-NG?~v{#d;X5z7_VZ4f4yWLnL$-G}jLxKoY>-#=YXX z#}`+Cukkd<-=@xWWfii3p2;#Lq5Q$lMrt@~#R%swwJTvRu8NDGp%TTMd|oSp?jCA{ zH2Mb0+n}Gt{G3WQsw$HDrei@=cKXpUcsr?vEDb4@2tFN5AFp=hFS!SD~IEK(* zC5HhvDaGtHy=HL0IT9ZREA>_UHp97UPgR#vv=Lcs+zWV6=gKx7?bjR7OUl>QeDCAD z`8bH9p+fr!i^G0GcoN;1)|Gabtz&q>#aDZ!+mpr@a?V9!mw53$@hrF@CQ*;%M#@)r zvii47P|N#;3*hDgirCRh7KavyeBjgWtHsm zb)sOLyq^VbeJplvwYh0;+_wbI#D1ZtKO{YTC?lPL`TlDkO!XSyhY_EJg#6B_7FLT9 zo&#R3D$zQv&p)|RG5o4nDegUu!(NtN&HbWc#lb&Q)e?<&87wwHkQkw2i)J#jz$nwy zXtK93ieAGFToWqxp#G&zKj*CbC^QdAhMGAt>_r1mYFjaSstzp9XgZhREV zm5pi)ez`r+86#wyqz4LSx$o!UuduW@w<{FgTC|3|+7`xruWmy2Fh*BOYU$_rOPW}q zpkosJXo&<*TR-k{1c!1pH8;sz%7vX7nNdHaV)rLhNyQefwDx zV%nwmVxC&h2M^(GGXg~X4d6ZI zZB4u8yxPtxHN0!HO#x&yd%Cgc|NM^D_BCRQYyMg+KujWW=Fb=+o(Kw8sddykAG%xmSFwWS#+<qg1CAbUE|Ibn`uzQ8 zdI5jIuU+bB2)qDq>0g(jz05#JeiqYa@Qg2-v%bm+q~&f`%pu$il{P)bIOj^#TSgqb zGaTmX#N$n8UljpI@0hAMB~XMfFbim?oTKQdxCo}X`Ic~!5@bn~Be+?1r)>DQ6_o#n z09YE|u1k|Xvk>v^Ak=Lmet_dSVqLxctd--VDTa^YLbg~D+)dgX8n8mj232&G+%`=d z-EnPhTCSPT9KQ_Lm6j6Lh~8^$DcnU^Teoou&YFx0B`lrOmI332e>7HPiZ1jV^Gk_L z`{$WOWY3W?8{sHK$(S@re3kqk1tTe&!&9^{INIG)Nof(Yf1EU??95YRH=8RlYmA*` z0G9JWZ{kcMMe&2AsPzZg}+yy<+_!4kR9E#tD|o(B6$?6?aY|L`BGj@I4yO{N<=Tp7^(Fu8CYo zZpqKZ3}v=RdHD#No{s80JVOgd#k4kEbqkc_T(LKnJh|2t!&dbS=CUC!HBP@bTBeEi z1uP+YPbpMJtnChVU4+EJ#g(rkoLnPdBe{UK&=+@&JM%`cS`!BrtWNb;-1?$RxKT9O zrzu8>56*c@wWM|*G9&@Z8MxgmILpbd`eIU z0TqoJKHR{eOjWF^OWDZl*E00A8<)hzPoL~GDBK8u`!qeYSy&Cpu^0%Y&s_t_d0u#=-hcS_`W68`v-w*e!luf{2B zyFyW4NzA)USs^kC&Y%XQ;#h5*shhe@YQBWUr~N|FR+4Ip1f9`ll7j)&!TwhIi|7Y9CS78 zLD72H)wtzT=9X4b8EMns@ACZpl%W7%Xm<;eqpy>mTrJ(YA`e!>KlHML%cf}gFtQY= z->;2^Cau#`uc9GLodhW8D=n^P-YEZx;VI$s2JLlun;$oSF^3*?&yw@ckn@q%5+X|g znHwFqsW5$azTb*=^$GYF2Vr^RsYsCT;4Z{ztGXEpwZrm4`2;QF$Jj!1Z z(rmEc_L;pU9&B#(YzKv0oQRenFU$f;2uIO9h;gqHcah;XSJ%4?^yLh*OeB4>Qbkm&@0VW`Oa~OV=2(&gge~F2jt~m+ibJ1kx)E-2|X_tyMnT<~rAw{0wPd z`c)Y-{o~!Em@v=Af#R~_)2oAwiFD%X8MagUK5GwcaultLRVZbZO;1CeVg(MORmaP? z1{XhUJ$zTOwU;46IQabapT!TjM$F74rg;$4%snoqEe`Jf-HP-mYS+cR9wDJd@F()R zxbc6%1Q3g9!Yd|U;YyOiI-OoWx>rxBi|f4nqq;LQIsou)AMJ@eLq8j1WChT_2psfKw`wgPU&P>$?TdA_l#IQVEYed!22CPiCaX|EPWQH0UEA?HyGS z?!@7zhmCQMtsF5!gDpjo@7oAR^k6ta{l81Sgq(koc&~6q@Yq0Hm|c4A#ywiGTq+$j zY%`#BuChzN;5=@byygx7(R$$VkP6s<6NfJL!Mj>&P0PHdmPA$)SDnKa?q&o3=$9q` z9lxlq+NC~lkvK2M*n_Tcr_{ab5i<*CrNdww@ZqQ6!xAXTHL`91ztV@9 z^1qs4fA^1!rx`yVFWUbJ&UjL*4yy?|C#PvIWh*-L^xXn1&|Pf-U-n^KpHISYCJXJ^ zz8v_UG1zpyPcWeh_3Huv%{XX4G)499_5gFowNF#C5QT5Sh*si5U&g_qklZ z=(he9^I@!)e6rT(Ce~%A{P(HT^zl`=;O?GtPHvza(}m|vOQ(@cUQTuc(f{53vp{~0 zTBfpAZgBYv0w@X?O5=6+9N_i8PI&q@4@%B5Sqe zreC)COxB=J?vMYCfEUZY5^<3 zMH>pHui^8^??nFpw8>OdXuV<@zH2@lUn_>;m{)BvsZpiA%Rs{UT9oQDP73E!Di}c zo*lFZj}P8-V$sulz`tH=K!@?0@dbx6@TmKpHBEc%WD4_s<$oywIV0?~MBM{8;r*#; zcjfb1dhJBnJBrv+1N;(5rcKv2<3x(%0?aKdBLAQ=)XUT zPxDs1H+fq5RseYLBtpQ>6Vkis!ocdqS#s`1L0^;Wz3(y?KrL$|Y&>~t!3ngDE|=?a zI6EN>_7~^dj94u>d+1WRsOMkbTQX$=#r!a<1>WhYA(6sB8EW^NbGt9r2$t+6raMKY zCjjYtSWpa>31nixt;r{Py3O*VzNbHqnrH#uXOseVnze`P{Dnyr>m9I{m%uwZq?zWH z4v{KNGqJ2XRG}+Hwq5GR-V&3dA3(7RbA74hRGZx!NYHu?f;4_u@R;?ze@v7fMQG1& zg|*<0_Z;J507h*!SQ+Fi%}#I;ze1blETr*n+1PNR>?1IAU!pvP!+CgITw928?A@dM zYv8K+&;#j>XZP}t^Ih1uGb+z1*^C!k^J;X&kCK8}i)}+?`1dl5I9D%%p3>nL?`t$KB*@9XhnAQYYQ({_9dy}=!62u~Yq-m_tNU8J~ z@{(*4TC^xHNH4uOV-zq1ka@go{}2}Rxm9_jY`Rfc?&RUw&6v+ZZ{*SBTs(rOBC2&1UDt4$-ft1 zj}^6cu_EQSkF)R48UxU;|EQ~P%tpWtbT$d4r9(&}k_@{CFR|QsCL_^=Vp~rQIPN=> z50Epxt}P0pZ~jEu;(EjoXK45Ije88@XKk&bxfXY%v+JEW0EcMRe@#NhrWTA>IdOJs zGrYM(kl^A@7nL@*^y3-7jop7P;0EmjOEct2fYJ+r;uL8^subrxW*}k>Y|Z>mstis* z#M&GlF`foI+h(YFKB{%qGrC5aa`jJM@K^@jb_!^O_+MtCUL?^F;RNs)-=sQBfvQ~m z^?)ud%~b`3TwsV*gAwV~$~_3iPScg1Qk{Vq`r|o@95aEUlouPrOU8gAA&M$A;qG1r zWL4opF7fchl4{2CP?pVI@|fBVo+$;@mi=|&m-w8G&C}d+D4G@!`YfDZVH@~#w8)y* zptcXYVmYkDrTwf&NH|CsR4|Irebbv8{pLMK4%LjOtP2(+D$aw%Hi`&0$X^F$3f{j@ z9qGuweWMBRflPEgb2j|&VpSnOJLIgXku8eiZf9oF=C}Y>;o5hTWRoI;eRsQNA=&Rv z9QFL?lzPcV$XIID|8Nb<;P>xs7h#r0nBx4oo`1X*Gi^t)N)fJs^OI?2Le&-F35_2a ztg&{%efI)X6onhS1ot|NlPE&*f21puoGA{G-t@HEQnskAy|xm0u3RGj$q5n7S?8e2 zoFdmN1nY)m?3=ne_V*#K^4;d@c_9eY7+mM;;A{eFt83)BMdXtFl{DZ-D>fkbmS(pKY&nu$F(m)UtJF>48L4j(?ZU)qz#xE9>>R*aCUvv@c9 zRmCP7xbIj_!d9@kBD4+1A1N(H5SSYbhF>azSyDM5o%a7-L&P=_yKfR{mv1Kzj)T zDrD??9N|4B!S64^co_Ux=25M!>WFf5czun0i2K!iRB{~#ql)#6$9_3E zG6CJ0CTc3!_E9J>HZ((xZhXiOUDT78me1rxIV&1{V$~=U@Q9y~q>h<*LJ|6V_|3(g zOFbgmMQfs5Cr-0XDBXA>;xo4$h~>45$id<3vlTxr!KRuPjYG{#niB3n%-^5Qe69?v z^-lMLYNU)nry3zM(f{+siTUu^OO=>>?5uDHph4o1qrO&K=d6aZtBdR zIt9ep!olzkt=jn#iN$QjPPQ;@Tq>_EtY&b?kHWgE_eByP^-4^~r5Cy7|M4K^`cd38 zr+GW;{XSxR&zEnUf=qdB^mD#^ZmRV>R|KKSQFP*IFM9(uWXArsw@CZ-iXgtz`z_3D z*By`vlt z34C3UUZp+c^F;B8d_1B+#k!gb%c?D(UN7OlxbZKut@lL`8`D~I1C@moO3$tKYOzfU z;%j{8Rnk_=V^W!0(5i?%XIrg+0`sE{KEnmYdRx9y;C`xgmwLr2&W<)_00g{4{KF64 zUXCyYhGHQp9z#vGi*B|zzR#2mn=QrO@fMjzx8M!x(GtXW?2(t8Jk%72biQ3qvPYAq zXU)+FCU~a#utiBjJ?Q+`l-Be?vfpF=Vf<_M(*DwdjR6eq>M2Pw?@5?Glmi1v+pF2a z1<{!og-K!fcQ_R(&?8FeK>f|IDGL7_45K-{c&4l2Hh)N)kywtCt%%n5=!RsJJnYT| z>p`jgKMLAfCRWVvr_m>+G9{hwA1s?q5@u8Po!xbKG=jP&?{iI^h>_A48IJQAZ@Bd5 zV#Igr^iKE(gR*x>j?f0vx z3GS^h{2$}ve8<1zK{*P(&%SB*ktXbgTs%McS&s=GtEAC5gcsQ`FXrs_5A*ZPc7^h<&DR60k)!R9* z$@~xJieuQAphSa-tof_$EU0EAmgQ)A;we2^q!ospF@=ZQ?sEF2)n0zun;b&a@+rxj zt8Gw57+0tYHQc!lJj{8)A}$+eWBGTBdRz1cW6=d~zVYyGG*s(acLCK)`w=}7aWs2F z>>pP22CGvgXBm%o#nfOOw#$8@4BZ^)Dh;_f))>M5%N@%6^m^uy(diH@a%)jlLVTxf z!MwVzS;BhW4M73S=e~jTHETz>0;n3@goDzRxC_htAJ9_jt#}Dr&T=X+@!e2P&r(ky z?T^Dcvs(g0E<9!%wf5?eNO|=QK?wB|#r};(6C0x#5eDGLX?K1ODBn*sjA2n^I&7v0 zEpS#13QMCZjL$#cF-jeN)CexMmN?lCWWOM!9j%p!;PO2j4(b*5sfpv_`DiPh4;@ce zBiR&oTLzE9pmCtHQK?v+T$@1D#nf9jNn8Q^GI;g+elBuEh-h;&W(+3Y@*QUr3N%j? zUo*F!B7I+JzCX zWQCBO#?-uVR3AoTi>S6YE(b1gou6mEmO0AiO;T1;W+-RuU(mHq@!5WF9hN` z*%+gkU49Y#*dJ4|R~ML?78cyf?0qc>!?>LzcwR(a$MT(Rbv-sXMvW5pPn9nz`xQ<( zFu%#voKoduHsL1no&8~~xi)p<9+n0*h`djQ73QS?_^$q&Qq|}8*no!i2Fx`~Knrlb zWXSY&;0?bA6ss?m3+p|3wPzwaKRXkmkMqaIDu?KRxLX;Ak`W^_k;#v2qBn>B(te|J4p+aOt^5u=(G+$C6MumQ_$`W z$SF!^n``iqTR@SfOGUo%d{+EW&^c0nh=0T3&)gCMj`Y9Ux%Pi3_wE1K5+P%g?8+rc zP9(J5M@D0}l^BXrE{#rf8FGn`!BApr@1o%*#B_TI>C+&$_(V=lS%}AKTP)bw?uv%)c|`Z1kgtYgYVs)Z#Q_ z@H1k6^;UdpNp*7b)bn==%}6SiM$$JaOTw@|{D338qt+ zbUTSGs7l(oW0#=j4PNd~`$TAiu|*S&qNUNIrFr)m>QeLj%f|+`AqG~1T343Jt*oG0 z*$A)HGjowWQ!Bsm#T*C8c<5#F7BKkRL%&EJc3Z0zAHz<-5OA`f-SW~wOymj*(1C9ExfAQc%M7=>c?DUy()Sc)H|~FWi&;Zw7tudS8ptCb_Jy5!(+qGp&&esOLFsr_c4|W*r*rgI)<4xRmJQn>!aOX+61ecG02?HbAN;(L! znbAb4VY@Ll#iw|HHq3M{v2BKj9;XrWXJn*tH}D0JEL0emGfvJL+R@7PFtm&6rMA_s zgiYCUWwP9y-_1pr%bFE66uphZ`>;@y)*AY>us~jgPo*sR)jcBZsm+(Ky|+};p`ZD4 zMPj6CZd(^iN$^XYMekGb1mVgABjNK{PFr3eQ@Gt)$e5dGL`7uaOx7D))f40~r^lJ& z#OYF%uPB}>X64oiVQC;!|Dk|C=j{(~i9Jfdux2Sz*02{;owr>5E1P~LN0fx&>>r-O zge}gdj-WaaG8E^w72COlyNaw`&*fVTT5k?u5~0d@A3pkC$<3 z>esm~u=;XhTx&ucIj_?X2M5?_yt5!u8;KDITw$GL-o4v$R<`oV-TJEsA{f*jV4l{= zkw;_n0%P-jy4#9wH67R}%C(e<_-$$}J)A0E^}*r_HBUd_4|C-zRQaTP>|+x+sobPG z?d{u`ajdKvF)F*)Q8IkIGk(aYzlkg#d1Ho-YUo{Q)G}y2{BlZ`L{k^DbksDz&N!(y zJXh^KMD(>~{j*yjU@&F_H93dsmb_Q11tIDFt*vuhrF8)(w>K^)?6}bSs`QCQ1##KJ zSJbDT2R#N!j4+<#yM~pi)?s4T!>qs~hioEGI8S}T^e6T zO9DMkT(*AIq2l*s35d|~gLacg%^hsi!p$9sBr{9xpMPUyPKPH(GJbq5ifj-3YUpkM zM%!QfA=J(c(^tF*2m~G+3!E&Slu`D?!dT}3G{17n+lT60<=w2kwK9T#pL!SSzw%;f zJZ^bgEp+J`8ExwZTiDg-JNMDFly2_>=l4u8D_uo(rca0F0)>x^&|g2xhZBeOf2Dr%ho8seq(4c4Mdw+j9%(m0JMM(W?&VJs>FcS zdArG9<^$h=xNKhQ+4GHLB)6b8Zg6N*Pv5h)b2?PtwHVOQYHTZCeZ)9|fA(V|XL4@pbwB&U3iJD{g_)WMu$W zdI2EfwU51{9Nrb4jukCiRdlsLveXtZNS3Qes^{wp{IB&>7L8r_3vZ^Z&^r-?z_JnB zlMASPfUS515^DJ(LsiphH(_uYHmSfj8H&ug^8m^d%Td@W_GZ4G=DR9JOb>iLm;BkU z;x`by>~uVp2&|o@GVJ;yjQo!I!%_UjirLk-T=+$rhX|N@Mn|;zJ%}G{SU_F=*phOz zt>BN^2InjE3w`j$yn{Vp@80pOvPKCZKqS&IQ*)D2du-|9hE`XAb7vevV60_6mn^a& zq2N7jloq+#d?%#gFZtSu1!vu<08llp@?y={jIYd2J2;_23O1Jn&aTuk-Fu|%jruF? zmf&ZkQ%o418m1I_UD4{pMx&9Ec_Ur1vlSscabgXxWRy|42U|8&&&)q9;f?3FQ=L8l z-anh;JI;MwT^6x)e;kyzlRd~iCVf`yg;L3!A>08mIJxP=`3g_bCG!g}RsXz)D|%#7#hJVfptKJ86~&Q`a_6lw^WSf{Q^>SUl5A5K4^HIpbmy;c-ftY+W4 zC4yu{Lok}fN;s0K=I5HtN2bK0%!(7WN|tAI!!fZ%sTOOw?2odwFC}HZHeC)ekZi`2 z*vCVR0ZXgp`+}bPt)Ib79nh0r`7}S$C3I}DqLoCme%A8+k_O)}JKacrdFmEeob5i& zTK!?L{JI~8?bd>;U`46_JQ-v0QZW>IOtk=hZ9CWlX04{ zvlQdJL*32h+|ad2uV6HVCidVWr=byV6amNE=_D6=30`GiU2OrT(gTH*T<)ZN3JqYD zP-JUs^XLmckM68$Uzf}XH|l)*cZ0Eo;PqYbfNS*f`x!&A)yh#7aur=>4DJ{2`D$++ z0e2ivq1e<^BJa$J$9UIv7@flev>k(^AX)7(0x@5xwew*RUKITdMyI1(y8pKwG*UkBoKZ`JONgSM12nD|$bp$CFi ze&B@eH638RXQ2H)bsAuT5|Y%=`sYz<>n1^Fb)-cWy?nGBG%ba{4&%uRSOOK){CZ9XV1s5hPq$zzhD$w7JrylfV@TrT8 zfGjx9H~k5?N}Ub0_K8BqxYy&_6~&kt#2~(1*~XvGSRRv!VHfL0V)m1WY3Z1R_M^Cb zp_ci28|5XRv^Xo|RLYzH5I6Fn7DcpvF(c$zWvu<=F>c6w_!S8CN?~H)jo&}t5%`Y4 vcLe@N5QvHw{~yuhyBhyZ1e|nY=7b`428={^jkS~wO->%i{qhRyap(U4PFLTJ literal 0 HcmV?d00001 diff --git a/docs/images/UnifyFS-logo_sml.png b/docs/images/UnifyFS-logo_sml.png new file mode 100644 index 0000000000000000000000000000000000000000..acd6b2a4dcb997baa0eaeb796b549979e50f4d89 GIT binary patch literal 17422 zcmeIabx@RH`!M$ zLB#qLWZ+54wP)ABFG{yZ2A&WkbQS&~^39a>5u&!&hlV;XO&_Qmy|7sb=JF+a{(xjGr_?>s3CTG`KHv7& zGq~1+m2=Q8S=N;VzrX31dI#Y6zws;o0XGjVlpYZN@MpLJML_IGH=miWWav#l|G=%c zHbhXF`fF0C+oJxel}Nt7bZEXDWuBq8RKKjiS8gfBlus4hR6x$WD9IfnnRC}tL#06mhbf7Ws;ul6M~aliwJ!(Crr>_BxFHGoQjuL?L$8h{5tI-c zdQXG9zFW_^2`r3M?21lh=a7lZ_|38EhGhECG@wNRF?N!v&_dSTZ7uR!Mjf#c7M*PX z9}VlNwO0>+(uUKEx(h&%-?au}NR*rMWzW*=uIhuR5NVPo2#PUM5P&!x+131{VFLAp znmT{dd5LpmQGg(X#dT&VM%ik6RI6a;Z ztdP9L#U)xO{{gsTfK~|uyBZ$`f*w5vYppvB$%;#jhx%*-v}KbW_#mV>1E8TnQM^1# zkDz9+fN8nq911}KJTStuxQ|U@gnH)cysWZSel}@Z2swHlCbgC%c=JTx3HlK5)jf0#`IY#IK@0A zR3FwSYxoTWA=_c30IU1}-M~v!1KbpX4)tQfvh`QMT8>cASO~5AE=_#hGxe^h`wjL9aIPn@AS9VHvqYTF z{@8b8BO=Bw^E}``ozLd|QeWdHjp*p8XH)v&A5h`d=p!$hUaivkYta{lIXTqa*Dng+ z`aR&!ZR)n8_}V9yUi225Prg0a{?xL-@mbHdl=)>MTR zX9=UzpR;4dR}EE0-oH`&Ekl!k=|R-8lwZ{tU1F$S7Y5;uQ+E%Fa*#Qbejbn=)3(+; zn1t3j#mugOYo8yJ9N~>xECAA(E$MrC-SUVq63QhqDbcO`{bQ(}26x=~QvddmH)f0F z*P7dmKNF_?s2@{!rjwYR5@?M_^OTH}UiFZv!IJu6bD(usegI%}t?4L*xj#^C&RQ@^ z^>Rgi=W3Dn>*w7?&cUW*6mwBLO{qx{dXRtDN+^WU<7a8%YCIm!B;=JEdg(~9IZa*h zb>(jokrrhZmYh&U@$?5+jM>Hddjic-V|`Lud~kV}Cr-*a_KckYMZSUuZK`qo9GmL8 z%-a2p*@X=zfR%yDk*?ZluNXb0TX?V4hy(mcPSb01&KF;)f8GJP^%` z@j8!#vZ?D@wZXN8uP;X^eZqO>$Ky1lqQ7SoX;E)BAw;F6t3Op z0pO&}FTC4YhGO+B}Lumhu9GpWJkZ|QOnbw z)z^WVQ}vk$jpcG3(}Caqx-1PpYD!VQdpN(&j*!UAIkOmG7T^srMgfCT$VF=sEb{05E!zQUW(D7Q zV5@k?EtQnCmwU-^JTWjrjBPydD zMu%?Y&v?LL>=}Lc-8vW2cqrcmSh;zQtH9uJ7`25RfpZ^>SQrKb+*(T7Y4K38Nc^G7 zZy|{-CYH*EwwSn>p{-Z-VDW%>Hm|*yvC5*RaClJfb;rR>H`hD18Q=z)K_>PZD%o)c zR(vDSJU;;6TyoIA``#bPx&iVcD%W zeaHK;(K|w`6I-F6GgyVR$M#g^_OQ!gfAP`Co1FuC;oS(Kmn#dnbxrJ1hT-jv3$BiK zywZa$@p#}ms(DEmD;d1QR~?~)2pHyHyGla}*DKC8k&$C3pEBd|x9Wf67`!|6HV+!0 zW*%@S>#jG}-nu-QXKxiV?$iq47#+Bmst~g#Y|^6c-+wC}5v7{|GZ-Vdj`#f5`|xA& zr=bLle1DJD;G3e@;;fQFPt%tOJkEam@Gu9%CIOAUg^#t8On6Day76}W@ReemRYb|# z`C4Pn-$Nue^(mJ<0MhIP)+YeJ&=VQ~b&J7-iu(E(r&?O&vP?Z!{!ydsykM9;yEgyS zLGdN)Y7CYh8O#Y7y)#@^%pl%GQ zd_QFYlwQ_TELc(25q*oGrs(X#tGcPLZIY|n{`EX>O_!C*lut1;af}T(JDo|$Y4hrG z`j&SIW0s!)$=JZ(!J61!-mc%N*P2C_*pRBggUa6n4xWJ0m6rjKdX??i5dm*87Cb%c z;1-qv7Kz)$$SBw^GinUSG1k43dwCQnT&aqnW@5k_`MRy|nVS3-u56}rQQtZ&;@`1D z^KXc=Dwv@;u-O$wKSRn;!bWEIVpL^ z;k8A79XLCcn(MO0FcVQaKO9jsGLE$U)Rx|C8hFwYo5fa;W=OevJwtN@L<-Ml0R_m1 z_sg7V3N5+&flud&R=q5lSj-vmc0H`0ZGAOZ**kgK_zhz*ArB+9h#DwML{W<1rlV&^5@(8{@H61l7x1Zrl0#bGl7-bW_Oyg8!zzKl&%4)|5@2!M4%Q;MKr6TR?-x+Z>l5(%a_iL=e?#CZiO^&v<+W zh#9bF1pHjNopc2M^aCbogwm^DPLSBTPcfBHref1NSFzjNknuEnS!&-Ywd*`Imcc*` zAxlabX`|n=sq(oSX2jk6{o&?fhWLv0UOQl}eZ4SL_8i7{vcFqvP&p|bflsy{ZgCUb zEKKD+*e5YRhWQ-&v67Kp)tkL44q-(3(uDF*CHYgFj!h<|EFF7-M4y8VJA{EFYk9k; zDpSDdjx!d;p3x7waZxgaQz=ILqerRoe~v+V1D+4n0f|g=pei&RQxZ{& zE&_cH~|-M1IWfAVIvHLdiseLU5o1kgxnqbG~)OW4Ga#} zzXF^n3pLaalD;4Q)v42)mAGnuv_%57kp7sV5fzSs7-9pFl6iHHj?=fC9{t#FxEw+P zKPunc%if5JRj>*h9IfHN-JXd!A*$;#`7fYM%7UF|8JRcLa08AYUzj%nc05qp2AQ1I z6DjKk0w@*75T}X90JncPG5HWVc8i6(>)FCvy6Cqcz?XhMtz9fu*OBSW+o!DWgvy8J zCBWk4nAsQ4LN;A&oo}a3Pk{y4%>5em^PC#Pnp~d@b6%)vkC8)nfX;N&+8ezzRYheD z+xl8Bz4<6snp(8s<_WOqQ~rFe00#)2kqzR};gNS&L7erSK;Ikn5yAjhh4hk=k(Ep+ zRJ&%nR09Wu3reek+@2+-s_VSH!wKo!DHO?pt6t-gk8Kuw00jR6f}ieUw*wP~&4Ngu z1xE5iNg*$SqH4GpJ(PHjNk1>^XjI#fGJ|`WHo2I=`{8CFis!>k^wlH>pdWn>OiR3q zV?_GG@J^2_Ic~~d&sobPmGJSGTm`C8om@zap8Br)YD9%N+ndb5kh0S1Pgn2Y%I*Tt zk-Y2J5a;)2^sy&2{rIy#D$YuTLB)y~r%UV?r4l{L{2vLGOh<)j05~biwpqyrSS`Ms z;o&42decR>y{~-tIOFvC*N;m+An@Z61btVaFpQ_arZVAxDyLf5UL#PXadEocu$CFR z1aE79|6V10A1j1OdQO0P-fjT%#^;EcaC$Kz`XEl=eQ z;=j1BncdQul>}o&r6tWCm%&b=wOH8t{%xd$IMZmn$>c3%MZ=w;C8JAxf#lxkQ*GGZ@ z^l_LVOUJ@XRT}5%!l@`k}*fBOT`xqWxPQoTu&?jNy2w*YNQ?nmX(fM z*C1uoGK4RuMzTBGXnz-FiDep-O|w^sHZ{4Ky}r?3AMOzN%Pv$6aXPDVrzlh{DlJ;VGt_xJWi7RbC=Hd1 zHP)|T7jF1K^}ZdHdsq$kFN37m`)fxwH=IMpI`6P%6?K!vZ*TVJx9%q`^hW$J0+Jmd z61sLj(kD7EV@yU;#}HVP?(vzqHO-BpJzG;1;*l1ZJ#?h$Jb*pc$_!4+E-D;?+CIMlmLJWo z;XW~_Y}b7U>b?zpBjm!LpyqwEF@eFH2y%u?LFr8P#F+FrddTh$P+28cU|Nt- zAhd?u;Off-^Rt-}VlWc096wAG{7gE%r*_9t_}GTe`t8wD|9AR`~2&0TO?P{-F=l975XWWlOMv zLg4vMcBuO-36KRyEsAGVmR(`6sP#>PG9Il56q70+uK4`FdCi~arUp!slL$BoJ9d_C z&&eA-+~%dGgN_vewn5YHv(q@JxR$D0e*KK5!>y@aSET4g2L>(}O&o<&`&Y~^?Xz6U z8=4*1KIRIQX*n$QulB(|+c^HUX1GzZ|2x*Ke0LEYbP7QxaD2{ga86lVXnxi1K)+OY zI!TSYG^XmT~Ql-UkhYnOS6gt}cn+2`L44k`>u zV}PvF05v4%0a~ucKUQ6-zWZNP$BpRpE(~Dx-agPVF`jl{!A+LbGO% z6$Du}*`DnjQLk^0QRLFj?!4Ja+pCacQpCc{&{mSW2+;$K@@L>%7c0x;9$fRh&YIC* zTC@L741Bum2KJh+9@|u$v*Xq^>{XKEf*50gbHFRVKsUuMoF^cTGu%ZuNCXMIytj0X zDRU1d-t5VwD^Mf9072Be@d9mp&Tj01=x3Rl{xnA8KU%*7ChxwSBDw)*8Reha%ls8c zm2<^s$z`5~_v5_lK&`H=Gog`Q5i?iJN*wbPsGG+kKyG5<8JC88L%D%)&|=br#FJui zIUrU%Iwnk}UnLj6v(XYYv%(NR=$JHiFI-%H5q2j*;n( zwS9+BWW55#P{V+^DDzU4QJTdX1zMO^$;50#eb!p5W9rD=SRGr+B52+bEIJszsv%d) zlT+c3SMGUHLst~4hk1FE8d(E^ymVnu2kyGYvKfOg3pZ$KN4rlkEix)%>TPX_x?KTX zx0sw?77wjoYj53Z#}v((y-s4RvYTac_Lk0RF9h}YpnJfNO!&_Houj+1dqZf(KKt*r zX311sQDQM|jom9X$s08%4{JPD2dGPvc%b8lVD&7wXWn$!Q7~GGTG=4TtL-)d6gBrgKyX6CWcd&7+qA} ze1U$P*B;QgpVoKit|kO+7k0<9OMwtQA&yi&%PIddWUVH8xFAI=;BV&|c=mc6(!?Dc z)fG-}%4GhYXgjSAs8;@fB^1NWR{%ACe3%*>EIkxNUpReVlplRnm-q3ZdEl6wMPtha z#R;NjI2TG-Ywqo$<960T#S10NO-$Sr5TNA~Q0;>B9QRW*r$9nZX zjv>U~H6Ru^XnzAsY(4W?q^an6YGkTEAlmNr%{+M3b~mqIM{Xn(DZAAGl%vXS3YzDR z1LRj_AiH3#R%@WA6c0Z1If8Y);R00`4I{3X?~s-ssJq-dEu&gh1AP-p#_3>fe>a-8*@=>s8@y;0~@7D zvo@n5xYtqLOu4KBfj1`OS{dm-2i9#roO2`bf$yIesbBmCBS^X*1KcH@YsFOSE=vOv zRg+a{%Lk8>bU*p`JP$mlsa`MP6QFA83i#*CqsKVcCFLr4;oROOr6)kBtMv>wT&@)`M3Wh2nxjfR9Qvj0GgD))c#T1h13u;pQF zeCn4Rg1}uU1q2Z3Ut7gKC7V-mBfhF?bg6fBqJDnSN8O7cm46DH^8EJ=P@HSjGSUAo zHxVsYemEY~lxwR-PaK#Ugk@fwPKTR+zE^UCTh z3kyaBGT{QZuz* zs#tX)0YqztbkTtTCT==r*K=;XU~Q|~@OxN!Yv8dU^bB@Gxrb{V9L0fCbOua5SCdoJ z@60WaMo&=AbtRpB7!pZ@zyPo40%u2F1=|bDt0&a=n1j%O^+S;ygo&)}m#ek996Pkf zb|s=5%*-7=B9#=3{GbQzWVUDl95F7p#v1H-vG_ zAN*99^hJI+Xv_~&6w2<;6gKko+Rn8R3qYcfai2Po!Dt&kNmQHi1g+Xv&XT z!?t0J{AsIahUIH1=psmSIJL)5TZY!^G-v{OSX=dk!V@&c>2MAQCKSB~Lw)2^W+xrx z{$9*mT25?L(sx*yE7vN>k+;A_tvfU6#`{elX^4}Rb(xe866#-X7StRM`_z+js0Q%2 zUv&JRSpb;YK{q>V6BClqwRDgHFms%-mPoIY?OIxviQ@#)FKFi3-2M?mmf1Ofk7+CJ z{u!U~?-M#UzXYixBd2e~D=*q&Pc<54Cp#flPNehFD80m*0w+@~oNNOSXk(3*CpsdTW5ia2K~l&g)odCbUIFLyz;*J zO~$fqGLx?J)yAK{xiqe3XzDa$$`JAO>b0L56TOzr6}rsK^_ghB`u3&dsby#@j;M`P zUhYktKF7&S~48w{f{3%X)ZGze=)@N8WzzM3b;Kw<<)Wqp0i^W{2`jr4V;r1`w z19-HDXmNF4$ftGcxW=_vjR|VGki~lc7_YROkt5$<>3ROSsh4xDE^dzsc>;pO^t=w! zgKO=w;k(8Q=%BU;7W2%J!T;L z^>=DmWc?PKwp%e$vYX{M^KH7Xusrf|wO=ZK)Py}Njj?|c(`!M1vLg=c%DJ4Ff~ zv^&nWQW2mRB174aL7)lfT-&);ivmM_yb3##7`WpY6?f<;b9bET(?kGTR@kx0WH;2g zautyhXB9V;;7`yEG}4=l-1|$dhi{}7vv6CSbsI3L#8>Z*WlHjATcXWrau)<&W@<5* zKF$Zti`o{Fm;ldDLVd2RWVZ<~G0uuU0~0-`%=~{eQ_FSe@*D`Lk{qVl+1dL2O^&e5 zIXPM>Z`yFPqp-0lvD*BRC5)_|fj)*@&pime-HbwPX%oxe1Im2k?VzFepRYdkana*w zJ!?TyLm$fG1J$c#5lni7u%AC1rU=W%<)SRX8?iq18X*TRdwl>&leCJfqawvJG_)?Q zJI}R!1k3okgmKijPw6aX_4!1MmHNb2Oy)ytf44n-e~910!lmS-r2n&L zV`9+Jugtaeho2Vrn|9bMf=RH$r|!LHwUyUxRg7qjh8j$8zmCLcfg=kHCQeap|m zPx+iKU!XAZrX}zfPmgpMg&C&ME25Pd^cy44{z;RCjy^0KV>j>UQLc8Wz1o%goSU}E zHbP#{rZ|FUD^PYkG#@NfTC|Yr;K|S}H+2>_o3OZf>v|<+9yCb?j7wRVex|t8@srQh zd+RtU0tuE)ahne-D7DfZTt+66Z}>Thl6v#c!6D9Yd-&yr&QvxrG0&+5LiyT?R9auO zecR-$|7Wizwvh!_pXFh|#=gR^uttA7-MkV$J^biV;IVcRKXwKIxj6~iE!9qH=dHrbg^x=BRvxUUmL6hp;F#kr)at0ye zvgFGvTDblaXCr)ko!Yi0Kf;lz+R`yN7;#f`gsG@5 z^3fn!*etJKIakvv4nb2ycH5OQsbqzM^|lKpL?D{E)pPp?O7zZo)MGa$Ic8L(3QvP> zfJ>tT);Ds$eWc}5)2`20fuPfPRgUaa$HjiZ?R`U$rQtGnMIm)13?@Q~>}Nj85F1ur z?(tnfr+ykgjr-)Evpy{Bw@o511!u&Huz&F=^TnEc@87Z^v--?LZtpswTZW0rz0i8VFufpj^r1bcT{ZU$w*TQ@F3n*nLa=nA zG&WXEZP|W?dSWs!P6wsHlLhQIx{OgFzxMfATziyDpSf3YQP$((nz#Dlf|;vPsS7P@lB{?2M$XVSD&JPDWk5(=#?_~veRqr6bAYPP;c-e=JlpyS^&Kg%L*elm;oF_g6qySf8);_=^ta>lN}JAp{IsI& z;Lic2&jV4CUpecP8_t%73Vkk!%Peh4`XT8p#%k4v{|h5_7>$De38_ z;II+tJDR6^S^MuwkBlLtG~6UBf0CCqY;aa__v2!m&U*iH3`*O9)ghzMi+A3kDur?& zZRBUz+3k_6jh~sNo2>QnHdce_5n#xP5_p>KPu`Rm__6G90mp^2TCCI3_^ zaT|{n^i0BMwf@X6FJ$x}g0|oqK#X6(S+}`tKH!+*@()w@@#F}546{oky`!fMr&9te zg;A@@6zIvpom!-7#o=?03l($zc1nj0nvUM)vh6M@wL70h+9OJxw<82=uMk5+zwa|Zrawde9skZML8qMAbb1pK%Pab96%>RB&56#QKh>KOq z8BzMk-Q-vacB_#MO>z3}@;z**ZvQ|q0{d!Y@p=F3!65`yPnOa{>1a#y?Z4!6i&MJE zUP?P1u7=+yrh{LTM6%YGY|Q3cdEa!(bt?3ScaxwP_mjQMt- zvXY!2B+m*{uVgT2RblU_R*J)& z%+}BC#MgIL;%Ev-|I`6w9c?_zHW#Auv4ukyiT^dHAWVyl8K#dv&+?FSv~?+Ke{F{? zt*;)VVKR2HM2MmYLTI=C}iPUK+<-HgI$owLZ8n=$8wPo}*M-M&KDP)+?h2w+U!7|OmB@%Wb5K z(A@na$M>o$xM12khB7kK!Pl|r;gp!C;*|Pw7^LY7atO$2Sc|%L-^UiFHQfi9RXX)o zIh7f>hkiR$TXC}V1|Bkdx&813C z!o*eh3={!hf|P#F{P^cr8x^(O2BOtf8VCL}$S?2?qQEJ~IR6+@Ke18^*}$s5gbvM$16b1Z~{Nmm#d*y-*4i5 z?VH53dRXK=6)tE7w3;E01u-)>5sh`@9xw@84pS((VpA3y%BX?)NepekW1)AMe6ELn zNf^^Cn1y1rfp9hRZsyx!SI8O7&Hzs2}=QZEY?{p!UoVb^IX(LC5fX=iS{YRu(~#@@y(;c3*~{wez@|P_ax@ zTMqp-Qh7NV21M)@Al8j^2@aJQB{>wiKIlQG$H1bexf(1-W*`=yR&$O7lL%2mR4U*O zq}Q9jm^E;%kd>3j7r=5pX&JNqD~TmC_)>HJu%#|3GX$Clkb+Eu|1LSltA!WJEQOgT ziHzn%@>8S$gKoFKV+m`ZV8N*~%ba}PKNqCojr)xUMi3I-ml&7V{wk-|PtJJXuHP@? zspX+LuF#GMC_8LpL<@ozmqRw69A{O>$w6?9ySr>sY(|5fUP3!$8s>o$%Mys5l0juD`tQ8F z{AHJFTptb|53yO)BSXlof~J4$Treh}3cT`XoQR=PAc&A51G-Q)p@LDU)<=BQ?=&}m zk)`CW`!_u!e?dVZ0o|Rr89hio5@0LO7grv(b`mX1nRc|KRp!|RmWts84C!B$2-3? zfp*NwmRW^lz%r3aIkI|*#wXZC@4{f=uNJ}23zp&n z@hpm71=mZ7p<$RD#kw>iLbV~dVAd)tl_1;x9Vv3%r+@UT7x-u$)66inW;}IZB|IMg ztbM7`<8!4>F1fx)^YBRyaBnm6qKghWHr-YHQ2Hi(%_dzIhp7$nqV16%lRxu+ch==^ z_Kx6Zmy?o@WY3cq>&3325ZYbCwuFdn}TT z>J+{{r7PZt*K*?qxK_ZexGhzh;@m(9R-q6H5Nr?Am#hq5E?H)$d0Bazm1;s!WfO~1 zS$;w7E*}sSP?t%)0#5e}%Eg?aXq0v2ttV(6}UwSBnCS-V=xv5ofSbP@rDbJRt+r|N=QBlZUsH2)9? zXS!>gaM!DixP8rt90WtN)3Br_U9n}Ec)p+2VAn)k}X+CVo?b{IFngPvMkdWFl~R7VGzGE}1@S)Scm9rhUhzd57-PXZb9?Nv=$2}44sJ<#Iv(2QQ2u*a~RyG0kRXNkq$3CzsK5!W4ru={u zxMi|zVqkfZ7}(eU-;BTmtMWgF8T^JDFyI#_ENkHP|1>}Ue>VUBm0;g^u~SAOj@hae|jx z=6~nDOx#z5Pp^>aze;gNj$SPI20eQTSLW}Q7Hd!OKE2Gz;ed17a-@q=HB^Tx;1ai9 zoN}iAJDWXW`O~DM<*sOHN4EcTtW3OIrn{_+rmSYIh_;Fnnce@>ZW~BR+S`20A_pC# z5!A0j4mXD>h~|ZW`;c)5>oq6~p#u%U(E8JHmJ|NXvqbZ7lq};3e?$R3YW+2_kDsSh zA(Iz|>6DYj+4fcd=_GX`Wp+WlnGKl=N@U%hk7K;#V@PAv3c}WWYHmte`hgB20F&Uh zG7`c}enyV5)T#6e_&$#Bgs?r=ne<&iTkD_YuBSsfTJQf@Ql>XK6iAjZvk&j8giJpb zg%W@XI@_b&luSr?~R;3TWHEV?j1Ch_`*k>{`qi4OXPgzb=sq~I))04aB@d8Ml>A^=-Uo;pdGd{c8fm=uE8){(oI$PsWWh@?-K$7(h!xQE#oc65o+{tlT;HwPG8_5B z^sm9rsTaJr(B@v#mSc8YU0cm#FZMR0KSq}<;u|yGnm2SV_;esm)?$jNnHun_e5*d> zM8nc?YRc@ex>n}H8G`~z$2pVh?=n%I~RF4yKd}e{>gi9vs*sp z7#}~ed%a1=aJczOiAL?#E4t(376>9u2TWEts5?Eb)woWezWO+wsUGtXcbOOzw~Pr6 z!pBptM|WQELsT#)ro>4NyMov0P9$ib%jr{3Stw3Dr+F=xG#!Zkbn?iJ^xv1$kVbnw zYZGtc3=S1@w%m@LJ-=tBxD8o3^7N^fz4gZ9O3tf)vc&1;Y#GwzvG13jj^sd4AN=hM zqrLIJA*0`WB-vNDGv$uDcCK0U>&2E1(>>q*tj>pB$RO07t{(ZIu+Zdnphz+J-IF1? zL}hQOs#%qgt+YFM%wcTlVT$(T^-Igo^O{Oe>s(KE`LIyB5(tj;vsaJJPDc`EzHB85 zpJHn(vuhTow9>sEGPsD#k}=jNTq`~>2t4FC?iYKCk%BNOAdr-ttUR4Nm5f`u=3TF} z_=#g*c;|W%?vg8*HY(L|zZYU~V44@!qG?ElRuEK`Igom&t4}1K1Qa;Et>zGWnklk5 z8?SrOY@hD5ZC2h9$CoSRA{MK{Nw(Na@l{c@i0@XA(?Y?-G5HHVX#NUN3G2Qgxzn@( zg8#Eqz2M4z{1|66B6wAmTQKO(_pqC~73?~M)x)ORQ#q<^i1Q^N0W{M(acndpZLUPb z8FE`+8xCFy;mi~Mmf!!0QvN@WQjpSjt;C5q(_rZ{)9z=6kX3Nf9;^nYyXVCtX`y|1 zuItD{MB^N#D-mb*zjd}GPwt!U{a%0=IJ{@ z&G7W{%n0S8%DFy;&nIcYyWa++pqbBbjbSM+WNkqEg#gv`UFzf0QXk>ACps2TQ?0PD zIXjo^^D^f4GddAp`d$+;sXR`VFDm^w85U_w?2yA>-wAzK*UDAoHHdbNd3Yf$9V?b_ zP3*kpk89U@Yt5LLdj8t?$Q0R~cy-!C7hJ9|Lp0@YKSxq5o)Kw4dF$$Kj$8cCQZahA=u1nK!=yX*@}XC;!u{U;b3vQV5z|FuP^L3^~XHN*sJ%Hx;uUmK}gJ zIRMbU`Ii7x~0kezNFQXF{5E@bgEWE;e<&R2Qjq4agk}{&t)}9?%0VvQ6~l z7gJz9G+-)lt0|bb4}~WcKx7qo4s=NW66@(UIn)F0c(=Sq}NNc!d0prlhTs`7j}uSApuN_ z1GbaXv?ne@wOHe%_kwaG{tO9p`zQAfw~`;(;CI&82~&a|M%B1=Ot-dR^4uwE7EgNTH+bJ909E zubUkc+|8}bt`@v)SdjU|!|?`p`F)XDO<*Mx1f41X)Y*snp1#VeT(o%?p??K2Q7#?a zz(+^_xiWAo>U}Wu+ZcBYR+#A=T(6bY(dAvPSyY@sYGJm=2Dve*`MKZ&ZVV zzYFZ0SUd>Sd|-R=jm$dq*SvAoK9D5}!FAOHzlZlAJ>N9tqS%Yy$5YGlz6LnAQ8 zC1W|8!ndzXt1^iQS`vgT8= z+Vh~$Y_i+mxK5JrKUPPi|$wZJ*>wgUyF+8Zxi~uzA!=g!tmiW zJ1c>dMnkM6t+8~Kz4VK$1+5h5mA)UMc+?p*Ay4DjQjH(LgRhjaHU94EyN#K82Hch3 z)6hvycq^ylBl6q@RyTkWFn9}$B5Cl~Ixo>nYrpM_{x&rw6)Q+8{E|H|9GF-SRQ z%6Kh?bR>mv90P8HHS%>q)=Z3;KiD-ZB0Ptul>Ka zuMH#3)@!f!JLC$YY6^}I#cHYl`A^t~{_py7%6`2^-uyGX1LcY7f{}czq<%kF!TiPl E0JbE_B`4iX=^8^h(jhH1 zx?^;YeJ=3zzJ33N?+*tD&H>Nsc|GF(xIgZXYkqjDu1I&5`78uMbjnJPwIGQ0*U{(c zQ{a_Yv41@wi0ZB7qeoAbA3eI^;AC%ZX=4UKT%kUpl1jBtue_tAKauANaJ%5x@HFAx z&O_F;0Qs*^2hQe6d>-1MQDY9f!l?MYcy!d{+x+NwZjOJ>jngu6@vI3~3{(Gg4u2SU zU88Jrf-plV-!_oTjE=E#`$(N_&eZr^yTw%JXAR7IHUr0S`I6A4hm`Q9ckV^u$Od3AKx;}70_k1q)um&kkZ@7KS@E?xL5 zCE!xi*9%Dmg@6yYmlUNB9tF!j;EOT1bKmLSmlIdRCTKDKBZVUC&%*7r?w@t3{=HRy z`oeD-W|6@|nUbZtGozSO`9)C#En0I~Wn6L5%Ee!mbOr{c!k!FeLsX}*E{&g98#PoK zW?pX#q0{f(cR1&QD|fbDp4gyL>dK9>_w#>bTqM`Oz`ng~g&ibFx9*ame+HbsehHz$9bQjOH$aM#fp@msjGsFGQIa{vGSG9Y=m zx^Aw{Sv84XKdJOeiF@X+>ED$`rs>x|3K*Yp+VQS(dK(c&vw-EgwRKpxE+vmZP=Uq4 zhhKBP`N=;|p&MVQ|2&hsM8)>M*TPQE{_okR$8xlPp8DNBA@RQ#Z(scNzYE>?;0HE# z{Goi(<$uqVZvJ=4|7;2U7evs1*#rFt8xZs#UPAvB3JChILPGyh1L!}R1^xe!8iWad zgv9K#vzr_`HFVX4VuF1IxH)CxqQX_vCn9=;C+6-WA}8v#vAdKgvzUVTgGQ`4cRhAL zK^dVgl$Va#8c9J}r*g^c%cLeNG0U%O5c+%L>IpmC#mR{Zc`+(u~q%3trzAiamemD9m)w^-7|}L$C#W zHI-=ckZ3DnHPcfE_n?e0(^GqfKr~s!X7fBlBC@kpfh|<_=_)hRltPm;(@@>vH?#WY zl@X~Z`&6aXxzkt$e2h6h!5klnOaWiR-BQ-|ayrTfty~T=&Oi_*t;6)wSSn>=tc(&j zmWn-k>=cHiG~NJ@DaR`s;PRHPtUow81-yls(htJmE2vRef=lf16?xt~g&E<3jQ&?M zVg>G6SBhO?g*f{}hl}Edi&!{0S}j5dHt-5rl5nMX8yWN?BP%@eaW0?*adego-Tgs_P?aXjEV1`nHqsP|m3d)Vm2^`uASq z%*zShzOwIY@#i&srLJ$E|7ao68q-q3*lE{hnveh7zxAVW=9*9+N-B62yzUxxw%(Qs zs`+{QODzou7-zAe!DYJbXkgF{1w|&cJ#0U-gPvM$#Hju=*nCz`aJ!f6FtU1hHhnO* zRn=NXXld7|t%T8+m?=Vc4)T+=)(UnO`@P>YX1vWw>z}}S-VOXy&ECaf&GGYLBds@! zUXYu){<*o5=}y~LixO#U|JKWE70of${AT%u5xZ?p9acr)LQva9Dz>)JMwfWvD+(j6 z|1fq_QL(**j#ceFliI2JoS*ztF`fqhaHb}^c@tmYH*K1Kx)sKiZ@sL@D{m=|9`ZN= zm3;|zZ;)u}a3BmU*e2#)A8F6PRE>peK#hxq3qYm*l{;x zS(npfC7xhBK7CKcT8fL_*s;cThjm-E7BO=U{tLGd>OwUbSO};HEyx!V1#d%sjK+HE z!1}vAD9i$9CBAiWv$UXZumxo=G^nT{m`G1|=I}}3(%alN!}iV=Jz$L^8;rN;MVa@; z*Ms)IJ=BlBz2A7K`MYt31nXiNd`JW1?NZgcXUY{;>rPJ9>F&%e?X+^~v}`+y1_LjF z^@KDdJ{KK~-9rd6^35yk8_8C$-9ces8Q;7KgoJD!x--?c5~f=$lJ{P^ZwePt&`Fhh zUCCdcF+z|x;)H}Zwg0N7o0moo{?H9w>wmruMBpi==!t(vxW%v<|Fn;z;ltA{t8O2L zufqDM{8WxoXa7~ofQ)ziQf+ptnC zwb_^+;vPm3u&GMTTh_f1Kt#@+h< zvf09lX0hvz*wF&6J3IZmlMqH7-`mjHGB8t%Mw;WU$!iMTfFM3$ssxpk+@{4X(Ng!2 zL_Q9B_bwJTbFpPJLAJV)6Qkua<=5n6H{B{)`&;7w-kgeoQSOfz9$Iu!L8!mUhg-5T z(jF@sgYQEMU5i4G%qKI@@JjGL>u7maAG2;EJ}k1|8rUuN=&v|JD6pcu+H+cH^v#7| z<)0M7Uu^Bg^`Puw`{X>9JZX z6N+yM1Bb=>ie}>CaC<5%3%vs$z(6P0F=Rep^+DFsDDG--)d^59FaHvU2bUUS^lUlo zCLPM64_9uGS2dfgti)>G0>ra~hv?++yXX`Ox{EXm=>#M@q@02Hl559M;KDf3Ez(r@ zdK3H-tvVH>YGp_3{J~~!t6_W7P_cobhqw--U4INfMWIJ>Gpex>6EN4fZBqxd4OXyrHN!Nmi_HD7s`Z` zw8t!(wyfJqUOLCQub|pNNaq@=ZS&`#u8~L9tM13H5LE4RLc-#QFJ^&JV~gWpvEOrQ z!Z)`F+BvH+$vKnVUNVi%(>991Tg6BKh{K|EtG~I+GPsk>?=Lf*uQ7TGvjTo!TQt~= zYsO7A9jLfX0TaEw{A;(Djefh(XD4%D$#(kH*J$F-eFTE}e9b!AiQj5Y*g|kbw+%qZ z|Gf4csyQbbdwzXmbmGVn?8`%~{t4U8!b9-uelw}M_7*;4({m-H42vAtJOoMUoRHw} zekItl)sbQ~GiynIs&bBQ8_<(e)GNF>S2tUVW(t2BwyrIrd#CLrOfEo&XU!1iGp|x4 z3=xOMGh`bM+kZtwYf7z4@GJMyjX(^sG~=!%Oy_&9q$0+W-}GB$Z1Gzb0%|^=+-(Yt1}<$R9xkLLFdTMG;6v6tNGVd% zJ7Qf+rzjOeO%BMoS`Ter<3Bvde}|CLaWd zEd6xJOIoo$;IZLpn`;#ShX_%ybNr0O$#(Z40|{_v0KU*)o#71b%hQKj9M1QD3tabe z&epXz^BGGUGVtXs)fs6BN|SV(1fFQW^lP_Cy?uR9!aLsoyZWh=6z$a!h8h~*m`|Yk zf#L}qFggE&7ajwZeFCg5gO(x3D-P4(pHS&p<^}h6Y&Z63!{u6W2c#l%GL9*plH&nP zo>$)Sz+-RyihLrOG}#x}P99jTeu#D5v+(JdA?%D~?ff8I#_Aw?7>U$eIcplhd|Uw& z?bt(f_WaH_r6woz+P;QNG*6dPY%-U+o2%I2j5fk^?Rnb0=5S1j*99q#;^e5!-MI2^ z-kk7gqvpJxKHGhIsD`so_=E(etz+ee#uiCOQjrCeM?i$GpRKvmXXPT;hiX|{>@hH^ zSh{-|n&|`qGJ$8vp_L@fxJ2%Pj6RP+YIcY5Wc65aYpLLo8hJdQA0!m=B>?%m2F(%Zk{V_$DTs(+zb@5E-!cC zgh!Dkw+7+}28;QJL)rji=S05vwPleHd-)R=Wq11RNS49xyqF{jtbT+qC~NcT!icmO z09+os9MhG}6dN8WUPQ5$Y_m9=!@fvt#ka~%Hh!;n3TBRCE_06~Jm%O<~>Ety_hYua|U-lWs7riKiP z7M@a($LB-n0nR)q_Sv|-Pv*7wOudo-xO}(DzQu?3;gg zq6V9pEvP}yuDvZv`E<1N+JwrO2oryCT7U+?|3P{}n**yRg+#O)u`F_8epP65~G_hMUFJUtYufGUqF7&kL6_Rv$I>K$5Q4SiR8sCcozrTPhEW| zevt8we5ks!sLF7$fv&eWfTVct(+(e&HbpU;9lesW<)xeI%eMIN?`0NT7MtQy#lkFk z!dBR`Q=8_o#mM>McU0e;pFzX>|h(jK3OiCwDc zw;oZAYFYK_11j5o^VwFa)$lPSR3uPlwNml|IynKrOZph zR(avZMAaPPwZ^eGV#Cab9$q_+byHkGAPCPg5vW!%GK3o1zhgf+Ua z{={B?;mF8gy`mS#VOLv6I4k|u_}xp8_cD#DCTJ)AYA)1S>s;H;Q>PW9&( z0Incn)-bt(l>9_RjS>UYC{k$ z$JjK_;jy~y5#O!xf~;#thGAty7xI4pq}j=C%05yEiegQLnWQ8we6)gD zVLQA>O~H}tEy(uHUQ4aXDC_HIL4m*f1*3>g@s}Ff{W2is{W)dKm&f8MRgJSI@V{j{ z;BX2k)UI!bof{p^sJ?JwuoGrscz(I7bp3l5rv1Bd-&54aTXJLX!jb}Ko&skO8kUFF z=b}Y@=IL=%#hwo-{hR+iQ}90V_U0Rr`Mjet>z;pLTaDS2J*f4z*u|>RK%+DU7UyP1 zI^W74*Z0j=+qH{ZiF#)A&`20ECHx4NjnH>MD&(rudQfel;|OTV+@Ngv{^XL>Z%_U% zEjiB@jH_8`J+Cr)W>a-#eQmm9OHj0Je`~Zo=P(ByxkrK3u;>bZ9cYP`b17=5%C*|6 zqBZ;vmJA3=>?$txI~@Ob*WbHp9kuB~c>sEEWDIqUdoCyoNtN+A=SrPSb`kK;P<2v4 z?DSzTN6vQQfxe8#9;u&X&(gF`FwjULipgNB+*q3igMtxLHRV8yx!WU_(m1O7oI~yX za=5fb!nCR#j(h`WxclPh7CNk4&V!A|p+e8iw)Yu(M}(A);1ZWH@hKe-s%o!6ZL@)2 zk{eW=WfsoyPxa2T$mev3Uh(4>hzWMKUr3lAt|zqaYfv=aO;jgI)z72?DF=xUN8%F{ zj-DhWf6Zns`<$&E6IDNx9n#$4sEs{uLq!=%)UhhB-gtr{Y^7BBE-C&r@alL9sAk&& zHx(J@{%d4+wO*1$22AD_xg+T{RO9LOAmV01uMKEbra>|NscKDp3Wc4o!w>naZ8dsM z6Pi6Fgd=9YXxgV6l6(v3yzsZ8sBhP6`bf0A>TGxDEY%`=8Z2Cq&u<|_Na`Gl+M_6y z384LerJsOS7*UVT%#HxEsId8J#@}W;(Nt`i=`>_;V`*9yX`Vls)2lbcr@wC2e<3~o>s`s|VryTNa*<`YbJ_mzU`#Yum65cj9j3`bLd3BBDQ$fMc=~vh! zh9)O*D?t_Tf2g47MRq?w z->S}ov1QsGKwibK+tjNam=Azu6k{I zOD}eFB64cU9KM8x6YtSD_^LInHZskV&)4}3R%7Hr;VnfH8}#868IV+%0Cb)m7ts0K zEveB6ME`F}Cc13>ftyzxH;nX<20BD<>fH()?g_Ug8fohJk^Y_AS|nh0aQ#ctBA;+O zGuQV<{zQeqw1Gs0n~lE#}Ze-V3pt$S4ckbnn3b+MdULGh@bI*C%A{4oD+M zk>gqB+$VSVV|xs+ilw@b>&;XP2~}M1OHbEcV3*)z9zBi&^@^51k@Jk%<*eKrPjM_ zci(D#^u@dIY8*&9yVjx^XWLabySyq*;d$hfA7=2F;B{g+CdZeuVzkyVEye4infoE4_NN` zlaN?fo0%LosteR99J*}(LGYc5k_RaM{>uF1ju&-ZNlCYl(qLi$)52hPFax^U z*2;09icGZ7Zq$zXF5BauNsD`Y`d0WiSTwh1c5;3^4}cQm6Z8K26%(tS!a zejgn*Rk0>X8JufocPB~rL47|ivfn;^$+SyE4c?!J3{kLEVCz&`zrV-0!TIs7uz*lV z3jFQe?l6gs8$#MJC2lTPc#x84zp&by+foioj#CAlBu}xKH_&P4)+~t_#E8bp6DMo* z5#>TKYm$Li-X)$7eW?c@E!_An9+Ut-xW=ey_Xng4dh^Y)E@9+T z=o#Vn#mMMw)K}T^16%UsntOv>_@qSsOv#4 zDJIus#BiaTvPCZ+lSxB2H2qUAC%P14h1nQ@ZyaoLWakE?u?5xyc1z5fv3rD`*ENBl zp~&PWk=DA{>C52en#;x8XNjBp8x1ib^RL4-nA$hX?}H`B+Qp`stKCOeu}+GE}yJ-*24Qsw?G0SyfIt3}EWg?x+ZKa9aw zZNEwH|9lY)^TclEeaY#{VRSwG{iyExdn&5suwo=``Jm-+w1@DEKc6fY$s29|z_P{H z(#@VAT)E(3a2_H$y8HgZ>sO;&df+~j>$D=5JR zoO$#!{|m-eCeoZ}yDYKn8|C@y*<`WV`Suhl=+&F5rYjk@?)-eL8+xzjTX-Z8#g@C1 zPX1~X1^am=TG090y`-goI&mRq%Drc;@@&;Vs^Egm`I%04b{{*6YU!y)PnR`50Y6;2 z5KR5&Chy%X==`=AB|a#%a*$T{a*g@wR?d$m27D)g`bebOuoUe**M~91g`l44Ui}E}x+ZKLuf(a- zHs86b7^K+vNcg$#RrKK=!;-tv9)Cr_cBd9=;mnQ|jp3VVVDsK-6_AW9)?sv!R~vtD z*mEu}j0tgcJhPBhrr`sd9Fjcst5hL+Nkw0_gXp4C5Tyy$t%4bNRd{oav8Lyp$YA&H~`Ptc~>A3sa84d65YDp;_EjKoM(kDTB>q}d% zq3}(y_pueC-aWpgG~8TnWfG}d(_9wg6lLDN+`Oh82 zCotPWQWr;Zl_UhI{HxlTgsM6;X(|E%I-2ic_oej^Ux8Ua(^Q#qtttDLv(90s?{*p` z82}HX*9!|)d0%~73;|aza1!{v#zt4yOg`CCruQ~mcEQvScG%P~74)2M8Ya6JjZ9Vr zD?s$`O$=;ZH0C=$)O(cuee0W)*+9@uLY`e&@-ORV0eVgjkVI!-Il>_#jqdz>vp+O$~Mm<=L z5xx(B*k#A9x^1yh$dZxVqIFd!DyQomN5O+51fWw>ayEx;<0|?hCwnG03}y|BM3i$(BGa0X+1XAM z499_2Bm2pkC4T?0lCY23{aU?xh@L}Pe!XiM0ty8R!4p*%Bj5mkXP4PePL8m;j8I7V z@%|L-M46xyabx6k3$<%xbWHlLQn-!de%do4fPtTnprez+>v7)9iuLFWSIYF|bheG_ z7lyuxvCMh#`?MbR;5z$=UR!WBlYrUCpN{fTia^iH{-s$)tA|y955`TwC7K-og6POJ+bakXtZ~mjjOHx zoF~DBO@ko;t*2qM;|l!c=u8EDX@s3Wadd|H^ZspZEnIQZJ{`B5G+&ND6j>br5DT5!C75yUzoI2HoUzt>vzGS%l(iwdh?=X5z*BYT+ z&ZD25xduNmZhx}QHf!yM3Tv6y^o`%kO|IVM!KEC};fV$lP3tGh>rzss99RN)bmHl^ zRjPCXOU2h7Q`mi0hAcLI&1KMSgEtB=3$Jq?$A`q}Dy0k^eh|xwL1SZ=ia4%&t=5Gs z8C4&M;0wq@x4ZzN6fnPbFQbjvivGS5_Zr;Cq-kJE-L`w*>LfGiMng4+ z%?LjmsadkYfp!IH8M?TwqCuN|&1RzOiS;9m1WFM*VaBA+nOa>Gf?Tm(m*oIR%O!XO z<60vNL3%1rl*>>T{N0q@?W|L6O(2i!6iHWCIMmNnCP|D1NIbyalfNfoop&}4#xEF& z$R8iY6*M7`X*~LZO3$B9f8nT{*>eiw9HHCtU&segJ#{_$!4EBp%)hj+H;O*P~%*5E}()k7rYaoqoeEd4FdVWo9YXoe#bP}3*At;0lP$=-LxMdRqbF! z)VHo#kC}bc^VLakkPofj{5YlAujLs-xOjs<$D4w+Ro@a@ks+|;uU_>SdvI!>AFMy2 zS$t9@CDTc*^L5=E@u68-WpDx9b-@slNxOjgflNKzJz@Ov|qb_Ue$`rBr~n}e)@ZB5J3I{bxj!G=J8$P+ca!A@%~AS5{b z<;sEY)?RB(#muR-1i88k2K6TniT7S1D+Cl6gQb&04EEjt03K~V2gw=wrkpuCg#U$G z;(<}LC`aFiLun}kJqWIV8zfcl7;SXHMhr8xT213jE)JiMm@!MQQe_Zq6f!e=@`*Jw zhFrby{lOOc#t7YH5FW$;l1;i@#^zjmoz+^Ka7&$<$ zR!y^BqxyZ@eD~kt=BJjheBF+_c3(hn!wr($d6^fYeHx!lnWZNWwBKuQ*6v-~fnEvT zQhd!=eevn`kg=@u@G29l@D|)UHbg|7_DP_}i(u;%uO}d{b7dU%ENkO0L4LPlCI5uf zvt3kb;R3_#k5z_GX&n&GM%IEdK0|q+vxU-&r-^jzmT3@Wz+dBJ3exO<;u*t!5XlW} zx)(wHoKGRxoB!$PJ0~`y>dOtXvv_)8d6e0{?p&iBGB7lf7}&d!bdHc;PueY8HK-1O zpcdC))nkNH15pa9S7+Mg5avgoow480V|yV>a4hASAzM|ITpw%y^sC?ko(Q&+&4Gf? z^ltNHe|$}~lE^yI;QojPwV}YhQxi1%Eg0P{#dsfvXWQ6FTE})gn~^b-f{)&&hVIjz zJbGir*ZX#-I{6blx_`~AztXkd91r3c#8JkZD`dP~1+Hn(;#(`nc>1R^)5y7NWcI24 z2cM+)yWY+-oEx3Rob!QyBK^HKymF(4y&&d%afD8cLYLvs{-?-R%t_+e3iR3{=f?MS ztUwE3c%N6RqK`A z8%>r--jPf9+8N$VZE({%2ooX`r8vcAzxfJT4@W2cq@TYp8`*Q4%_?G%?=ekO18FwP z8QTqZXnT{;dgc^leCjx6j9-)oGyi@m(_1UPx zyu7v5u3=Npcmu)6QU-xcZ)$q%8Izg&8L3_@^52v!FKt{R@dSSu>F=}Ts1xu@ydL-4N5+*U zy2XYODDxl+pS08Tt-aZ_k*e+>q|xXim@>`$vFF^-`BYhtHUGOCje^x|o-y<9aXz(2 zS@y7Qm>1X7rW}`yDeyyLZm5t3L2rhlb$MsJgxlc9n+Ow;Owi>90t*!qJJkl5BrtbXb z7l=cVa-AV8XFR%p>SPPmmo3%_o(=u8LrP#@qi+RY-eOx;N2W7#R>qp9 z)U{E2!+(X($fPkC?ZkgL5aqlI`MJv-LzV|){=~w@0FCt5-Bf{dM2jxH9IVlPHF+IN z_FDQ0kFEMwvze61m{iHMYm$2_Z`R+=_MLhLo2B~Fck$GFq-)2@H)WnA^LSI`9>Yst+M%#jZ zgU+*T$x}W)Ncf2xvwkt4ugBO-+m11x8+c9yt+*XSv)|yAJzhhp!%R>GtW0l^*Ot$1 zwPOxRn*(k9%wF_TeLDOG&m?D?bywE*#8{C?-B2Fhmv=mKbK^xzCV3!Z8Rwt|YmSrM zk}YtD=JgOz<$r>j}A8S}E(HcU;+POl~cCKy&v`(XV8kG=pnRw!wF#beA@j{N2Q z-gxom_ZKRGSqgLc>HTG)SPZ@}%Yx7M0f!j5NZ<%1n??6;9lw)rC;|BkBs{kB8!<*t zMtPXs=o^SkOm};}uaO$B=J45y`M1_*@z)qxi3l^5JG;JLrMU0%uHP-y!JugKU~^fBtD=EMrWq=rGpy3<~NwpZH$itXzz4a$VC z^#n)H*!uY+ZayPR*(PicMUt{&UOT6#|9Zx+7fm{4VCta(YNV1{M8IRYK#%ajxR0i3a&_FdyH;#A5TF|rQDD6GX z)J2b0Tsu@|Ul`=~J?=rXGS}|%J?lP$o-Y+`nEipvv`NcVjhMWwR_OQ$=Lpp#3)c0w zt8PnJ(Xf@*nLJ&&W`=kbPcN+Q&eD&XUaa%q+jxXg0lm+eRBGs&ySn0WXntbYRYjaljT`TqTCQ;8SY@H5bJu|HO1a$?gs>N{Syas~;s zeI)*zx6mQjQx?lHIp19>Jp(V9bf4Kf+F0k0!>hJ=ole=HXKk%-YW@h^CY)$u09s=J zE0VKK0(|}_5G9=e{JHuOX3YvdZXU(dJn5%gXmOKvXw6`vCO?@YAHemvT|h z&0v5C)tV}rA%y1(aqb%PCfh#B4DsFpC>qF5;ZFjvzP2mMR#eAK9t$G&GwnfLud83 z2C=Bn7#zT3-)$iaL{XXDQPTWl3W^T}$=$kmRN$as8&Nx#Tra(&O!OqErU0Hmj;7?0 zd(Tmnbwfqx?;F-5IG;;5h-IWs6db8-K;P{U=ICf=1yyqNx(+WPWpVDQjae0}ePx=53_( zRlhh2N^8ItTn+LkvRRh5#b#cRocbAJj~w^2Khtqa@u-!G7g!s#k3d`a7JIMfh_P6T z4cloeR4kOC4~Rxx^~?Lbm-mV|AxPMiD1Yp=Oxkuj7rTkS48;zqZZUyYGoC!4aTrum zr^D>Z3Kn9_&jC?#JzJD7un9@4JH0>EpYyI5$dg}`KMI`1$9>-jHmEmw8ufQ1BA#AE zsf49i8Qj59A(+cr$BHISI0I38Gtv5wHRsbZ&n^Yy=9)&+%z61ty90Z{qwycTOTBtP z0kjm=iVrb66H5&_+%Ek3E3lg$4Px-kT(clU_z`FQ{Z($yYX7s$<}b<@6;5`G3|{C< zbs_AyeV3DtKqQU1003%e%H|TH%cv~o?|Ahw1?5p#U*7z|zyaP|Sl|EDQS@dP@C>@O z?Kh&{|6NF@RVcq24dy4&a0bC9JjfRw{^2a zaPwZG%aAN6K|OOf`3M>~Sf0trHbyeDkBY|nixl>@6VTV(j{*S)^wM2Nmk;Wa4hl=C zvvF|wn4(DXL~;-)rE0KQr%S*xF^d0-vj9X`J@ehlf*vo~NuKPNvK}$P^Itzcg)ioe z41BWYx>(V$#`6rAxba;O4;}bV2Z*9xLct)KO9)2O!`H)n83Eh}g&78Q;_Hg~J{*+D zphp-tyZbemYZVwEIksu_KTx#W9H?FEqA2g^HPye)(+5xlm8$s7`>Et?HivD4JGJ_F zGAa&{F2=@oveo$%(DrF2u3N~$q-VL;1_f_^uKuSf`**vNhP$;+^>_|$x_o)3s z%fKDJCs0l3A8WN}A|jd7yq**^vJu`zB2eiAf$pYD-cKw`5Au-%^^=moSehvF*y&qp zok?ep^wE2CeU3eAUOoq-$Dzzvw%saR7uj=W+w#f`K<{86z#apl6B^_OFP}h*g0&^dwyS{M{xU@lTgdmW3psi7bevU ztb}O2t6!Dt6cnBmc2lOP?>srm{PWM$|2qqSOg1sS$$K3Js3nj3`+#*vlf9QGz0LlYL6HExqB#*z&>Eg<#|T{2-UCg%^lFiPr6 zHrhL(9GJD04xcVSil5)kAHf~F=Q&B3@h`}D%lXD}uGEZG=;r%`(Tfxg%#*f=C0csv z%<83P_7ix+iL8->1j8b6PoQ=F^{1F83u8f_31&C9$gGih!pgJ(g{xltr$JJd z0mi59*+yfWhtT?$wtw10B#8i~YOfN>mqNH7G8i0&VZJ~98!XETM49nI%gq$J#` z7@JCs0GcKQIluh>WHcTo5d)B%M>%8If2xIBV}sN)!@f%m1_kwR!CM7}#OeR=GT-$o=*^yTFCLKLYhM+AO@~Cjz1#mZ+~rF$)g3s{tzSB`>DUr*c12B6 zg<}CceGV0Q%A#7sFlO<(j>{IZNk?{i*9(H9_Te9T{K%EN&o_s4qRI8d+M8J4;j16U zZN4arT0J&DnsUeho9h4|EIMQLwW_IzapY;3h80Cu@`VxBW7K|rOsM_q5EV2e^2f2Z zJlPLZ=1K}~+WtG^%BBHg4@|F25fN{}C7TMayb5TF3Ml-h(3l^G}gfW7ps(tgZDgv#P!DOS1D3N^XtG z9_zI;EmY95$MBzGQs(M-)C)hm{IY$Y;od&@(}O+&#tF7Q?Zd7fbu@L! z;hIY_WCVl|$W~9b8boPtn0(wDt3Y>Dd@Ltt7kS25a#n+O@!Cr%)``ri?JG2FZ|zdu z&wjju$gffo;k_GR=}y(X=B0{6JO3Z|(nPH9FM!F6%zgt0Xm0!?Txp>pqQu8P9prLu zF>_jFZ2E+PH?e_~c{z7>OOTgm$}C@5G_1i%Mo{@8DCEMe^f#ORda*>p0haWHaJWK>AvXb;qe}KqJX9*JeR$WHZ-bz;h>Xy6)9FlAcCZl?%wTDr zxkaCy)@D+3&jJoqHz#K%e-)VXx@RzFeVJfgC?YMn#R^O3zMyE~lI)HbsCM0=ju0WsL-K z#EZgHV1C_a^2b%BJUI^m$J0eooJxZod+2no?6TqU?YnFo1l}tdDFfT0HU(hJ8owNB zOFslQ3|kvKi;wb#d)KQ@CO2KRNU5_|?Vn9V4wBgCt@W&3eKxWS3dE;|Bp(=AC#c0d z+6jW?JcQTbxcMzZsJeA0OMA}f^sGs~UmLFX9$E}RZ3jI$npKr69zKnOS(KN7#y{W> zmee#^0Z>$aAH-!wq(_%_&m6R!O3^}^1$2Gm^7K-AWVPgLzwYAx0)3Hs{RJ>4eYVZX zQXdTtkQIC|JWKwoJiK^#pL*cqK1H19IFYzkIWU_j(GBV8Uab&Czg2rFM_m@2q5d`b zxZORkM6F%Y$<^P(H5{3@YL7Ev1?~Ffy&W>I*pB z+DsNH50+iu&bCRj1h0_z0PS&<_;q^_sAcC*JFozq0eI$3yz8;0sdA9po93Y|GH3jy_lSc?uhkns)RoTS*czM z$VTyDJpPp$)@O3tb#*7jG=&->N>GlQH9s?R+cZasr}4c6&RlCL*A&EClov52wW?*C z!%x%hASz8hS-B!ZS@e?bLCsR&L9nPq2K^mB5YWImBFTHT_X^uFo|ss6)1dry87 z`nM*Z+}i(65Gur`*=sL+)EK>g5|wIKmhJW{rONBpbnUaHx;`rc(>9n7%oQ?Fsx4^@ z)xo(6%saJXByHwt&0xMOnQfJLwF>LnrNglRV@2opFYemr6$Qh}ehT&;^)s(e zb=e}KL~lNqf;U&X*CJ?uy4j7ULs|ho6|tDr))T|Wig}Fnnq-g}3GA(I(xvvhlW20> zdejIiuRTvORYV|2H!?`4=HKP44WngF549_VL|*Mmt4`$VqGzo(8Sb)b{p#YLKwY1> zu%J!(VL-J0WMeHhoIfQNzla4ofDi`zq9E^qTK(wKN5`UGnI!dJgo%-*?;G#x=KSK$ zU%NIn9QPOO(vq4zSKg|`%?&;D)*=5~Rv}gp4)X@n*}!^G_#k7Z2t<}BV{O?g<22Eda7Owdgp$)5M8F{xWtpCvvY&7^X^s^cjukrl>`ijd zTx)w`2=7Vn&f<~0LL<{Gvsp9b0q28qvtV?v(LU*K0fl-miz}>J^0De=H({3W>N+~w zb2*t#bm{uccO>mkcE+vv7>R-jWdrF|#4?~;4Fh{*K793l)!}X*rL@?B-={;yXRNU9 z>tEDR)Tuw}cWyybF?<-QX$rFYH6!Tpnr&zOrGYx+3)+1^V0jpG>#dI33F*B!>Ow+X zi`4!BK;k|@Kbdy+9#=fJwNTsc@)IdsNic`r%0W7qxFRD%H~RO?AG0t4;}u(>^2x~Z;0%;CLN|t6z1|8fibka~XL!Y$*6Nx6YwXEnGAWs4 zP#^C8i?gC2EkIBQsm|hs-^-URJ-_(NURh*uR-k?Unx~rn(+Gq}JxL2MbmkFGq%iwD z`PCyw(d6W_QRa>pEN=bm2l_3D9@)>B6d7kfx%x^AAdL(p7(BCFaJL5o9yQ813*|9K z#md4K5GNYT=##EO&rOG>j>bYyn|S(+E1nJ3)To7yh|f09^8IFVxKb86)F9iSU{Cbl z1!WERC^l&KVUp-aaFjywxZ&A1q;8GOi-mgG!Gen)9!qk?Q&3Yf?pecByfr( zGyg1jwafMvHY1{10$7iBDP1{GvX>aElNoW82H&x7vYlYkZ6F1G{qiStA3Rqp{GiZi zmsiLyU9ZEh7f#}H+f@R6nRTAx>RKTNo_4h)Edd_t;Jt@VElXKSI%o%<)Bc6)^D94WeaG1ZEb;P{w( zejc@7MMHr0QR;=;ZvGG{_CFlT{xV9Q5zNV(u%h)@QqSjxEx_R;%tjp?n03l8lWp%7 z-}xkh*?ffn!~YFfzN}_v{{q@dZk_JTQ5~xW=LPg)??sbsTnJOv@$}ZZcYF16#5j)h z5YRAY+clc9WU^%4Gt_~!4xe!-IEOCX>nhF=0mD=l!^6(e8h&cHj6)9TQxW_ zYsuc{c0O4=B3e`s_zX1kc4g1?yE`Z6KZjfr z^##L0e;vw0oE@d**y1t8DxiecoOq#sbRL7b1;j$JNVV3&sOM;L5m6=$vvzt1?iNz({yW#XZxy)y zD%w{4cPnu=&!0G}7!ichE(f~q&t0XG^r=9qK1bIo_vxqm#`Z4ib9Fn#dfW0?i?26v ztM>12(}nXEAG9S!7XuJnT8$jQ(1NhdRlU;kbyzw6|1tI5@l^KV|EGvjcuFFof$Z!p zDv>RFk3z<=WzVQ2D?2-zV-=1)D`amcyX=u<=WzI5N8j)7^?SXZ|DN3L`*VM;>w2%x zEzDZK^=RSRQ+l#32?CDO$Ra$ivdMDo)8sNR&6HF69lV+n4dJ6TrIG@c)rx%=Nm|^_ zsW#L-^)Xk)z1bA+WZFW#W-6IS{rYEOK(s8Z#UxF7A_4chv7H~wB5$Z?{D>eCCp zK#Y@!uvgVohBjp|h<=E-)w15%so1C0UfaC0S~DnKb50hfdE)1@@ z>A!yIt6x8|(sg};O0yWss`!Etbk(9Y7lH{GethhczeE~pep(kr_|a$Avg;4mJl{)d zdum_fV%b)Aw_5FBV9h@63aAZ}33ax0(nhKI2$McKQoyBs?|c==p9u zs*mCh*V3aE9hEfh?Ct=i5#J#E_PdV+ndpI`!z$a_a8PYFITJ#QBhG-C{kmJssO0((K2+t$&qS%o@AD-e=?paD{skg^NsE zniPwdSQ_-sDB4&JtK|j#X3h;fm+IE&)nE4CIy)6T64Bu{%`Z%o6S=y z{dv@yhw`Nj5XWH3;v4dExx9OZs!9Gs{GijibwWiRTYxH&Y{%ZcsxQd1J+@LgV$SmC zc^i;+M!KmTqToVBfn+-&w$4jcteaw@Q+N5c_Npa3(IeTG+T(4*=P}P*OR5FEt4Hcf zrh^x;dcooOl+W!wobUf1qZROCi0t>YP*3^br(N>Y94-c`7XM71_BiuUl2pqZsKol^ zuvP=o^l3G&x}7F-Jw?jv+hsXE=gIs?`_v@$v>)E6CqlNW)o$!mYm`<~;5`JP>j$-d zrDJ%}m=j_7jqT+LcUfWUhVD$Ksd@LhyJ)K>mtJP?{>@J){_dydZvTDneETr-d1VE8 zx4-_U5ec67ORIs)6|dV}ZQGiuA2A;+lGT&f#SV}BEsgRX_>TPQMgS7bt;T{T(&?pR z>mp4fbG8mxX4ZD*jt;Z*hU!zwI`n}~C17<= zfxt8rM6nfVw~Fd#joF<$GJeOZ#wS?gx;6U3L;?s%M&5?fv6w%i!Q1V`e%cY!2aam} zr#_rInMNO63yb*W_xP1olccYC*zSB2O=}s|R;5hL3e;cGPKCWaV5r2$+tcrzR9{9? zXSuLWVZmUlQat3zA>M|;_sGi}K8s=@!#!}0+P2g&l%cH(w2{uf;?Lhyh4fU%*X!5a zAY`2NMnH}@aUq&0Jfc58b}OhMa&E2of^nfwf^(a&=*V4X7N)hNILC-*8+ead@|~UK z@^dpbeXI92$e3Tln^S&2|1w|?GmoQ}U$ZB3Ao8|slK;IAMlSU=rAp0MX--K<=B>P1 zbzWAe>Dn#pZ|ivL_Ip&}$F-29t`#Pc)|CANGxH4O7j0`v`WIBznakpP>*#fJ37iYm}lL*%QD@FhdXc$I z0c8W0KFDh$+N?U6c6vvja3DfYPZ8R^UZ>4AI5dmb_oXH)_X(-aB8FV((E!Ah}L+|3*e5&H! z>~db;SahuGEbUmTfmT%8g|k;i>UjH--nn31Ur^;=FJ>nZU8jA?Qt$v829#GuN7ctZ zOYol`t`}Z&uHrPE@h)83m`}X!mDhbVnm|ZjRA?7!q^_q}{f#?*-BEJv!z1iP$gx1< zMkiB2iXGQ#>X%hUOJ3S}8e#pf)WY9^7h#o5#;D(GWd&L?Md3GU&y?%0N;~!rFjpwR zpZ{VOf4}+jSTBDHP9{Z8v`^W)DALqy4n)kk+H{Bpf}mbIK@I6v{s^Qlin59}9W$~q zl^}JV5xV_qrnZLGPH~q*p|f7m2eC2?n5h_AwJ?H^7x%ELCE`V(-o0+Guvr^PmANi@ zY=s*mXf!own>9OikM4KZzlFODJM=$H<>ksnB5;E@Ei?ov*NgI;T9)t0ZH$C&eZDf5 zWKe>W^82A&SSbYZ$g?8}vFL(CGby5LmhbwIR+-N7ZnQeAC<9>X`7EpT{a;-SQuL~Y zw@NV}ZLo6Y*#+XIZT5a1)rtelIdfD>yMVblZX}&{QXN~weAECbzUVMA#G`+D*uov~ zOJ>)`cC*%A`nUv8_OE&$vFZP|NM2l@ibh8b3y&7~Olx&nUPOkT7K*}ip}5mro|)yJ z&{_~=X1@evz+<#M0N5v=4>@~5r&n)5DJ2y}A9wLa`G`n*oS%j6D7^E@zt#+b z=`LS1i_j|2i~vEvgBchIUxFpJKKO2c>iwa;CL;1%@Z zGJ63K&*tX^7n3-#MWjU!Y56>C*$V&-SsT|}Z7SMIzL!qL`bQ6zKa_{9KrNONB0<^G z6W^x4Q9zTe-#42#q8C>|QV%_m&Hz*3rrw8w@ z?tPLyoYH~;hP0t%UMcFb1w~l&-PbP7 z?#dtF;K}B-<6>j>p4PG8k|BvVcA=S|iUc`z*;fip0u1{3`SfG2T&&0uD~Vz7V=(eh%(gC$g3G56Qyovh1V(VPkHn3I*joa+ zH}6AH6k67o$^G#z#yrdxrd?_M>k>9tFV()u?F3nau_c?gLGmGA+@w;LCY>$GBgXG* zucoMn{SCK?AeZdBzGxb-h)Co!?mJuvs~o#R_#QKS-Gaf8t|4pkZw;G*%S(@qYsPI} zdzNS)gVrJDpW|cl)4o61vD(wv#$Pw4u$W|4#C!4+$jVItA|Ll?@x0V^%*JrU$o!y% zLwI4%{#;+c!g@|=_Urc40r6+wlIkAE_U8|u=Np|Ur8OQDrW^9T0QNc59dA&d)~%j4 z*f~Xpx@rh_p(LEM^D42@IkzECWwUELpEY+fTm-$kRE^BX=#5p4a?n_V7gYGR`+{C{ zho5Qklu7H~;WLowx>-iUs$V3 z&(5_{`QD9A1!o!9w116KGe@=5%PfHwRDHLa?9!|ftLVRWAArRG0Lo#9=tLzvi24Y6SRp?L_jM?U8<0*b5*K z$Vlh(l;$bg=Rv)LO9dv6^SQ^mbpCx^1AHK;m~B(cDaE+Er*edi9Um$Il2iJ0_N@Pc zQ)s~;JC(&$kUP>_chI}bKhyCv*MRkE6`W5o>AH_MtLR9hC-u6oLtc&{zTrcJ5jG=V zecBbFL-?=r$L}gsv@PekrgiZYd-jWx$1S)peV2EX&`1uq);8iT;ayX zQ6YE-nW-H^DaX(T#;*tf>ME#u>DYoolHk%}#FcM)Uh0z$=Fduu#~x@Cag8`_F*_6h zW(RL|^IBT-`@pO>Xt8RoTAOVckbp~%YoEpKDI#l_`u(E+Kag<@9S!Jk0KvgxyN=E#SBkuA)%EO_ufMqXk8b#aWrL=1j(IKo0DYhXb?y*!tdxAIltvPn}FWIteXYApQ z7tC{D+B4D)*%gOhEe@Q^o7MxdmSVonf(r^W#h_@^t!wEEm1M4Q?Xmsi0XoF*-SB?^ ztny2TwZUN2+6!8+ccMLbh@swMevK4+|19!5GN|1U=%^U!Dm=}!&IlKxH+}|71V?_E zfrOE-?F`)N58FV7S*=1ib!0s0?O~7Y@o0xuq8R~*nS=xJZx;WqNQ0f^bM@lGlzEbi$y(n=-_>?Mld<~|(S!)MHz&~5PFHn-h7qQV=NY9i9%9`^@f+`P0`IJn z*90ouwv%Fs=c%@C&3+wuD7FMZ!TQcRB$dYS#(!8}#0Yvo)D=?_eMQ|i%-CL$d(_37 zZap<}oybkdufPfLImg#S(#ksy3`G?Ya9rs^Ybd05z^=AN^W;($tyHG|C5D)Dq~T?R z1MvwD_z}m5=bXxGysmed```dW@vL*yvu|Vav(HOhM=9NW3`!FjrIbo5(0=dHO6Z;e4WD3zL-)aLz(qci99OfRiQ{1_FVt%l@b`TdvC+em89X=YFXyD&Pg}B1|c`vS(V76Pn|21T7=(zDU6WiT(RE*D*oHwgT6aYnajW%1cHUCv3C^1leeFm4wRbeV*o(uAZu-ODvRHtKvyj>kX*j|VJ}Vc6 zLOr%pFbDCPM^#tAp1_35V9@&DucA?Ho)ZkZiTCt>gy+`7Hp&Mia+LclxSv7R3O`@0 zLa-CsfS-4xk3*D;QZ0zjASO`(tZ-dH%8-%R(2dXVJnnJw14}_*e$C|nP?W6 z&9Px+E&v${b@`g>vA#$t_WoAIX@(MB2=IS3AwXESQD>^-&O4C{5_Wy&9vv~M={Diw z!DjXvnJa*vGK@MCx9X1l8tA8fU3||fUgM4Fdz3Btqqwzs|W53^Fvt*e|0WUEET^qp%ISoWr z`xp>;7o{j@P4OTXAnj3f+tQDn1xufzQvRYKUQ}rK`gDPHA6@h4r7*kBVScm0<@NX;)VMFp);X_ZkVhbi9r^cB>Vr>5?P%cT3{p!QaKZ-2+4WuoAY;)nN+ZO))*BZAU>rL6EeX20%&- zsAKnc+sCU~hh;3(i#~iY=^BOXi)i4{+=w1Lul$VeO~i-r&|8gn=RldRti)Jnl}!&+`y)Uqc5_Ni-#3`L;gPvg7VUG0 zJoYwWr%*52Jk^Wu`jrHGwe$-j#g zEPTqBAO*d178JO+_vqN&obirLD|j92uU8(?#`LCk>q0^oTm_U(yTioTJJvrBY`2xI z?41nFH7gzYNG2Y?pn}MvNNa4YrtU(=vET0R!$%d{4eJMsHfU{a%@VhStE`bvg`F7L zFC_~*gc1fs-TM>3^63I0(KW^wVbu5EUEks&eDhQ~@|JwMnL?zLb-HY1l&l;B>nAg0 zRo8{eo6>5G7S#%F+HPwuyYubjJ{J7fdg=!&Yf;LYn1eHasCp(uEgE|BYZg5kdbe;3 z!5KsGqfHbTkH}>kS7YKei@C8Wv_t`SbrO6p(ttp1` zDq^-1mZA)MdG7qwc>TrRMDLRPP5nfC8`cJ zk+|QhMha_sDw>r?smFFtoX#v8Y*}W-IK?3RNuTIf{B~{hGRJrc?=Q$SP?+pk8%fo- z_M;0<|1?}!(9L_{m*H(ISf+zT9xsru+8<>%ntsZsYZ7F50cSlwng_U}boq@6yihfy zYeTNfE)9gLE9%8Qau6l^COOEa5?YQjbXg@XEiB-AHZPs?!mmz{i6JcxKDW1*h^i78 zjbmtYH8<0^T%9qb{jNuv{1Kulr)C`YJ2uwIkNZfJX~NMe&K#LLre0g&Rph4XUHkbm z&6!*z`saQvS);b(rq6L7T!!dulj~9aqtjqDn0Vcu9~5A{i*toHAJ#!eP;TN8X?-Ou zX6|CM&a-5t;>c;^_T?gFA7e4}mqOmKhl< z?q;K1+szQQc>bBZsEu@MYFFoZ7a7m<7#$%1du7& z-oC>{$lQj?KfNwC+V$9*1ZNRVH0RX)nB?UAq=P*t3}CCaZwqwK@;FCG_r#{4!pEl5 zYcP7zBG0lS8V-GN?HGw?Uxf+2-(tj@4|Z$}-5HU$oA0$2_CBnHBGI4Do!&o9c^kt| zV|gvrktB;Tjswjmwjw~D{c87R`4E!69TMNSN9uB{*;y#yo5^?fN$g3KGMm*(E;8~- z=c|1DY^NqVIf)St@Vc^KiCad%u72hG_nrq5C8}2K zW$X&|i_&rMxoiYRC?fZ5nFDKD3T3{`{j?jV=}{bN@XKfagbps9A40jheD{?JUn_3G zd1oF=`}}xyT}bLU=01_b>`6^lJt;lRnD&TjK{~6Xst?<3#fj&(A~>y+Lmo;UYm`~C zn39|>l{w1b^2Cj$4WnG+eH$q&ZzsJ4+rRK)=GHzzwnu573a4W6*1iVk9)PW!9_k-0?x9wAVxI^uyVHZwjU8lxn!pUVyIzAhWn%_27u*skR&9 zsS>bd2l89*e>~0KyfxS-**|*tqVyKijq^Xg7nA9^4d<^T=}^`ER8a3X@b&H>MY!?2 z2%Dl`I&6ClrMvt#YAxiV#S5aNPXT5!+=36wuz6fq$_4_>%7*f(#r5(#<~JcHX#8zc zBX>}!*e6k_=Rr!()D4gDSm?~mgBXDjnHU%2cPV4|-!_uvdD*A`)P&8*R*Cn!#(LP^ zg70}=-R^(vc&V`au7|R;7}_Vw>Ac!|gEGr3;qzx(Tf@`)`y{oD3>riq7K~|@I95d+ zB*{oAeRd`yLHG;Ii)cbm(kv;zQ{Q}tT}>%7{m`9FIXFX+veLDGptJbD&@*7i5*+KS zMpaGye?^_Lg|NGDQeL!$EKXqoeCGSun9X7@PBika+OJl>TlXF1Ik6YDPyS|A=NYJ#VfptuL+bgf9g%BbR(DSeUAMy z*S{ZhUby3*v{8BG9GoNc?~XBxy#2HK?Nh@`1#zM~cRg9TzP03UrO$9MQVUrV-R}PK z=};l&=XCy>#ENs3cjku@5AkQvtiFGwIzL_t<&e48_P`@@^35^tcPhPZ`B#7&;OU>Op1#}e|NCetvH%(JTzXK#)TihVHEZ6 z_UpWLG7w%w&zTil@kfnhP47#$@7@)uX#2tO)O)b{S-&g?V#+}EiuEP&{jY8XW6)X4 zqYgxpBp=6MFI<2>7iS2sFt_1&$R{;gd_-6{bFE+I<4~~+T76tU+IK36vzvl6(3DBH z?#kYlPwx>au5a(~y}3SFAc2YLMkNI_g4;X;95hxmWSH9?Mq7z1Bjj?aT=$;vh6Sj1 zXQa!fPTL4n7Eg{uZok}&)r~A(uxO!XgMr~cpc|K6KSH-d6QnJV(n>~=jrUrXxCCYW zB2u)m=$D0U>OpRZ=*?gHxRVU6MQ@^)>NgJQGuM(P;Mno~^ml_gIoPT;ZyvmQecu?m zP?DFmW^IJ$_YFOdyZ>5{R$|*rQw4WdY2vl<2u=kWi}WrMLA8hZT&5)WTyxAf7$p@n zMyq3KLx1!{Z6puA4y@~tdqkonYja_MN9|3~;z&z5)&#@pH9xX724GhMOO{|5lsuc; z3koIkSNbZ?49`#W4Fd0@!2=w65%IE}S}?g}%!8VBzkbJ3<@SU<>9;(%Z5HmT|7bJ0 zqY}qw8v2f(RndF{ttH#wj`^ zRm63Fe(+Iqg2dl;zp+}OL+9Hy2thJ?e1r*k2TVDf`n+_-mAzRJ${m`@IS~P(MZM4f zGgP`?MxpH4HZ7G7o1H{rn6UN-7F+Hx4BB*M*WpttkJ&KAd75i?emdWUcR=VaH7A_| z?U3})6guJ!%wrb-$db7oG+m>aLzp1rL&=?N;bJK#1ji4z+!ga5$VPFDT4UVM>Z?AT$JcYyKbwg#JCdlwV7>&k@EB#Z z^>`#oURj78U#|gm_XEswqL-1I(X7o?M3~*@-+f&(t-E7UJkrpC~`!eu9a zzIrBCLlV|28;(h#nV75GtZNB9zsyM&!y;|%e?jMet%%$TdF0bCw&PPIWxUhoRE^&(8U`nX|I!%|7A zk6a@*KaIQ0a{Ei@1%DTr6GRnu`8LMV7bQ%Jnen}YB^p>i^gPPIcqBTHQN`naSC83G z$E_N0?B3SRI{Gz*R`~(%uWXcHf8*s1_`J=)aQy<{D631qNMot0iH>;c`%9LK8kX$# zoe$t&;U}u!k=dr`&cvxg&XcB#yw#~rx|oZMHY=(o-JyB3E!Oz@vl&g0k7vrPJjdu5 zl0I+2Fh)SBbg6^cA(RKYB%O+}dh!`U_#%JdE6r{JHoFuP`>K$sr4sg@2LX=K#cRtS zcM21cj}3bwa*o-Nkx+CgVGYvjz^OS=BXi%F%`wwWD+~K~@z_4>uwkFP;owoB%hDhj z!oFVYKPW_hr!yOX8=%QO)vs{1_Db%ndR_O8KO{)L^o@zhlf>8xHdpD)ABs^ibCIu) zf;l>)07^SrQ=NYkmtFTQWFhixN zSBb4sl-Y6hU&co(GHWofnDZi(y7p_df*m1>R`jrcT6C*E9f!%hdcAejaU}&7CoUZ$ zu2wLhi*;Gpc$oPk#(&x*&fE&dziZK>_9OPhmoT*NFYomi{$6_I_a|-1Veu-_rF$NJ zCI7lrO2#+-&4Z4SAA~QN6xPOyt&O5zRG&5_>7M=c`MEOp$V;S`wH9#vPNbk zg9!m<6$y>ZyorR1G4}9YX$yX%%cpC~wpTA4&W8Ccei_WK5?B+oYjfDB1>XgjonmSM?^FPgU<^;7+zA1Zb zdi*elMZS>=lMmQFYH^7AeeSclHj1zS37^ZDmx~%`OkL5Kfak7Qa zoU1h+Rl&!c5he`NCAmKx^0m(nR)>CTwv7f_NR%s@CaLkn61`K7vNH@?TbvUa9%$Rx z=fb@;`ipJbKTV(`a?)TFt}yb2;rJ2+-fwD7Bzo>hiBxnpi9`ADlPJ$KC!6U|6&tEn z`ra6?Ia)qD?lgqFX1nC}MGerlvT1w}6sb+Jhq!7KS1yy%6B4B_r|wUw0n=FCCT-D` zmwIWdO2v%7!at z;<5TyeGYcD9g95yvfjLhf6ATjbdDHpRk)6rgl~7?R8|U$Y=yP6O>-S#(bLaphD zGh{lt-;qo$YQdEk)ISQoAJXiR=*a&nr~lrd8|+mrH~- z*U)zDrGJZ|op(ph=VQyf&U04MG4ge1Tx@>Dn5oOJ=I0qfN~zKwl?}YQzEyR3^;}7s zpY(NXmrBBo1o*T=TsR&UHw94U^nG7vR}MhG|LwX;jM5&v(4{WFr6Y9=y_6zUK;03E zs@ci*@%=LXYdr+%HD*5lt#*bKWn-#y z-*Yi2*XMl=?6q@mU74dXJIyLb>U^6AQZi?ncu#nm&WF!)H2W}8b$QyH(12$VOD zX}<3fAW@D@3Nsufw^+4%=P;qT3LSe+&e4f@?g_0{^_!^aMG%Kqa zp~MgOP&Ur+4ue zZw7x~ki^y?cC39-HiJNiWOZL#;u|>>Pm_STN5+%Bc-D^TW~04Cn6ER0-t^=muRboI zjHFV=js{97EYffP91RN|=b&MgT@b$q_duYFsu38j&$%`pZCV?32j!RTk-->p+5E+Y z^^K~TEr996%0AjxoR<{%cxi|&vh8&~ap+5K+aKS>4KE(u4kR{7H+TRW(BZBey%oRY zplHll8!b`jbr(9+m2rz3?eL5Vfen%B4nnND;M|#Q)+`!e@hE9&4@fl_9rau!RwS$Q-XM zJ>RX#7-FVYN!MJnF7Z%^)BG*Irat}mF0MJ=cbHnL*&Gqlyn?44lxNx%2TdLy!QW8= z$&aI+cr+(?pc;nr<}D`bzk)(eKU-u4O7iV|(oVsgL`>b}a!TpG4*E#&!(a0PY_@b2 zX_2O4*;8#mE7*Mw&o`QpMDwvAicgqi6J=LMnnHN zcf;p)w<*z?CbUj(vCYnR2OAH4d8svnmrr zRWp6$)URLQ6sWt>T$T0NAbCcmG;ipjY7(c6TOvQ&)9Oa@!gIbnX${#|!@j(jVPwH6 zL-Pj3mNDc>br$r*kf4~EafejA5QkjGTqX8l@_0Twn?RL7baZYS6eR!HvLV!O;14kW z)_EudNYstXEOk7eI?)1KcUd_oeeY*1|Ad4T5XeFMI@)fv@ep|&2vQ97WrRxDrQfhx z@g`>co4q5qJfIjDd()h@pY?+fdoSt_k?hvPmMHeKbJ8`G?as>?q+fkkU6*kr@X!^( zNNuQh8IVf7Zuc@l=X;GAP(2XYNqkhwu@F$e4=PU5WMjW=>J@x~2{bItwK&h~f%~6) zccLYqv^MXT@US6Lv6Z2C`XrvfBO`VEoYNI6#7>7sdfoEy6{&9L-&VPt;~d}^NZ4+Z zDe1R%`&C@#r5SIIEI#g?F^`B+LI{6 zd4@>=r-Hr3cx_2ra)(0FRl@6=ohL%&)ybk$0J7&6R*hrIo>r)>D7c;7c89a)G%1e4 z^UTye)s+J7eKiTO?%ARV2hfRuIXck+kU$_h)$kB-D%Ms6bCmpL*BLdK~1x?jIwI~rHt z*INK|t@bHM=4GR}Y}=K3tHfA3y7KiqzeXG1vXE}4_a2rTNgk8PJx9bJc*Wt@*}E1_ zIVN9c=ju>S9bXX$FE*I6KWOC~WD%aMxim4((sTMu`|~Gh7K-U|&$X~_4K{J+`QyTm zhi8Rg@R~%5?gYN7?E1u^K)hD#WD}1l@s_x>Vzya>*IVKrJJSMB)wdqc<5$@R%2wFm zx+wz7HAQiPdfXB;fFM&cr5U->sd#mdJK1`U8_GnzqB+AXzxC3@CfGyV=-abkJkha` zFMa=EWroo+_frXtCKufI6~4+cJoHiUa$6lCKkIbPDLP-WwnR?&L)HP4yRs(U4iT9z zas(ndy?-c)p;|v?B_R3{e6rkiN{P~-5o=zazukK`H^XRlX{>&uA~5p03O`w|{Pec} za8euSxarTc&#-55ryze^XYZCvZA>xGQqIhMhE{?hh$zn2UHhi@lzd4hTWPrT(qbkw z;TL|2a(fC6wkjY{EBF|Nv2Gu-H)c>7`r%}Hy?Lsv&Sk}LRANVHhuQdmI{H4GZ(Lx3pNfLn$0*E& zlUOlYRW0p!`glqzu`TiC;;+phMpiCfc-TYt43xfhOHn(zz7l)?O3vd%X8F>`j`S!N zce^_Upb?jHD1Euv4A*j(ub3~)GIPnS$yEJr;cN+?;~E}4Y^d3h0*-fjlMujvrH4Yr zyd`~vLTyB^_m)Stu8r@K*HLPV=mr1E6Q88QD26EUyCKosrO(fg@@$0u z6p=);D=tPTXPT<67)e!_BkPE0!{C<+%C!^~&56cwR9E0xj%RQv57-{^TT2(J#;sa9bleWlXW3lKYTfaHUA(kH3;9!p5T zA~%f88=PRqS6#eW5jX4KX9u?mgBU}ei+V}P(O(SR>qEK15YOg~UNRPev2=XMb#?6P zsdxA->(4pgrr!NoHGj{feL&gPSfHfczb*(q6!Qm;=fyA(c4#@TtO%09rwb5GOlIuj zFKmSG*1B%qQU~FF%l!fOlvL)xi0^7RM?qcrFh<#Eq)p;r>vhK|#L2~rD88z9XF9^g zbt`JZq=xH=2sY_IKQDIdv*0&S;vDZY1xGrYrwmbcZ+%Qp;&`qT(9#-OXe+g3X6AgMTunt1pIJ zXsXEiz6RwMq?04A9MdeSENFH?5_9LF9q!PPD%UH zWJb}CMgQ)bC!}?A)+&Dw#Cgwd29R)0c?1BgJ>w&N71yKPQpyqFpR47F8e<9uss7Ky z`@H&Cfzn%=cCuZ?wZf#!A-?S%txpa_s>8L%?$)^;nfv_BPUEqHvk^^KN>lyj1FtQ@ zy$Mb6Ut=|B&S;6oAK+cP7rw@OL*?Be<{RQBz$ee<@0Wq-O*`ERnpip@;r*_N^>#G- z$y=s3y+jr_a(+_f_V)19WBXlcE=6T-`Y#pN01cp41m$akf->z_Fp@aG_bjB;|C#9f zdg0yrs{Gr{k5%>THtS}fQm;D%?47%VO-W-@TqHD=ON_2{odE@Lur%hkDU&x(W%pVaiHJ~ z7ygQICC4rM;IJ0HV=vFapBYXHA2xtRo-LHxtOG?ueVj5o^g371X^`tF&t$K}WoxeH zFK})SRRO0=b0u_N56+nfT|SBQA>8izyM4u*d>R{xM^q4Ai9lGst1eH>+bD5ii zghOEx$(dc3Vk>f9v_Cwl=M%gtk&|Lle#OfFCqm$Ya>;W~@ooy>+YecnldNb+1-4c4 zIQ|e$Vw*Fi&d*b0bFqE7ejR@|Vww?lEO@oyN9+5S1k!TV=g)#4-A&To(#Dr4LjVqI zx(*vMU+#AeCdRb(<<xyq$V-j8}j^U5b`k$vX- z)IjuN)Z7@Q?v=rF;@xG-mW?hUR(Krn=W-}3|9i^d#~{* zAx%(~h8wQl+ySKT`w(juNFa{w(a~T>TK_ujR|r_>+P1mkeQvF+84Ycrgc~oybLYN; zAQ{Ald;-)1@m&LGf_@gWrKZ>za*EBi*io>)B%*FxBHHX?;He2jq5CO_SfmJtkNlY1 zr*GeC<4wdPt()mOJZ-N3F31k}$!krY5Wf`e@=rvy{d^-x>?8W~%F)yC{j8{^wpq;f zPb$Aw5OGWpwBeKh2R}MK`ziZGoNI^vPSoOdg{Pa$>>qIM87?z@C=?ZZfvg_0rN0Q1 z8u5db2{ie`)PY+!sn;`pqvh`gA2}SNBsK>Q6ceK#5({1}KmxAK?8zMxDPx-vfLC;bVb(rbAs6?u8vTuUd+MjfX2E={F^ zLPn$S5v9+s`7|L7djdgnJV5`r8(^kow>eP?^fzy1U^vts26JBP^^E2O50sMeM9rN@JX?|?EaOFSMuT3giE(^C=OKIy+!C1|DYL7!ed(HM2a9BC}|W|qGI`XzRiLHi*Pn(Hz53jw`$SwsGWFa7@j^q*n60Bi(nyODp!&A@_)?rLAv0taA_FaR@0Zb8sV&S931fcL%rDXe+N=;6;)k zvv`{1kT^Y*m_$iW%a)#&%64z#1~V38e83sC)I@WFG)H8B(GXgw$z>PHrb-$Ck;LY* z0l*Mk5sL+F$#9FYCNeElAdD!Z>zY=FY`46A_X-*Ofo||i?RO`PhJvR+GXQZW@}Y5; z!Vb$4@>=S}w{Grb$zHm+Z<0IR)0mKwqnpt-CQARF@n$Cl=^}36((^dj#0QMM)6y|L z*Oe*~Ft%{E3#MRqEX`LUKG8!o}+42pd_;Q zDnXh7B!;UCJJT5}Bhcp5^)BHe8nLQA8%JrpEKM{gNdkHNXO4W0&6zAYb-`EFbX>jA zD)x8dWhzGK9_Xk+K1!d<9^qEIy$%oZp_V|Bm6K`ois_6RvigNu*q|kalC}dWX@|+y z^%wv5xrV-dAKu~7UoYOCLGRbFTH8qh2`??=WuvFYD|(v7V*4C&@J6ffXecJ1g*WW5 zWyh-FEgiHN(cmO$25`b3Puw_>A;Bwps~(N?D&^3RV`RwWSBSZS7yEQ{D$^KUR%+S0 zM3R6%AVy@7!MR^Kb;t78t+@SUKTruWUl>ZOPltD2UGeB1eZH5{nt&bhziTCRycs;N zBsN|dtF;s*u1_4(W3M2P0Tj^fKJZt!b78Hi=68mF!zwU&9N9)!I=ahj9ul#?H8?RK z1wvjY%1A*FE2E3qBOIOD;7UX6ltPjYcc4QMXUG%?Xaz+oI!vL!1Hz%=_gKLPj13oB_*0onZ1;3ly5AE;P5+* z1hUK0mh+wl4oQ!~^6o}?J<>1LFr8{}sK%o{?c`U4v$ixw3I|0_QA@h1OAMfd^yKV; z7K`-9Us^}R^liwThtZTACS?p}qMP!E5JiS#jjPP>P8IS1)YPw>`>tpKK)9nEb?Jvj zn11-@InktP?Rz`b_Nse&Fx&D&89kFEumqRl3ztBo6E!XmK zVU4SD+hFS3doSc3TS%tz{&z1}#}=L+&oDo_t` z6|_MXpk@%x*y=Ie?(bb}gSA-a119(1y>lm2G4}l<(^tX9Gg-L>p0DQi$H4-2?29)$ zOXMFmFPiFkLQk~r7^_m2@#;6(CRy=&m$#?71PGx|!kXCz2kwW764=cHC8It&avgz_ zvAYb`BCnT_06U!kn;6{W!~7CC_tq@xrm)BOaZ7$OJ4jkat=YQF&z#%V+>8!>eROX1 zjQt=4i}Kgwoz{eijEC2~kEp_UG{^|Gf=_k>3MAh1ovWF{0L2Sksozwd8%)UQhXGrq zic(hk-}#^u$*-anHIYNUy#iW0XsDq#gO73mrny?z!p*McreW6WH&02Oxo_ff^8DFy zo3hakI%7K!dAyPU&!+AsjZ5YXt) zFsDrqLVm_+J~-p3 zhs0X{YN85cNfInufu#B{Qk<>8oQFD!F%x#h`<`=W{AqimvR&&7P}&UeEmDL+nk^0K zVRW|7E~7Z}HztKG9gD4l?F_#pkZfE^G&|78jU|V#j}EuD82eu^os5brY+Mk6YIvKG zJXgJ_F`T~^Jhy+wlqBp6pY+wcv&ES+Lqo+o<>TT5_g51tk1D}>#Uo%N^|%&3;=%*Gp-SWQvV`6aq_f3==O|4GhTO9Xiwo{Y&uk^A{4D>qQVAoqvO59h&aMsnnB zCGdc8Z%$tGd+=w8EjqU9VphGBr6;j302YYkv+)P_#F$-duF3$s2YL-&jj69NsAcOc zoyMk_dZ9cx*M@e%>@LtkE8L>fp@`!ERi%_O$OIOGV^mmpke}wxudUB!#k%~rJ|Y>p zcvHH6<1-&O|7q^1nc{{`&A41_cf7AML!+b_qd${R8$L9ub0JgCA%}y@!mVo69i$|4 z-MT4*P?>a92Na{J+h0xLiUPaY$7^G$Hl6iDX`jprR7d;mX5$Q!)JvN56x})XYl|2J zE8s2)n;6CYhI-aKiA`rYhMFXAe?aEpGwI~A8I86LpQh{7J8163V6lHfBnQt&T_rfh zF?(>LqCLIc@ns+i9lnOrmyBsC2+InY+wYUi9-5?tpolANrG}%mwl_T_q9Crt)=W6! zZwdk86Ic)1tIZS`XtDTp(t+xp1#+{8E-NmY?;n7kLZY5#vh-AGyI<`BEj`ePGe#A@MUtrdsN3`u@^WxwZQ%E z&HqosgcQ;hcC#mVxA0!nBhiG0Vh{0#=eNX#z<>Efv=zjgbK?!N=luhM<8*v{fxcS^ zP}}_X0^=mnl&L{`zu##I0qdk6ALH}FO=w8g+!yDZAB!_D7#IFvu*-m#g0US+GTg-< zjI8nv$D_?Sf9+4?uc?aC=ul`@=A?MW-@~u6KhG=hUE89y1KH@CzM*k~Em|XNJn5UU^44kG3Jp!@eik z9)Dg3aUwkAQ%dJJI@~tx@L+}nC~9Wp>RH2>I@&WQFtIpqMR1M-P3kXQN~Xge=Sp)| zbAkpB84y=yR4?64{f|Tr_53M$?m=<3*RdPcWnMH+K#3Z;z#k* zBu$?Tb+6cd^WRClpbzPCbA81SsA_*{@CPB3@Z#MN>N!0+R3GRn-ik?XZ5|B?J?$V+ zcp3hYLK5L(M`>0J5>p?SI7}A!A>?R_;donhTG${6sa>8JWqpO&@P$kAfBF@>WAoe0 zz=;rcC`b1kD}KiRq9A4~0DmT{mn6Dc;&A4`ms@e+!fA&yC;K}ayxJLlj`=ao%3{}| zwi~kYTe7o%6lFAK^j;3%M$7e`S`}ffKx4#j;brlr(-2fv`g{uR7D->6e0oy;GhrE9byO5{y75F?z#O1yxp zA+A``kn*4Ft&|{(<5&~0FIXAY_FW^+v|n;-{L}eb;M|YplK_%RjM)<_^U7pu2h!MoYWv9r=psN9nZo1Nc`{N_ZGr78O+i2N(!M zDr$FT-_5~!gRAXZOuCwCV*kU_m&a4Reea(}jdVo_QIsMykalIpL=iZv)_BKz1Fjy^{l-cuDw5Rj*@+z zUdh(Af*cW4t~f|ODYe5$(h<(2-sCpWl8e`msj2H7<$8yNBBv-d)07{c+2y}wN<4~e zn7gvg_xS;ENju$m)U%m-%U?@9;r70u(v=J5dXM1vnK87BA{Ov67bF+_-{M^H7?T_u zSsG0zHRuy@U!NeFT?+E}lKzj=JyrN~)k3uL?`>0^wHooD?abuIVwikB=PhttG(31N zR&^(O^z(7ST&0VKpwgMU^mm!fF9khKmd#W{pe<&v6ucUUT%)JPZjZk@o>0Ig4(nB2 zBxf+{H(C%EH4Wrv>9E{)YZ1YClO2-DLCK zMp|X#OF@yVJrD0Ghjkn>biY818vBSD#%7kzL(=%gh(By>9*c5)l?6?CFlMgz9rYkyvh0V?I9(+J2BGM=N4#AKtT1muCiR3 z=hJPVmj*Dwk8+$devO~$*p|uJZ8!7$rn6SJx z$`y4Z`oJ?907t4L3pG#Z42g8CPrq{-#~TEu84Y#nX3b3D5jxFCv7J@l^}1nn+&F4* zyG#Lgu-g*YfT&KZz8~74sLab+@VE|M|CP1OYWe!o`!@c0j+Xak?ult50&KW<^mSnv z%Qf-BS}I7h^f(3&Z)-wt5>(W~ECducjL??aQVUBrQ*RT~APb18AHxI#y$=QX(6Z5H z8lQ%ZzQkRl%iH@`6|XH9F0sPG>-DkSIyT{4y3>$hZcp#4DDqE*$?7%yP5uaw{au2J z2n_QC8c;kl#bPIaeE8{?Y{oi#Bx`NFhhRDP(S!#q@Vrq2;KS@$oweD-SL$G@CZLg* z{}RP>W+$dT9A=vuYWwY#I^K(Ew(&S72)Lgo8ZI8lr|^XEknaMC(ijjpe1z|de#tEFCaKtW zzObtllc}lu+laS365NKNdcl{hlPqMr@k;f^m^e!7>g&7XY8*Th9?U3z5vqo{B3WS|8)mJd)GD zKevOcBr@fW9n%a!@^|wjU`K)))wirv%2;vI3085x#{E0Qf$078hbpZy>^5bCt)38Q zqzT;9FvWfzfJU0S3fQC%UC;CL9({ts?Qr7Fr*AYFec->O7nZHgdCdG|+W;H??=@NB z6(nL70y_txqao`+yBO*>-K(edU%i!2z8^M)${p#&2D4iRiNhZBUjRckH*>B?2XXmM zl5X!4N)frDGB%=uyCTNaXrgO>EXU);kg%%{t6<^fDgsQtW4#92?Tpy{Ur%&x#$`Il zbPJT}bW}+|yOnyv>Ybz2b@`oUxu<~evsNxJ9Z2=4)GyQY%ov`b1@-=u?^SrEX#$0c z{Tw=|{MPyk?lAe}+QqQxZjC?dM^N3#e)aqeH=9%w%b^(!6hGeto`Q9>@w?FPY3*vn5ShA9U=%1UYM;BIo&^-fz9yrq zIUfRjKup1-6qbFE+LdPxWR)zZcj3XsN(l(X>OYaOO&A1bdHyD>C(-<9fe}r6v$tw- zrv~ims6a#dEUaFW{#f5zyVlaO)mb%`s+^(yU)=Qi{2zlk_Xhs~Ut(E#b7i8w<9YVL z_k{*~c1;HN6luF`y8t#9ly!$nd}fS_!bR9k^Gp@XouNhEF!U|Cq_u5#X!gS1U5@jI zdIxrPHh5^-Qz>z;ZYk_x{pNlJ;^&Kd@`Uj`m%$%H2FA{@*ioPZU7_OAmsXgMR-b>< z@I1dabV?N-FN0>6N}M`cR_5)|eN$37<=xx-3e3u^j`Bd=EWF~myrDRaQxub;WYi={ zefVC!IurEkeuR=Kg1siOg@OmrM0NA5(}g2?XJ3Xvf1p6&BSh)3yhW63x^@aK%K)9{ zZezfLd~y+5o0lV=S{FY}tlZ+}`I7kQaXkU)2dfS;4wur+VJNxzVN0Yr~&%oginR3n}|TQdg85xT72DK z0++exjH{WErkjh4QrK<6_TIx=)HGf1t^*Mzv>Jym`~=j{{vwS>kGUE22qP?k^|4x$ zugn_mN4p9UD*>PEUi0oOCH3o`z|2> z{jA|KEh%&)@9MD+eRt0B45xocyzFsj&*1B!&xgZfhiA`s=dyBOFXtI3{fTy$miaSo zSUTQ3@-L%WmKN{6HTkX854OZXBTPwG_KpXta%Iu)vRQWi8+^vy51|EOK8IF! zNiMVzlQpASGPw}!O23_Xi>FD!Ww-7=YPiO-ScEY+p)JJ@pc={{+I>&UP;(YuBgj6r zekrSIJIG$Y|3r#$ru5rsyXHKu^`fDDF_k-EsQ8LRP-w;HHX5-r{+V;h;iomF+HhrjnJt>=5bL4oRauRLAA_ZN8=iED6S*(d1_%T!gT=6or+2d$c~iFM?C+1T76o|I|{ z(BvZ2?~GF%bg#;t(?)Q(7x{1Wi@Db`ZUfa6yz=DuuEtUVq4am~cVRM=uo44ng+~tv zYlk0{JPnCZiN#RqPSUW?ejc{$xh+C)Im<2He^OG?H)Sd=M9pns%Xep2O{~HDt{pkz zy)8@9FxcXE58yfk*9Jzq4&2ydGkWcB^`?CvkUysk#bEqo^|6tew5?%!jb)fW7jrg<5SPMVQE(j!v18Fk?_clug{u z?hZ#{JMM~G2&D{_nYI`Y?F%4_DI7L%Y1W9*R-3EGtQJolZ55eZv6|Z6sf;PS&J4dRGo6o$ydf()Sle?NMD%Jb~c7Qm}?{1$H?t`%J;=U-DeEv zPBVL14tmimlBGCDRj^{)JQCXQIs&p%%SnKc<)QDrBmdYt5G0zV4*k`&T?| zKbO_F2l26t&UIab!@-hw^WPt_3%7EU_w_}H#No)_jS*M420LeV5(u?>$HMb477=P@ zu!p5$p4w~kFI4RqOQkUu_IWPpMFYp!8j?P={WlW>hhk@*;xMrA{q%7SP$^rVRgwR$ zB5Lbtykg7)^Ggc;j&b3tNK;AjcbK@JFMHOD3j?Cr6!NWq2-RZ=h z!+E|#dX17X)fB|MHPf(|H*@+8EB&@CDHY?vfiqJpSJA4hwqn1SoS)*Ra2u&Vd{`e) zwEcS6!P65Fw-hxAi>R0l6dj2rU(_;)Um7!bP)bRK}$H`2{3*%)sIUr&6Kq^Uman z8*rNIs}6k|8>uxZisp7Dz}a*cc#+>-8U~9~32Y-zYm9rFzT^ay&^)p}K7?1wOK%PN%NiKUYtERv>snB7q_C~LJ2}!l6Qk=oG3K&1pU&E5!$BI9 z(ZG;bb^s?rQT{A^>uc0q9NfS+5!dW!XsO|WO>U;~s@y}H7Vwp`=zsA=v=Zi;N=AaZ zVxLqFS4+Y5Q|U|6gws$S4S&P=4c>y7)++3Y#gnd zyOQlP#=;>``g2}imKU`@L9e^(T}E0`Yi6`+OnQ1nnI+Ci@UqBKR@qn9)QRVbTL(YKJNwt9-7>1%gy*!|3$Slyq`8N92IclHv(3Gbf0%n@U54tv%h#1j zWC4HfZLR|LHx*@z;HmGp5oThR;;NE!_V>nwKh`K9MTBZ{lipd$$pnfNzcB00GAB(i zDaNMuD|@~Dtc4hVL6&ZmixI=Zt#f}rzSuf6;buSNj4{=Mfc6@Fj=O5kl|wI9 zK?hRvy2F1dB~wOxcZkC6ui(*?TYPVvlxsV!k zHF@!mlBtGJH|}Ze@_bWgmX*b8X7FTT3AU<=YpSBms!o_CJ`bQ7$;>hE$~f@<|UFm7K~xEnvRbu}kEXRLl)ug|59FoLKg5ewTEzemcVF z4c|~VNhnEly|VK|vQ1~{*FZtUrER)h%Z|QojpntEpq5+Qb8-QDal=Qe_3NB^=?7-U zI;mmcVacojXYPPIY2GZg@5+*6357fV5f7g}iT^Q^d!K}ly;W;{GJ6;LZE;%_ajt{C zlKJzG|ETl^r(t+;55gbA0hsK$^$-@}RC%>CFM81W=5ZG zCh3u^B)p)V)m0n!Eo+Wm)(p!vDX2Zrk;m^TT+>)Nb=Z+Uy;|GvvXIpz)2ocAd|ZMufEY7 zAP_~jLkfzR!dD$5l5}q%vmaK~xQ+e89hW`h`i@)**RVc6P?D?=omwGFrl>zor)$yo z+n?noNW<*!c4fXq5+>ABm(Rk9mS0~NV}lgBh%a8yJ#-HN6xBApt)p#gMlYgOnOgci z4zo>GwadLRw4Z&r($tOMWs7vd+O3>i7kY8m$ zJuskEq}qBx22eu$a{Vs7bb@S%;FST!!?aOralIiH&ZropeV0L)mT%3JL13BNX3^kQ z`JnLbEs9Upwz~`1ab_ zbWE^^O);Znc*2uj(}eiOP4gSnsIQ4`7mYowA2Y+aF6OWq2|aG5^~9r$L5Vcn=9NRv zJI_><*Joqc8s2p{Pn(oUYU95!4(-Vykss1-=xWG(9D=g$5l* zP%bQIE?yt`D2~kb*|5*$$9=M)2n8h&q&H3vhgzT(h8q)Jn`d#vDd5eGa23PC1UY zIP{G!_A0$5m}~B7x*Qr=Ji>ltmp_0-Xj=Y1JPWRBY z`gAkFkUwt2TTy5o?KK{O7gWWr-$&gx#C3OO4x7Bi_>m4=FHVzQ8MAuowmRB{^E(99 zIi(2Cp*ITfwDQ9+<8J^8;4{RzX}5KT4O01;-yN_T&5^OkCQAw7rj6IjJ61x)oOaPJ z3*)}5Z6n7?4BRi%$b?2g(RTUKN87=J!h_UJds_CgXW3AwZ&DO_o4$^$HGwkV;udX* z!wt5dGpewO|616Wx)iCMfUD4Ft@q+Oc*WM*6K`=wne&?;4hc&@kebA-M$e|bEuCa$ z5Quig%C^7#o~4mLfD|Z7a3gZ?r%?S2a~Uo*_rHaU%5)IGDJvCZsrT63>Q?Bs`z$FT0ZJk#OzQLQ~ePmc1UxZr#2dO;6- z`E+@jy>j&{3)x)mqzvWe{0>;igW>F>zKnNuowttdj}I0-P1Z*vAtHAq*6)eOjjsE@ z(?yVzWWQu72{`{+n` zS73E> zKRD+(I;a8qzke*vPb$MPMyP$yblAm7x0^o6&8hBHHJWoZ?Jec&5|$^AA9O9Z-^{B# zBm@|={-7qE8ZQWcin#uRkMxHt{c8sWZ{(YN^7+ufY#fEo>O5|o^)3ax%OS7YH1CGy z=(I@9yme}Wn%4r8zj3VRii$K7d18c-o6(&6_C!@%?ao%*zz6v;^2B0E3_}2 zWEHh`VXiPy2{%*u(`^o49%?1LaNc+VP-Uyi+}j`c=)Sf2&&;~YO6e8;iAyH{1x}e) zl-}C-9(6EYyRaHLt8*_k*Lm#Rr04xHs@W@b(DuYgM9| z`7vAAyEVi)YmRQ%7JWHMQ{n%LG8l5aIEF9#UTvtV|AaU4E$&5b!_E69Cp9KI~icA;!&+{uXWtuh) zYAxx!l}cg~(p)kVJNy@t<{N&&K_e}&CF!}pTUdQ<1ECs0zL=qCi&LwWGi-*JA5aMg zN4(Mj*!`Wbz2=O;MzBVo@Wzm-1v(sGmKp0`ZRv%@K=A9_{Dv>SD+!_QH!YvZZ^DC} zPKpC34sF`ju3e}>TXUMsRNC+Ii-NPP@VjgA3o@Fl))3xcBI(3SLG<%&DoU7*R?$gy z$#v@3w#^ZBdq6W&GpJ=X?4zJNETSpW+NHbA3J2;GlErB~>L6%L+=l@DupQl#illUj zKY9|ZFH-tGy8aU`410K6|J=JHD7SA9g;j^i5d8i%1$=1DL?2{(IWK~AI!hGPh<14{ zQs*PeHA5q1F#*n2*^ZP;9H*sP2)l&2#14Rvz_%*jPuw3#%nyarMTrjB_|Z`~T8F@Y zZop=KE89CqETRTk*7eW-`Z21ve3dgdD>LndQo2l2gjO&J7y7wi)y!*!Wx)kxuLY(* z6UK%kV ztYljyz>`M4f|(Pja$WemoqygTQGps=M_|QSz3fn}i18KmPXiEUG`ByzkTV<9!K5r6 zXhuqJCYHDhezzDhUa1HamJh2r=ojwR0MJ~q`I3sYm#KgMlfy#PMW-L+k`PND9X*3` z!U@zE3krulesvOlWLN_T#oId_|IF%b1=g&5R!wxC!2#}}Ki`FPM0m++MA)jxkJvTO z4CnIM;xHDxvWoxpFG?Nd?gY>JAhdHi5cr8~VH{g7T;FAyz+O3e+a4v(bmg2A zbKBlTT)vUFoEq z_4*6j94oxMgt93<#aWT*`s&5=zg7qb2X|s94IsS#>+wxV_y)i>s6hTkVmOqt|Wq-i{Ua`Y#nxx(q*3=e%Ep z=|k3Va7q4On(Y$ENYML@Q%r(R&dJ=|#=wDZc~v_Bc0rs7%9@@_o6@)+M|8NMI+EZHj0twJx3HnNghEEYQlFZqUR=QJ=7+?0d%q3`#ZG z;iYZA4=ewQ!+F%CWnJ%V6oRC}VD?CBtjiqK0 zMauX5A}*_gH(%RhN~V@BI)%p4Q*9roM=d8wwmnW5DFXk4Dn9)r-V_4ysKyv`8<10l zgJR{ICP4p$KdzS|d3myz-=@N9_(UFwL25qt#N1PWs1>I&)a3X%>&}Ol$508&U#qub zS{p&0f-1hdHZNxwP8`)p_kL9Sv${b;zHjw z{3s?ewcEf~(16&OmZ&qZUjCPb#7Fd>0@i5Xqu^~M2H=|T=eKd( zypSuRTZ9?#TBNRRcTXUq+QbW7->xlz314Jc-hhVPKW{ zbrtbaD9hhZT>1kn-$R)IPI(Z&+wR_!%gjPf_``ZKWK+Dp5ehX7EYT^xZ7-6Th}SC7 zue0-RnCvPw3U(9@xivRZ-v>G=YBL1WfBq+Fz`qgEbw!wMU0;6m*k8awxR}i zsYgc|{W^{CpoOqjAwK4mL-PX@BVnSbAuoa<6lyNM*NaeQfryMis>blegnp(-}abA4qwwfEt)IUaPB$=1mUO7K-=o#siiXXlIAU8)fgaG)y&B zrsrORPda@MHQYF=_qAK`?@4)dY1?O2aAD8>vObVblGzFI&@#_b;daQ^%!lcV+Ais6JmAWRc|P}gje1vJvkmULu(ptt3?F&OGcSx^2kFTds~0YpvixPxwIXfHBr9(La87G zWcAq6&7q7wiK&@m5%LP+M^N-54IOsY{~MYIr|^O>;5j&!7g2^)P26wOy%S6svg8g3 zHK|+!oz>5HPXyP+zgS;0sd85t|Jgk6JYzqs?VFy4h|IRw!ggVyw(Xr)+>@{2we2(e zFTMZft+`4GE1&;mkq~;hZ;(IGlL2XG;eQ7SoP3v&IWazL$I`8oEP|E>wmZswkU6KmziMf2I zKlppUni>NfX=wBFWM6XTtq1pLAoQhAM2m<2d^b4*oda-`v0FM{_kU`iPiaU!I8^#H zedj5UJUJ3SlUrPvYG4s5g9V4KnqCM7sI73uv9Yu@%41b6=Ku}YZ=D-zzcRE&c@!m{ z_}L!Tzic;&H=#}%4265mw%6wvs+Gm&Q>zTpe%VF2^CKaCZ!9EdTNe&ITBfeEOiOG!W?-Ayz{oT5f}n&wf3^?6jzY z(3*I`5x|YkOVWuE9CJGS-urCek~;Q<`jk0D0+&Dc0gsa0Z1u$xI(C@oDJJ(T8+QCI z*~AqKish#g>ssTOUT(zro<+Et879tQq?|6rNzvwxbsnLNCTCV^ z$;11c3l8Bk>|!GD_kvSb*rc%QA#f~~lIMf6o9 z)W%=%8cr>VZJu62@b)Q$1HCS|mt*^2~k)@ke1H-g!s7aLk9>=94j%Ja{xUvx=?!5GCnO z7G&$=p{7TeXdej4T;UDgbiL3eGk2u`Zk0f(@qLs$Q~pUDh8*sC{4Vx=I6p0=xwK(C zWcGFkf<#}(3Dxl!Gx}#ByO*yxI=|RU%hJ9J8Jz&Kx zRD!TP87a6_ySDh%Ip&d5PahhGCfZkee=`zZpjZ^tXr8qd+c}FZd}X^aI(P{R@r~#g zP~+3&M%+fmnvFK~S_|$O%uUgg>+<@r7GN;1fs3ES53ido0C*_u^SM)FJUU-854=_7 z?qt;W+p|ZIQ^R$I0{f>W9`gn%X9_XlOF2JBo-Kr2Y3pfFH9DcQhvLnVK9>e$XMttIi2@?70CSgWVqp?ua zB=vHNhjn&0HV6hXx>Y@K$WCq%bPq~mGNCNHWz9Y(!F@2VZ(b0L9R zwr{lFob~X*XMQPhRffh7I|<6*aaWSm*9)NY^X2D!)IU9#&}-{S9q&_O?*LgrEwDd5 z;Muz0L(keK6o|}BIda>lW8+4{yTF-EoC%%^$ z$2rg{cWsN4+ltAkf!9bXP2GUPMDe0ugE8g(vk1Bj$*k$%bP$k3r^XasZDnel1y8u0 zIRn0!PHG01Q@(fDw(&KEbPld^UNXJxNH#A0mKkT~nr4Kjk}2AI1Lt5`bm%Ove>$ml z7J~^Xx);#bD9E(nL=V}%VcUXbcqNUor&E1{2GFj2I{N^t84<;Y^&HkB#eRgVjeaNe zrZ&9o4HYFL2h_Np{9#{{g8)EzED%@R*_e2a3jr=&5s-jr(T)|vjm zFr?z!S-bOv?h9kh00(NA04cJ9CIu|LbXvzDZyeJo0R%IwgkQWN zb@!5e+7k~+J?>`sVrKN>B?WpqM&OMCjEVtXH;L`f9otkXLYzszsbM@vPJeSW-DCX| zI{h5PV`=-`n}OpCB@+m0lHBWG9=P@ltw zGtF* zyS?=LXw^(jB%XLJeB1+=LuzqjZZJ(qoEaJAoWSq2AFAl23+hy2);5l*hfQ^bRAL2} zoH!hDKmt0(WkCjwAPhN`=TgU!;prjSV^0*)#1SRHFLd0s=rv7QO9vYg{e3g~CR*9= zK`i{srx(0FP;+S{f-?uEo? z4g0PJaq9Kx-;PD2>aVlJF?5zkKG=@OMJIgxne^6l-A`J@uuvm6ACIh8Joul>0#JEUV`zO3ZqzJ3 zXEZcl9aaujC=ST8E(#&{Ae%d_cVhU}=jw?E4=E;vhfyS6Ofbq)I^;AyMF^xSgV@_2ar`re*Li$FC%2elg{2vfo2Z<6! zRzHq{i%g<1rTq(za*ySirWR;9Z0?Yl7zi7gU*`I(+}^RTlVl~ss!*gU{Xz<5wLf#g zAyD7PiU8f>`ff9fMZ3si*VoTGY1H~R4eS>O?T4|TLjXtMAQKQLZSWh%Q}1vhUNCpC$l56LlC8=;6H+Liu{=hbmAiLxyPf`F{(Jexe zMaU{5UY5{ISA!(zj)DoK+|Z>g@nqZMK93rnRWdgl(1RVtZC$qCJBP4q{Z~|hrOqmi zB@WOtG!XX|q}mHzR*abRWd{mfUr4}57^*0<=iD=z1J4>wekfd6sHCysx~RZlE>^;L z;}qZ3S5e46~7$Ar3NMM;DGzMmlZKZ4HTKOpEj2=Nt_O2YQ4r`ASs2db0; z>zcNnGGP8?>^aX>>_^NiOx}LADB7%)eiEOVm9MjOYe-2Yb_BSIYV3bM#Y35pwoy1L zFR?L3>G2h!EGZD;AYcOR5w+y$dv1sGXNNXud#vN5K^BLJD9f;(ryEt4&XbXTbz-mj8|^d z*%UE#W;c~+^yU6w7-2q#s@|ye0HgYQaHZmMnCx6`j~^^&2ljXvwF7Wjs@R}rx3VMw zsQ8XvRVXEyS3Km8T=IzzGU@6 zQ<7w8>`}C*(?&-l`m;QC46#`U1&u%P*9EVqD5axukm}OK&i<%VdR;cXo#hNnjw+Nk zB`=Z!+5E>R9%-c*L05MOGi+l_YwUPX{y@wYLR_+5IuGo8b{vK;m>AbwTX`Dr3UYw! zGt;b>-+oY$Mrh6S2Z418y*DB3qPC0)8jw=Jfl@p(C9UhMe~)9cOfJQ!31aat>#AdI5kR^klTg4;(xHe2}@ylG%Y3b z>y?qdIFX>-~R%1}Eat4P_WlHJIZv00rA^_4<+%#4AFRGrt8Dky9d7Fw9!IG^H| z%xsezhL6%cFniFMJs1FaYBdln5>F;9E!S`MhIitlV^jn70_8W3qeowl;kRK$RkKfV z0cN@6`{StC-$u6Jm(*(*>WumKh}F)O+aCDq2yI`5O+lJ}_1JFPI>Oz_%t}<}1nHZA z?F1nj{vs5{p4eo|V};?@07X_GMdhB5Ot|q!0%4jQKH_hS&CKHwL7nlu0ft8RKvP|o z+`Wqgd5V)2RPa9e8+9ul$d-Jd>yH|6RLOL9*y)UzHjFm`VUoW$OtBC0>Cro+46Qo& zW~Q7ve!wy?B(t?>=;u19WyLBEEw0>Nq(#<|zNoQZC6VG+=+T@qPI0wb^D#_=kbA3V z+@9q;o?{bZfo11{M9fK+3W~>ojXwO(96upi<`mquW89tm4jrfGfNzk%Sad)83#23s zCuFcG4r9Dgw+d$;-@_MW0X8( z1X5p=Wh@xF2*Emrq49p7VronKMFie|l`!%p0UqV7EY;fn9+w(p_My2&#i7hEKfY&| z(O~eU2q)tI{+)za*pknxh$2A-^H-$#tjFL*9HM&<3_Q8=_(}XfAX!H7iv^v8X5X!Q z_e^_tY?LnGK1SC1NUkIt<(}`yf=E*OZHBW$=>`_z**9>BhsREM{O+?ty7^UyLWdP2 zeU59!Bjl*!Xzz$D!o;tUhenEZPVqsLnQ{5wk&68}{WplBQ?E?2?FDXWKO6WUAZ0{z zgNp7-O7pjaVp4=}pAB>)VLQ>ge=;NTf`cR11JglFZh)!s+)!pW^Z({BY6~g`uh?9! zOQxE9R)#w|Um*@Wh(y0$%nD-yr6Z_sSX$8( zqx7olLu;;dW$G^1`r0BlQ=BqeLkcSuRI}GBiYk9;c`=HED;NgQ)&6$Iyn?H0zo%Ox zy5=Q#r1|$pkjgE}{L9j|i{(rs4O(&UGH?fz zbBQWCck^b`5Xi${#C0hzI1~3s3<^O)^i75yZ9RNKU`nEL;6dw1w?5?3)zLBr=|BNg zalZ%xbbQ-c#w%>wBa<`VHTs$rqJeMZr!{wvPi|a~=d1jEY=|DI1r5J2qsN@>!cPsn zAVfC`>MO&&RyvpeMPdmmm(H~zKIn#23Uk}Ru)@-)F&<&|qX3)t7UB<=T4MUdb5N}#HjpEdDZxhqh+PjD85+!I z?;4$xWMqtKA9n>h5I?H zoE3D)k|^+4q-1FTpru+M8O{we%?FS{d41z&XrRqR*33F6-{8(?qrzBr*U#chL(cXI z@tF(L_y1IkFtR1i$XdZ2*6Zus+i9i9lV-#Ju_rdDY&*lAYP#zbrM~s$z6SG|FRk9X zEcF60_~F@WWVj0=V?ydiUEo(?-y6)%RscJ@t1^wTSSRLq%<8Vm+lL_Dcn(q>oif>- z;4w?JMC9*w!HP}&gLvKSI^t2bIo7s}s1@x?G@yY9@7grJw6lqB^>kZ`71R~Ub<-Re zU8F?i+N0gB3hekP{sL{}ya7XipfXTxH{lbB%Obkr&hlEY-k?lFKi<#&Q3$AU zZX62-M~TOVvL>fuRk|`O7}-Q~YWx(v5WA8)Xd9M~+wgbq>>=ezy8txZfKEt<4f=Ib z-Yr3Ai;kb6fI+Bta7-R*Y)y{eB^*Br$)c(qD4vp|abzPe6J{p+;?%LvJ5)*2pctXk zW%;|(Con0SqpI#L?i!$EQP2TV*X1XDRb)R1vrIbsJi0HVh*6LP<#KYNaY2C}+6rQ& zh>nJ_5 zIMqa{v6dD}jVQ3Eg>Nu~WxkZkG)ZLZC3|bOUz(m+%IV59=j_nlAmaw|~clzy$cFM6qd8Q2?z%vxXu@sDk#w1{_ay6_<)>0xsq)-uJ zr%uz@sKSxI*0zOAb&BAV=8xiOrp_q=kxVF9?e4+4Ogbsy*@0Bg>Kl)8tH=+izl_ z?)V0zXPv@AyJFq)eIzur{V1tvcFp>le(APtA^RKi8}M~z%N61&AJwN}X2o%aca=K` z(57R8h4lRRmfG#Sd!X%ZYg5JU=2Y3)7t0nl$)N3 zV916~&E`DqOQ}K{?_ZJ_#j+s6bpZN;C{8c#z8)|Qb+ip6-t4u#XS(jYv{!MdXS?bZ zp9OL!T(u}{ z5T-V}2`Yuh_*ABcJjTu*2FDlF;=;gzJV^b4&1)Z-Uket!(2TZcqxb}$m<^?m|4lAV z91$n;BouYd%D<+uF(>)kD2&bXjCyIm3j80!r(>)RBPw`aC4byo%h7Lyld%W4B7GuuX{xSPJ$}}$Hv9$9-i4m>RJU_d7*{O z%n()OUq;D-qb&pA8Gh`e8I&}m&OU%NwEE-;kBo;rs*}@(itrnDQg|xR$JmEn;aGmO zedF`eHssFOAP_yaY%AZK(=sWU*ZR?!qCQOq^pxgcVru1DUDSNlR{EiTiAwCjIWUG{ z6KtVMOqp{>Gi}Pl?zY)A)1UO2Ftp~KK;toRHeTthw&}jd)a&lvG~$AT+r^uwKR|m< z4685v1Gqv49AZeC0GP_oQEpb!m5K9TOF4ylI2-RL)U9k9B}Lh1FI8?|f0Jaja|N-A zgbxualQ&n)41idQ57gJ-GO!TC33wZ4J>-8rT&pN5$CnB$LY9W?s2rUAY^TXn^)e>N z2cYKT)EzP(@A(RDsOa*bzRPDtdKD6Q2yKixd8nA78*dvl2EB4%`paB;QUy$9@C}<%h>Q5A;t1r;9d)tBwKlkegK(F7# zUuKNL6d!QLwDMcv*St9B>I!T~@V@Pukg{1{i#BvqJ+1qBdwr>V<%x=XgJkao1dN+C zHAeIn0$;++;8?Iz{#ZRsb3UFW@({K`uAwrMy53|m5$mSs*PxStIfbr zP^HBO9pa_ke$zmiMN4B`>dgK}ao8M`P?`xnC>vX})j&9w*&4MiC+51Wo!QEVT=+Ds z(f)eT9FyH5s!N@(vkLZTz?X)6txB777s-VgML4knS+(kRC4=TzT8Tkhc{DDPyRw6S(u*2t(U;Mv{Gh;_@ zVZO@rBQlj~2%sC2JxvC!iAednI>NBFy+?yfqp$zp0;7B!S=-|}^QG>8oI;O^jW>bY zLyCM~*b!K&$Ho~DeOc;+78mp(O84bR91_eDjV_8H9Zg;w-(w^UwfWUK$Unr#xepf2 z+Da>EWRYFTmxG&sCZ3FlCye|66Gr8lWm#ZvmumP_?oWNQO4xrQ>IYH|(gHFC8itD&NyuV?nZFVY|f zK5w>ARjK5AT=4tHNlcDh9}L40PIMvmj`fjB%8I({1v{5-OW&$Bi0^d=C2bqw^KPWwkEUc__fKOHmTR=^2Y(B25 zwmh+3ONu_-lD#qmzNt|{tdsfj|Cm-x0H~^5pGbq1&zZfD|GreUuQEteNg5@@z%W8E zQofyIJBJlBm?<;}Xq48JS<*YK8UOoEtHS zdtuEplH>LM-Xt`S5;E4d zv$|9o+D5_=P+U=5bPs1==Rhp8xHBXx{E_#Hp!&WtK|KeZqS=>W>W|3|6_`b$VyW4M zAbc&20K^-wKAdT^W6l|OD*1`%g>=)Zjh(6Zpq}|N$jY!aqOmb`@@E%c@kHJo*fo(s zKUpFwsao9sX;HLCg!k^49s{C(c$T!tYM`7mH6QqHwi-C~?acmb_bj*>{+hQ(1nvXk zhgt1k_QW0^K#S(YXc=m+doWR}>>TDhTHTE_B)SiInhoIu>}PiSjsBzayLQVz?5PYP zB8>KAGOLu#w`V0y4)6av8bQhILdwk@iy`=Zs=F`(p=2GXbHeNF% zzfLajsKq*4WS~P^TN|%kWZOrK@V)$;HQn`1>D;`EQO?>v>Uipv7WIlltV4VS(ev=| zu!YNdYT_}*f?pkuksmExP~Lfx>D|7LPT5Xyl)^?P*gSu+Sqo1aYEEAvrR6Gks!WFK zo@6UeUtMsfsgbt%Eg5EDXwOqsv5d3m)d&xxh4WPVvF1h6h!UJ{RBm9}L{bc)h<*K8 z%C}t_=b}yB4#H2v%+{d=oDZzc;v3b#cnq)H9(wb8O!->R(7wM^l6pz8!zQ6nFm9=_ zH*y%9md`1~9UU&B)O0^|1%{#bx^bPYbY^wq}3uo_*cX)9NX>0>}1V zcp}p_PnGgKl?%VsTa7!rdqVR5$C{hMtG>LlKznaz&lX+-E7$wcmGv)Z!EqflvlsJ341|cOB^adC<-n7~n4N?>?|} z65S``njiau2w$)8@52nGJV|C4&+EE@1=_}Dc)-TuT#LZRc)VKnLF~|Lnu_~~i4}Q- zHk#@v#AQ*R@Eg6e!S$gdY<8{aK1Y3TRXuAl7L!(+w3*&#ngM;_C`Os^s}GSWMs^nP zKQp7x3BGmuerns-HT#6JO=pO2H(baNEMaw*n=KC$ThgA=Sv9*>%Be!Ieb6ohvZmeq z*xuN|)-2)Y1D|(VGS>+Y*yl>>wi7Jni+DIG47IV6FgOPXIpy^$mE*|HZGFXJZd_4K z$}t*HOQOMRahtn`l4UJ1UxU!M#KTdiKsfFFyFt@FTRz~?gEorwj}_ zdsPXA$6*)Z{2%6}y>0o8TrQ|b7qsS%Nj0LT^)&SX`%(IoVOfT`;5pftIQ=MOznHZ~ zvh*y{u052NF=^V{+n`aOkK5k~v+R{QfuABaH)dMU|83Mk#bxreP@U{V$mn4^x)YcG zGwY>|>*^UfW_2(XCfP6qS4~nNh=i ze;Aq-H{d(D22f-t*)Wp#BxT>g&3tSoEWGFY~0abQ0J_~lT zhM2(9W<04s%5Qu|=f#J{+snEE@IgVfmu&9)rpfq}tU%cRunw zgc)=BHrb}nYWxG9ox884RlWV zic#WdR$0^bW7sc5h~vy_lc#$5d_^oE;l3Hvadn%K|;R z*s~|zeBPL!w9tTxIA23^;zYoJS;~Zy2HT6RPNnfRY9#WAj6HZ#2Snr%NDh%}cI0Rq!V_1_aRl!zY@ zAd;tvz8>h&EUR(HuVpx*%eBc$Cpk6`a;V*ife^|b6oATV%m67lbQTxto`YN)AZ)6&ttvj;(L zS&a_Lf>yFw%*1X4g2k?KC%{nm845>!H0lV6RG$D6nr%$kU%e)D`dfwxr|k5pTH=Cd zTS&evC%g8DP*B*qAif7`?wIqFdS z&iAGC*s_%4w%W*;n^W}>SEzXzVRc4zT6{OOD9qG&>zSw^(Gs5HZBc^>yvArCAiw$j z^S#7@IrUOn=`mbU1rnfYl5w9j)=O__Hj|mW`=l>uf#Cg?-B+)u-QbFAlXQOe*-rJ2 z|G$MUv%;S4^8m~NiOyt#Ptz=3HWZ{Co@^yeR&b=l?c6`U(L}(N0$xHRRA|96NpR`I zBy>8<%Bw=%2JaRk>WdLaLvqK1bgB~pv5s012U-LmBcB1QkuOibv8?q|I{fHw)sWo> zsP?}k7YN~^oplV7q51K-q2fBA9xGcij$cBA=S`JQ@V-|(7oBM zMI_X3{J=OqxT$}F(uHg+tr{v;=({g6)WC%o(XoD2CX453-kj8V}GoRt+&AD|ocQI@`kR%xdn8c6Mf_ zrE!$V0ovaW#__4AJNl0qFqGjR5~nz-nB2;NQu47OOcHrBr1KP-2`tkp_+maUxuR&a)xUlQuo^O0(aT~?gRQ9}CJezqHH?Qr@c&in8 zXrMqKwGgMA#Eh(71O5*9kl;G*rzFCGCdBMBQQDi(ho&O4KA}>RuNlLvAJ6%s+uUkp z-~g3&n02?=XJ?vQsHValihz{_XZKBTTEE1(f6!C>i_5BNxJDSro24k~pLnSM@)AQp zvxVLBM+3#1p59%#Jj11%s&pI^h`t$B<)JS7NYd|ii>a0t7Pp6Qt*N2Cr_zEc28sTe zXYSG9HAaC3i$umXO)9yIBQuLxXwIi-P2K!m6NJgB$rYv_aP15^fg8|&@GK^t6r}J) z+z<831xldlv~HqtXeI8R53F{>Tt&s2v7pkFy4M9yu(&bf?hI_znwvy0A;HGYfK}z_ zq5*6a#;ptO*kz+w3-9Wjog5}cmq=GNIZX?lORhZo0a5%*nb{$FxT=k=cp6Cf7EUC* z?{7Fz{A-{u1k?mTGP|~zCpf+f#e}64NESUV;EBB>1>c}{n?4x-esqc8jftaX(dzVS z!>bqAOkIuatn)1zzh8%Yt$DW8v*#4k)1V+lt*)w6So0M> zjNyWRSU0!6s*q}vU6;T%=V-{{Bwkc%S`x|2D{3pox)WkdFaU#%INzMXW&o857Zz$@qRf2W}kqEhV2rJd}_4o=V5<$zrC z`l-=@6rlvR`&2y?N29*aa2M;+O9r)(uA2(?V0y@pR(2oEH9K7|mi;7$^8Yi6Qu{Q* zc0>gZ8J&*I87*exf6uElSl>u->Yh@|#!TnuL)wok=H6(dJS;%bC5 zkl_Rs2KII#3y+xE+y1Cjy#Z4>bB!Ry_NW&F=Y-VA$#7NlTGw7N;HGVoaAKwl(FCEm z0&!~g0GuQE<9tYiK-6q*E=X!{bTHMYw-#Zuj$Io)?j5HQ%mO8SU|-?Kl?+p9%^yYD&iQE ziw~BGAx>F?`HF>Z1F1=Eser2(5Bj;7O!b3Z@$m-i`CCbCl$qRV;n{aHk|viG8J!wC zI_U6|)i_LaJG#z~xZaK1S6QuG-$>=?uUmBJw?WgOWRMB`-!#UShf@vM*onWA0M&gp zIi|CR0fgjIbj8c@@A0u!9u|wum=o(UaS|_!+9T9h)WD19ZmZO2Uxq3UiP_o4U?aE?IBAc%-&%8v+RCagw6O z_y2j3##Y5GYRq zeHThZ_&vb;ou5Y}`vaQ-iMhedZfWA+p#itmT&Y#l95y7BaK*XQwm16wI!UdZ|Mq>i zaV8v5jkya*#>B8HKNY*PWYj;vl#2@17f6R#H4+W^v<+e?HnzYmb3*^oLL1q`#0>*% ziNw5BH-lf4P+g7KPOfL*eFj#rU}(rG`_Cm&|2THolWKtBFQqZ(`E2fJs%2?a2SRRpeBMgThi@$C^toqcE2fBS2^s$P1M|I0ki8n|-(+%3zN z-%DhmC7`%+#gnQKkQl^Cpx7)=ey@5f$wJN?wi`9&=K2GuQ(GLR&-9V+{=ph-7MgPX z<=ovIAk$p-*Cw%}^5$x)?rGUS689-S-O=ETW4rNi{Bz>4dir+C-eYZIytsxFtcAen zDg4e}(Cd{8L$QhMe0sFTd~EGG(0*q>Wv$OttHK(RdDC4_hKI_u5KhuXDX(G54!BCQ z=4&Hql;AAHLvzH~14Y1LAjy=Nzm1O&)|Szx9)|C=$oj@CA0mbl58Bo@Nj2%bchw)uZKqwOt8wxje^f>Z7JhOLff?N16)W zefffrL*iCOKTkajyiaOxMq}0T%PW#TKHc*JH91kpHEHGV$-Xn&krL(rr*h?e1pY*G z*Hle2eQ&1H2H4pwkl3WMQ?Se(Cb`g2YGx7d*gWcP5(7Hr^apev(YL(V9zg^4zX)MA zD@@M?59lEzW)5KU2{5(-UfHVW+0}EB;l^IvU%SWVY=tyzP?xiH#3PHJ(vw*`F&4kb zud0fuB!khVtt;2WpPF!+%>y#se~C(~m=J#F;(YrCO#O z{5DwBLi93=hG>c#seE6OwMf?q&yF+_Z+p#b|Cvy6AG3nb5JB?x1%@(Yxgs&w6*0hE z#ycHe%>VU>Ahhw2p1f501esxOMJ0gsXD|39Bs4gWH`?;E-e2;EA9?N!RL84vG}%+l z7nQ8fWLj;5)s9pH8Ml-{t;A^XFYTGRPQ&%jK)K<8RNji+JD|5T(c!KKtvnwPC?diG zWH2Q}F6KSPZobYRg z9%9<`u8b$G4(_ z_W>wo^XF3bU?ll)*HU9(Z@nMB=RF4 zAlrZoPWxVF$iI}`P&$mPV5q_V|YHkq&Xx@$J$?Dyin(mvQh9Ji) zqI6mWJg&|N+M59BhQyO7fugPDuA*;y8=Dcq#>efZnB!ni^WoKtYoeAv%i;@a&Ka z%}m`;-*XKOmPncq_JF)u#AnbLb7ptSjf53C^5^0JVwF#LxGIBivd~;TNgDu^=JOS_ z@spamW8ySkj#`K8zM~qgG2+VMPgfP@5WsY_tR5a7Xm$qMV6nP2vN3?b7`L{WE)B#f z^I^WHyV8)6RsVe;=~~}~`o~d%Nw$V?5}OYH*chY_=6ZW&vnrGmfe>`HTDM>%TjeBx zZZkg?(-~OJek1$%lg_}0pT!KR024rxVZy@gd?pPUaSk?u{8Q5=Hiqks@Yk7YxK-5$ z%j!VO;dP~#Db>X5%)6E){yRE_+PA1C(ZV~kAaDRcUVywHCCnWB2klQ5zc0pu$LTBt zwgq34D;Spkvvb;O5sb$`v=nBpKWJeG=ph-M-vc@eqjF*Fy#gUH6A=n(QO4(QD8U%R z$(+PaI?se`hY7B}%L$sV>@KgLT$U2LU8H3|>wwe{aC{F5qgGI6%f?5Cj|5UBoYRLd|@c0p>6`QiE=i?YYW=hj^l@v1^w`&*cSFtETiYo8jPWH<7J9w$Cv&}>a!18}Yu9&*Q;|Z(3Rr4c{vOQspX+f7#@6u~yj>3bBa#hso+9QrJ7X zFT#~(hbgB!&{LW8S%9jLkmi&4iCr~nCd@l3(_HMCA%K$W<(^lSnmg21iMc#`Fg-jE z1JbvGAt^AWk!I+jYh1~YHp;sMRQ1})SLromH5EmvA8D2~YT~&K#8v=O)B(t5GGL+h z`kl=TC<#ONp-JGkSlm@8?x|o%8#n?o9f<%G-7SrnLWIMdXl6IOxeg-qlhsVPLmHb3 zB+ekL2R|;jvpv+`P3qG)5?{oMurSVmx)r3XmTXp9NZM_5d?zj5`u5sGM^2S7Q?OW@;OV=}!6v;Kx+t5FHR{@LDWpoR2p* zWt2OE$A{geR|72BF52ZvZ~$?yUjFRf30~j-X#cR%z+?L3l(@Np8R?y;u-vomwkj|Ttw~= z06?Nup#JlETCS?PF5lY*U>37uButqCiIKY0rh*~ zo^=3$GY`h8HPSRvlsKHa0quO;TCtw2H)!;?>6F zr(BKJrz?f8G8C9?NPmq2{*!07U(?AYuln@U;*LsNg^BE;^RazfVg=)lPpKvVcdosH z(=L3sqZsW}u;x}9D1hTUUVB|uQV0cCe7v(M?}%FjTit>mKI61{F0M1SNz9E^ zW|~GJP*=hRc?L4M1#>Fzy7jYLKY4Y8-zjGo#)niiN)nWSuzbkRB@pfuyKhDZ_uiD~ z%)0JQ7)3w{uIf~EIjHq4z$H`Ryy)9aqz{jmK%vpkyTili0O;al@;?B_hrDMy8Ms5} zSbzwt5mjFWZ-RFoBK801D>5govWYTVT#%HIkmekwByZ9P0Rb~UfXtWJ4-|)NRWEk{ zTHN5B0j+g`w9vCE&d%VK7IL;qD%;j;p4}w2`P-CR&wwA$J4z-D@?^eI_wglGRUbKK zv#Oe2xp)^@&UIK4<&KB|aKy~sRusHs-gx+vXo+Y=nSBm=6>sg5T(K9JD-1t??M{7V z(XFj^gsuE1m&+r|ubl&ck@B;{cS1{$Gb__Jadkv}Y!djP9C>10LL$}w0SGLqJQX`T zw#ge&T$jXlou~k?h~7jJH+&mR7>T(BxJ~gC;=2C1h_{#h4MQxe*@xOLG zH%@PIuzerSE1!}1$HZ7RYv2c((Ma$ZK!mgMmxD1OeVT>+ezu_CivFsP^bgKUmQB(- z+!hr$4YSqha0A&a7mW3gxu^<;M%UOq>sJ*=Hq{oottO8K86pa`pv}u!v2b}q2}bfE zPg3*xZl?gm(UUJhN3%1jmMruBUJxat_VGC%1rQn^uKv(eLTGymNlg;Wy=XNBk*#($ zVW5`wP-#_&P44{EjeELlStLY8SfLdRqyoLoR<@{|}Dqe*m zYz2TrD-xTt4oaO^Hc`n;!L7y}$h9V5`9lo-y%CX22ix#>$t`P`(V2C3+{9diq7@8G z5Hb1vO6@SyxtML9?=gK0cdSRAkO7Ze;|3K1K@SNN5$vD+<%Hc&k`$%moa!Y&o9?FT z(RTx)&;jR{I2;?tLp0rJy_g~KCO~w2N9VufOkkxB4_S2nbL8BzteIePo1Nu`N@rGc zEG*M^Ct?dqf-sMnPjfxpl|gZYjkDNR7N5Ywf`3QABZ~o!0%;i6byX55B>Q1-uSCD{ zopU6wG@(*qHG9qCj!Tb;!}3A(Ro#?36;{J7Z-6NE*ufkzc+SDT)b!yLq~1j+&e_og zBPs?P^5nyw?~rQiraa^W+{hyykfX8Cy)%35GMu2MY;*SJ@u$JHaZL?*5peL$+AJv| z8sO$5XawvyIUtD}c?g_g8!{jFI#1Djl&_E-fp?RbJO$#5bxXCl&amIqL*U#lLA=Nh zJcM-IF&FE{O2~2;GwI=w2!IX1bP?Q*WIiYIwcNH|@~ZrIEos+$D~-FB^;9mL{cglPY_ z+(QKW&&1rc{o+eBASfMYAN9|u!WZ528D{c9-zp#FWk%l&H}>ZvGpYa!ZlWuxk}qm6 zrd>w`5`Lqc*%%9Q1ctGJzjpRV$k1^u>kiav;f(ExA_`dzrS@#_k_L4fb##Q5;1`QS z|M8me+E)r7sN``rhWckt$3fi#A5xlMQK8}NZVPHS1|FK1fEJS)^(OzG3$47)P{J}Y zG>cdPnHl zst5Z1M&)?GV}}Nc3ybppKx^I?!7=`~;UU=>u6Cpv0>B9`hBW<&x`v6gtj${H)WzGK z=pqIJ9R}+O0M*+f1Cn&1!hHU}18z)nvm-U*j04cqO4pB7WfoK@7b6np_;Pwz(xpS6 zYNZ$Ms=F}QKnQ{_ZIQDxs)(^vbE|d?U_LE{fH-l=KYU$K@l`-d_3pJDfEZ@%e(Q|A zs&BsSD;&Jzu(B3+>wdK^VP=O3cfy@ZCfvy$?7vNQB}F`-1D@q zKu9bVCf$YhL~H<#N!T47IN#FaK4U*jl!S(aRaLDt?~wVElReb0BP=OiuK5s=S71`l zmCh6UVB+Go^ZDUeuUUV!0{=JcFYDP{{8Dx@WW+hlWOSq)D*zi-`N|w6#&$ zaA%wKjrz4!F@M7ymB?uR7f7}}FvKantYJ8|oJ2{e`CwNx@Lw8JOm*b&k`(Qd=vlwC zQNFX#)Tjat-eJm*nE{95KS3+crhK>agIOGGK&!KAX&;{&@rgTLEU0=(D%|)^(=)2q?Zh{~ z3PUl7@qIG(jj-I-F7CwDlA;E0iZ}UJosRyA2apjfqT6)s0(2J=Qa?5PD#?GB*4N9; zei*peKF=LB3#%gh;{{T@b;j<@b{Br7e{L}|Zw`+vXYdrK>?D`$WV~2%I2#jT2!L#V z-9+hywyXGJ@N2ICd&{>N*=R&ts{gvKvbCtd>+4jfL*NWs96pPHlLa&DT;P+ zMMXsGkZWu7A+GaH#HgInh@!8ng&c^IgHY9bQghV^qf+gVo}c}NlcNI}w+<1&y?5=# zKLHs6=bDSR8&S8_+bd8vd{PFof&54mg_#;Pj#rPs3-(_cj`&c54|-Vu29dGffQhj6CE^TdjZ!g>L-Va}Pbu~-kBG4pe|a=OcKR^Bl=XCpWWpx~ zHvrf}+TZ>3?2bmik-kD-ugXY|t?PbSsQ-AgZKgdXL2@Q@{uhu3i1rHnq44mLCkBJG8)?;4SXE7Nu72Dv70?! z*PBv_-oTgPz^;M;zZVN!V_^B&(Uj%FLPVTz&U(DhGK5zAH1EJ3G*vSY-O_0qM z0ZG%w2Ov=1N#qMYwiap(4VoL+dOgdG?|hz4CbZ<(7;3!RX;<4Lt<`j$jc{j{H$&Rc zSuG%Q#kpj=Z7g|kw{5m8XUIQA0fn?OgtUv1OkV_etcmsYO?6C523ww8()U;%LSP4BeIOG7HroS+=x5vY|KW%e5@s zS$7S&?N_Q`0a*qB|8^XX6yS_*qulargiKu;5{NdzKu?*L*dTyFOE>6#$o+$Y*dZ+c zo)t!iZSiLsz!`4O9peHy?5DrC<=3<9K}9+llXjKOnc>^3Xdg=@5mr~4qp&Yo#{Hd$==PpY$m!pLx9=-1TuC&)(~s>9R`rb20TSagw{-t z!vfXScaY85HQ6K_$-LW)eBJfKD4$~mg0S*+eF9Q1)vC{}%cN2l8X>4}MvniZ?t89;M-%Jd(bj8r z%s96zTVRDF^*f_&y`F;Y<`)*M>yQT7{KH;?in`>G1);zKVEuQbP#F(MA1zCvKqJrR zIQqENX-5qfhO`&y_-Y8u4g3}oHN%HHrAk_uI@9}~awb=`>bz=`Fcvrd23-2%SSRF$ zIOLi$Y5-ft-T*m3bPnMHB<M~`GXbvLiO5JGKC2g;ptqtl`uelQdU4|Sv6F5q%atS}tEATGPJ z34F+Pd8FR`m3G89f(#yzQrVV73B5AJm22~Rr`tr_VCaOrjD=>r49DQhWLkbbuL`tb7eoLAl_ zRvHcG)Xt#hB|(zuNqgg#H}ZTBy&daECyU!E;a#^!{|2#(eJua_AN{x#hu^6{+g$6n z5nUU~5o3l5sq?Wg(Ru;o3iGJ`50$SMbmqqj7Mc@s69C;0{6{Fe^bcPXy}9GL z(21SRh=8AOgp+n&HfA?rd${ORf)IG5jhJc4k{iFpZi*e+p)S$nr|AMXiHTIow*0KO2welAGPnTJfYr}Np7GYFPng`dEuyRJ{L!9k*U}CKwNMSbW~r#Jht9@1I>IsF0?H-)t#Z5 z3W@pfLspFAz9l0j?*Gmife560r}q#d6oBpq@ooU2FD9LD0V)`G+KbbT3?T0aIUW+u zpIx4?{9BZBcz6u`T@+bFp_cf`Y!x*rh)bRRjvo|Cg$_k=z83i9Fio77kSIrLIcL3u zuzGcTU<{OBQ@E$_)M8Hyc{EiHK|f{+;;WP{fn?XOTuXrkvb7 zjj2Uo9+RNY!?NrN(U2aD&ZtKCfBMEIp+9`9yIB0IdHyM|CGbY#RUt=!*g3DV1Cx29wtrEwQ<5(5a1w#4NQ#|BMtZ zgnH~Y%m@E%R=0kUQvHSTx)1;DUof>-b-+$DdSl@Z24mj3dG-!ZtY|hx5Bv*BaOn+N zeY6seQ94Ec>2G|*_}4vMm$py9(E3LCHRoczobG5y?F+KtL#lK#ol%ej3$m?j&X)bz zd;uYFw95b_;M_O>@;YqaNpn&{n4*ZnxK&}^i9VLwPGxS_g1K$MFjE*yY-|d-bD%Om;Zs2B2nkVUpXU5sfV$lv;64& zWulC67?CLgPkE<$Gl+<6S24LyXe7VSWg(lz;V<)+RLc;&ztu*x6&2dIEQL_>1%!bV z=^yrLYkPh02Mq_7_uAU7B-o)H9?}eARXg!808AZ0g}Ccpn>ks*}<+KO{~0v}yLy_9g0?_%EF=3@N2;5Re$zWJnjq^VkbUezDv|0oOv7WQF1j2;iw9KC z`|jFh`^0J&2mX`e$AJ_>Bg(*i#)g>v9b@WTFQ1S~R)e{-H*sL^dy`DTx373cz#DBB zU;l%Qn%<3H+pLg7It69~tEoz5NzJ;oBJ?a3;s&bO>CEp9X&(E7q+)2Hp!w4?QSdO5 zTW-nG0!6#sgEQ+)i@ivAp@wZF)O<2cab6{+DjklCcJSK7A~T_gp=l1?jC14sbt`>{ zX#?O79|UetdHC-DSujc#iwC{L1KBK|zttqFW3Og->#K*UA3?tQL1GdhDE(#e#;wUA z)cUYpB>l&Bkmaz{{8Xn0@+jTDS{DbXsb&VQFdkdKsK3Lmc{UT1ic#H`-XPDDtNnGS z$85-CAR2Xo8oaGFW*4rQ$XBX5(BsA4MfPBE-y$HpE}&s zISBNBwJ5dXT?H?+^>YsuVUNk*`;Oc?%3selMxxvDl}UGgVgkNbRQUdS;PztWEjrr| zjdxZSutBEyqxX*UeXmrl#`@kfgrkTnWwB!6(|l>8SHweS_kCuIL;zCJ-%air1pBLW zJ=K_LH1AmsHKPu#H6YBWAoDbv5CCw>M71TbYxKEWfu1j{F!X#IlxmaBwf{ z6)l_A|FWrH#@=!^-EK#0{w>42Hrl*4!?ZTq>*$suUAoHEE_dL;n}8>e{O&b|Eh@|1 ztW~bxG^ZJagM2cwnw*+?*YiNn< z?tJsvPJ(U~@tmTh&5g-<-6Ym7(;dsafw1oP+2Nb~s-CS;pwa#qC-n#_*%|%x#t9p>(#!>Mg8_7c z3OCC`8Do|e8>f`2l1C`g^UeB}hR1(aO8xZ8ZIUJI3h0FQgWFjy+O`E%$L%#{Gh9?X zwI|$mE=p^zE+Wz`Qhr7@3ae$FF9bPkMps^q0`0XSrB|SXQvdNAho=k;gS@V0jd_Zy z+&ua1qGK^TJYInb|LAS!_jf|YaSAf=M=3rhzeY-vGp_&QdtML03zHBNokP{C?%ca@ ze_7=kORTK*zWQ4hTe7G^t}XV=kE{))6@&3&blzqlOVuZSKj`)^$4q_wSbqybk99am zxbZw$!@p|W40-+x(*dN76o&-TdEkEgY2o?%_Ij)4dfG#1iepINxy7{&`dL`Z4K8H% zBk|#(Mu%ev!BPO@gY8a?XzXqnTj}vdVU`;m@sTB^R=u2N9B^}R}*K+oZ zRuQj*Y#pNn1i33&1s{VIsJDy2Rl?p;9d;r!1 zDnPPRo;=k1F=-n8kf+LiRZ;icKhfbNK-=$Ls|DynC|AZwe0<3hLl4t-N~QZnraJ|V zf3SWR4df=pa`%1+`psIA?0KuIs!CqG0N!oZiW4WD^GesF`DVEXB7lyVbHYoSSa%{l!A7 zMK^J6g7`y?vK;@AYWU99!iP#P+6M_SgF7ar!gjN)L3ZJ{30mDkmTMP&3 zzGBdo#GHel#lX*f5w4#FfH18chVa|1#i%i<_A1$R@?XQTi)X>P*hN;F+Fo8Vy64AJ z$IFQc`4foCio%PokMsXY*XczR_yAZOBxOZcsp$RLZ56-n|@Z zLA;6DME5>yA`VYLOjH)VRI|p8heD)QpzJD6Vz2p;_t<1g9QvLct$xUt4ujO8siNYlm*LB#8^0hy~!qg@7ANNaTZ@;j012;Hq6jnk% zm*u)%nLr$DyRAj(Dko2&qMu_8-Slo>Ufo_ha>2F@n?{mlO(>q49kof35C+Zg2oLMM zP>TB$IwqxrLm&Qn!YHTSJo2`|l9RWGOR$Fv{5jq4xP2Mjnc%_&x!Db{+n)A>pDnGd zpKyC4;M-xDuA;YFnO-`MJJo)_56*-Nbgv_#w*PiIdbvUzd*m63ffrc5l}Gio6-VlR zw9sYm-XyyB7RL4T?L{N7UHC-rQNzSbg>TZ~8;6zL{MsffZHiGXw|?Xvhv#1tB97Vv zJe`C}zq_9(@~dH=bUmnXSg8`MdIbM=ua-hJ9@vo_Tsz-^ChDGY@Y?x->j_}d2TlNqX^4)v)81*vhW?NS!P3&^&c3C04gqPF{K>{ZvpBbkUI+?=DzrHxp=Aubm z*W|2L9dlTW*&E3xANIZwzH-0p_CCYrA;nq{fI2`A?5DV>deVXi zKV&Ye7H_1lM})6r=Igr}O-eFI;5*ycz#mp*)4YHG*s080O&xap z2V1Cs&U;_f(^7TwD~t7}MP@_M>{R#F(!IutEy73!Ama*}MhZ4qkP|8o&5yze7)qAC34c4_ZwqmX8( zMVT-BtVLnjqt7A^^Nd*kEu^| z3y3VQ-!12Meh7tsn{;A+uUalhMFTSZDR zPGEdhc^hPxs-5v!a5S6i-1L)N9`AW*ST)ot`K*)cOmz#L+!H)pIFj-FyLT~+;x-gS zKD=t;d|Y?C?Mk*g`0Fv!!X?>wLPZ5Tdy`-weXvn@bk%%>XFA9q340zfituKGYRcwv ziWuT^%e^jIF3ql2S`g^zr5sNm8vZ;W{EdD)qs-@Am}&E6@&*1z{!WfC!b|YXMt5X& zy@ud4*$!t@&1=O3ACEFq8(cm7Nfj8&Q&CCs1e_YcfE=P3T;LW!N3O%UT(wt4} zZxWBs5iA>q=Rj?>^h2+awWa4z zZ+vJtW8Cr@iW-VC7kSTS7@^FZ^W1$K(DD&HLQ-FlJpme%fVen?p(1CYWaZ^*gbjwIjANZwJ4Xb7uOt`Sc^SoEno8ck`# zee;>pIPa)z6j`~G5tjhpREvLy-^k7msA(+{j4RhU!_4` zpm(w#;ZTECzLrlxNvwoRF7p_T*~5c$O+eA2*!mGzj#Va6lqNYTN_6@odX!pxiqOjY zKWN-9Pu5Anl$$aCxQ6PVnIB7q<8K_8ZZ>}&&NM+z4=%_olf~?>_yjs5 zxzvB4&aGM^qQJy;5-(jhP0VHXpT9JioFU3o@!GTOV`>H}+Bv>(-*5jZRO9wHbCO*c z3Ej`Z9)()nlPeP4;gFE!p0O~nx=?am;!S%iB1}SR)G_wDbd-3e826r zhwIC^%guv_9r&ju!U1E56kKviyFT?xLS-cZ9y)^$+__~>ov(Vunvo={sW7Q98;u^v zsB>lP^aW{_+n|4zVm7=Oq73O`p_`bTW!+zi(UqghJ}=y=vT<^))Rtb1>|VLBwcMs< zL=;WtkN5W~{B2|7pK7PX0UI_QZ21}?M)kH#GzXIiHLS4cMMqB|{zFust}(J-^_Qk? zD^eA~GzxOBm0tW9K^*}15Y)(!AdHRuX!v}%e# zxv7-R!hLe`Mz*OsduCPo2K!nPe?ubNYDPM3XLjseHj7bhqetsoh@|L3b6NirBT!aP z9-veo4tJM$9mkHa^WY?HPi^`2Ku)e?>Gfk&X19;Xj1q>{+<|rv0zdpLVe~5RIYbTK z=e2)Cq)CA9LZ(q2o=V0zJRp@-bN1@d`4GFWyx2H)k&_j&wrb3+(;Rrcs+VXaczG;+ zc_$%Q_mk^rYTFk^RK>kaCv5)_{26f6|1*9V9{D2yCOI-sEaj5c8x~~nt2QQFIB!ir zJpR8-1hF#DJ`3RspVKVTg?=%C*^ zBjav}C9E8Ov;PE(NK2@*nj?{ST51if#E4R6%SKX87~j`xTYYtUL{Ylpwqa=YGN_!z zw-yinqUE^~7pB}4^EHtn?xe^bX7$N5jE5ZB(@yCz1Eg@>G0g-c9#X9S?l27SZYrwm zW9PbMNfXN8lH(?Pv>r@duk!4}E#%t9$Z*75ecsCaBAHLd(q>oePDeIf7l*z_z~~p( zr!{*2p@=4Y4eq+34fQj9`_&%POXCTwJ>dp5hW-ubxsi_c)W{Q4^5N+q@Rn?zPT3-% z+gi_t0JD0``DL3{XcwOtAuBz8HhcJmD!K`S3O6_*jr$%wNk*bwru$-p$*@-;-EX@! zFY|{2(*h=*qjCJ^zmol!WpIqzYkq&$st!*l!~BIe9qu@cKG~K2?ALh66LZi| zZRXDr+8cDFdxAx+YWl2~%Gxz60e$FEc&ae<^F(yY;xW0+4{ol#RETnb*)u0A`7W@2 zTM5)v1pA^OMGJ$?0-Qe2?%Mut#>Jn-%D&2q84?6B*?ZZe%sS9CVZ`>0ROY$cVdPC}`~93A0McRjYc+iOYB%@kcS7h4$}5PHcMl zJCdCZZDg z{Wzm4%FAXNo>^EFraRh}7Q-8gDskPJMHBtzP~xENpI0nkO25LDVbMO+a>VG?!K!R8$p33xi1VHY9_3r*8&2*uQK4)u-%XM~>F{Qy=z_L^mi>uft~cymS1&w2Il$ zOi=>-c3o7{ZL6RCM+9deZR3QRDs)8*&j4{+YtVTz6oQ0Q7tkDa>kXt%FACxQ(OyZ{ zif+q;1!h7NAfw{eIOQad0-UsXro+SNQtj4EQD%r6ED^3`F}R|T!lK7;Ap6zw+p=v% zCA>#cK04^$j@nPmr>GTpurR73@vPu3ytGd__cn$tNm>fbuT2g@|F8=9F6`Q%DlWo> z+fn(RC~47sv?r&oo44-oX6y>G7DMt$^J5Oq{!b3EBJS2c{Yn6F%;>R3;{1$%v7ZvK z-Pe2CVI>+dU6kjksH6-wpQiiB^k~Fs z0=Yx=aaL2|gB`hfqImiia_P@5S&yG2XMGU6m*DN#)$P4ESh2{iD_Nr}dzJAXhs^`9 zagtZJoXF-kfi&UKKE2OI=g$RuKCaUVMiYl0Q{g@O#pk3%;@}@|l>d9y^oaY^yXAa> znyQ``#!csI9ff&JkNq?odllbE8;ZINPLc~8mBP+{P9_DwY$lvz<^J`cmideq z?MpdU+lUp1<@oFLNc=gY{EnHdOIp#)YaT&N?4)}=EJtj<@~RE*MKx_ufSD$W7h^9! z>42a9g$Ev0zmT#<-p_w`)|CUDel1OqA7uLS-+U( zIaOykAIxb5@+Y56P}xe!_j@60ytCy!%O-L2IQhh9Gw^Y>34bs1_jbIP7`I&=P2GWw zZJpN3;Or{z1V?Ry2hQ0{b0y;LS=^s=J!~EglA7s#gJfZFaL=Lqp3U1n9;5c@Qmd?w zp#l$)HDZKEBb>aSAEV+Frj%uWJwt9Q*X!zdk@vpAKLzPtsG&!9boD|PH{xiK0~iJl zmAId!G4$P4GC5r39lkbZr^4;XF~kr0_gVrC+y~k80MCOjLyh{fmA}4gpSqRO*hB9d zab%&yDfgiAX{b?-`seN1qr9Kx$9s@fxUMBZGjxx#=)E@vyELbe@4~`t%c}UVw)Ef) z+Mn;oX0b3F%Dm+rs;-vLt^r2w-J%<{^$j3sI7liy-X7!~};<{z%MliGL#7(?Q=5Va|IUQS{$s`~KOd;V~i+ zO#?!{l!hk}hJ*dZST*4l`mQ}J{eCP{Xw%O`<8EBYYY%UgDW-CbgqS@_Zi>9~x;VvN zvmHBq8g?o~A24ZnTs6(NX`P93v8vJY(zZQy#jk(ehdS)DaQT9(K25BHSnsG?Gs!dW z<)Oe(^z~!cCWV6x?m8y{4Na9tIfV#jfk;ckx2<@5F5S(d+#=XB;0Y`4Mafv#lLP-21#7$EH8nQ)}S; z+mIu2k3Z;n!(jI6hoIFc!64-y1=9XX+T8!OcdJq{P0ir5Tqh3!8ALkoi6!Zj{-e;J z5b$&Gee?5QeYLlmKyHr?hlJ&)*li8dr19C_1IISond@NslJ+HWDK8Y=3z$@DQOVtPLW2sIh2%ygh+R%bVwgMr9%z~64D@&($Xy;4bt5WcaGov?z%r+ ztR=(D6Z_eF&%8poRYB2Vz1OaKVPbPn{Og}qef{~xgE|tOP8AvHae5gI!4WM1rg&KL zXrR28a(CwdVf2r7Lhfy%k zi)wB4e1{7AxDp{up!L`Hw!NoIopri(4e8uYBtSosahzVKGZV#z z(_Hfz<7{av3r(mBQ`aHd7ygV9!h+ZEt4z32)G9MYhWe<&nfR z*R&EiaakKuvI_34_j7+1?IoCS+5*=Si6LB>>{CHP6lvi_PkRj4{9A)g7}M3ND5aly zYTt)kF;TutQaz&7&TQ|3XLyxO)@5+T7 z{o-$af!toH#4qOarOAg&D!ar@0ci@wCOdu^MJE5;I-^mwwAlml5BKcby>W=lWf z-Y0)@*YVs{j{+q~($u7(Ols1&phki(g7{{E+%~MB1m_8Ji|FYi&Bs3gr8r44A^|rLxeck9wAdOpnKoaYDp**npqakhAhkgHH zJS3IFf=>T-`1|q!78jH0zWZA|RlYou58TC0K8v1BXYFo0nO3y}_rn0I_nz>)QC0u= zBZnmNyMHtLxAT6GDyLs)Rk?qV(M-uQyk@x{!awG3zNCh z^qN2AyTa_*m$QvdJBaH1T23UQb$I^gdpDe48-m+DClwV@1{dBRiN5qbCam|}dT6)r zf+9p^b=2)`8J|YI)|f2l>BoT48E+?$2w!8La5!5(N!)-~eLXW)8S8d#(7BL>Oi*&N z8gJbJcwA+9>)>xuaTZWtF6OC?i>sHTW8*cO>)Tjd-1Z<=rOniyXs%to@C5VkoPe&01(`}BG>h$gK2ZcySsp(2lvr#=afz@Y z^m=K~jZwHezz$4kGKVA2pG!C$9!C#w`kkY?)}9&PMN$%R-RFDeab9GHW1M0R{IC(O zauLzK5<4icW!>vrrUwe&LGbwC2Yi&iWwpcUspO^B2S56EY#)dkzJcO`@r~7D#KCot zWE=+#5CHqK-Y$F%vv-xmv(;Lt~~b#S99)rnL@Q;ni`bBI9;Lt5Jr}3<({j zXTx^!;!|8D2zv3Ej?5IlV2D*;#c?2;4aiDbxZ}pqVk5{T4vg31CKhkCUuh_4Y4v7k zRqNqakm9POOVKpmYJGtlcSD5-3`;KZr#Ih9&$M+tlC9Hl> z$OTXL!F=SX{ciF;Z0=#?5Be$neFxr(xNXm!epnz*y<*eSY$0{`Gr~h>sTOz)Asn4d zXmW`dc-{jDJn2chIB`BB%!_-C&nIu$Kw~j{;^nVA2FB+7BoU^=WZGeMgzpJYq@5*k z*m9JgNCeZpaO|_c%>41|B#8$6ENpirg*oUd;Z4AOJhvkee0ONVnyM5$C}ZxuueqZ35Nr5%1b%E8BcxC z&ATJm=VRC(4viGE#RO?z=&^)7Up;wnWRG}pJhpnHnidV8&42NH`g9<3wDf+wPD?*4 z7Ntcrve$(s^2j|jfmgG*fgKlD=dEJ2L|CoA5MZ0!;Y3psS^ChckySDI*+v9C9LV2# z^!Qb_-6^k^5q}|NjYKnLfi>Da$pahx^7bra3Wd6d1LGwl^UD^Fz4<27(VqPYQ`XIX zGOqw{%~f&d+GE14s$s7WvpI?&W?~vNT1vFNhE@LRz2L;LdciUr`9))5~lK<1uJoSKO)Cq+ACR;4UOgO2X0JzN`obuDiVLKWBt>ZkROUvxT+7E zZ)#J&hy@Vae1;?8q)ZnqjXKBJ3>|jT{6H$`uM>ifA`D^NZM7zF5Pt8!DW`SpxiI(+ z0DS0izV&v~a?AWGS$fGg3 z-ZLSF@@Q#fM-D%9a4EJnv(c8Ri=IR`zIpdHZtU)Gy~0Gj;be_OOjdorv~4b|TQcc# zg)@Eve4ig!n|)_UH>_@T+)iu@Mq&v(w#op9mF*D|s3|sL!v3JyQm=k$w7wg&G7cQ zz|p0_Vt5m*G-*$Yr~P$DO{-XpRcMUZ+7OANo?3@EzK#Z6++103JPw>~x@*xNFP*2> z#50332=g6a)O3IPwWKG6J9?(fZ@ah$BR3l4*PDr%@q0l zOFS_*g5@|)H&y$?%E;&%hH0SnHgek*uf5*1Q?(4oF9sqFy#XriU*E8m`=Kcm0B1Ws z^jI!o@Ajhl%N;D$2L$Ixu;FP7ULqSP=&*O zo$vM5rzJ^C89=@kQFc;ge0Ju?-_)duKu8v=eFU?7_>g9^jbgOyd@}p{d*%UH(a=SA zLrgb!uL1AZ;yTpv1LEFC_%k017~FMQhs@#2xK^07dlO*aF1VT1R(H9N=qZPbdp1~_ zUomcV_u5qH`(k=XAXWPp4@^>n`1f&;G=mxpBh%VVS=cg>-%YuV(`Jx520w8@J&nUp z|KAy_p`~&d#&}5@#3wX?{j1_mW)z6Ffq?NrZZkfPtma%!RwX{t*_eCvI7s$Wdc$D$ z`R82Lc4kgYMj;$DhKqat!e2qsDpAAG=ju?dfC6iI^76c*6bT?i6j}~kPmP<`I35^3 zb*F)Krk-^t&+gUvmD*0X>&26C6kp%cD$fkEzYBG2AaH4k~K(Vj7OkT{4kLD#Q@2Lvpj^bZm4Lm=UtJEkVFuSWK? z0?MG>?#OY{D6N*n?VE)AT4Owz<+r;cws#$mk+Tz&laT#xem2=2^J88s_4b`{)H67F z>tO^w0W?ppkEN=N&+kkGO7oZ8-)qma&k9FpaaRJ8&DqEudXBO%<;h*nk4Jw2YQG-B zOVit{#C=+WGCEIaY6y-}e)3zEc^s>M-hf-h-B3m@Uua+kc( z&vN*~3nqMB&B{pSck??|f(tjk&IWyCJPkOt)yVL*(DwBL#osaiQvf)ugIBfId&-pw zhTSL5)(hnPK(<~1zJ{V(uZ7+#TD?q7eu4aL;rr0D&?mi zFi`FJi8m96+wYetH1H2~b-iqvD5gSG*b>Mp7;63A}YzySh@C?ZPPzH_M zi(oQ*C-1yj60e3AlW%FdIP!B<_%vIJJg5@&mH&$cHmt9)?lCkmIeK z3oZ}#MZMZJ+d4k$@09L{GcaH9n#3#=bTrmN#I7TlZ93)vJ!O99)6`#+f!!C#$-fH7 z?1=+Y=Qfytpip?u`)?6A!+#d6{3(qkEv6Sn1{?JOM+cceseTcp6Oi5Fik~p+7EP|l z@|Jb3yvDyD?RH;w2Pd-XsBx3|GT94H}fZzkf^OUPM!-y28$)aP- zSEK$`Ou3^(Q|52y8|}Re!WVm!fS+}C1{v^GWrdM~Sl~U^x~PW5pL5zaqPlW{?r@Iy zR_Nz@7oS2Ng(BqQN@1>D=8xR!A#L=OZAf^=^}!3~D3ZIX`v3W4Mrl zKqh+RLxDs-s-(jr{0wGIRBZm9^g=v81C(@{-YxguN!J$gVLzK^4S&1@BmhrHkdk}W zMI@R#QL{(>$49hia>UgnLtik;lHc>9eSDuJV9uBwQ6Y&M{Y4cfF(2XEN|!x(XjV1Rm0RT>;^ZfOq_J>ot{AJmg>roxL1wT_$$?l_k_x ztDi7>yXv1v5o^OR)U?y=rl(Q(ayH(9U+84~#LLG^DvM7M0Uq$DXy)Ji%b5CJ?Vul< zX6n;eqvn1U`_Yq91{0d-27Lqf%jsR2d+o(aTA^0%5m~GPPynuJA0yB$|AG7fyj9vo z440kQ5V>{N4=L769NL3`x8$UL{v1}T@>S4~*hwjZnT9N;L=TtD!}SJ?>#VVgqbDtI zq#*>@C1TZJY=s%#R!?MSlm-of`ZUU@!qFhWs=Qlq|HFFxY`9!7K6Exwo3$*DW!ieS zu5@Vlzg~d43)i5htvvo7n+I4cKO9;k0QthN1CEEU(xb~2K8w%YAG!`SxRC8;(hz)U zVHvcsc$;hMk!^KT>A7ZPnLZRWOy#{O5PeQp*O%jLfqDGe7vrTW@(aGhIY)DU?il;l zUsU+G^TFg13#Pq5v2>X8O@`BrHp(3DAB6SI-|iG9lM>>D+CDlm= zaNJ*bmoHC9h{jRT&@{w@Nt6h%A?EV}Sl`8jMfMDm6oP%SkQ%5SooG!+Bn*0%PArT> zN@Wb6EOCEqWr>0ZaLrLfu`|a}oe@}2=!s*(g@dFyXa{UYo{drJF1Ew=u9L^QcXjP$*{t*)q}j?Vi#5c!|&J==D7YG}Cg z(@UJva0muD8ge+7vQWpPf$gZ(eq9rjGEk7GC)KS77Jx(@c_Z%{;7>!*ZeCWdxgy1l zIA2*0cO_SS6+C3fEf~aVJDa_X)Fz0HYr^@g_2&DB>6?2eWin7xL-C(%g+BO=?RCUE zsFmy+svrsd&R)u=8x6^`<8P@3A+4nUK05IDC2}%Tkl9*eyn*?y=kjhFp{;>SR2u_L z1J+yhT-{qG+AW3)>fWDl@Nt)48RGq?>qRr0IBXNE7Oqh=1bsu5@xpm41^r4L4D&;x znFl|}1wRl(7=(6Czl(aLipP3B|GViecUZZr*lHq(L>XOuq+MZJBf!z5Qygn|E_xj_1xFQsui3ApWwzg@qt_XJ*HwMLYz z8(I6g=;W(hOEq3~BZmH>^OPanH5#@eGARnbc8nsKSrl`h)WeG+KSF zI8$E0oyFg>ipOaAXo%8E=}$u&jLEPq?+k}tRG$u35}mI9xXWtDrMzGMbZ^u4ocj<0 zp{xRRoC#V$H|lL)fhAb(2u3a8%G4zB?_S`sPD?o4GXHTu?a@R1z7oV_ zswp_J4Lrzk5b)iSDFc!ScuixWWJ=Z0DPoE^Gx>62^ekS7K)OAGeKDqTJhafq0PBE1 zdhynf6=u`|0sS%w&6N2*xSnq+gcy{5vsSM^LDS9m1dLxP&zpQn$mQR$@RxUnSt-bH ze1*8`0i#9;6?%c)@Zz)fKL7cM?(r-PQPof!q{li#!bxiw62J)PJQs4vBwBBZ{k{st zJ`8)N%!lB*Pc>VJ>_@h-`tRS$C~k6}Gk^o59~f_d{o*CV+t0Jq0cBMPS-<8YW#l{W z{Oca!+Y_f9`wDn2A>3cwlv5Kx9gRUi*(&2*YpIHwyFKd*T}!KSXuNmC3)tsiWX1v@ z)eZPcm`%_E0c-hNLhVvJ9$LoEvKWn<*snBi&X3atgwP&%PkTg@;UdSDWQr+MR>)Kq zBaeV&D&V4;@CqjVF(7q!AOaV6tc;KiJJ{;Lcd*|@@jMO;jrsZM&rNK;HATI zy!!)8TD0X87Pr3PWhN(m9hqNu=2weJcv~Eqz)2!hERA~ois<}_r#c5HXT-vu+YLh? z>NYQu|BR_+f(J*Bo9WRQhs9Q$OsV_RY-T-lnuAK|(&EIi;Wd%qdG(7Ca;&g3 zlP=IenE5vLkF<|3ZBZqy7nAoax7xT>$|Zp{L>V1e59c=#yE)2xdK6Pmv-08WMo z?wbq&E#K+tdmh1buSBlj)4IrcO@cWR^e{gs&u0To?izh{t$ULu)2Cxbo27`Mf%=OS&&ojYI}1ro0vWK;8@L8 z#dv{>`3n@zH-bDL-e!9*ibkKc&h?)l#fJzysEIAGe5wFP-G7_+>ckx`UoqV z=aC+({%#V{$7zJ|&&&i)Fi2 z8sE6VPPxsuIRI^M%oH65zk8E0xzRAWL#<^KP$t1KfyVnTZ$m3AwF91mjDZ;zs=a$OJ`p=&)+FksQQ&wi&oQ zix%vCy&&4^1%a$*zbVZYMge*D+CoY;WtDre{;9o#HTx|P$DfMn#bzB)tmgx4p_56+ zu&`{tFZIE(KI$N;`e43#(;Ch6)Mw9chet5uPM+B2KEg|VJsQdtMD^&quQ?eQ>zU_c z0kZJe3ez1Kr}&$Xqw`V9ZN`|?T;1*vV=?N*sdzBFwUCYhHv|Q{-jwH0oG+Dq%UvGF z4k_&^iGRZIfXX~sO?r4wdh;)c2^r{(?j#&y@W)d~78|tGUo|~Auebc1n#FA&s_iXm zrxeMZy#t^3Yv}2}s6?)S=3?r*} zQK!BXhQRc^xq89@##z^6`l`kRiL1+9z5@wmzwIf=F9yedeGYLQ?j>P-&SGbnkJ%_yAZ>-j#s zFw_r7;*al8uV}T;4rZP19a+F1A2f61`Kh?46mS}^pSo)T#5IqjTYs_fw;xF`{_S6a z%flZI3qdP=w@C;6$+b6dv(N}nG(KqRrqqE+O(091oXWz~?s*2Vyt=&}ZG}C$<6*@?%UkSm} z;+b&*B_$l_H{q?Oq|G#J%c=cI({0=5Q^_rv$vNcDwq*nk`2L`5s3P}98z+%KEzR0Z zZV}0Ekwz3iu59Gvd`U|seC0!``pUR}x+uN2WQ4vg82uESKbQ^kfkAuESQ^s1di3M4 zb8#-DjkaH{g&n0XtJA@+o^Vty>fdHOHcn7*rqk(cdS-%IetP5{Qh76|UYC=TeNN?8 zCX@!w0$5z%>x0%ubApOcaaif4p`v)#TwJ!}vGM&T@jT--q!%|j1anb_pQOfYmnYIa z;TaOVq`wyUgm&uT!1&UCg}zTi`8#5)|DN*11orEz7P3Wmiho28mJB~9sOekEa`&rw z$J;&=L#13kQx{Di(C>%hz7<-GM}{S+Ss8#3=;%&4_F#;Vq9f|KHrT(rEMIrg!+0%O zNc{M_2&X73_*RAC&UW53Aj53y>A2$Qi@XL7(T^No8V*nEG$H8mPzO*%ouBJ)0Fx6a zsIH)RX#)P!S17)g8H<>urHQCEF|5GH0GP3U4!1TUw=Hlm!eMfBcpztNtoch)G?lvX z5#LxD4O>0mi8>dT{WnjQ_v~JKkfms~Wuo*P8Zt(xzbsz)RToM5FgZ?W_y$j83W`7| zDvAXpt1n&myE^_P4i8?GJkdI?FxRD6S5jU#rft42N+zxM1NmIFV;B%@6GuB5ZN zSo_xCIgzINfOxHSfU$0m1 z*xdf1;;}e7B6mvei(Jk(ncWMFdcq6INtzp5utE{2-i|VZa7_QsPoU2oKVpQ{a0AsQ z^+8Sq!g;21s?algbR1hpJx#jYiSy-+L;cfbZ9w#N)i8mq5?o^yse;1c?ifjUIRRh0=SD;*E|63m$hF`BT4$^=wCn3yq*A31(#4R?k zdQ7HrF#Je>fX?3=Y)5^Gx;F+9Cn}I z|7s-M)?v41+2b-Yl8UA1ZcxpHB>SP)Yj?pT>EAY52MWZ-F|e1(*jG-C*n#;4i(Kiq zP3!D9Mdwg0BR)l;wJWa7Fj1#TI@4pNQc>mbTQ$mLe=}rf3{jO%S`tn8HLaz;_H1G@ zNw5DUYiL2XOZ~(Lba2L_>ki_Lt+>yHRA7}i7=P}fQKj!5onl1Ap5?4PEvRwaAOY#? z%T-eR2a9#?73dRvsI-h2-6jUj_*fR!{@;0&I#|TKv2`CDKmJ>*BurkRNiG6<#j+jM z{4nm?^l*&ZKiJm&9E*lfsnk0im=_(M=zhni-)5BKV66u_7Es)U6Y~TYdis-c%bc2sZ9}AVK~)n^2aa2~%xfLLcady@d`{vF{hhoz z4&WnOEcaTg(0_%of6JkPg~!JA*&M41jZdR*ff0yx((3~7a+wQLnTPz<`=lYIK781g zFr*s3a~O~b5#uHIU!o-YS_$@*oT#_3rx&bdfpwsYv|aWwnDu z=uc2n957rzpg7GB zZy994)?ImG^s2z9;{*kq9@|dYV~tD+3CVSu`RA)rDi(k!a%^d zcb@@b>R)WWw{Zo|#Nr9?C@s1sD$}@WBkGy#kiw$W-@W}f?B~K@6I(G`@-(Ap-aH0%f-ikG*jqb>s7d zKbOBfCxY@&v*7LD-qJ;-s>M^ZPB~kM zw;b}1r^JZ6<%XefsRr^kI}%wZg*>tP;D`4F+(vWyn!`uOdcHAovr)O9ruW7~SQ*MB zqV3D-!Tn-H3J&A8Gmu>pIDVIlPvz3O;A;j_i4)MF8qOe_6*R$K`kWu^OQ?G4_g$Dh zB;#L#*?^jGtqKnSnf3P=h33`Yh53;q#_Mmsin|j57KY2Nqh+hB1I^$dNoym|ejkUT z0}{Rv)Z|WBc)`bsv*i(i_fY^FxO~Vbtt!RfNF4YZAAx3=Cd^~Ea3BU=XQqUwS}AitnkB}XiMG{x zjNZHkZoE++77>`C#tw%t-mlwhL^A=Dktynf)_S>5f^=u3i~-I(K+W@uF1T~~I`l6I zuiwjJ3#J1rJwy4aNsvekvPwW zEz^fT7_DJ{&tFq`2T2krZ9#6JOIJ@Kig@Dw{tC|Bs&a*dCHgp?kQ;HeP{UKSzK{$;j)!iVU^pj=sL z@AiXJwGgFf3l{N>M_f)y6@_02)K-Kn&0zwk9Xgt zOgHQ%yz~RqyIvI(ndWco$~S}kuGv=ba?m6Z&Qp?iT7h5uIiF!^ZAa>O2}x>S0Z1S^ z)djX>yr{U=0D&8P{D%WBq38_`Bvq`jjMz0HH%!GNrmQIRp@~fCAxRt43z{M8Bp;4E z=mZ1HH5glfZQ5f!M>F?ZJT^UOwNZc@@}>6ZBYzcIN9!Ow8L%ousz$M)#7-vjGJQI7 z0ZfbmO=^_b?_horp>KEnP6+%?Vh>g}>mo6rU7xfzOlDFFJN6XrxYOgrBEU25n9pNt zLD&7IAy|5+Xh1OV0|iaCT;DzjK=D2gbNUc))5`~FmiU`kGf&F78PlqxZQZF$M3_vV zu#Yz_9F)L!uYA?w`hU_P_KJy}j=}L}A$t_!hKi z_+WkXH0<~zeS|t4LC2P_lE(($&CyA#PAsU>fpKDeKc^u^X1x11eHf-|ji7(P?;+p! zpzt@BhXdeWP;Y`Ad@TmJ+aJe?qGfcyK*%m*SK}oE zMgtvg<8^ExpT50bNZOBV6@qTG+`XW<*%9IZK3!08AW3xQZ~f=w;O^XP)$k5oHX<%R zcbi8y3j#rC$aRse6#tuR8MN}9Krf??@JUNSUSBG3;rH*nXV2SC4>fc^0^kO?QgU}P zy@mQ&&yB%Nbi>Km?|FHgwf%lYoS5JXX3VODD~=rwl_kaxkmcJvHMg8 zBXG_rT1GLuYtps~N20J`YOii7^JMRvQvGsk`oT7Lp~Zc;tx{L$!|JRweso@b0VhfjE}GqiHNE;qnD~D9OZ)t-OsTlqOoq53z*QfFSUL9%9{c1&00xvy2w#! z8@WE!-nW^`C_Ef;xTyA~#wG5K>8~Q8F4mu)hl?+L9_jMWzt(9Z&YM5mjL2VdMkJ#B zzPywY#r6B|Ri4v>`ct(QiGQOoP{(RFlVY?}P=-@Cm9}T?6r?$0mTy{Y7_}6CcQJ{3>xt zr*@YCacEXiElWxq99)(&zVn9{XeULClF>IiXTq^%?WG#(V&utozS50y4o!x%R*6B_M!s&Az(yq>8I-W;0+K zV13KsBLV{j2C4r^pHyhw-6DvdO?ylXALY*eZ?n?j7ra0xzQ=s_ai(9E^qNc4AI{^y zeQ3d?_D=$-FQ2b_|D(lFQZNI4J6F{6nqzCuC>GtdE`elZo(R|Ia{4vg5WjSs@23-l zS-#(G(>mArY8?vYQslww7!|P~M$OQhMGyQ4z&eKRO=ENJb3jw^neOh_QMomHa^-S@ zrxh@y-wFuxhC}f<0I{|%%EQYVVF2u0D{GYY0j=Df;nc8vP}MlN{c=o5SU_5dIvoQI z%eGsPoBKv(xUw-=2ubVv=56K`lC zc?Z%4?P;`4TchtFCq%L`Keq8GBOXgu%! zH^Ksi6G%f$@}+n07o(J`;A6h)U{ObAVi?GHx02rzimBcVTj6Dzy!DTA?$*^zH#|Xg z_U0q+IPkwEH0CVhl!chOrxzRb|MA6u1P1fyBUtq%kO1ungIH*qv<*l!34-gO;PY3v z3IzPX;dmGol;fh@15HC=FrVwCr!_k8qBY8$*Pv7$qv(z}w`72NAHImKb4y%n*hjLd z_1!VWa~emrz~qS+vH`y(=Pa24L{m*8J`7w>5rKP{KIhTbVk7=n(A4zVa${1~C&9;^ znVe;DhpnkGBF0yo>SfDm(gEvS7>I(wxTA|^7i1F)EqZ!+%E)3M+$;xo!#)3iBR-1D z&u1y?lh6zhq7?=7eF&JR_kTa)iS8E~H3L+&_H~LnmDgBwx_fXJVKi4vJ;;xS9?5Gm0U%1JoSaI2EE_#)^n`gIPYuI`rT(u z9$p*@DS&4oFfDYCb|_n%t^#n)f_3_Bmd}Y#BPuH-$i#J~BhcGYwNC&|Qh@gysCds=^y9zHFWT zQYX&nqe_H3^Lk{s(I~a{9tFGvtIjcIGyNpE%m(^Zu^V?D%GVEo_JJO-G$$PUTJg_@ zTIA@5?7N8pi1`co+*T0&?`V)sZE|=lO?bP%r|1r$98TaFseo_-q5U`Fybov1vu}Qj7#Qqlvic#Kz3}Q*r4t`d2#(TPu-(mmQ?p zH_R-LbKR1Wg}Fh{_o}Fr#vQ?(DY$J;`)cuk3NMC>N!~0PG6rQ{`&#^}phIaqO_^+w zvx4xb@ZuHx+UsKnp~j!lTMDv=9CTa(bdjlU7pQZ0&()WYozn^x*9Si=!xy|YGSS+U z53HV_!Ya=uO@M;Ez37_&h60Aan&8rZaS`Q%$douEEXqRDZyfjxgz6j~ zy_>W(x(c3SV+x=D(}kIoMf+&E)c^5cU#zt6Rwv3sl!{hQnc!JkaccH@ zR_?-_YVW!Wn4Z;Jx5+msbacT?T#|S=MkW=Nk6WEqrX6ex< z3xN)FZHDNRXnq!RThAWu4~0&9SdM!T`M;(=(NZtWr<0*YgsM0zokCtjjQ)Zzv{f6D zHpedgn3Upuc!@(=$xS7lvApngY=+bAopLsCa2S(fnw(idRDW)JZ%Dq$brvO7G3h~? z`s}+?DJ56PAV%eRCgNh`qV4e*Z>!YG=$ScNi4m1^Gb;hod;ejwm-SZGnq(>)@zla6 z2M5%I>y1n_4Fq|uvr`3>2}ud=Sz8K+Fltz)13!YSyyrN(+^_F8eGu^4?C2uIhxsEL@ax+^X<9T z{K(h7x>(zTZaJ01gogsVhev`~N`B3A$u>K0rr!7A%O}y7HTx2x)QqIrEdQCcFzc$s zsEJ)JqHKw*5E}xAv6?_f@*UGnhJ($b4AI~K*ep5I&v4Vi-%_9snt5tl@uIL=CyVfN97beZe`1jE;3|2 z0By`Ru1Fiq}mx0!s2?Z_#XcRru_V zxI(hxqSCdl|KPhKl`2z}@sb>~* zj|8iOab#_-2|~vlZOIjEXiE?<%^^b@gxV>C6A??3j8VOkDTm1Nd$8yFrhNQP^o0Uf zw&x|Qt+$hy3K&(^UP|!lw|1Lpv5dyZ=Z912;=5bbv}x{IxM)9!Jv( zq*%HQ5VN3iVhZ+_ZCN=6S!OL4wu2gbwS%!HF+hXU~aI@!4X~F1;3OZ@D_PZ@Mh8 z3qAM?T(CQiu^S7oCDQ0AR<(9?&wp=GPCQp-^*SI$?$`Rg$%%$ zX{}$Bo<>A-8h;GbEIAt>y z^B|odhy!$!3mbE`ZxX)M$Le`K?ee=m%XA6hJawW@9%HjW%}-fc{*ol9RAC;VBl-#-J*7^?ESJQM9N7oc5Nv}vojBE?rwvx z!+c})74DYBLamZ^wUw2!@45$1z-ZeeWN(>KxBTZ>JvZ^M3p_c${;A}=m(~7oVS9Ui zupq;gyIbpQsDNUeixk+hneVsw$)Tq&c6j}jjOn8?VWA-L=WDD^t6=p(8}}}Zp^bqHG(=0A533iy_B(I$qsBAqMK+Koi)>QF?g z@LQ>;t}y0BgB~;M^0c=|;{Dy*oYZf9Bzus2;mBzG#<4Fynz81N)=G|UW8GopZx_)I zJ$S+g#TG&uTk6Qle9pCceaJMQqtBaF=(4r?1kSGn&#whf=|qVSguU_M;rorkcWseW zi#{$ZWUQb95m4&y9=!x80k*5IuO5uj+Ke0W<(e_-Pwe@!#o~A?O*WJ%<%clV%SJ1# z`&$%ule4TyKo;%gdbG;$%C?%{V@5NToR|+D!PC_0X|YEDF7cZFFLz;{$8IUlNLy+M z)Xh^sZ z1-6tp#MHHGO<(wO50k0r<;b13*KFwFp?i*mAG`xJgMpLoFWW8dy~uZ!HBYOs zKTxX@R9r83;z-|##f_V+PV7M-41Nqh&Q2mZeVuCed&b9iD&a&Sda)P%wUmAIP8-DH(V=^G z+vb~(pajWTM6*s_o*K-fmkhQWGX1XP9F}D+IWY#i(m>hs!F@1vc4w(p8e*yjlm4lc z(~Rz-r40TPEZZ&d+55-Y1YAwRved`BeFSRS(xk6Xtf|ly)7AyyLSWVpYA$35#C-Xv z!SR9bcb?n3ko_JmL4r=HMNdL0kvw6*jtORP<>A>JG$#kVf$9`EK8*b9Js)=J9=biB zrB>#rKPSYEpP5y8tbG+equo>|e_`TzmvV!P$>5>(t72CZ1%1AcE!q%&kI6NKyI#i| z+YA))_&Pp)N6gdkRZB#gK0mft#GJcpJImBB4@lZD#RA$_;5=;@BrMl(j-dxuN`%T)=UsStT zEc-Wq1P4+L%5N&cn=QAe2RTwbM}BZTll5(;CRl<)1N*hm*`QhKj~bbiyRzvC1eM1I zV|c_>!^>4G1;ayPCk9tZ2*@D}w)fk}r9T`*kyKkX&pP)-mDiNJn;za}Ilh}DtnZ~ShSjV6%g`^@D849Evkj;^jmvKfYSRlO~K}{@}U-oFvE{I z{hJDzj3E7yXY=8z3c9;Shs|Lwd3W&j@%dmdpA}h@Ze@kZDfD(~nIAM0nxb-P^LO%wk{f&Q?xO z10$Q_FGnD~w`I1)Dq*|9j)GJ^M7w39auIyjn}1x1fW-feV!TVKUsAGmD2mq!?i<+* z5;At9%%+o%dWNft3{mae%h3%n;~NA~JjRR`d$%}Maj|FeFB+{Z%Zzb0VeqyxU#d2F zQGGjP?tv}<#wXmzljD(2Jt-+|pDd=#9vbxp!#(JF@g$;-+9!gpK3Q;9R^XG|Nt zp8xQBZM$uyS)Hl(5y@*Gnqt6xfT;hW=_;VATG}>^beELUB`w|Eh;(;KcSuQxAl;30 zNK1osgGjeXcT4|sy#IF=>z?CXXU^Vx=AAd6=bhR+nB}vX@;J@H$MrG^rV!uirZcK9 zufso;3^|KwOj)$d6Sc$WPgcNWefYQGu522MWvM2q>3v>u#k{AaC}=&~ly{mQNHLcz zHoqu1F&g3Ufw+2{8d}NK3wK_JVnhi|y&=uzMJ2U%5M6ZRxbF@aH1AX-$)0vusw7n; zv$yY8Z!Gw+r*M^F%mpiFX$atPij?5Vo+V_Q?+OkA40$Prt&yYK(s%EhhT2Nxi>C#o z6V+4N=+37CqmNpyG>un$8EYxL&KGg`&%)N)MnL(x5GXmLI$a=RCUmLKUXyz&#ruQ$ z>mq(NQINKgpBbwMBltVK--5s*{g@j|apSa+IU8!&o4r_O?3Fv}YEplMLw5vS6a5#F z8m(WgNk!NT9CrzyD|lu0MrYwM-&D*~Z0fpC+b*9C%vtsYeDN@t53}^Y!#9h#(W0QJ zoF`Gpwfx0m^a42NYMaA#o7XIcpZ96(QnGlzea$*~3Rh5I^+alMSN-mS2CT|7w*fu`812&cZ$!W^f110j;*MF5c=i@7Pd<7@!Ok(h zo^AX<2m?Tl>I+#K%hw^)C7C*f3qr7zDy}3XCa3fxLo|y~vvCPt4>+&>WrEYOS^R0z zKc*?b2xc%$-jv zX94_=_Zefx-FL#epMbwLcI|c8PKJtY0^(KDwcc`tvP5NON#H}5!S^QP5 zH(C^?5y&#E@EU96FuiuM*@7-j+0^B&&^>ok_mpn{aN-(*Lon~*PVwmvfUHLXh|PA3i`Mdd?ftVI4XFVf9X zfcIvK7evA((#x1bHHnju>O65>0*!;dE?RGSu4fRR4hjpzelc8&dGB!m<#*>X@tecn zm-rl6Pk>iK0{MQ#{~34OFOUjUlU&V7VXskukktR7cQrW*LOHwaJPBYbT|#d{z#hfACa3o(F?8#u#@|~!YM`@N$iF_uj4(J}|^!H6G5o&LgXadPTo2kN6 z7&4_p1K=2erix}XVrK&ue4kef`G`mg28RC)(O+pXh#Er1k;KZU%e;-Ps>cA$wLr0X zwyc=I^p>Up^n%!|LWDA-OA0akYRRx7)(s zg$$ED5loSIX`%wdphef$L^b?%muE}LNX9mBNtsS^jK?Dm!o3?1sbQ(it{+GMN%bA= z%)?UMPAqSU1*&;9K?otE_C{MRnf0ZLWhu0*QV%&1yZJH*iK}LE5#s(uEyrzCpvbGq0zG$)%+joMpk#`~x^fV5MJSa@6 zfh^GtRNnR4&Xep{jV^&Mjpof->@fan3pZ5kX3@d-uZL)|0s{+-XlLh)A*y_oR3&cA z^L$3TfD22W!Y!>~CZbA$wWlFV(4cBSq4{<7vg3js9z;jgYm{E8^;mSq+BZb(AVbm{Z;ZetKa^e>I8G0$ z597<;cwU}wT+7{q^H6cJZ8*uy6ym8~lMEhuGtgbLeo5q5_EK7YzPhqXRsVWl(@Wtb zaQ-;P<5m&|3|jc;WC*xozHE9^3-s59{WttvzxKisErOI6STHY)S?80c(szekVS>7u zC9p+R2O>|dX{B(P$RCA!L?xOPO5aYl_?s%AG)C#6l^y-)q6Gs_7Up_1U^W)V|*DaX+c3$Hrs~#cAXGs zw=lY9W0X&rs}+wqHYl(#)3i?4=|poj-+V}IC6+o#ZnNq68u4(+{S)M|!HZ6MrYSKG zq>dVnFkGYA2F(a-nJ7taOj+kcg#3x6e)zuVm%>sE#gv!CRF=e&4aJoEyAg$&JFyn} zB!c+Esna4tyB!IE9KKu~zI_l<))cbU$14+Id0!2)B8)wpZjl^qSGbn5;5NRuL?5PI zK_T?i_cpcDB=mTILYCd{dCkFS=OD}r&}wADgm8;psX_FP5H-hD+GuC!5DrZvE+;Ds z<~&+5jFn`+@7*_Ium;h=-fS<)M0k!TtcE>$*#F`lF9IomM8=5R66P;s^3Olc#IQ}5 z8TXm=jr5&B4Q9`iyF( z5on;JKKSbWI1U^%AOXNvwH}p$?3$)u6Ku86H5p{7i7F5!fxb+;Q4ZYxu&=hj1r&@$ zt05tP8yCQ}yr_Z{8Snbb-|08JLIz>TyDgfW=weJHsVfIFuV%tkM+b`o%_I$c)^G{( z?|V2S-z=u%D|nKdvX`R9V(rJ$+*-jLo+#li5d%rKW%E@@Vu+~u!h$SolA&xyCYAF; zzku$ayW2yY@4UoN&fIdTC=^7Xm{z4XA6pSHBw zB0UBe!%pv?y>R|ZEX1`u!UU4SLaqWb1^M3IHNTgVsEDRSGNEbmZTCW~Y6;>X$>6Od z_3{)wl4IiO#|xy)A1jjxJ|DmQOrxHaD^C-lXbfS@+RrLkJXHm z`}%J&1R)+7mU&W|e7X99{rm*L=xd*@JV|W+OMqA-cRZBZ(6wJyvD0mk?UQn7enaiI zcl%*?^W%U?G_I8dzUsYaR-fa-xe=CK=wDP={WSHQ9fFpPH|M9>qMR`yD)`#FYU?AZ{{Ad`|Jx8uLW< z$l7@)>~pEEq(_YTWrt3}+5in%e{0YM_lNaGmxa?0W49jb5%G>XLh5V=T%WS_dUFrw z=KDVZWpmsrRT*`*FOP4}-+WtSY9W_D=(ZO)kIle}qVj}n0;kL$psqpJ>WQ3nn%si7 zW>4`Wa6!W2bpYNjP9zQl%S^C}$+7k3T(4G)JHgFbb%^sLEc8mog5E~G*SmO4w5Aml zmG2+fM;R8^-I8;ARpHC2`5Axm$J|(6-p{0@83%?35NXbALKM;=dN~MGz1qly@9!E%gM-|_ff_e`C%QL9KV;?!+j*4YsXza|R6&OjSB9^O#23P5`lD!xrhz9{ zcyvHA_4d6^k*i4ps4n2N^={HNx0y{Mze^z2IGs@4> zI-Aj$W5sVw_YMfVd)B-Vpbb+Y8Y!{offoxai<(Gzv*yI1@1(=yuJe=-@oZo8Gn}_T zI_O!V@M9NmR|oC)7Q{sYxj6HgiISGTib6f~^Y{E=UO&{5cj!Qy(k}~dsl(|Ule+Ak zEXH>B`FLMw$8-EufY~qBJcW6kp4(D{y!`z6>eN#o7+J?PLJ)3poT4H8TZ7l5V7;fL z7ctOHN*Dt+j;EisN%0XR5%JJMA{{ug;t91V5?BG1wL%*n!dd<2s1;8ZH>w#v$XE|?)(mr@mlNB!YSvicwTu{HRcN0r=wP^7&LWjFhVoAg5hG0Dfl$9YXl)V7 zp+@?~l!z4hH^lCC+ml1|g}H*o6X^Xh{7G+@Qr^quRwcl)8ar{(-{fshO8cUQEg^^+>0KJtL^%48M&=k&up?^ZDo3k7SYZM?B$0Ta7*?x`1JB7B zn`1j#4@1f$h}V=|!w#B7j3ieNiwCdlf9fea!5uFU7e&2_F|YsJziqS15KdEC!bFbY z=R-MMc18XU2dVG0?nE%`y1*CWa&cP6XIfx}15o3nk27wB*8XTtrF#+arP{lqLsTmm zX>rW7P@}&{)b+Pe8YqyZAs0eo=ol*EZP)#E~*LLmcMTGCx zkN511J5uYU+E1#Ars+-{fsXG$oxhI67bov5)P`?)EJ!`8eVwmY)Ni_85rVcEGpS~h z6(hrnZrEwq>6ykFXx}iep7o>O-l@#>tKw)kX(iF)Q=+yyfqrs8IL&z0PVb|kVeB1I z63`JKiCF!7g+a6i`jgY$41Ren4@nzMS!YR)E9UA}C~)r%?bR^02u}R~M$lYYSONtS z(esJF6KL9da~-W~Ueb$u4CT1*SRFIPz7}+`2iFwIh9faQZsXa2`!Xtb)6=33pOeLQ z-R_gjYo#XAoZCe{h1Nj@YMuC>*ksRBALbvo|G9#|Q)w}Y6Ct4#>~`_%4=(SpMK@as*|GCg{ zbG_G`E*rS3RN=U0E?Q0}h82Akl4vdDQA|Ug+Z#>aj`|+|O?gctEMT;N0MA?LFZ18D zL<{~r+H(B`xRw=j({~XH;_DmA0gCm4PaDq=mTDnLe5$g59-N6Hv7wv02ZU3lS?mBs zUUnirY56$~bSD_yy<8Es11(OqP=#5eyQB>q2K1H zzQS)u=~H7wNxK87I1~1S6)}p&==xZojClEpR&c|OUIN~Lv!gVl-xUkX>ogfinPiGL z6$lpv6yf1z?3&Ubdww1qY#|OcB5OE4bk#?YB^vEQJ-IYT^QN#=UPFfvWD3i^a*qpL zJj*FZ5T8p=I<*-t_0I->_*(e2aDU8)9dG*BSS-Hg=<6eqT@TpChvw6N;c0%(H;`w9 zpHOP`(!xXlcr@U(U{}KEIMs=Amyh~I3EOE4L3AhO`)kKNd2iB7;hNNXBaXgUGKB5DTiJZ3qKZRzdIMC|%xI!@iN=LQ!e&%%7%Y)hLAmMY6Dh zuqN`Vz>Y~r5Je*+gd359l*wQvTG8JbwwFmJzlr>)XQJaqAU}@>BI#NE*uYuCg{kX#yJQ{kxzjDZ1HIKq$L5x%6}V1`Q;~p0|pEw7J@9T zY0&GwMs>ZUi2$(q7U6?+-WBkiebj*Xg4Rm`T9x<}THNHd6-o{LB25_JabETfXGW3t z#Tyq=Z57j@_esA^GeF=Ggn4M7bvJLzN+KpWFA_Gdd8xh{rf{2P z1gUW?m|#w{sPvhuQU zCX|t!{WWGU^8S`s3}lB>pPiq$o3&-BH`U!`(h;jG(s#qIEkikA&E~&^7(h@Y08dQ!xU%_n-gwhSGc!;sEYU(b2b=DWZ@tU(MF8 z+|Q~M#?z|{2T5{z)L|)0vf&ebiXP$`X8(aC48xXFH1OE3+KTmZ#P)MdQL`JUD6BsQ z(8(vw;BIs~IEbjErF4Vr+=>judJTJtu5DQ^5nJv2RMt5VEY*80Jr<3PkT@s-N9ZFG z4QGe_@A-EH@}&;Emku?Uw0QL!j-)?19j>{HZS~JSj7d?D_Q;f+L>3z=7j2$~++)Su zzZG1~%T}>{?g<=P%qHxF!fR}#oV~U|=Yo}de;7_fI zGb|v8D{7-a5W}a69p}nmRv18okMLcA#m`+fEoJC^c<^O*&!<)Hf2DtA(2R_#e$P>i zpFWiiV|_^^!bC*YM4_KdU04W55drmaLnvAb;Y)n{qL2Uxfc>mcAh}B{&C1lkVec4b z8um{Q6c}WGw-MA7*;YL?EM_RM;x>ky(?wrV3_G2jZms#clDzzV(HUy8LG{sBRI<^$ zLHix==b*-jQHgpjG~b6kSGj~oM`X~u?1C6mF#)K9Dr*&ogZO9>t+pxyx4X73+bhB%t^=~F)MbvFr zm_*^UYZ3RIHLH#z%re-9xge&_DpdLBmR4*1IX@Ah2;{k2;`!XF;T;o82M)M%*Fds$ zM+uTMqsZ@8?Pjm{%K1~@JP}0d_vt*E(muw3Q zseTW=Fe&|Jk91M@`SSh@B*i&a$IH-A)ysF(0UXTc^IH%tvi}uzmAdewdp%zpF^x!T;}=`(*=$ zrTYrXKmQ>=WG7*8utEdn`04yjs?SyZ;mIk)B$|1~LjrRZX8Er)PYNgGs0Giz=z@xB z6MXJy1WGoXZ7h?zj@&@g8yd>UeuHGuSvtTxfk^NJf<&wC#sW)b(P)`iW#qve`pqpM zl$(ae5a{pjX2(n)EL&Pl?Q56hs0;z|&nLzCwx>L2G<7wm?VAZvK!bnZL!M=elh@aL z;_|-9mDP%Yzp+5N#%@ecrw0uAlDmQ3Xfp}+%+S9);Rg}8BSv61fc;v(Ea9%b7(C{3 z%~Eq8W>VUy%E?<*5jfkNV_#KrY9{z_(wq!tTve9_dH(OZoL-%~{VZ9*ztzqg`U0I+ z7XhqCEh8!JQcDwgoJeM%s;vJK-JMQS>AA;uyHsyHlsDm{GPu$DfmI7HcmMgsDNxXu z^+X|VPgeBBOM!Szsk`co=(Uf`jCPnBpX4IJbD-?i+d{V?+ptol0byT z$Xjo8Dq4JvHOO?+RRJ4@D4ogW6)s5k6M+K$DZf5yu$+5`8>R~T!Bx#X6la#yip-G^o$QT3;x?P;u68r2cjR0+phS^UdpjqE~w31r-jp=^(XMh zMHk~PWYfK9GjiQ^d^#332y|fma@MG^((%HLftgf9;!gKM`X4ySa>WUfQW~81BmfTj zP(1?T>i)aKEL>l+lId80GKsX62lU(Ks%%ke@!Ci(-0yNKafY#rKWNnOICEQavD8~38*_R3P3rZv^QFP5`_aGG!`LR_XlrK(W9s*GNv~yEO44}<51C1EnM__ z>|kB>wx9Tp^uXv}Vlka(J*Sp@_Q$_Hrm?4;$f4X=15d+F4>T;<3cx$qHhp}Yv*r@9 zrGPY?Fth#Cg$o3Ayb1ny(hUpH7uW5pqV#s_iU}0*H<_ z;U?}>Qwc~KS(sQvM8%*XLjKV8o9i}b}W}X9|n>BV5avN#}a{x-M|;thhsbbXxwpQYyv5 zmY?XMMk%)IUPSond-@-op_=vqNL96PGYY=}Z|a#llLl7K!$rpWSqq_BJ_(2at?S3% z^4#4fGtaAUzET&yw}xP>f*>{9@L*Y*$%Hdp_jb>CXf+Y`>>tT%WNNlPXf7ivItS)P zL#*I7htF+{@$3>-Lr1nnwGBv_@@Fp3*n32v-^_}5%rj?(1ZiUe|NB#}x;Vw=`vXb5 z6rY-pM<3N=*FpJ=r=`{IXD17LU{%Ci_HqPw&T8dg`vHrWXaeXR~#~R1-S}f@s`p?a`nEA5>Q&=n?Qte6TU>vg?i-!2`)2 zq25=NO^q;8v{Wt-o#d{teDmTcB`c}f;M zU=nF}v5&^FLGQ@^-Heb-I*(xq@71db1Ea<7)=jRJd{krqA5=)b#C}uknDbZhIlaM! z)!YKKiK)9e+MGWd?nWX{2~qvd$V?Kw5qBaS+ta!TZtQzfw zB1%EgnZ_gX7}hRVS#-b+|jt6*A>CMBJ>apdmZkQ}j&itQD zQ%;zOegEh*&Np50klig6Z`_1{IX3r261tc>Sa;U8!5oh>)T@oyG$+vE3Iwib&X1zh!Qw(n+i?ooFqd`B8C9m;rU zod_*l6BF)`nD4VPW&^PsZ)qt8JT#*a7X}Bug38r=e`wJB!>)#F7Rh&oo{?=zoCq!K z&gT>E(?9;5fr(-fQ|YDVe-Gyzg7{+(G|dYyIQC@rLEag%1y%Sf#WGIBg6C?n zKL+i$&6AO$I)66(9F2yE)}WscP*kW4yQ-F~aun$j@pUy{VEAHD%jcxfLb0v~2+dmP z!gQN$`o&LNO-{;-34b|}EKLP*`@3Mu31!bUC7P`6OChA74%n#p7bzBK+8AEC5cwf{ z{LAAwh{K>UPJWf`@Wh5=YIWkO(qR*y*_7{B^}HxunZLTjDxBWreg|4K32CoX!v3A& z&gr_>jb&^cE3Nwgi5ywIviqdMVA39+Ug1y_lBEm;PWU}?agLU~_ce^eH0)={>r~B3 z*{+cxp?vg1(1P_Pr7W}cFzbqyh(3sjmwQZgJ*JwJMieWv)l3eo31nrV8UA~BiBaZ5 z>Q=D_B8$-i?^%EOr*Sl z(dk713hQ;7{jRk-_}sr${cqXkNi`67qtM)9Ew1kcg%IM@%ssT6*5+o7`yag3Z*V~J zBfAt}#;+-TLA3Sfi5USXZfWlsukpk`BjZ#OoO}1NnZ!XHzvHMrw!fZfTwCHBj*4J+ zcJ*nxyPu!x5t){3zDUa1{(D4=zWF$}=q%J&i+bq!#AD!VMouN$Y?j?e^E(-lplh$@ z42GuVFZZNt&LBl3rqQ7ahmPXLajoI^dUy_Ck)R~daCQ@_M1)6vEi`<0n#ShI+!y+L zCoSf=L2zQUHoLi_R)AtFR3!syOAp{w#ZUmOL6m)OznBI<`cbj|ds14JnOLuu8c4^O zRDKqj)JE@$Yh=LReB8nx)#g%c@Rj(am&2<%aN|aSrO6V};D0ZRplu%EiEPI}Jqf)& z$lURfW^3q+i$z3ORY6ag-|V=!cw4U3&-X+1G($(LI6~2A&W&&{4GrS>>ekvX?kVT* z<&KqhvX$gKw=8{X4kXQU2H>H;pLN`pSke9IlMnqWiPu1s<0jet_ar)L@T)@bQk=i> z!`~T70ND>MQBiMr{{c}$iN3G$=8d$9c1H^aMtt<+lJf87ur)D6QKih+}_RIadCAA*mS(cuV- zUy)fld`+3h6O@^E#B!|ZcE3^?w9B(PAyRI&#y=1zS9XKFep6@xnz+G_zo<3?0@S@D!Q3RQ$@I+qQ6>eXCS*a37&j<@7v)9%FO|} zD(|)4JFdjE!Eg(?pERUb=n`K_pF%KN{U;vAwsgHH+t@$42@M`*xE==91VRgZ_!8|# zFEalgt}Ta3&T(^0BGl|unnv1Mo-7RzDf}!7qK*Xpfy%!f7V!?Rk;7JqO!N*t>ZemR z8Ty;h235|w{d<%8}hYE%$Nbgv*-U z=o4-W%_p`)uLI!rE#5A^2H0J%gE;IS#zv_$&xDt67!o~3+J*0|Y8u>F47NwF*z-F{DGeu68mY7{`HU*XRlZ*@=& zd0xyRI$PW#qhi_HihF_f>UQQr+A*C~K~I z+7S3X3p?aL%B=Ao^f8KJ!U2FIl*iMM4X6|PX`}twTI-0a!hwynWlT~K4+sDXo>1%O z=&yyh-xe!*++@-kxV|O^t2eQj8PRT}g*BG6*k9i&ksga){Fd@O&qD1G6MLfbyQpTN zOnc{#++xLp9;GhPQ(=RUo#^$H@-v)ftewnxDG<8t<^f6Zn)<6boK${lM>wphwTaU4 zUQg?Ujc}{XJb4%sbOVRHC-yE$l4{iYHQ%I{)w>4YLTqb@O#?e;zsN-EXj@C^U9uU; z;$lb&*$y+sU@}K6y-xhjLGe5M+*ohBVLSX*L*K)v zmkOX07UlCd?X!~{rV%T_U!MWoGhn7a7~i_s!iG0;D!Lonalex_=nSKeeq3vc7 zbOjo*YdPe98Ch^8!oAIufsm1Plc~^l-by-u-`JQG6i>AY3R|OwLP0^V=q@_m^96b` z*o;pkE0|uydYHPtD!lFB$H2$~3C8nCy|zKC2fscmRORQT*6@%^{o%b|Pta^7*^sm=TT`j@ z&ey;bkhSJ}VNltN>oY?KsvcgYCng57utRlylu!0NG?-`V_kE6+8Nie+=?u0tH-@!P zGl0asQ+-rl3oR(e4mD_P@mpKa3g<$KyxgDyfXS20Iq>JL`X3em8ctNi(2$&J?+4Z? z@L5Tv?@y0M5S?5d5CLK29C_JshsdQqgJ}emEqi^c$$p^3b_9XwI|!%>nciO+$7{NW z-2eaB0S3eJLl?}iF@`O8nIc+bisADQ#!Yn7)kIO~uL+6Y+O;vX zqm>bymC|F91_~EF)!m)%H5o6|gPt>f|A1kLzlZ4hDa6wcM?k7ByFlh*Vt$D$SihP) zbS>p}urde{yj=z#bT~!W}R`mL*l?<#?58>{4UX$Gm!Q#-# zX|CYO)awI&r5w#LUEy{T>LmCF_Lsjv{m}1`g{7ODaMCshsG#pVj7_=UGe20yH$uov zDbEkT3oSOkpH)$7H5{IiTZO&NYwdPv5JLBGxA2dc}J`3=&I~9 zOQLFQEhKcim7(%Bghg|2__JiF;@gncYVL4hfi{AUr^DtsFu8LpGoN_pW-dOBI7zP^ z_3L)>7lOl=YAvcu2HY}qjB1>um|DI7mgAdF!EoT4rynE=qfS6H}US!G4nyEySi!T>90jNT*sQ~1xempl~-hA`5hsQzYz z-G56;P1SWdNnxV---KwulGtbZa&OPP`Io{sdA~;`CFTp}9AMuLn5MQ|hR^q|STSsc z17#e~r&+UkoM3{2f+9&%UcETQ_j#wG8XS+z($?oiM>j4m8i9EHkwA|O5&SXDMz{q~ z$%-+>iwLAkP!Wnda-w#!1b0~xn)bX#y@@d?6b_IOC_Qk~1mgMD<4xZsM$r=OW*imL zNykS_2+H!k?2)1K0;xtt`;87$iy!_RFQwvD=F1-}ey1=Ld5q=JIv>S3xRyKPp#>)1 zoe&!MJhWGo@pTe_zzZv5vKGeZh5`b9WY8oGiT-xqA z{&r?I75o|QJ}y2X^qJ(n&W{f6#hT6A&2$=`s@_E2Ev?%#e5Hh}AfSd;{T^5suQ3RJ zd!h_4ga5x!;y}MSQ~=0fM-bi}1#A8zN1PQF#15-ZN4{hG3I%UY<|)7Yt(C^`J60?e z)~O^GNsQ4q#(?f?HdNQ>(w6J{z89b)KR1`j!W|K2b3O6aD^bjM>kN|F1*efXz4Y`* zhTqrJg2V?tV@dqNe@oPgfqgt}hIWaD@Fs_z-CP!vSCzBb6=~Q3CLIn_g>T3 zO2-Gi0Dlx7TqO{vyDa7*m{_c~GxNui0!UlaIpr zSy<{#n#0`)Rg|CnQR;{$5B#!~=@?D&wTL`}N9hWUw~4>|Ta)vx?cd<+6{Z0fitKh^ zOq0`PAo}*|?=M5Y=K{O>77b%nA<`X{$e)W5J1?m`$4hWs0e&HtB&oS{RRwT|*^i>I5g94+z@+ zyzu7s%bsf=)s)`+hL_@<03g%@rqJpA=EK%+^_gO?_QiQNCiXr0JBq zTCtQ%pAY?RW(+0Wj42@~H3Di_*g&PRm)|Hn%-vlreqsak(B|p5>WdhqHqrfqgPu~= zV$T)5JdjL+c#ES3?0BCBsfsu3acZN%)S{o*_H=l}n7Y!Xx(#UQsE6N<6c+%sK3s;9 zra(IbDPUfl2n%n}4~%?>I_o=GNvb=O>s8$q;tklm=W}xPXe}BMDv`vM53kjd1aQvr%~Kb>KH*yXfHir~vD zW4cMxwoRoG!d*h~gH^xG<(05+!WQ2rUg^Lor0K`(MU?^mJVJ38hjgmsJh}8ujWNmE zLeOymauX_qFIC%4PY9Pk*5b0?%$;^uuDCa`g*M|a+#|8Cv)WF zYz)Rxwpr1LV8SWn7={5IAPth!u<;?QzPIb&lXCVh_po-lYhHYW^}BRKrNoyd+;(Zc z$Skp;lT^_bI!9Ug#&5St)4;@q0!rW;AtDLeilDWoYTqcAx~`3+2LseQwrgM@c4$f>gR*p0*Qv&FMNnQylR4>XO9SejtTUw(*}kM z*e_HUA%FnJKKrw`Wfb~p*N$9DAf0ulnUVP>-&v0SyiEy2c|3wKDC4@a=Fk>;eQ(oU zBB+Yl3a1qGac0)IA_K2omNL~SK7+q?E8M4U*3j0V(2DK$MBtZ3GF+^`aqHP@>^8xf zNNbt1Az<_2?ac8$zY1a2y3v-Cm!bX{9*X?}RhpF#Q_6_PWS?iAL?dzbPeJY|nsOXU zBL>6AiS6M!F=;|7QRb)=N-q6lW6ibm-NB86?FjCn16K^<(!1vht4 z-*|Uo@<2GGzVPXg+_3E|2d?$%NB1AlW}Dv(*$Ix`!FElSD^1!liUJ=b0b z9wcFTnmbX7(XU>S5&N5cTe07PcWFjU6640#loVsD zCu@6nkD}BBl7_=FqOxRU58DV259C~?r|d!BOyc0cCO=fVP@C%e;RK=dw_yqatp`az z9M5*kKebYG*PEHc2&POFJFq$~Xk+OlD}a9vm~G;wWQB#*T!i58dNB7R^biuG8eX2P z7$D;8@*N1jjRYLb?K%;GkPg*?U4FN-1#P(x?FzX&uQFCX0Wu+^|3NkmB~N_=%F#U{ z(6mIEPHJ5Eztfj2^IA$%yIMpXhVpt&Fdgwl=z+r?zv3@VNYn_`!L@+3`mL5Rd1yGHZ$|!8)Jr}iiGyQ!o-UEZBvwzYvGL) zf5xe^b~12w<2~uJOd|i%j3k4ITFZ5W6XL#wh)E=v;C;?|64dvq^7C1D|6?c++7=xi zE=19lztBO&3ooC%7AhGPPR~&eDlT~OXH)>|k^5*xezzEAT(@9UqxD6}CGndT7Mq(p z+BT?OjYYj&cXe@u1}t}Au5bK(1^sR(Sn?mO)+C?xLGu|HOhPn{7ZM1sh7`=eU3%N1 zqgcEt@2)U^8y=DXqTKKpagfv#2$tYX;&1}cdn+9^2%$2!ts{T{0n5DCgzdV*Uq@jD z9m@W`4`=ru%#tZ@p6-W}-K-IVV-mph6SFG~Ng#uXoIhXGAX#+7_A{j0sY+@S<4&)( z>CL3(U~Tw{ZyyV265?VnOI^k4qyVK|cPV1+VQB;i=!fObwbL#+fWsRFy~hBp2d_`d zEHw}yeiUV`i5~3=c7^z&k2fg^t;mBCxeOT!^7Ak`lGRMNYhUu{abJuT-O=(9?sd=5#eZDY}2eLM#NRcIpTVElJ+srnikBKj}dZRpSf4;|_G9#Rlz+~sq_#EyW z1%9U8P&;bC|MDgJiGW-8T1HJ4AWbsLf&!9+M4{jmMQ%$e;f!LR7V;}1<-C($ba%Sa zQ)^tIPmOF;oBfOin3B){HgA=x8CV3L1Q-oowGlE-8URC9Ol6n6;I{A&)dxpz(jcFk zkteo5p#xgBsA%X1*X~n58Ua9c31f@xocUcgVia_`z&&PBzCg?mz10ICWjr@(fSwn1 zGo=TZ{`|sPC^Bj2<;IUMKJf(3&jV6M@KSax=vs1yJLg56zid(y`S&lu+h$VLFj5V!hnmU;#r znyze6TwRsT#v(D-UIQtUj9-}!t5gbC;sqw^BT+qr(4`}UhZ<4yVsQnZ8mBG1HhgoS zJeLXs0(@NlkUsvv#EIVxz=A!#e$nU%>%^y~X0HNjHeSUe9r1Gm_<1fBc^{rZqME0Q zo=U*)VHlwI^GZg2Ry!MVyR>04`gkJ{@llZVzJ}9?zv%z?5Rau|@3ts`!o(?2Hu`crV5J}sPJP9$oGTHq`C_f8xH0RU_%YDD!3 zWRBtsk5L1;!r(*9$#6Bc^D1VeKtq(_eU$af2jS~uGx_*Uxr+x{2EO#P2{!1!2q+|~ zZmPP36@~1Cg;u_X1M{ra>kpTVmvxtnzxXd(xZLE7Ck^ueU(NRJPd8xg*ufJb6WX)yMz*8Jv9ja69HPihTJ+1pbax zXSI5qKqrgka$(KyRP(uUslLUpd1kEqbCscOee)tPWzJ?{GR=Z;Nxe+v_Rd}zR ztxU~RgMLEu* z?DtLo-j1_RPD-lUW6d!iny2bRIosXZQ|z<)SnW^$d;cfxc(u1`dQe?@Jkp>|6Kl>1 zS+VA4rRtZy4^7HXzt`GOTD+R4@3xdO=J@RlJQnqu_8Ws16|3#JOB@iJyBR%wZl5>$3UDo z64X=BXFgcpr#iT??BD7v3^P%3v1`IP%R&ZBj;>GB{^V=##n)(}9d>O3MeI5&Fyu?)m#%qw@L?SwenVPg)qI~U zo^D%Rh^aso%7+PW*CU>KvA>;OlhS^rq<(Z61`4XH9v<8g7_!|v-sbqh(KpdeY5#)! z!2^32>huw#KOAot5btqpH?z|e-2Z5^nH#v&g6A^&nMwQt42NQ+oGqOE;ro#heBY6q zjj_zMfs*LzlfXqXtHWCH+1gJ=yS^a=D5wBgVgYbc92m*WoV{DBBEz4%-v>J9QnRk@1aklQ_uc0Zts5BxV*>Gx3`U{qHP z{kik(^K*A!y1yUIql|~vzhS&ClPA3vRFfyeh_nQ$^xzr+*2v%{z&jD!aCFucS?|lr zJJX8^YRK2GF$Tg3(sAQU+_x)@S1snMrG2fwA8=e-;!?A<^4{Hg7eW zR@Hr<3WUYM#6@Wn1))EMOdB%mVgnd5HNLuHmX4-7i? zs)+!Fb1HB|x}}irmi|Y_meuz8doBle7TzH+kkwi(_uqzyTe{WEo&G@zMZ;_%>Gs0! zm-Z6dyaeMUbtx%Vi}g*C1crCRW6&o7S|TutW&}}rfC_v7>$Uo01(9WE)3uvVSKw|c z=QUVQ2V=8D^D`le0V^CbeiM4yDejeqswD^N5FZChxN3ro>R+ufIH+KFLO)6lcj?VFd!Y+>f7Ssg16O6?zbYrr^G zuhoSA_7_!-7mT3B#zHM$^UMeWZx}9ATh`kU@4nkfbZ4Yq#z%R%6F%S8r>x<&seA#I zxFq)Nm6LqlwGS)pvsH#JVvtS8wY~Nl1^g%v8CM-Ud);c77G>c$L9D}-3q%C<$-iJR zmLbl`Uy!N9e}uowIS8$6F=jN4KCeg|b0#vcO-mX>zGbzVb12~f`=pDhSp*)kZ&(T3 zw=X8Xo;XusEn7S_OF&kB$c$@IhZz+~uQ2u`9$W0)i4OJ__^OOt+iehdhaxK*99}pP zuOJ!na&arqF-72mY7@8Lqz-OZ&JFSD=hp_`5kbBSB#?keW%HmNGTyFJ|6cY=Z3sWu z&z@na9PMkPH!pf2kJBXz4}bMP*+AAEd*YX=N50errxSpMIx$jYPV+oyPSeaIPEZMf zY~E2lMVOgM zkiRbxVc~&nvH&9aJPv&DVY`|+Sx0@REGlG_-=Z?q{2lkQCq^(*Tz1hH`&1F9lEm%Z=3V7Jzn^h+5_9a=`0YYD zcc4r}xK$h`iwJCbN`ix5%-99}Bk}(UAJgmuV`TJw_;qH1;GOE$7R^q?{00o`vc0OJd;va$SJ7*D*LxeLHT!GM zk7^0(2x%s9OXDWEZBUuvfb_f4m;o z-tjPwo+_BRm%hmKJ~mZHTw|2DLw8;kxa)MduY<^O7;_`L%1$orRDTxlE*RP+5F!HE ziI{0qk{4=e3x>n&92x{;34bV4i0*SEAL$7x)N?Fpre$5uI@j(1(+T_pL;`S9xvgY# zA)2G^k0dxR6$jY1GE(c53s_b(^uviwD*vl`=jw{6-@YN(B z^T&m-l$P1b_l~ z9(qGgodM{`UTZgDl30(`9Ns3B_*U=%`4h9f@ZoLVzm`L7J$kjcnfc)8^5LN{u-VGN z`BVksfnkL#`%m|wkOu1sONzVP18InukoRk1bn-=)N9Ig84$ntMW#cspzlYF>Q@`AO z7oYPj(T`$KnVuEVRCu!r^4^$g^K+yEG-P?}XeD2y1P0)=69n@6PB8`npcK8-wv>y01GDA8?%tX+HY9u$j;3nmY}-<(fJwbH@JM z+nnH&FyH+<5;za=@Qi}|+dlhyHLs}?US^@-X!y&3Dj#fbGZds8TBVEf?)ZtGdYMbb z*APO6zOwB5v;R62tDjLY)J@hI#Fnvc`J7CP-@D(>silHF5;Wt3&D=EUHE|AjE}L0= zWy=@|r>7Sw2%-4y$T|N1}}5oc20+L0inpEvF1&76qgOUymS z&~8yLNjb)b6Twq*iGe2%dKdX3^;y~Yo=%)hpU#S24<&VpW}o52j;Aaz4|@(-AZo?R zo#bFh0C#U@%t%Us2gT{?=36_4ZhS>?5qdFN71N=u9den-9<<%Y(q>6B1fU&o;@vdc zf}P+ORKT`DPuE-VSEQK{`Gc|8Gx=;UIJsiTbgPDZF$3}hMnb9(BGCqgj>D%+_jUc#CI0GO690b5fN*NjMGD+k zIssYB-Wiu1>~Lnp>Wjf6Gk`GgVqYD`lMW9Tr|0ROYY&s?`MV!;%0)k9vA@s2gYjR| zFVyDb*N0rGvpG}JJ;UVx7V%zyYAls7{*4&ee`cooII^k!gl>EXBRD8S zQv2CdV-(___sbEg?C`cZU;nWe{Oqua1BWO~- z(}H?!E7kM@=ESZDe@q`q(@4F4vHKqPa>16U79-bbV|cunsC~&=TJ~Ka4$p)Bp4t2T zI9W^tGmmCUS!XjIw)P~<8A5=RtP;aFzC$f1ZEl!fnZRwj&=^OmjBtEXd$KgEX12wt zi($CmUz(_bJ#K)sk%R6-_4|7Uaq3ooIS}5fY)-&ahfsXf8xjq6Q*(LuJe&c(|bpW>bC z0X+0i866DdLVjGwt(LU*HAniUQ_%2(548|n3}KX(frQ$TshD}#7`=ZUyRdmetkA}uz!PKb3Vo)zw$-80&JxW^~A zt%(dM_84`F1!uWSj1?A#2wL3hTQn-VS$O`~n7XSO<#kbt($L4{_fRKzXSeRvTN@QnSAVnvM5q!JW^w-=cXI)tefApx;D0*N zborgIPc+C9Z9jO>N4JdwZZt@cx);=Y+>UNleps0}R3JrRQNQ<= zb|Ly>+*PnV8%>C}!4?{`c9y;{9c(bE$hbmdG&rD`eqE$*jC>X`;%5}=%PDe&G`?uF zo@366@J$WNUku6++aq$^-Grg^YcaDc^^9Y&+xziALeJpON^+FxU#8Qko=1F5P(~}r zS&;HcT`n%TtPFv@h)v<|zA|^gQ_!u4t4GM1J&~#8k&Xv+LQCg_LHw`fVX)n@D!h7q zy6d)rLpUMav=8-@OQ5*4GJ5`ij=hVFf@p#Uu-I5v;u7ALli!ugW}f#SmC~_PO}ElX z6S2Z)n|DllZ!a4piT&tVK~+q&mkY%G+C6n?BZ5PoXSAY~^xtA~mi`8M<+7Q$LDD;o zR{>9)NWFjXuJw|}E*jC^XxQ<$Yd5nA8?x%qBj#}cOSxz{UaG2shn7b_D! zRvdyeg_crD{jfo2t{dF&Uwov{c-1?BFS|OwH2nU9J7B?X&Z_6xrZ36E3jZ5nC7?4e z?f|m;w{2Y{GD;|fcg;J|nwCpPQUfre*fdx=7A{nngO*vJG9+h-fo#< z-jgQ!m%CwQrZgb8%FXNa5PJ)AMApVQd^FJ|9MJ@xwI@s>vdJ%IK%_kB3e0363&JkpE3 zbGlONN=39zj1#pd1KHnX=pSG~{(j9%MEHFC!k1#Rh?L`iZ4LlXHkP*#cKC3t4qQ1s z!BvM7(mInSw`+Ru>HH&moW?*V*YbVm!d4%+Y0%S$D@TQWz9XH4fHVF*qDjzcFL3BP z$F(9dP9;m?S^G_UVy@6TL428cu@_H_zH#gNMR3c{B`hC+8dByT>O;vgq1LF{Bf;D& zM2!8Bo$(I@Is~OdgYM!L-ir$LUx@4w9asd+y>e$|FN@mbet+bh454X-;Wt2og~WPz z*N)#yC&(7qPnf!0lFf<~1s^8q4xfiG92cRqbX{7YxNZ@_;lZrDc~b@@3I3cv6#pY_ zQu(CDs8Sh;OXYoRs`-}C{d-~@6lY0i5qSLCawXoI0SXeA==1iR!!FQe_bRaL;w)$(AEmK*N}17p~;rkEL_Sy*4W}65t{af)o?I z$9X8_@k~#s2((=@7+&46LN-|&Ie&v#B!1o?608gwlVx~$2YYeK(vl8quLpVUb1r(a zOorV2A;fN-jDmy+THfrVNS_*YY#up~V#{|g(Vy=?Jpw$^Xn24GT;%tWYvS|W;mkt|ip9~Qf7kQcdB~{EbV4vgf@%n=}dN0Iy06SgY4d^H$ zYjEFJfC{WfEP=0LnvkyXKhbA87(|82ZoM!N>AINIof=y2Bryt~6sPkBK7(Dy3%j-< zPNzw`iRO6<-mwC#Gk#2e<)6c;*JdoEbJVSk z#%^>HmX8IfPXU_7q{Mns3Yh?+AAQ}s{Egnk!%pIoDsmVc2^RQA5N|FmG@rRn{nim5 zqqk_yroj)=SpbtXB=??wxc9u@tk>15ENPgpy{mt=Ut9S-kJMg zJ-C8fFx_V{$d3i(vu;UCgE{}}wkN=^3K$7BP3(C<@ihI1AFG`cLSmihM+7imV4=L& zlt9QSedl3{Q)Nokxn+9kr=CW2`mxCCYoDr|0zGy&9(LULZh6jFC9F}6?}v9G%R{U6 z;}x=g2yrNiyi9@)9F&hXP=WBc6HHpEOhdgyWBiaatpC-k|3>m@J;vQihe-6k-Op#a z3BYpi>L&_V)T2|p_wiL9SHmPzX1z#ycG1&;)g2$NVA+3PKzXQ%o~EThF=v8ho8IZ< z&%Cb*#Cq27wEVp^Lx+=0gJzNObn=NnKO7$n@uU^MD8&2*jb3`ox1 zI=U-!83IesOe*xF6R{YVjqS9~ts3TXwP>wA_c|{Y(->b3=qST|1=AtIjA-qI9!*}ZYF!p9M5 z&n7Rw+{9JUoG$9LVM-Zt@obh-%VtXEe__Fpq70QOV-o_R*h^{%a2ngZdQ~5QkCR*g z4%_IgH7D#^``>Tq_gX-eYZcW5N;LrRhOeJRWiZi!#iwWL2@aMY7b!RAnWh} z9k(9MN%iLoz!0OgUOH@_9OP#AzJ~I0+{qGe%BIlN#mNJ4@qk_DZu|TmZ4&%mbkk%L znJxQJIEwh0tXRk1ZY0CB%pdhsQsgF`e&qr9k-RBtVNe=AXG~yzv*v?9(>!ow^RLGeC?#A&Cj!=LH4Gd=AB{W$9s4izf20a7KH6PYOqC+sC9Og3OH{| zS=i9Jm6JjFqarZ3%1PfSm)Ri>cw)igao?SV$nDH5NKxOe75m=<3!}7(Djk!4ZeeTk z$x;IE@Vz`)4!q+Fp*ybjP+JT03ZZcy1cn^z-@Gwe^!7F%SaMgYR}cy_K?7F<-jj-a z6Ij;F%zbn^A;Y>UXwL!g`O!$(2&4iG%=fgia=8f+KhKg%p=Vv}96Ad2F-gvxY9cYbYftfBF(_1aVL=yc3-a%V zTkbf_Zk5_n(;+YX-YU}bSYTWiYvgy^9Q+~`j4O8Z-HgVaSP(YB$JhD@OL_m^iUE(w ziAsAumAq@>;v)@|o$VYc1czVWRSy$VW=QEk$SZ=e&&8Z+s>2Q|aPeTQgPljSj{Rv6 zSQzB%)RZJ~7xHj#zY&(Y@7cuLmm)`cL^EnLGacS;op)3(ajWyOF)6}l!=@VZJ>t#L zzt7aZHd+HE3B{Ot5bss7yQxdNt;<8Z7&B9g3JGt`o&eFQO2_D}^VqF$mHSei8IU?Y zqDUk-M9e1syF#`lK4jdx+=>B}YhT-iI}-hL4kvJtS~JsO@_e9{BTdKNpLbUw^P+-N z_>$C{l0ZJfspwUsE>w|#!8WSCYZ^@-VS?k9M7;bfPWH^g+NSO&9H3UeMpoYh>tvrf zbmzJDr_s$Hap7uj#YvS;LHt{ESPwpr2c1rLSHBK{ZKaNdQHg@0OPJKaq3UKiAvLCV z2dG|Bw86&%R2}=y{`>_p+i8IF#9ou`CLcayQ7&jab9rkmbuaG%k~5k0FLI}wPb>j; zamCzR5f|rMWj$WoIi;@pbnv{OQ4?}N42#p~<1H1l3a>}qPZQ1=>zfnmuk7YVSea{91@=Oh+0D;5+iD+u4JM^0_-h#j&XA)72dQfrZ&I;Q&fN z!F84fQ$ zS>0{w=It&Jre?J1ILjzD?;I9+vW&l-7A5(IT?XQ%$lMVDRuuQDRB!1~JoQj_0;LI# zT^`a7i;&(AG9O=qeTiIJ^XYWYLjhk=JDq~1+y%fgiMJocPG?UAeguH18uXU(XEMA+ zTs#;X`*yOd%9+#dOAh}Sj+b(Q`2q0IVkZBM;H!tmvugPw*a;5Zzf)tgF})uOPq(h& zA4X;oF@~(| zl|=4>U@P$F$Q5d$JcCOsXOfmAkxJeuE8Ln(4mL#1Rc}JhAND?~b?^3(7_>HssBkl+ zoIoI$q)Cg;{&oC7Xi^4U5j#8ZI&TlMY>QmexHI1-KjBe^JYm5NP-q%m9 zo;AIl-OChBL_Lm~#&ttK268~T)RIY!Gd5I{exK1K$!EdM!YkJ?}3; z-hehX{~jPk9U zxdiDg*Nbp8a8Dn{ss22#mB5=$u|V z4+*$M@uK%?J(#Ef|1lMD{LT}h^DCt{3HUoF6X?3`&jdtC_P?v3j7CAff?hXO+)7y)?W0!VeGYWvZLsAEE}&%OUsu)IJ-z*C%VSiUwTGvDX->vS zakfbnVqGT=IZ&M0`fq|UeNe>ej_Pv3>pt{5({BBE3c(%t8~s7xEf}=~{?&{vxBITX=?F&2!{!nXzuqj!y3|ZS? z&pn{9AU#ibYWQ7w+9ANRZx|zvSxYdqYl;P5;Q?-RXU+?d2(m5=58*bA8%@HiBbWt_ z>&#$9J^wH0JIj`I1BW4XJ0F~t<6XE{iWl{ss7pY9_%AHTfk5U@tam)~>)C!u6{_Bs z9Yv_j4X8b<{Eo-)FtTCtZpPj3gt;E0QhL}2WFO9KIyl?_jL)uqRaNf^BvG&^0>*1m zf{Q+#thNbV%@`j+>CmU@T)gOwUHG1QYcCjMJ%V?QQ&p;{GxcT7&e6VK_5f6-wGEVb zOm_x`C@ti1P&z@v#-kZh{Hfa1F4P_XtYsyIkEzvy1*VJ96pN47ffexI;oyG-t_)m! zidMPK?fjACRV2Xa1vVy=7Ab&rWX~y_3;S_d@Jk{wqwM@a;G^#w4Vvj8KhyI1i<FZ~%DRId92Cj^nSM~Bb z01mJzmYR@qwlx_S0Py7sTneDB)BBQE&#)(f5ct?JP2l|0OR+qmN$+)w=hQR5z%MIx_kJi;y@~yEm0cV<=~7ptn)*jj zeP^-D>d}^S6u4K%Wrduy8Yzuq=^anX5t1_zef6q(QefOCL*xJ{*AC5RDL#WgRwY}a zCvgAH@$S|vukT+yKaUP;2%M!gZRUQQ z5E01dtSsu?twz*qql@YUy0nMeK&f4Br8_6`=^*%Fvj@$Gb_Mv;s2@J(O$ifn=#J&h zB6M+Y#iyE|vpo^X(CAZ;KX;d#E>?8RU(MiQP1`^lPr~kM?5AqSKYBdzj+yNCQ6eE5SkFiD9TL94)`q4{fLk+4PeXPK(ixAujLg>Ul z$Y%26iKB(TRF?-iqGTh%VfK)3y|zf*M;9ST+63^J!u~Yu!ngO&RwA+@1p$G@QiY}a zTHrxZ(tu6IXihu**X=l|ca~Rzu|rqwD|d3Hnhc<_`Cwzr(-tLZ`7H|dI%lw5i}Ply zZ8sh&t|YuuqrKD6fX;e+aTSRT2WOoz3~P)MVP*QlyY{8n{0csw=g&TE!9`^_?a^5sg$>B^CfSfRF&C}$X^gL!5Ij@sdAZzNkG~?^eVTjx9OFvPwV3Kl# z>KxzD7Z`)o>s#f_NuYjQl;rd03iw%rh!(<9hk2b5kijPAiN;I0vt&OTuc;}`F4-G= z5XhH03YFi93d8p8C)7XGXphR*vGczN6u2%2W1cXcv_Q@+v|`m!dh(>+Kf&)x9Av|` z(}s%)B8`V*c2h!lBepQpSakpMB#rLzK1T|b8Z_1vRQGTGfa`-&VDYtY50Q8L=$_E~ zUNSc<{Azv!_q#zE5uQZ2-#OO|PydxWikH9y{i0V5Gr;$d$A%f-@i=~S7aYL&aWKYx zu!+VTSz2uro%k^$%-s7!Pm>)(^U8WfZu1AYAEc^3hP)Z19Ou$|gByCZ6RH=*0D5Zq ze6=Y~A&-n4P{~=|S<}1PG>zc}!*RR@<3p%+D)t-lnH@*IV{9W2q5~ckwnA{kc$0Eb zaY%f;Hg`}YmIW#{DH@=B?i+rE#7eZ@x(t%~IMM)>y|Q9MyMPS8i4P`UdOpW!uIf`) z`QtjY0<*bGEhrcuBfbEidS=A)g=L$?(dLu zF`)*0%fH!I>t1QY$RS!b7R6gp(Ct?L(IGU|%GSIcovRXPzIYRVsBlt@;+II|EF%G3 zRb=-l+NA9z2+%=tXura{hc)1mcT1++507_5F~}Y4!jV&h@5qZ>XrN%aBrw;Bpwm@O zH$In}TW&=`(C$gC2nb|ChX-sjMT>qyV7Tc|o7&_xPmTiz?ZU6{8yl>4_ibJ3xSY6v zAsDNHy1`v=^qIR7mq4|la@nY`Hwg?ebycYGcAH1U>?w0x0kZ-i&2|62wq#PtrV=-n zq~uBKj^Lca?Xb;3vwRkxSZ-VA<{1nj1G3}L67ut__4WETlTEY6khsmZOVqLWTrwz_ znBEFG4gOq<5LE{``n`(GYdhFf%McOY&2AkgN~^ylFxP_?UQM{6jKLr%fMxL!WBfYL zc(E+piEok>Xb$Oc<9_cV^=i=5p(+_i66W(C+nXSxx%Ko%(R&Zs+_cZXHqEAPM!3Yi z*P+;L@KBlnSPZse@Vvj;fD8{sVk177B=k(#;c8crqJg`jeY~1b9sXASrgXb3y#@8& zi^6M-5poPK$H^6R&6>Waau7y>!M1OShXI%X6d~yCR8Qx*3Z4hu&2_P0%;WuZF|Y%S z><_uZ7+a)<&C2eox#99)M|1M39W+wQ#n5iXvMFuQ9O?Pus$m*3zl&1t$JZ`O?!^#^ zo|=DyVTjk;^_QTzgg6M5KB!C0aAy_JrmaeiM6)%dU1&-r}r%&vYB%}|1#@O zH;1gbapf33mnncLLCjQh2p?+?4=4gNUP{+RV-N~?kqwV@xH&JZNTyTN>o?|_dwb+^ zT=3RhHraUA{n`Qs$o(KEuUsx60_q0Tm$$-Z)0`Vuu?+Z+4c`k8#&|)QXG(rpNW?qj z!RJ*z;?2$!$C+r^4x)p@6m09 zrS{EH)rJOa+Z#iDmb5|;H+Vp47?`0avN-PDix;e8Mf~WwpB8nX{NS5nZ$zn3T;Qs# z4AG;qng)&wrZ`-njPge1_!eKArTBTgvV6*zI;PVBb%T2PUyRq5l7;`>(9EJXbsH9% zQrJ9-zPYe0Zny$gCTC}6vY+K!as`{x-VztJZO3b-Z-F!^Ijh|2Q%&?O{UnV9MRm3~ z>sHr2f6-eD|E*Uxy^r}z0NT#`*N^+VE~?yxE<^+{%kP7=zXAXej{7Lp{RSni)MH5# z=suFt(r{&p!(bDu^57rq>p1($BIS;{e6BjQNhc98&=CfKnzQyWfq8pD@1S>k-SV7A z)8Xt`X+p|DZU)qFQ5*`jnwxTD3^6nDiok;dp4o*!oizMJ>iUIOjn$JM#K1wo_)d{> z+_L^{@=9v}is>Ntm{agcHL>lOYOhtweviBQn`SxDukyOc-@|iL+Ej)2N}q}YNl#@_ z>w1qv-UJp0WEMM?rBB4(L(vyJESq%#Hv5TSiO;D_4Acj0Q0RL)dR6Zg#689Sasj2B<66ez24MTAi-=Px`Mg0^NFVB? zu8Y1D*;fw6{!c5z=Gs_$^+Gi?!M%^}Q~r0xJ+i8zUckN9J%CsKJPABGaWwqwCIs=W zF7Ul&-DYD%lY?MUT-aQhMiyyF_*eP&5RYK%l!sYyXXW^oAApDVSmVYr+;O^Ujo3SV zoFap5o1NzJ9R%PcvSO}5EZg=qQ9jqBu1m%}4^Q#5X3!9RD(a*LJgQ$`&c7UtptxY` z^X>x=+%6}Wo^biM)`0EH&n+E4C$vGT;+4ow4T|vj=!9YWC%fUKWylNQCW@0Yf;*B5 zcL6to-gQPO6$n}j+AZ9RGWuno)t7PN?~(w@Q|Io!3-`-mDa|fzShb!L;iD(8ugAnV z{!U(n#DNkp$NaJ~b&Oet7Ubh5(eKvT6cXKJvNa)l)_8%Ge<1WRC z=$&h?JS25VFHpA0ZFa~%@Bg*7@KCSm-O(Ec5M%zhJ3OHL@2p;_n7|2YA1TLU9wVFA zFHdRi)PZXE#Ah!%NI=i1ZVeG8|3$1i#D!WI;$sjv$MhhHpwvnx2O}h}Z{#gI=EcN$8n7z0>Ny*D#H*-ZZwA zAg@8tfvDR-G4T8&KOvu2Q_~QHHRCx4bkcE>l#i?HY6_(lr8^cnTIbj0_;X`UVPwPS+NKAKNw9 zUTkBx!hta}nzk|tpoD72K{a1o^lY3A=EHZD)WB2<1aOsY zV)tsV&jLn`JJURI`CYbf?_#)n$y5~>zzH6IA-O*FEm^S}{W#yhe%VmU0Nc4t2<9t~ z9_rHde*=o{@z&!p$VD4?^Z7Zt$Viv~@rkbLg_jU7?|g#_9uIQqn$8Ly+w=1m4A|1U zsD1}d*F02?wGp0b;P4rF34GkHRTO3KU6`!40mpr^(~z$~CK@yfdaN}U388c%pnt}f zB>+6E!=$ykp(zaPq>UduZFvLzRBjvF@kK<%CJ;>as*Zq~vr*G2>7q8EcLyeWC@G>2 z^2HH*NLEAD<(*nI&jFcz@t!nbCN0Oe42q|a3Vb~*j@woT>5P)v_lMflz7)rf{m*U! zPFApR%JA%`7LYB@k_@y9#H21qsb4n0jD-i6q z&#G7)n5yv|77{odi05UEE&OQcS^BBOv(B#oMVWC2E_eYl1R>Y=yK_LX0sCb+y3|P? zfV(xqeK_BPXac@q<4}I=(j+HOoHab1&eJXx*B6)Z3H;a`&K?A}oxh^XqwVJ)-^tLg z5t=fMSapmI%l>s%ty|afnU)EB3y@OKW9>Ms`PxAPTVX{utx`&gBFLM7GK?W2>J8H= z4mS@Re+|$I`aM3^@@V2AFnKcn$?EB!p_Sv>XSpdB^M+O&oT~*lT0D3Pk0k_`$~gy*u9|X4^#PqmvjgJ({|yJ@gG<9Z119lC8GtqFIT?$-C!? zGflKIP&!$_EQVh){vulch%Lw_c8 zhkA_bGgXNquzyHW^KAd8z`}eG7LCz~4=s>rR%VL@gHf|VKu!47#6?ZRCZA(R4dzLINa09Q2344TamNX%v-v{<4c(8V`(t(Zc_J9r=32wdX+6DY*LqyWgoJ%o^?SXa?Atee zE)V%DlhD9^%z&X{H0k^JXXgm+eVgq0zJeRbh@Gu72qXz~7+FKa{+A{SVFF3=P+CwT z(TcTeHd$6N`!66F+>z=-0W(~K8Gqd>`#pVwMrMxx$hzffmxd6!tjI$gN+5lzX)mpH zdL3$4NrG<+ttA%px>0Ivvmo3sutZhksqzG{ZKj(7E`1FU{7RN;QdKjMWUg+dS44PS zW@ICtV9?M&UDH8hXw`$%Ypd5cY`GrlUcIp)R~FFmL~b@F@-qI7xC#uE>zZa{^+Bu{ zpkvI3Qa~>G;fy&0aelaWhr`#m<=X%1w{I6Z`6Q&(G%aTDu0l1W3kwZdLVQ;KljR&5 zPozpsmT#BFg+cDMvb5;(5PRQUpx$E5_FhRn?nsr}HfTsJe=>Aq^O|&mV8?hY|QG>+D7{+^m@6#Hil=KFHhN5_RCj&{XL-PzA2MTbYyPdSEA zXH+fB$Z@M$S$aC=3pt(}G#7g$0E%1UiA6!wa_jSVI?+Z|UA<yHz(hd{>>GVfWf{e5-`|rnPV^x-3hVA_wq$a zt1cR{O<&a*zDNyTcgeIw=L}b8`a>n$-R^N|)Uv;QcS6oUj0ESw@7;N+1Z#CRLAiR5 zJ}VlI!=00Eo3t8a(FO>Ta!<|C)O4UY$Tn=_AURVs_WR|6MS7&jg&h@r;_db#0c4vjfJP_W)ZiMc@FXv?l}b_J+YEJ4 zh4uM*1~oSF3GPhOkv%Yul=S925aLW+jXHE;Q`w^gL%k z6o%j)tzCt^v=sDZ(GyVAe;6 zA6*=veB~^dD$~S%$fv@+88=mRZ9Kubwx3aVsX&FdJ}TGSPwyoqRBHL^sq?E*4iKDq zujr%g-_)agzY6KTHAxvDoAsLv;;x?qxw4=RA1j}*ou>miAsO!w<(@W|dU8( z6(b6tmFRhaJJcBUCTbhLkevj@z>nRL#Q%L|W)C=m@g?g1RgO>ESN z1ENsJWm6E)bMCkWTbH@foLaD4soy=GgXf?+ZvrDjwzXd)E;wT2vBeaq&5m9u%t^|A z1FjjA%Cad#jHtD27AzV*@Wx=8_BK_9n ze>m2d>u61u4+PwD^-kDx>VbTNldF<1Kh7^9E#JEC>x!433@2@~^{A=}G@G&hVAgn| z6TDd^p^_(|H6Oi4W4ie30xW2AJ6M+m1X5V8uAC1Ccj8 zjUFDy&wjv2+SvrwCZBw1)0Il7sn2;7)wJc)r%&oFHZ}q+wS;Lw6`+MIQlLPB z6EizVopc;a5GUR%QByn_XyFDTd2M<-udXEu-#_EO=Ks+@T`Q&B1$kJHZ9Zh>{e%Bj z1h0G3R~H&yJg{{Phht|C@`(KfYW@F=)_>XGOaW1CL|r4r28qw$W?-)ki+(Ejcp4b= zh&0!DDeWzmx)AQ%!s~86Bb8^lgW;Eg)bgYaH(JN%zIjE+lm&IZCv@6CC(Jc9k$zHl z!EJ&r40NEk3Ye?dhk=OGv;sTd)Du8r;iS08@YI2iXxhGP<;AmKGZtARN?+>pcLVo} z6hyuPJM8LWQ6Fxy7nyD{`M>inurQ7^bMQ#8u z8|!7Sw-UYZ;RQU)a>xQ}tD)2y3fzzWSdMo7Dzy$o>P~6&5XxQMdQf~|6jyqr_U0fd z3Ch~mHJrI(=iIgu%Ghx_AtJ<iYR`w;?4Y{if}c>cEO(2GJPIe(;!Nq8ECwAJc1-QHS3eSt_IHX);%y5E1v02-e1Vv^&BYZ(|G3S9OV z`pUAsnqJV4$x+iS*$0_M5GylA&AJXBpj*yST_ihD?r*^8E?5?8W#Ak#9UJ!<3i?We zAbTEb;)=+Am`UuPB*7Uo^8Ncv+~ewD7E$DeQI#&rvOLlI3*fiSrVs1dLlAOcp_v&& zsL0}~-d;VUy-CWz&p+wPic9d6QX*gFf8Hfa0P_8mLD4Rm8j_wsZ7yh6)70CS;tz^) zB}ESM_B;Y(_pHYG zg%1ehgjLarW6LDEC87mFSWGYwT_&w}LRA0*i0!bJ4DNQT8|*+IBke1Ddv%TWehE(h zek^im=`lP6`3(Io{$!8xlB(lk3A)g&*41pdM-JShpwR8Hm0j`HHW>syW^`{Ot>TRS zDR{)Ibj=J)OB>yEe-1fdB*5NpLolQE4`nFzPtOYp?gNM8)^)!he(c&_IpCbWN>LjV zCP4kiP`cGJ`V*mE4-Mx=p*Y zRu8sKs}aQ=EzFgoceka!_tpzQR-ix~!jtBRt!Y}bgfC?q4|gZuMA|v|c8)KqKB?~| zuZCK=~ays^x3!tey zCkp8(XY(ruodzkdFF$QxopdkWavf44972_z$lt(i)#pLdk(-aM^yq_+tkRtqNe!8* z2@b;ShaJU~yL|r-&>M~W#G#>i@g*c^^;@RoKJRV>7PW*gJUu~mZng8WVDzo)0F=#l zkU#GsS2AZk@g41IYyY64@4}A28r2+B5(^XMAi|gVKeCoq zmk*b>0@Ay8*@y8j_^2chko}q^^NuEQ^Que!b)H#Ztt>et_GbHV{tUz{Q(=e45-e^G8$Yj=+41RC|U!&g*Bp=8eYIq#QE z7kso{e-`EJ=DYbRO2#uIS?Kg0L$)*jNZMF8G2-=IkqgsT<6j@MzAqoPNvJY%yLrHX zz|E58i%btSUw92T>Mkues#IQ6R#je*!6(}BH{!uX!06q(zqHPdKi}{&J+-N*KfHAN z)H9+`FAnvD#FAi(d2xe1~dT&z{R*!eL{0S`VyM#wc+pXeQh1~ zib0auTbNYN<9PIz!HKUJsG*JsUuzflgfICch;9>_)Aa~#t%YU zIV?fA59lqwf_Po`1S}O4+oS=1smwNHF%6h-1_}}}5jVcQ8`wo9Gq{wOG*ZpcE12Sc z0uTtCCSpuq5CSh>$+DeT1_xXnQcaj`Qw`e_pgwsJCLgWF4B{}TwYLh1#EhAG{$fnN zF|VTZsjYW>o-P(dV0SLugzE)=OoGvK|0Y7n3>yw49z~fnr?R-&Qrm}_bIzd~vLrZP z1Owjdt7_Wgo8@=9Jtd`1tV(SePqYmpn}rxl=4@_o_;}l6j`ZEtfrH>PJle>egO}Mt z6a{&RbI}D?gxR%h>?XTwqrM*)X1oINd#MnF+DS0 zv4Vm~frw+^ZXIg>aJP`oY0ygrV!X%Rm)Y7dfr7;9;a!iL^%$?7GH|xqS49)@<^O0^ zy%$Sj9JwO+a_wo8w6MK-b61}L^UC4jWzl~Aj~nz;WRN>c`|zxySK!8%HhFjT9qy8B zWBfZxE%*N@eNhs}zvz_SdF~v#imr zhB%?7A>=d-=IGOoJMA1=u=M!gbVj`j#ZLwZ^fv1Y6iFm~R&pK~{CPX_Jbtn+ovBn! z2k-yZ$>9bo{*k~-rTV?!Pp{6&1ySk?uqzFN&xskqaCH-p$sj{uzCXoJvQXQ=7=xI$&vus;88bhiiByMP*4|Pjgcsjd&O^u9_xK;Ku>i_h_>0GK* zr1RKJoht?r)5Hz;R%Pu3e-`X}a(C?zLwooWDWH^3vp$jN%>!plQ_wy`<9w6icauf-fB--hCj*bVhfAIxT({fDAY+ zCUE2T@kx4TvCKNs-mU&v+ana-=AD4PCrE4jIC61gb;@v(+jq3H-f`+6$3ZDQ_pnh4 zvO{0KsGKx-R+4+kCwX7ryR?L^yi}l4)d&=O$IB03a@Z~_`*6Li=b!_^j#yaxw4vT= zYqH9jAHF1oXsm9OPe@1|9(mF}K-#=H;wP(zcRZdYb9CV4Tv5`Ztc-_C%UPgOIivGs zLHbPZ^}n+S4vy`{*}^}e;03tCKw2|PMh4)=YHLP3)2FazlQmd>8JRgjjf@(op`S@H zXT_1?(1a??S=njKlB<1x-SQ!*Uny1~1*MLqxS{5J z2ef#*L;A;Uj|nr~61avA1|!fuL~3~29r!nHOys}Z^Dy1N0wuG?pC21}dhB0CXt(nV zM--RxCtK+`%>h1KQPFfTD?TeBy=#=;`P+6{I!b3v-b;SaNfxeEkn@Mh$VUAmDX%k! zTsoN*c`tBx!@6nq4f!I0du?QJ3y0B&I?pBk;QPnsxAR_Rm+Lbgs>U0KT7%T?JJPSSor`4D-&&> zw#Q(lH))Kb=DNnE-PQApwiL7q_J;1&4IxQV3Xc@s39%KxKd#v}2c%67K`+4t4fD6d zZ$tpknpV`@%SnE~0AS7wx5)NZ#qO2(NDrwPQMDV20IVb(l`>_V1dUtiv4H~4%L}kf zN$cp2ZLs+a*W+qcL;D1>u#T|fvU`Dl2KSnVSpZ;Lb!^-wQ~GppY8L7q4Blu7(!Bpy z1HEGX42j`I#fZkD2<*xgz-eVVX_d0)avA$XP04Ylw9#w3)lUFI=4-WFgC_iD{6fD=1s_JldXoKJC4#^@esBdL1svquDE>Q1$V$-7%;aI@{jQ25uwHKw?H( zm$9$YHJG){m%Qh3xML?(lm@8Su(R{wk;c+L57Mvx zR+A5tyZ9&lFWPUO^mcB4{93~v=kd!^@>d+@L9-<1u#m{|>cfGJJnD{D&(YDm+3aynmX&PiEnn`fm} zX7!7Q>&>0Xyb)2LTl|l$%a|Z?Vxfzj>`#RegehA2zspmnX`pOZD&X6H7%I<^bFeho zZ|^>m0QY~O6WBZviCn%w%-8x@Al7su?c~=_I%bk~0!lD@$%>KC16%^+EP?lS58nvi z7Jq>lkp5d#vD01u+|b^adYUnL+vB9)y)LS>KYVYqrsLeu zDbOHyvrUwh3ru^c z`-_lR9#&8WD()P-gv!fH?^SQOX@wZ_!Ap^WhYoq8xnK4vFB(>kG}9eqPxS_#hh-{& zye$GBJu_JWi2gnIDGj`+Jw%>XbbMTTlpr)>8Ap3*ZS6cvU?-0l(Gc9qB$`yE1-QzhPkH_9Nt+_=#1? z#=7J1rw%4ez1=Zf_(%vt?`zP0Zww6fQkRN7H$e)T6T=gpIJQbJZ{Lc7@~&PH-BX}V zIn4SdukGSPuTVK&Z&PzGb7s`!OFR^9n3tAIXnCVAVlp7W1VE>sui*a)DJvb1BqC6I z?h=QxC&gvl$&#MC4(8WZCuCYgFTsSazNdSzRcqs&7(*B1>Yo=U)~YV1~eKq7BEU{l%H3;&G|wf#9h$sCY2TxQT7 z&(Y&h+mQMTB)8XbpxtBSK&Qp9cuf!cq~O8E-q(%gBE-G} z@BwpqeftGPm6}75gZiTwZ7Vh#|NV_h7x#1a5!Al{i^;0=D*1qN9jjTpAHmT4VaSSyv8AK9`&TLWyGIE6g@c0= zt?`_-%ry`$8}@krL`BV(bdym*f><+6(2&Zlx=B;;0IoNk@jdx;6M-@8ezxX?TeyGY zOI?)De_(pchx-D?xN8$B_^DLxdW*wtn{Z9P(awp8Xy`x-fFOC`WqRHnT%CyR>tunxSo|~Fg(_C{P zLZm5bZcEXEo$VLt+vkU$7`}0i5MN(soIzL4jwrP|M2a^mBoj%4Y~m8-k(xHq<;E8# z#vuq2w2$DrU1m99Vq(_el5oQ%pcBZxV=xYLy06QRPy@jR{lcm`)esb% zM2E-p(ChyTJ>T0am%g8mPAzL@DmSdZu`{E`9{p*29X0&~w>^FQg^?y{$}`hqJ3ycU zo4H@{CyYCcbzT~^if!Jk1a(e|3`A=rITH-V+1UlRW>f(fmAXzoh&197S7gM!dpT<9 zxsRi@e{_kkJr@-qvk1nj6%}oZVV`L;%z7AZHf+f-gm1p`^&2|pWWEdwpmX{!WLdEr z9+w$Um#3-D4wmROwRWa9;+j@BQ92K@o(_E^g0T^-1AlXPtMULqYwTy2 z%o3d=10e;V3$)GDZ6yj#;5LA96B-|LM4|+*^8&XPT)rbtY?0XT!PRfzH1HV6a^>cr zLbML!9RK3Kex7`iFW@A+6f_hoV`TaBoIc7=6Yz6mur2!%lTyw;<6JxW7ki|VJBZqYLQ7bDGk|tma8eXWO zy7Ho>z+bP0qK-spB)QtxS!IEX-aajNZVZ)ur(d|Re?qG&$kaJuzcDW%=&R;1drBs# z?eGMGaC8>*oWtgqYORSgmNI7vFE9Aff(lW?o%y^} z3Kwq9SzJH9cfa{35?R&BCE~3Ep2;I`I1r_gIX`2CnnkOepq%wWRN`Y|UWseb0{yAk z75m?xfW3)+i_srZ#x+9`yrpG)xaH>G*vVZe*PH;8m$Aksu<4z>u6uyu*olhgZk3ug4owE4?S?*Yh7bHTF)F;q%bk@5$tD? zn3$Yp`pmmwCf*8V+N1i0wp|RimvNM|?xm#PuEW?zap;*($9ptpe2|5G%Ad;4E;VW& zcNI&T0Z~)5;<47+J9>y~)o_zh@k|)o54eh5v44@0d}q%T-r_fl>dw=MHys~8nf1rw z%nHE>g_NoiP(tp*9DMC`b;@j*g{gc1UT@L@tjdOC?S*a$j>gAN@Sy zibQ$&!nL$4CRIiB{gb8!K@Br>Z8r%w+5qc7+R%Pp4d65fVqJ1&M;|G50PNSxr!5#H zQG>McCt~mEFcl|fyb?rv?GQFGzEjrKt1;>cBJ*F&1v7Gc%vK!gV{|MdpEdf~uPWk8 zI{7ZAO$#gWgj&^1uNoAJY&vo^G}R&fIslKjt~kbZ;`!h#U+OnY`W`SfO?B%ZiEfB< zP#hV^tDJegyMGl*k#IpcWulh6zKy7OO2vILR$`A}a4X*6>GZH(#=nzD5z9*C4rS@T z_-;ne08c_=U6Zm7&Fw%6DqL8zFviyMy6N{xh(gljLSqkoWekm?ab`BplI8`=X|H z1%~x-oW*5LhKu?%!X=I-Hs4%3-fh6GmI7pV;@U)v`e+b93#U4t{RU54gh2eNshi5S zx5m|}6b8M*(|rNc<`HG^2^B{oeP9KXg2%EPhNMZ00Hx>Qoc53!qBonN>D38glOaVr z&isfAL6H8^4lyZU_m)NwgFI$9LD$bgzfgn!^oO=DMBd`x%a5%*sJ@r(hjK+{4jvyo zwLlXZWivG%>qnx*p%+K`yIcOUY#{2ZD&Sh47y7ZZP(7q<2@J|KzO3{CY+!(-X~9U# z{^wFr``Y?muIN~FbG!S^UJ6j}E84X^{hD9nt@HP^=psgLKAPO657MpnUM;c_*MR{0 z-#2|Qx5JWy>6ayvuZMu?DZJ7}O;F-Gg##qjhJQSE(-Aiq{?R*h8W^63x*XTaR^Qlb zSF*A9`s>2BArNVk0gu9gt9pKTILi>m^X-o$vb2~}mbh+b9|6#}jF0TF{*i=(j$#9# zXr?e&o9#Spj1?`IG z@Ah1Pul|<5a}fONe+Cje8F&!yPyh+tfW?+{_3kX!58_be4GFvL zHubfP6tuC7Al7WLMU7j&^&Wc!g7XBw?t!c>p_G<0_^zO4b>j)(A3;Wgr!>zX8KtNE z;o~t6ZyUR*!LQUY<o;GF zjmcT8wu4NN?7Q!;4g`A_e5k(B@16SX!JgR?K*qmoa>^g~@)Mn&Kg%BA6B$@tmeJJ} zm~~CJ3I&Z>C6d*{7k%7GZ?`#@Mzp&3?rvTImk@(_7MeN1VHgBzYT0K$YW&!`B7Cg+ z2(c2ri}BTTbvH{(GOjChPr6_nLE*Q6r&ycryjTbdT|G#MqO^&@w=BSjFz)udH+5K2 z&epi>oRJ3^r2%Ls9Ht}%bV7B(ir8_Q_h5XOUgYIlK;YOcD$C{t)c^~$Xxjjx@o707n6t15%Y=vg2jbT&Y{YdPZ$UNL8tAAD^X}Vz8@<>8$P7Zl4TDSn>DqFkbIzDV& zUaxR?EKUjysq%v3fM=aUF3WfJt-py_aim9_oC`>)>qg^?6N|X6BUt&b6%QBp>p?fW*Hd#d zYkwR@S*C8*pMPWN!%;(%mxMhsmYfZov4NILK&IFAVU`I=S8`%-;Nq+TaG5@o=n2fk zB0pd2;Xhdf(X+6S=<-O+MK;D&f_3ei@#*xPw>=BnkTnt9-;$Ok?AHSo-Nra5IZET zcRNLETp@*GTGQN%zd`g_lSNIrk1IMq4|Mq=F&(Ep)ae~c2h|@U#Wle7oljZ7AZc(< zT}#o0g7mj>Ep2FlT~u8cl~}M@{oe0jr16x^ZMql=J=Zq7`-I}?n}*`z|E3KIOW4K9 z$$RJJZbbFN4;iyMlnA)lNMOa1T3Tq^oHl{W07k(qo41d`#M(EpuCcDE<_E_dWf#iuQqSuqM&L;&|G$?w*-<~ZRz9a z*nf8*%1v(f+&*2HG0ut^(LE{2$7~p_b}lB(cpqDU90#r2QF$TgMSFWH)On#5_{Dur zPVoZ~-KJ9cK8q6__)dqEB)@g}i~nf}m->hR{wDpaha9io;lbGYnuYr9_s({6Kc*vx z2usV7NJFIK=rMW>)D`XczkCmGIleQYY=}yk=?&+a!G|@CylDkW>c3|R`a+#t#2Q4u zsQH*1uB0uDVjwy)9p9+9FU*45!YQHeozAt&ta>&TFPJQ^7EImWwc>E^O-p2iF(m!G z0y8kO<0#m1#^tYa6UJny&jMQM_<@=3q{{*AUz2>M`t9VZ*I#kj;97h{W~Um8D?LA? z8SEW{ag=5AjCUyLgF7)dB@%$my)$Mg)yuGG@tA=?;)>2rWq_3R`U`*q1$dDf18$_mKn? z9hOh=%@ZqesD=)<)nl!1GqMe}9>7efd$OGSP9^>N^zw!<@}&%JilDI3=!4ocwwRp{ zf!yRyCwEw;(_z6BI&&C&+Ow?>Z^4o>v4fg%>+FRtRe?sTHs}2{Fw5ka5fM@Cq5fLv z_RUzOt`Dde6FL=8t%Ame%4s40gHiucmbeR5H^v5tX2Ft)u{x>AF6SbkU9CcKvCz~0 z$#+kcK6XwmZ|vzPc+EbE;|i=wIG7!D$l{cMVv7jLK^DDhfAx0RN7yAd_xS5P_CC${ zZWC%8%#}b>-6G7~mTp@;M?3uxk7npdSQ|@zLXIW>xg@7q)=2nZ$6EAzJs1--F4a8$ zXN<)bNegHX7kN2sA4&i*uzFzTD%>{cjE_A=Ud&&_bR_=^6x*)M zWlDN22m&BJW3f4}fSc}>63JI9?vgeSIFVg>ughP6%8Xd`{iGA0pZ4qpA=8plCB1z> z*vBJ)iD~Mh6hlaH5&giHyGHzbH43ySh%~P3@lXAz&v&RlbGDcMs{~p695?HLblB>- z%{$O0SF(S{bb}Tmr+*H|$%9P`Fl4CItR+3GYSv;{KvtS7)V}Lk2iu6WcfA02vnf1nSfts7rzHrj{G4zm+m=S$w zObi``=FBR(BToaADHp>DZX7C$%Kx_>JK$vCB_*L!6nK#0m_=&<@BE(H@oi~$D0C5ZaX$pZ_blNX`-m06~j3{TM^=fEU=N~nb zz^W2Jm8C9&8JwkCi?a{tHM!rssC41pMjd1X+z?Ry@*YO(MC0pC;OUmR9`X+80kcJU0IoIN!IfJ?dB^W+D zU&3Zus#!F+NuWwNd+}WqD5wBHjgx^A-8?n zKv#_CRNcpM7o5_nSo@Grg8R8henrjL4ARlwkxu!SneeCMX8?C7>Dg+G+=qOsYW|Yo zpxQh6w}hH^-qWpmxM<%Ds;w=ho~C)hjOC>Ol5dZ z_o1v%b@@r3;UdJ&klr6EhPEs`FM!rFrfP20VwT@b?B|S$EcTj)q;po*%OnOCCSg^ z-!UXBUxhXCk0|G8df_Omy;!<}PdSnz(nR`u(a}so=q4tEw<%RbNm6ZE1HfvKD3TlY zlNFK{^q`>V7}T=K2fC4`{o$0FnRX?#DvR)RPaoC}8bKsBj*@p$K{}B-hSdb#x7FnX=L2(ssmkM<#Cv!0(E>tz4+=_Y@o%%qy*c z*g|eiytXN%v{L{yegEqP;0t#kUG}1z1h*SxbSE+OoeSFx*f}^9p%Nj5r-}bXx|uLx zze*h*7vdQ0}l-duR;JKpRg)wFw__Dyn^VRH}j^99C();{4L z-%H$_aXWEH30lzg`lEIj6J$z z)XgqQHwSKx zKo8iT*7ld(p%-nPehZ?P2`y8%Q_Y251!6!}B;Rud)D0<`|vb8i93d#UyU)AQ0GkiiUmd7?OkO9RLNObowma^j@X z_?C&o;I0B6@mpIE0~(sTtP_D0N!kofY9&?RttwJ^3$y@0k382`d zQdk>&LVJkNOxxH|VqdRUYjm&Y?;^JcxAjk!l_XPLMGtUYXQF({VT&T?PwQee{n$l| zt;-Z#ST?2plEb8s*82V;6HwX47eMS%Z~7(Vm>l{nR7&y_w_JOA?K=jCC&;NWIZ&R+ zMQ=x>QlYtKulp&(J38Fs0?#A`Ihb(<1 z?obpQBfR$~^41T0;a<)%bzRcDX{~kD+k|EZfA7)@F=)T1;@APa<*O-RMTTGVRCW4Qu3vm<$II ztL?M(i!v49p<~GR6GK7<^w0A0YCQK?cElR)lUhm~u^40*a!}ysHA!BUlLPG>^r4Hw z&9@E`HFc@Fy8b*Zhv5fEJP(2A*5Ka%#oyDVIN=ki*_j0&&;i$%B&wuCE#|4UD_OZZ zxHnP?PhECPa%Y|05e#%Bh~C8tpzZ(T8Rxn7i!&11K-)HH(!k!Ixl zk7j`@M)^Q7PzEtTC7VsM{PiN#t{4FtNP#BVj2~c=9T!X!0}KM+Jwl>?bvd0wGu$C0 zMnV^xx0cELpFhOZwUZXe*K)s&uRgsk{wPxa(mXrLGaJL*zx~=Xi3f$@9x0i?Pg-G{gL$8!2S(HL9q^RSdfve^Kcp_~u?3f3z8J z_)VE4Cw^5G7sT=69iu+LgX95YvK9mx8uV@?I!H!P@V-7Yr|$)AJvLZ4VEf!jTH`V^ z954?#Ev@eL1jV~x)joyx)}RdM8X>N+(n?&fQ|M$K%F&r?4Sn;pTqm!DAMf;Llp}W) z;W^+0Eq^`_;!VXLck27+O%73IOUUOzPS^hdK(^0j8J>rn;k5c_X5SQ0bCHSre4ed& zBn=9vq}7Bak_~PH0o4A3@rkr9_~VOwi2ELnC? zmP}BD*E4;?7q0q-Y_X@3Gq$S>GdJ@Y2*oR{wjGIWE~&m}-@l%fnEF2pz?&nY)lGC@ zd>&c{b`R#(#a$RCf4KPlb;iKpJLfyobZ|fHhNSiFS62hW&PnfF22*Z#^?|z^yL877&H zE{uDX6(3Iax1#Q1jnE4-Dmryt+5)%nrQp8A7J2%miLNzf2BW1o?l2&~3AdTc7^$z^ zKNZiX)h>~|+Yj5iB5P=wnD~#cw1-8aZdVPY6dn&u?UIj-)$AhQd7q^D?X}JRCnleT z8~42@aGq`O8#RR&G_Kqt`A~n!G)syscRd)ThlLxR=$&Z$|liu zls_TqaXkz4oVyIGd23$rqHlQYlL@rHqS2RcDW|758y+HbtkezYf7=|-*)$G8?UvJD z3LzSpa_>Er_3d7=<^B8}bxmdAs>8c`H{l;>=yr&wDD~&O$#~ItQjw+a^6C`-(QBFa zRI3lFd8?ppwU`Vi2ixk2iuXbtZw|2EB!r$-gfReT^_+RlLQqhlmr4GHP-j4Z*do~S zmYMwu)yZT z=b(!AhVhS#WHi3lNHoYwfj%7I(jLcd-RZtep8aNj*E$Cb-@+~(uqTn|Z7#5IDJG$| zk%DcKKzauEn&ao@iA+u2b1QaK1d#$P^z94r3 zi_R)G{upPR&=UqhJ3x46NB$zDUKAa^SkotVkuY}eooasTU*=%juFQuKf0^|eyX$qd zLL!wrOGgmx=NlG`~qbu}-) z{PFeq;-mQ+4QOcv+Z2_(+H{;o6sGHnq+u3q$PN271Vcmm7pVp4B1*h)Xt{G9|?WDp>eJhd;``T-t zS}vkEZYn>6W*j&cn`d7&^nLrbW}yPlL|3&Vc*)**K*Q=rUiRRKpImiHKMXO*oXE~` zlNcmA*dv7%#0a^2<<6`hLgO8ZA>?k(g&^rP+^&3|HM*5a!=Q*u>4EqyT{B$c^zm+H3=D`%(%jppC3F$ zmRC$+`dNh>i35kwy#O6r$hJJx7= zus5-j__C0p1MxU4r7-uEYzzm-&DP^k3j*ahUbFHczt=W`&k{JpqQB|hYunlB)!v&l zJ|~+O79hX_-b#ujBJApZAuD6=!4)==^kAn(ziO*=nB>rfQBp&f!XQEQ2FjU2vk)O! zHOO31$iZ2fvpoKt`O=t(rskTDlQaYbXxn%YB7Vl0lp?kRJA{EJ@oi6r{qns0=Pry) zU@pP#Ed&GaKXBT03Vv0k8=@?_wYQJ0ycnf!NgW$Y+4O@pql|`zRIk>0P0xVF}B zePJfVQa01Z*Z6ab?(P>BA@|@5X7`WJXlgrcC^U0{Aw?aF|8j=u=j3uEwz64E8%u+E zoCTvUi>x%{+gI%_Zk_Rlh61~MiZUnF|IB0Bf>B-#sa{DPkb4q|COzad%?LL_ z6mx1LO3Xtu^eWbBUa2k7kC2(=JjeYiqG8h6I%|YZ+yJM8Ob7dcXDF@ItQLPc2uAp<+ywIP|GbMqQrR ztlP6{bjdgBM(0v(*e^BKgKg^F%+umcs_^l=;H*E)@R#=pgF{iIiNK0b{Mf!9L=kfk zMFtU!!f4S5k>&uBTi}M0xu$iUPzDt1NO`^-P}Fpb+E0`iV9JgD6YBL{0#{w1qD?r$ zdy6z}`217qUeIwT`IYC9Hy+$-aSd0Ng@%{oUi(yAO1roGPVW(4AFFSca)Qc{#bsiA9~LR7EC-7#D4Wh%$PR$kd({y!$r6c1fr3d?m3l zYV)JP_c2_kvYlXZFw3FBC}awgfDmHeqkN+LrLK^xFR79Gk z-rsU^x&%wg%~yn*;Zgu!h+6rlZ-0F&WploDfE{-?inbHhKi=d5VQber7yY(8h_?J- zA2WNR^vijQ7>eDt#r9ABv#W*EsA_7{va#i&t!=2LrXA&zZaIRnC9<>%2F|`fegXyg z($M;yzHPfCsz)Jq;}dqXF?I-Ev7$9awlZ(_ywV#B+w{hg9)}h?^vNN&Dlb8ig06>J z32RxTCY}_`gA@{u&~QNKLndCK3%W45OwrGm4Z&!~; zd%tqWp3EoFcQ0MK0?XDa@D$N9V!@66K7pU0Bv$s(u2^VE`D6o!Lx|wxgg;E|ohY!| zg2oqxGeSO}x9BU0XNN)G)1~*5uevt{tY3uK=CUN)?SCj_!Z_G5J%%+Ey^&4x6iLMy zx4Hhoghcn3gl*5<%AuNjvqY7MOGA4GyS7u`w;_+g_S!JLQ(G9l3t;11tR>_zWt7*W z-u%q}Qh6u`UNrrDS~^;Foe;*Sq)R1YobU=H#sGr6(P8n#&^-nid zcd7vl08>q=C?an>xi$22G_tAmItV|GHq`sS497`D1$+_`avzixBvMSid~4xLh(g=yo1a}lPZUkX_!O4F9(%11R!%{c>aH9xMM<6 z$p~3oCa@LS*xWE7)3YeBW8oRzOh2Awln-44%)r=Hh4 zep7OZvX6sv&Jt17*ag|^H{9yt5eUNDvRSJcwLE`0IaLa9iHUlKY`!caPW|KCaUH80 z>WfO&;uD??f7p&VcYd@fI2-ls|2xo~#Q#Wc>o=!@pP4dhqu$r|rv0;#R~Wzh#6L12 zmr3s(#>58I=KfgPWk$;3arYQ_-uhDI{%~(1sqMuS!wJy~O8+Va^WpW}zUOfn0RkbJ}8quaZ-qvoR}il%e8`u@xVo z0ZhRqKmYkdP?l;U$Xk2G2141I^+QQ6&YSS%!&V~MZ?|gfK-Se~NJ5!s4jqe+DAf>r zn~yK@*LI+9knspBkcMo6^6J#{f|SW(S*wiQR+*7pgpa-68iCB?!B391+JP%CQzs(| z%S{BWJ;h1W@P$cj;$bHHr92~II4Te@>FA)%BJxAr|4T!Mg~$)=JFC}cc2qTXnDR|q zR}UktAjr^d`*XC6XZ)2w=j-Q>s4)>-fc*+VOgXqdbpjWoz*@mX{Pik3*Q)~#33kFD z0Q6A=)FO(Wy4>j_dhnq5epk)#@!5AM-cB)FmGlqn8Q(_j+MG3(Fsk^c4vmZd+EpT+ zignYQWJp5hG6kVL&D!`ZG3b^e>~Crrzc7ppIe@r`=Ufv5ma%yc;+=VuzWr$H{0d@X zc`Hdqzg=FT&7VQu$HR0cfT*m@{b;W+j>Uz{-UTyrS!t*D`Wv=>&u)Oe`%ii` z7lPEsPP=y?6tI+YKanep&b5viOeB`;50H8ut^Qr)nSA$#PQJo=#5E)Td1Zt6wCk}- z_Vj_&xp}+bXw?3N=6_x7hS7gBDFSq z75$OYMamKH^;sXAa+W>}0_%bmHalMeDSF~$%O^IBUqKE|JLE8pwX45aW4fMXhQ0wp zc5%j~%8U*NqvxUua*81yXjB&FkZ1tegfOT^G!FKCnIa1!u#c$g;?qsQ5!N|cv38`< zX<;ZrG#$QM-AhSM$Ddgnh};7ypX8Ub6wV+kRQjVfKf&pcEW&DP8DO4NcSYvW?8hp_ zJAGAGSH*)#r!JWrTJwqiqpSP|A5mOgJO^6HyPP*cs!+$$g5*?$qAoI8eMO*zo%l?UP=KN?x3Ltr0jJdhpU`UlG z8r_>tIfVhr0c3ML{!tpMK2ANeEMT;`I2+Sn&N;bC466P)rPjSQzj$ILPEJ%SfGrGQ znid-AqQU#gAx=;V@t-e48fxwZzDQz+b7jG~K z<}iZ@2pL~b0+f1Tyq!I_PHo}ju<_W!%M-=%5!GY?rQm>&D{zKvDh!{!0k~ls88-a0 z;iX{4(*E$lsX@My2GT$qmc-fnHh5R~-??|VOS>g00Rjl%Z~k*y{=&nCz0LiD?pbCr z!lVF@8(}`|j{3o8Dhq8f%OrmWomrp1T685E_-1M7;1@N7AODc^rg9Vx!-M&4e#F`3 zm(<>6atc(L9DG%&)?w|iaYyEt^W4LuWnj&XnYXUpt}1AQ+e9eytG2)R@3Hir%9E7q zO2viueuM<4C%;$TRiJ~s9;Pkhan2F7sIi-`+gIi`@%iLj)ZgaXF1{E=Bbe8mbIriZ z!UcYN0CMtW1P#r$*DwCch?w zB(-o7^mrgEXldbR2@ zWcphgh^1UQj3Q}WMG{L^S9i3+U@P4WNaNB5>YCr;SS>hCR$Ujo67f80WJT2C+93`A zuH|>2Ewg;(<=&K8-LNUIDta4)8r-ZqH^eTJFgJ%etNiPm1vd#YP`31%U>F0Q1t& zf)1N*cl~#dw8NGZYdj*Vp}a0_Rq|4A>*hix$o8KKfTb8FNV$V(?>e2obp~&7->WV1 z@3@SoEd??s>~RUTbzVL(=b!>K3rLTmEhm#O{OG_t`*oQ75Nwx*;JPwO`H6zceJiWG zUb;(h7O2~lV=&J|bOP0XU>&i5?4QgRGB=+7HW}fGrSFc}{21vLO<4lymV!i<2i;H1 z;c)l?QQ!R~^USE@_T05xkR?(tavM%ZlBv1nn?}yV)WEhm(j=y^&63gy7V5>cZF7rf zGHpXd_ggEPUpZ2pvlQj8_A{S2vArGcjizrxX}V?%?BUqUBZY4Cs>-Hj#o0^-GPB{9 zb&;=j%ZgOnb$#0ph*>r<|DW0K82bEjO4;f73;oKonCeb>7shNw@Ct()p8>m%4D}i) zG|2L7&oKTNL#{b;f9PijA$h%}YFqJ_`Y>Iua|5KhQ6`9c+ph?V@c>R+&G*DYg(88S zocAR_PkGuh`aa+0(;PrH*j^>%`9%pM0J3=V)jvZlUbPPy3vAFxtyJu%A70G+e=>?E zp{yb}2WT>&o9mPLumh$O5r@2z36=)Jp#7Mdu3&2!{(g%uqlkgJ7K=#Yw043jm#;EN zjGk_VyH~iJVe!C>_2~IGzQ_V2P<}(B@xz5m0 zY8&iCTEPxt5y%TRu}Kd*slsO-6iH&ax^iCv=B}*b%5$N!>mq&OX>!Z25$@99kqtrw z1mv7~LDcdc?(-2A6YWoNm9nDCWZKA~+IV%<@FR+?EIj6$Rhb_hJY%yFTUj%7anm6M z)`!EebGf$NAX*^D&$6AwlXiKAScHAcQ_KoX^a}~*-}h1rzX5UTUoQX^x#lSSL%(`i zryN;<7(QRyr`9VDWVYFQgKwNN{|6{;7@5DR6>E@6Ei|K8bRi9%CKDnP;Dvg#tQswU0?(J)w7l z0}xTVNkBxCX8exzL1wv$%Pcm#2~yK0V;v;`h79e6Dx554_Knd#BYbBUZVDUIvtqP~ z4SfiZeN&{%S+k;d%&E$Q$-j4|<4r>|rtj|~8admxzy2j?`0sQyWOf*g4uor+o4BB5 zAYD`tJ8QQ%_SZI3*Q=LB9I;h@b-*_m&jkI(JlDwwSFMZv%@eCPvzAQw%;>Y}ENrs0 zV%AgP{Vlan`L@oSOod$mo4T6A%m%E!|5FdLN4pO6_rW^*@)-6gstV-7>>`x;RLAX( zIOKpPE@Na%2Q6sn6ZRl+#E^ zx#5kyyMZ73b@ltRnDndX6KK#ZH0xl5fb;kmc#Rztw^F`9|#MXSe*O>I$X8&p z`As!pp@o`i>-<}ZFTF8T=K5r<9&cj9ToIN=91q&x?FYJF95gJu_HZ`p$a^vY$q1ajI^gsp5KK{(N;wC{=G$EE(e~kf(aOCZc8ubUa?oI#nh&S z9~AY_(I(NE?afb04uB#CFfih6JD@kj`Uk!iUXwqKjPvA@RPMc9b_>fJvV^E#MB5h^608)=wz#5&)ssVT^zK8$26aI?`%$^#SQ!?y(4q6@7B? z^^0qLMEqaK-q6IG+Tz{2*FDxFXb)%6w~R}GDh&brI_;hIvN2`&zOpsVVJGv9K^{HK z$*V5sQp!GI`^(b~D*!9u|8h1_h7@*~R;RnPYYPLQDfik1!&HH&y(oG&YQK{-ZOQ67 z!-|++b}9RV4o=`;O+-t=sbXyZ9akJ$*~Dn3mVb$%EZO17Fd2IQLB#T`1nFylS=sjoA+@S_J}0%ni0HFq;-KfGUdL zr6RR}2Ib}xb@WX42CrIp38#F^%FCn$^-%q$4xqUM>^Ys8SUa^>^fbyb+(i! zxCiw0vQ&BT;^j>xXeUNffggKeYLFXDQ9(Hy7dPk2~CGI3MKuk zXGZsmDq3bZPMT{ac@eFz2b*dj$`eqr`U(GvJuj7Yg*80tR#?G#XvlTntP(ECxp9NZ zp<>qu?FWTeUcif9yF8#=w@fQ#cNOsl_75m)zopa9+{fQ4C5mi-3ZbDv(`q;|(U0^) z-S;HR3Kf6MX`0v*N0YRP*j$m{yhztI54j9e#kuTgM>ZX%%WmaEL7`1zJ2}xn@)HLJ z|7?{fKX1poq&tkS1$mc-Vy7g#5e7gVsP4P*8^2p?Bt`{A&Y}s$hdBZY09?lFGd`|V zSTpoVXeIPLSdYAH{}UwpcaOfAim`HVWORUj+41~L$r-1Tbi1>{(jitMfDd$Hos&9$NNdO3-8u(@M$RyWX=J0wd> z`|fa*d2V`V;{#60+b&q_j0(AU`Ws{9u&V%Usaf`(AEXzKYM z)_*vqqVGmP6X`uEAlnN%-!g>&1ss!S0MI1m`ZNEVePrcBX>4v$6?Ge3eh9N;P)nC> z`{N_1MEi#+H=hKeV*ULUdqNTeft?pw3Ot~C(pOcP$FI~b5_c8?g8RMGmkzAyPtj8K z|9wlAL`RuT##1MWBEb9kht3#R<3%`O3Q=PegsG=)ck9rWAG!*7XO~(VQSWp*ZD1W8 z?$>$>{s1ML&!@C(?d)9%ANW=YW_PswYGfuiA9kW{)1k2{S_QkuCB$y`!DRT1)Ruq@ z0REJ6YA0d*Pj0$teq+ylp&OyV%xBDM7W!!*mAOcUTVi1-2}N4BY}-A`)VOQx;=V;Y zFWi599k&m)&!2>y6!_|D@j56;2=|I}-C{*MlZeR&Zb=nl)I$ljryci0t}b-gS&|S; zsZ>O?Z9k{U`T&+-n^4+Cf#Lt>SVQ@!6J_LhClW{-T2 zDw4s&+JZ!V?^%7bpzowO4$*fHq`3BvH5j-WqJ5n5%fI}{XM40qFwk&`EsdIVzF;+o zAX&J5qQUa#swsFA=p_mGg>yvolOsd(i63tFEwllT6J-1Or5XZD=6~$S=6vs`w=CDj|&QIju6;VwRZd>5f&apx&GlyEFCec zElAz8moNT6Un}1>>8{(6z%aaPV+%?^LW+V^Nsw9Rh+*o&!H?arOJ;&MS z=#0<9Bs}&f#ia0PgCV}n2a14e^0n6Q2t>nFql@WwzFw*X_Ag&mb0zfk!}lXOnjF29 zacvfmtAmzF`{MM((Zq`SXY~q{;~H6IAbZjz`C~F>hc@fC<-w!_xBn1>5>mUn^!~3z zxp1e*1olpg7o)eT+)_aD4*Rq1B$%4NQ{khc9^TY~`H$i&>-IlQNCLe71BPys6s&#- zY7axl`2750{<|9vX!4meRk?&2LOxoE(OUilxpeaAy6>|u=jHJ)7A&tbc9h`wNAcU|yQk1i_f_AZ%QQY-wy?XLq!z*=i}zfDo$~IBA$fcZE2Ei*fAc z$J+};5E!B)9~s^{o29Qy->k<9esZ&HdOIfw9yj9CTXY{LI)O7T@QDw7xQ=}oI&m(Q zX!88wC=16z1$|Lc!Ncr!O2b;&+_GbCoI)aT5AgcUx<`{j(9q*m1M|mdLgS&s*<*c~ zMlTcREjwV8v|Kyj^=yED<7;)4`ZSO+dVb+V-XHqyx)JoCp5SFsq+^KGdovVc6jH(N zmSUDu%HzA&1b)y&RJSb>^>GIy6}!}du+`Mesp|o+FbtZe))YP*rd+#>4+<(YUT-A_ zzs=5C9Ssg0-USKY4ADQv$HWA<~oi26hPy<=9y!x}mEWu(ywLQ3RcU>yb(Rg9; zjh$~z4wyfq_%j{94#Sa*zdmviCAU3eZ1}vTKyGXrHUk0m%(nWRDo8JQwI>F#719*C& zNeh5*!s%~Cu3kUPAMLjoVi21h{i?@WAZ!B251>`6M=&TX480j1+pd3dRO+{PE=esw z4vP19Bb@(^4=GlQkG=2&)4}SyW5_Iz!<>y;sKonwaQnzlE%ruJw-;v=+s|| zRhIU;irmMYDDZSl`9_D{V6W3G(Fm#AgLTe<9j^&=-GlBiV8gjkOGnc3IIxwe&4z4A z&&^FZ>Ut7>d>9mcZ1e4fH0EUCjAmNDMZ&`Y_-J-#CQXwO7VvG>$^euJ(8J!$nzfv( znucp)60BnFuYfuowHhIm2;z|~UHiL=Nx#>tT;%Q_@Z^0Rgv>%g>ja<;G+vXE=7xH! zZw^W#eP+hEH(ZT}lYO^lmFMJMP&<;~2qt`##GP(dNt8=HkuwZj+qiXbJ8a4NI^))y zMTWKV5HG+{(TinVJ~` zo?$o$u87J8fak-tNcdrulN4lQ`Gi1@q=|KY`}ahQ*o7@%9V~y|fGJeA={_1=^(K2! ziWRB{>^2}}%;47=DFak(P*>>2I|*EuPk06qU*0`88Cum!*PFXOd1(OAWGs}Bjst#S zcu-COR4(rrCAAw2*ecmnmfk!*y+DUJAFu=KKXI(bCch_8BPo-TA)ZI*a z_<>OZgMII>{$j#uc8=?UX~<+H5em$kp+7XI0U`o5J>NxQRrDKH>w+*%fYx%2`761cKzo%Sskf)E2;Z3W*JhFNdrTRWdk*{elbw4jEy1_KRK`9iEMmQ7uqUhI1h5;flQ;cO*2Q4|ZI|80O z_^-PnCCpE^)o0&Y_f8z-FJ4j#QVRP7SG{r+w!9lUrtn$$X1H?Uwd70<8@IJ`Y{a}` zQl*8mRDz9&>0JPwW=oPkf+vAOK>|!f=`*NW-Ifyj$s^;k^+%sk-=%o7jO^b;0PFC} z3>!ql6s(FB*+$Reh50z<)qMmJq75BP9kA<9j6xmVZkW9MqxHVrs4tx-CFUy2ft0vR zyAS&7XJi_9^l*F0vo1tL&fI*7`@w=6P0YiX^xa%PF7{0Qd5dMMf5@qo5aM*Z~NNiI2^8wCOlyyX@M~m_%MJi^|p* zb}*A{Tiun7Zifc!$zP3Q-Tmag{UHh0-wtD9c7;v>e!Oq(xfF-Ehli5N*bZJBIz~4m zCg7a*6KIW^Y4PY#X2O9GQ?g!6??Sb7+kPm@@I_=q=L2xFjbW?fZ z8f8U5^b4{TPD?wi9d86n5}CY@a;yugcKl)IzI%>h=`6;_AI z|5mhnLTB`>7JwO)@J78pn)ZJ7;^1bYd2vj3|7kknT;xN8PwU-> zHp2sDoB&9~3~ZaG=Uu?^J{Seb#kg7)aINL1|LO0K%qR^x&dDP$W$fKE7_zkt``%>~~!(wkEb6!@lk$?6q{~kI6PJhDL zNv|IQ0Yi6dc%V=LWNx0FBZCxmC?>ZRSs&*QX2a)0>+bxQL48oG6aGbhPfd#EYv2Gn ziXqpKE$)C>U>Rhfn}+ewH^4>t%{ALi`Qr=ICNp8oqV{X7_UIFK0y?6lf<<6v26ac^ z%7MkSTXWLS0g}J`oL5Rhl2R^+i^zw2AX*Lu4E4hXi9H%{|3NRCz_8^yh!fi1^sv2; zDvuoe4i8IEz|FB7&29w=jUlIAiTtb2DtTCVjG(!^D|W9|aiiK)qv|Ud@C!l8>P*1E zWK&{50*{{u&$MJ^hAl2()8RH%TX1p|nKa0YX^PrL7iC`4;@u6}rdKwtM=ns=OAG#eINl2-+^n>;8Y z%_w8UTC6h0S!FGmHaiNH#+C*^iSZU1SSNq+uN}X*Y&rxdOZ$EDZJQ5!z{nT9pI3%? zx2zX7`7Hv#L(SX@&fCyV+OV?!?p?)p<>19(0)U=K*QLUEpng@#0PmiTXiv2lf+IL2 zRQ`7}5WI*E&IsvZ^3zUf!BW@CM7Aa7Im?AARBKied=YH4iz5kfKI5 z;OL5^iu$DZJ25-AoW{kRv==VY3ZNWF!B`7i49G2knwN>7nSe}el;cB6KoHg)mH{(8 zVqN}1GvD(w3<~{DnNsIl-dyb&y?vot)9h3d#T_LZUMWBT;8z=Tzj;f}UQ`WeHZmrJ z0GT%|AISCW*j9qTrSP?uX&j%2SyYZ~>*e~@mV9gf4+K}np&QVAF3@|Nc|dJcB;NExP`nf)>-L7TLv#u14spAiCqlT*?vK-Yp#E;Scq3JwfLtL8bh}}61^xFcpR0DC@dRg|F9)P;IU|U$s}O-4 z--saCB(5|PIc~ff+H1hgBebnFtNyrH5E2B~?>!C^)0b`(SBs=M8kHA|XeUXARcU2AY1jB9&cA5KZ9e0OhL> zn&Dbk6+Zbn=9o**u(HxL?*+OdkXcdFj^c;o;0qp!zz3bRLq3%@hJO;0BR8EYJ=}U3GxpsnJ zn#fWPV1Og^APFk=`-`>y-eI*VbMyl4s8!t_I>6p>G{U(yB_=_sA z<*a{%=?m>j-fmvspXdl}+_RZswm}{iMlV@~Z8i=+PwvQVMOOC#1Uuc5T{2;k*y4f+ zA?W+cbjju?Uu^+YE9Mqw>r-%ZyT0~KfMkwe;=s1E#ZWkq(!j*5-?c26j&yI2MxcX5 z*OjK*!FNpZ)6U3df?Zb`K|*ZQlS`>YaHlzoqWK<+)=pP})Yo!Z4|W2bX-it@15$Go zp{4#5$e-$1i5pg!aeTOhg|MRAZ9(h$ieD&%W*nD}BEu5L^>ZPk(N| z02hpZ z8xDG?xn$~6ASy9cQd#Z;sSCtbqQ{v)Nk4H0gz+6ZmQHdOR1UO$)=T#io-k$G)A~U1&~quSsx=%uKUJZ~XcijE+j^?R1G{LX0^UNC9K9I(%bVox1Io zoI1a-u-P{JztC* zuWvbA6Z_@TdZpR@$Azrz^MN(scn7J#*#&Za|7@ggVFfElXV`fJbl?HvAFv^wIh9L@ zp`xAyP-6J;Otp`UzGm|Cyx~)>+6)c|6Gd{Bx+s1|xZbcd5!F?tc{02GOma0%59pUN3G(62#qa1tTG3pf; z(8jor7aG=_H17ncdg^S)AvreezksDsUn46}V)6ezYjvY#-9LK1ct_HFe6;^16;Vqt z{Q|%nX8S{4D~b&(?;>Nwg#rd|)cTHtLj z)vOBnk@jB?RO^G#1z>7bT2?>YtDFp#QFE9Or%?^B19ty z%(h4t$ziSkO zZ!OxWMC+;7%N%DHyQ+XITk-ZdPhDuI1IK=&Hqu&P_Wiu}Tt8v0qJbH#39DFl7eM9> zZRh`=1qf-@1lF&&4fQN1oYtuq;=_6%WgAq{36WW*NAel}FojrvJQk34+txxm3qfqg zpf2zEH_OkFj&Y*D+Zi?naV56yt-!Eh=An}BX^n!s;MX0-sEItc4) zD2$)Nm2EK82OD6vh4yCwuo5Y?(E2Oz>d7vZp3i-JJ^?#(BRI+~=z21z2hPa+J61!s z3A9`L$@^pz47Nnz#jdeW^i*?+}N5e=DE!l2{;y3o6f@%DAEGZ zSUbKuVmu11?C++OTT&sxPLY#luk~BqXrLec@yyl!n43Jmq4ks|8^oA5YJ7HGxPB7I z9d)hW=`NzJ31x!N?)ba!+JZQFG@w`H()a13t)6 zve;ZntIeSu2t%sJ+iRhCc$=_ZM2QV3Jo6h&|B*SId68LAIq%^UsS8 zMKdD#W;Q4F`VN+nw0nbXuJHdEq44lh4Gr>X!XuNlc1YQMZ1&1j9zEul?$nqBhH&L zul&}VwELCy>-mI4M>xLlJlIn6b%3K%y)~qRUk2l0}B z9H7GCGaEb@_;_{7y}r+1HvSQB$%ctYhqBPh>Jf9 zXe6EPRohK#=cq&9I&)8j621%FYMyqw_?St=IIF=CADoB62LJEWnng7_LW?iaGWXxv z+}(p(`Y(igjzo8(h61RB75n#JbAJ8+5@@f5b>v*{!g{xR9S|O_rvSjoAVt2xQF8u+ z92>!_o&r3I827YF8F%uO$$h=g-YY{MSBHFeLIA~g8=(NI)`k&r6Kyp&;c8#hun-v{ zY1b%2EJgalse!=cJeYXRUo1#|KtcN0KWGFl@=dLD6 ze5_nc;4)srv=%txHWvi2;RI7C#54_8oKo^YLd1SFVaTp-wA!}MS{alm@UV`pnmE{0 zjjHu}64WhYhct-oLTuzot$>MTGb>$sq;qSh+^xsuiTFl!3!A_83u@-?LrSAYed zn1vI}3ZS`48O(LrI0E>qD7%9i%q);hYZAxgUC>=a3bLG8eGFhJ>r?>M06EEN!B6If z#e*Qf@xF=wZ@QVLS!O(q`OA3Hw`zUr4YAga;wau`R)TN8E=2_Q}hSImsqf&1h8 zJHw`Mpcjl#0@wPF+KU?qG;`s#Qz0cWR)$913CamlG)t1Rwa3U;VMsq*{{VWa+)eg>|Qt@$F zKAg0C%_aPe^$F62|1zrfORkL!($5y7QoZr0LCbc&l>R?B9TN6tKZ@0pwQjT!a3$7T zYC<&4ZhdgjseO~6LtpYZmH#@VufN|mK#Cz753$8sCM2)&P;Z1LKXr?vWXe`T!a7eg z4zdQMHf zg{)SKJxwzmjB+dq6|06ytC;up6S~DYHt@Vr$)Sfn^YtyzW#`6i$jyFFQU!fEuR&7! zOLE(EgSLA?nm@RwU90o>9Lg^`?>JAUOx6-<+j6&v)^F$fG|ZxQUz-$xnwmOm{@Ju7 zVTZW-z(~%yfplD!ja+1tq=kR~VC6_)bfyQYwt480z%w08xfljG=_E=)FSJSU@hQWh zjxZ@HDWO45(!ildC7Y7J*Sm|+Fw%+IxDbC?2SL0eiz0WHrl(adE+rM87e(h)_ME5Dm3BKa8SUp&EN$VhO zcU(ksgp?U1b2Ae1T)!wOR^>{&eA{QwJ{yM>DeT?3%njrlpW%Z)wHf0fGueNtH}13Q zGc}^1aNBP;nBnjuy+GMv`p2) z{J36PD7QtfoEFV!x}D2`a}6Q3&6kTSVLu&GA4&aUC`yn3i~B1`I>pTBsa~+@blUY? zC1;0lbC>UaGJkGJm?wINnkShS|?ztVhH9H@2qwRyVfv+XyUNW1*p;&f%Hb z{OBd=0%JvnWerv~UQ!TPpUhCcg~Rg(e1od=BEnWd)=Ec5NXjlRgX`k7bZ>Uo=d1?M4(wN!tyVuj8CFFma+mf_}Tl^um zFO)7YG}Iyl+&f`%eeCdAG5lxac8tWT*m|BL%&C?`lW)>pq=lZy;!a!pcN@6h+BFUC zqL=h2vsg|3-9msclW#$amYJ-@oDGtq8c}A^3z-F~M?#N_gG{cpvDBmNy^y8|mbNDKOb^N80Yh$uy*v%=lrHA}_;?=-s z63|Xa`itiIBrH(V&`8H+)dxna@K8)qSjK63Wp^xDw@C|c2x-pkihUxA9G-vd`T{mo zm87s`S*)VN{#vqV;7G=^LW0qmbTQ5kY)v>RI!2Je%bb2BS^P zcWL%*o8@&HzOf#4G20>Xk>}YoqYP+?UVJ-Eg`hbe4aW_}7=FBVRF`87Kw#I=Nm0CQRJtWk_ zi~ZbpX)FA0DAp*ILxKEBb0tJ=oTMAc_IxV^?=97%q-I`yb@I#V%V--A1Z@~olB*W~ zQlVr| zdD{pJb7l6>t`^%i>G27zggJt!YiSHEdgSjX^xLH6dKvF6G5g|kOas4b877@bmAv^y z1d2nL%k{gDCTvoeKn6yet*reMEW=PdT|gUz5Y!{Hray`Q`s9p%f? z)A-HZ9#%Xr1Mq9&DB3F}9-X0r2Rh`oY@9XM3q>u6`}!(QODr=$d~h;gt zxcd(2M;Ag;q|o`freYWvS(#kH*P*7+O?WK0hM;nnSkrmF)$fa!9Ay&=bB|5+PfDhm zGn33Q(_0g)>-bSimiXQB9IRPEpAySwG0W6Y{0X0j^&(K*2%rHhN-aFA-3rcb>c;8` z46f}C*tM1ayLR~qQ75NWQH3@Nq72<_a(_cxTatL$)Pl}D%eFyxFEkXsi|5t5K=l%S zY)j5Fwh|UF4NP1K`PGV!OfcDTaCTGOBVO)SFz_#(z?a||t&(>by(q&Fo@WGo_Lbme zQyc7JKL1@zVD5nZP!as)FU9jKihjXP<~YIzwZxcd#H2mt)U!}$1&a6e5-K^QUwg`a zdaqwImW9}%+tPd^6I^L5riRU=*Yg_rb01dP<+ay~B&7CDENEr+sPrNxT6vFCnAnGH zY@EIfG?Xn0;Oy=;$4ZKW-dBYZAv*u#N9*(qrC}j4bH&_2F_otWT-nzpoy}x0NP{eq zLn8#XYW`#hpZvDskhiUf+0a1?9C{b{vbN>b1fAaaLjmO>DW_U3tnnsOnum(bI!sY{_$@T^5H8sqx7~^IAdOwTRoHFpI^*y;rc` zZ7%EqCs|w87d;BCaq#g0U4M*(kRNxNgz>Ox@e%2Gh;`koemt(PqJgT9wee^eE{@!A z^~M)FmW*sX#AehDw)6E=pe%udgZE&EZ_4=@E~y7{^25niicK#KZBEuc_yv-fQwA7 z^45@f(m};bNE*c~C&H`N2NumNlJEncWq_(MGN7Dl&OOO;6B8vRG(Uhlvz-UAP6Kt$ zE_2?TY#NHMO$3Ve7n#m+NjQ-nla_~cycX*xO*FdE^f4Yp$#b|Co+Trz9RUaxqgiM; zQiE1bI9Q)h@34QpOC-k<8Pi{0>n~StMM`^HmYfRT3*@0^!8V8AZT|SZ*1@XtmORDr7YECUIR8R=OkSO@_}Y1ewHgA7 zB4Z(=-f?%&#I;|8&7F#xLjHk@ck=jD^F6U4C3u8EnJJ(@bo?T>{SxO3io1hXn3Qhc zF)IqZYP62*o>cAr@c@?%Ys3+ehBO~9)`+`Xp4ZTn{-Ce->z7mv@mhFX^b>}A=WqmPJmNn@!d^O9)8{}ofm%;Nh1Nx3t&0jK;E>07?DJZ=DaQzNak~%4zdW8uO z77Pd8H&ZtenE+>vs~5Vl%8~kYuraKSypsnp{TBw#g{jYpLdlzxlN_r;9k5nuTS>D{ za@7`)#<-&_7On7ap%zn{$t^WYmt^3IfAoJaH;+bn(C7fV@oM#HCYPV(6;{!t>Mx?t z#-KTQTq*E5=H|uKTYLenRk#^xVw7iXy482leZ_Ezy{T_W0B1C0UQOT;s6{Eq5L(B` zrYEmfAHOf(TIHb6!6rCvoY~mkB6&@5#GaW%Sch_;6IQ2{@uAOvG{Ij&n?TFV!T054 zaUpSioK*od`<5Xr^z7#}0L0+#(($NSXr4h_7zY5x&^}`2&g|sX(+WmZv;d4tqx#L8 ze3G!NzBjeMZ)#(P?K8P#EMll0tz@SBs^Y<)*;TTA<97CE`{GLWKFKHufjZ0MA50Au zr7R0!=h^-qqdPYUXRJ+-n8SxXYr>y9{H5V%$m7B8>BdLe=3pfhQRd)&XJ&lr!&4atR(zj zq6{^yI|Z5zo;t+Emq5{)=72gRt37=30-AUv)mRuLM za>2Z_8#FRmjD>I{*KGc9-AInkbbuQs7q(OL%S=Mup+fKom6_053Rhy|Om#ECchf47+Sw&P0|I$(8BB4gRA-q2%I} zj@|4|Br2lYR=z3)`29DiB}9~EvI8=UCq+2Ko|cb#P(Ojt$1Vpn;}bs4ShOk4`8=nX zzqKm~gETY)d#zF_GQhen@gjkosZxOenRY??(HSum@nW<5h%I&B_?w%I)A3>iE%GcQ$PBy;jAA zGN2dqHDpcu$%M4{JCrnZxqAc32?jU>rzTq*)3{Ue~9eQDhNg6w5x0!+gQt zvw&QCuAKTr1NO-OJ~WgZ5pe@nu_q&rgKf|20>7`km`|wz&kD7e6sbN2bhBK~@sBH_2daZ4iMztQ#D)Sk9+whxR>|V)AY}PkW!)YA zGISCIV(!dL2n$pu>x0}X#6PRAjgmu!zx|M}ad;N)2SJC)HfiTmL zxzc;8^wgGK6dmd;o~5)z03$=mQ)tl@ZsbEFr1Rt(nCJl}tN4jCYjP&DNDM!P4yf4? zqqI)RNX?*3OMwc{-=D=wnhMkA6+`wWr!fF``EhyY!&Ul&$_1pJj8T`B-7$~kHrP+- zr4H+;)kdw1R0@#LKxbMc{ara2K+8QH{AA7!XIhfmIx0VrCYH(Z#V3PWJS(dftOkFH z<{(8+DXfP3YmHDpp`gqp(r6YSd%3wkF&*EIRHFRwO*od68E>-n^$X(--qzIj%?Pe zvplVrpw?t}+P1+%1g&X7T!b?ohy>3&kXMoSN`@DI>*^O?9a-xBNjnYfx?k@nl! z+1rtUX7tIaWxPl4D?{**QwO6f7j{>&fm8LQ%nzqB^X{)R}!?K%kunq?vNJJXw2NmP%; zAp~0QdWByQ8t4`H~qC) zQ6vS&?#xyMV|ujQq>nW1!8;I?3{gN2E_Zor0$#uJf^U9gaymC6)1mBakxPVBdp$9a zPiVB`k2iBCO#EUsXgI>J1W=pz7QOEQCuT(n|K7&TT{e}vB(3NZ@?y50qhyR$;S~iE zw$%yCcpxUYWex;Thg~B6D^6-}V<|GNNid0>#&}CvBsRoR5M(7R2nBH;4P9hjiIOCv zb#g9iQw0w_E7wau^Dw7OYv|$CyEwC>`DSea>dqhH-Bj80EuJZG$dx7~qECjJ(Jb{{ z)ulOgXZwiAhU;kprdUq_e&G%3Yb>au91OVlej<2IFaDrN}%LQceh*8FX_RX6&7O`;*02#i_Qc{D8pjp{& zHag$)m9Dh4UQOS=bsB8z74fya0);c`D7+mh+B>2^*nlL-m6>KE{vPrqN9U_z`S^=I zif)U5nJGA+*#kMCcy=(#T#fSLNtGzFpR!VTdE4U zcG|EYfy7eOwqYBA;f;Q#_4_nH0{{CNU?^Zo)|U50tGL{lIu{a4b)ieAQC0Pvv%FA9 z$+Sd9@IW6Xri_>W9eIai3b8ZFKiyCTWCI$XRNg2f~CBrKp2!N*`x7^m~FU5+}Z%SA3 zA#6{2^3nvn&L_NK{*+JYny|!z{t9np#QbXfwDQ&8@`@tM2W+!9U&`m1v$X`5HeaFm#|?*u z4s9ky{6n^L)RfqQCZM1of^(PXy(lW6U;+I_2dl$-HHq9JZD0!E>C$r=uMKj5;6lR% zp4J;vrU@40a|IIG-gq1_&TrbI?&4T=>R+!DCx^wlz*nx$D-B@*PMXG{$K{znunl-( zJk!_eof}(8Tw=^l%d(T6IirsicZqGhB7hG%8{>A-dLvTgGPSp4$$^SzkXZ&-Q2L6Z#Pl5M3I|htCJ#l878xKjBZq+BbR4c>%2>XkaZO zQj9cGUH0achXAs@b;ZbaK3fdR5dx0IGX34L9X&lzX>U4ZIq5o+p5}+w`=ymBc#`NX%XAFWqJr zz04Fk{|Su+S`b{S6*s%%lZKH|C_Y5ZIYSZ2DZ$tB5>RukhQVcibC^$Pn`sqpFi_0* z?FK>gzN5CQhW8ElMYsPP1F7}CN|Z81f_|&uhRwSt!@0!(UvzSK0m7D9iL|-!^#h7~P_Z$YEz=25;RtfXKI<1`@FJ%TKs+u_8>3RJ~C*ATq0qZsQ z9c_5zA;%Sn@kYDO1nB0W5vF;ihAzN4Xg%B#>lzO!<=jpMCe43@VqIxUK_Ni?fp1Gu z$Yl^f9&4MTrXId8RcgU(RRNPwK&YV|zOyYV2F7C9EO`YHYEPAjAACxYcos28#Az}=|WU#YKMDy^X zNavcb9ude__y%*;P}4@-ERV&cC0#1yy=McD7cku7jFaZ~2D_8i*_y*jTaFgP6|GXA zQ)h9B#@PKU8cjf`;+LuWqCIey)L~2@eCvBr;}C&u47)SS5$8+o{%WtkD2SU+&ZKkF zepO*ae{r$H2ARkw*Hw|&YmH>=HI5|4@+ncy%_k)$?x?$?vtpXKg0y!pNV{4$+LDc` zMlUl!XXcrnVLqvdw;LR6ttljq7oxi=+YYeB!kMN)9&(YI_zHw3s_i0z9+MQ-mYXq` zgR!{Uy8h%)3)%m6hfvoTi$Z7rtMC_`MCX4Jp)$tt3%=$-U>g3wzfN`@6|5NAUss}> zuMMGxq63dln}F1HwtQPAOm}qHj?e&9r}$1=!EiD=uz~**HJqb;#Wy0~l3;4gH1T;AR&vUynCA6#b9zx^=)8k8R2$X! zg0XC;NKr8ableKgmlW;{s$xLyE8bW~-2t2KS;Lt{%z=|GAWbqVIcfX`))6TY+}199 zoC3Kh{dT-ppt8f~4)>6iSUP9r4*IyQP^p|b>A%ErAn$<c_GP(P<0X<$NG-})!WLWIc&NC3ne zeeMK^F=n-?AJn71YrRZUqtKYM17;DjQdpH9RqXnCEq3!afa_oNUpRKk!u6c&6Ip4z z$tQMG?aB@piYu-j6dh<3W`&t$cU$7PS3@>$xOmK&7HKGrEdDAXn}fk;KueN_RWYR9 zk#?4BT?cTE9@DhW6CC%t^&G&=t7J($OCr-dfUeMdFMNc#_=kuVhc@;6zpm@UGdEm_kQQJy9$L9!HwXncz(R2am;xNg*N3Ir=#k%B94GP0v}8 zhFsgj#(^{pYS}LI5+bYTG#W48CQ%&zk-NTdgr7PBE}pqfti*OgJWl+KitXfLu-TvS z;g+b{^ed>56C!n1w0`G5S?(87^kzG^mDCJJpU&Bz#ILGA5P#>TIywKlGQl+p8jFXM zOy>DA*Ah8+?-`KMK$nO>0!{%|tn=EIUjvIuzqe_$h9VMDbsTe>K8s~3nwMqVYxKe2 zGxZtP9KJ366R6PKABI}9JeG&*7##t{NG!z+Bwo6AJHU1 zETpsA%S_`15S9!aE!Zq1Bwc40^`<_Sc;2pW5maWr>=u`i&vh}0osO3gV>+)pK^<3^#`^8%?W0GY zmT-&28TqPGT?iFJ7beejD%E93urk&mec?~1ZQ{?fp%Z6&I8Ke*DOu0(XRO8H{CM;_ zZLNZ&y-B>#)N$q`K=IV!x|*R{hw!2AG9d-2mjG?pV?65#9BuoHrrXi|AlRK^bEE@{eWzB#Hv>dtL*m znqhjUxx@&gZvBPNDp-w3Qr^GM{3=n^nN<6Dt}&i_aLvzg>Llx40%pS;0DzPka@;?~KGXuLH$+#Y{=rRnfU>gbCtNADbh#KJSU!4{T$9kn&`$iPO7V{JF zEb2qA%Gr*GRc!nWj(SWwS|IktM)mMg1lwH+d%b-?Xui)j`oDd88U*$!21;SNX*Qy` zMz_m#c4Kag{(j5?6!|?)rpoEUfEYY{&%qKQ`G zaXaGlTeIQy1)>c!_xGg$w%|&X=?VduAF78bsyu5$-(<5CjgrpxdJCWR-#G6>xfKKY z!7}>z_Gib~Aw0!7efCda_{EfLX0&4{ZjjsDFJn80Y|kNUKucI>LO9E6CRjzE@-#_d zWWDGZMbx6aNoX{ws|x|hZ(D}$I|C9(LEpi^92Rs?6htFc&af`~^GcQ3#hWy_1_*(` z<>ft^0qFprcT!{>oV)X?wRoVEoIU-Nt(B`dEMvZ6Z2}95=9X_s<>V-kw9pe{xtUq`e4Vq&Jp#?uOi@K2{1J43R#Yjq15KprGbaYB3-a3xJWvh;pB|9bAgQ z*d0#!kJyU>qNKtR^5L53pjr{Dg(omGiR6|%M`sl_eFX|8#U}-T0;us~K#ne}@&Dwg zM*OqC#MfbLykn)95R)sw38uR{&HpYIT;UMNpOinvP+^-ab*UH}gp**OYhv1}=>QEz zqq|v%x8Lp{X}u+kuhd7yPU$JmF5Q2_f4>SUvMJ4SJr^;{;T1SUh_KDdl@HeIDG|p= zV`hC)9LLzlunM#I=Q!}mBVea4&js`_n*0WC)TQ;Rsn4Dp;0M(RAfyQ75HyV94=Ow_ z0^xu$GV!WcaNm)^C#%J9PM8U>e4{O1l<#WW6UZR2W%|tl&UNbm&+{~H{|?ZmdzN8d zoYA}gtd6tzqK{Htq}Ib%C-eF{PvD0i1F51YTg<&s2N?Mkto&{@9AeuD+Vv7T-M@>( zrGP5}LV`UfnIckHlMThd+cK`-x#d~?PFK8TN*R)G%iQT_i?OV|EUDjb;X+UUol`=c z^t@zP01l+-hDGk@TzA8UIoDD{L;u?^x4G&OQYD{$*46gpzZdkhM4dW0boFx#YABBF z#cNyoG0DM{7P=rW-d2r}2y~#zjQ3abmeS*s7RisL)5@p#aCV1uzRvtjONir$zXZn6 z*4m+n&!(d1+S)$q)MM9*u+WXw`iOtijyj|0MV4o9Zn1|)WUx&~ZEzXJe&#({4)HsP zKa^GAwKH7h7^wGrrbO$70Q2v;ZNSqhx|+eHag@2}>tE~|ConLb1WtYPe;TdSJ_sv> z>75CV&T^6hf)s2vzy1w872x+FF41(Byw!ju9e1;YX8yK__;*F>9_;xorDJhC~|Cmjv_s>d{hrnM5xb$`-5QVl=K&}S-2PMt-t5EEG>_B4{Db&@aCdH&JuXlHz!_hro{ zK$G5xwg&h9*j5Z6VuGc!{@-sDO*L_U$n#Nuxn(xY&-esGpd31InLO_%-^ik&@&}{z z9E?DbwUa2BXlz13z$>eI6qwU^9hJz;qYGcm{eTTR0J_m_vFPbd4&UKg!hP65(I+~P zWF&w5Z-PjjQw2US8dlJbatqv@JOP`Hm;ai#w)Mpr{wRX@b%0(hz+vU-Pso!KX`R{h}cD2>d^Y+SwUl&jMA;O`prb>WQ?H4mGuV)#!R18@6&kE@nhT&*NP1+T#OUj@`H|oBO*Nf z7;v;KPrnLQ!Siz_dXS(4?pD?LkW5MPQ()y8*SC!b_yT?VlI(op=|0T_b?MXZoc={z zn3yozAI|Y@>IqM62(W4q$7molAWQbn^!N5klc*QDq1$ESL+`8vQOKAGq6d3VD(1C@ z09e2pnp*^t2}%YHdgw{)y*|h%Ip&pNB+#|M2oWP-Q-q4T_Tpd_&LU~K>oi;&RDPL; zouxC7gwcPaD*bFgk!Oo_aJk+Zl~C&Kj|;q|p~!zVnZ@-6m@cn<-tSmwyMWY;pk!6> z*m2GB+f}lTG3)dja_H>SB%sOR1Z;xcCPEvC+;aj%Ot8CYxiRI2fq75|j3OMkiIsZh zgHg+G?F0rFWN(Qum4$I|ir4Qt9zr!Zqcc3#QFV!u8eP%1`WRp+Lv!`E*WEQjGdAy#fQJW*NkpGKv7c_AiPY zslq@B0lkLnkyWRkIR@AU0e?j%p`c@nGZQppr3&T742zyCHl6)g55$g4!^IiB`cG4+ zlBwD3;7oI1+AjGjG4V_b&!Nh?anrVuZ@jX0VCJk6i7~^(0D|c zLLA_-|0Tl>bqc*sbi4jb+a*BrC*Ibyfo<{$V$p0Z7mZ-Pq^T-A6r+?C(mN9ju0%+KHIqOTT#BS8?itvYIHS4vb9tuoz&A*> zWu^J9!ZSuKVUV-#-XxdyXbddkMkLO(Nj%yv2h}6J@es=^!UFK6Luw|IXHMNlpv8_l z?{x#m6J!S}C2D@n;9leNW2NDUUjME=IGRvqw*yzw?fh>e1PkP1&7FMeLru~vE{1w! z!*(2ihtT1-g*rt?L|LBEBSbWg-6MfRVIwBVBjC9J{H_XSO=kYnRY*rVY!_OdrXTHS z!08{H?zLDzKqM0z7Rj8yUg2@XwH3X5;klJm`b4c(03Pg(My8V1o8@oxn1~NQ)Vaz! zK#Nu7Ys@#6l8iSzPYXSxYY^LCQvrPqJ5$6=U{oYK~TQXau(9ULP3W%PzHyng5JT<-HigwBtvLg)b~lLR^H!j-FOde35P$&Xa9 zBM3SF9_0c`q!l}+$@Gc4G|bVLGa09ryiuYo2LZ}YCdWFd&I!|etZJP9VojYM2c=9` zzR0{Bx(L~u!t=oG<10Z0HtY9NQP`X9D5hH{U%UrAcv!G~!=`}ZbfNy904R$!#ULe; z0+!5q;N?H5v6K@vW2;WkKK%-NUura$I0CN`uFT$^P2@=(|ZQk^9Ga zS3mtCk&TxILfe^ENPv>QJjY}!7qH)<^Y+V9FV#Xgli46^D@Kt(R!Y%-yUh32_Z;--h#kMF9=oR&~Z z1~#hr7J^R%R;TJ~Q|~y#l9?h-vz+dH<-@C*?sQ02x06Q!wAov=9rZw9mU;eRq;H)} zYzuT;>~^mv346#QNX6q19v@T_aNbQkl|>wEDPb`b>vvvT?vO82ylNQ#{>zw@^y_&2 zJ!XhxYxk#Q7x!Z4Dd3Hj^DyER%g;MY)e`mvW^-sSMx?=t`tZf(&DFK6+CV|^b#|kd z*EvYxXWJGlyxxB}Mrw)N;h{rPkX`C{&zP`&&3bfK86IbArS8vqII*`t*k2<+lo*eC zrOOYbqrPWnA{L9gCW3yTpAqI5SRBnv3%V{?WttFaQcenQ;PBY*`sCh(BiTNg1bgOY zr&|i^SMG9c3voah-M^{exe<*?(vyGF)i_MwG-6uuBy{I;t@$S(p4>V}-nY(=MR$ zYDeCBY@?H~oqPuEkF&Xc_MA%A%JYAGk8@TAW5KJfQA)tQLkY|2`|h$u9=m(uJ3-0& z;&p%G(F;*7Z-_4~+y(`@!tgkYIxEuevwh3;H=plpE5hBEb={>Yn1>=ps8~^$W2W6* zEhDFevWtXHm3xg?76N4E|4anc1#zcmRTvVV_FrcIG6=HyK`~yZdskay^{=;w$1hkP z9(Jxd3lf}on^pda4)ag6S-PcYlySzB7PmqjL~GSp%2_Cc$`^b!8AD>fZIyXaMea)8 z^M^ZBjylVjHKqNaKR@k9zi-4~UyK2X7Y_MXWp6?{A6|U&gX*Ax-8=JUC9FAhyWMrf z9OmB612!VO`HD@*Te-bd&`ExgJ;(@8&^tg}M8C5ac`&~AaV99ndpgZsQs(OZ;19G+ z>lbjt18^ko2xBV_kEc?E(D;FN*WG&YPch9)dx7@){pQjghw95bKu>uvcimr!07fxf!coL#pTM{hiIhzf5dS18|6Gv)?&-FXLh9hm3o-s0xUc24@3m6MxQS zTCEy*A9`ok?W`qB4eVe;Yk+$vQt0;13Vf2u1SE%Y3~(+XD)OA*a5W9f=df|j5aj~) z^k;NNGaxVQQ0=DrO9eE4opqUCZ(iG>uf6HvUQZ$M@ZmxA;C~ihpAC1G*Wlo<)z7QC zwRPD~_YDZEb$NvK_XJd-ysAzBydls~A{lLv(Y&2U;%J%;<*Yws11ha5oaIX_C1<~M z8AFH-YJOvWU_<}jnm=`Ovil zYPl;et7LR9N>ZZ{Y)P*4$}*BzLC{&ZY9)HxHPqK0D*6dk;r}X7iETcj)4+r5jlZSH z&YgLrm!>Cy!`rv6dZ*moGYLwXiEm3VWTI>*dWHn7PHW9_DwXuLX_NBoDLyQ*BM{R9aT_ySE*3gTcK^ zcGSSZ`YNu34k!&AFNP|%;O*T*F&hpdXK#=x^wcKTu)7Re_8;qDD^{RaQZ)DuvQK>S zw~N^t%|DYpKg7Fx-%`Lxjs=g$UXicqX)4Wq)Zfy(v`MPYrnQ~;c8=uOF`mh5B_jgN zzpBXe_>p|2=j6D6K|Z{7WznOs`6|>{PVpNw=MCP!S#kgR)iktT(2=t0Vw)oVs|5ht zXY&Np1d~G`AVWWrcmMa2%tE2pX5E<6<_224oCl{kJWzVKRXXWdpALoA^h@;y-z#Zi z?1nF|13NWA@;RQ2Hj?_HD_}M5_4Cd$2W(6o)}1@p{gxOAr6e!|vmDJ+{nw_fbavg; zWfBiEyHzvPu>P9T+j-;ULZSD**|SgOuEko|#Ex!v_na6<-{KF-xzqJ$Rd920ByBb; z+qVQALecPU1WX~m2BF)c01{9 z73B7R_CDB=?HV!VauP1zT9J+2rhAdNeBko2Dnu{|GQDek*5tzuLJ~khu>Sw*s6i?P z3;ju@u|{s|cb@Kl59gi`-*3eC}xH-xnc6g|DY;Y5*22C?mIbi=_qB$2MUTx)asg$$J51l^;;L*kN zu}Su~Z8QXYqWuieBRpv(^E zPE@=N=R7a(Jh^9Ebn)2Z$VHNVoDdG@nbdo_!$9{>Emn7*wT_t$3FqSJ9JSc3|DkgX z@3+BiUl`k!HMu|hwm0;C#U5^(TE6ai>7MbNT=V6{r2I=BH#J<9iP58k4hC<{VuaD0|!(bSyLA57NW9d>S}?6p>gW=cs$?N5jew|pYL z-7d(Ko^hP+vkbQIRI6D|yI@-Ag>cw0mphqva(oO$sN;jeiW3 zh^;4;njSY4`;mtFlGG$;_4%5@!$DyiwLwPBg2^^}j~&$I(bLlxm1>z<87_@|TWxuj zDXpfSUXq6l;cSue5bP@J_0Gld9{!5_rb0=Is3$pUombyw$D(APKD+sCiMz?3QQqxi z4^Mvg=4k-TMhPNl}$0Y z-94kq;Zr|ZCY7&J-K}b`i^By}IqfQK&P*v6ameuL(8Vs3#Rt+osgV&;LR-G=2C zc2n*c{ucZ<+s?J=%39s#$yDxy`mJl283T@0yIt(>x-LBLe#_K8>2VUo7#}p5B=)Od zxKJ=@>9y~t|IxFt>w{y>=d#^b%i_z{;sg$>K~1ifh;lfSswU7#%eC$y{gp0vTnile zDIaNCbSHkrxkQ{C%XscRiCdPSqVvzHH8-DrEKvWj4{@wJnj*K4CxEDuy{z9 z!KnH;=>;#+`(WQ7zP?Qby84Aa&sRE9WeK_pq4xK@3ll%vni!hmw*~Vuwy2lANA}I~ zHBhRdh2z4ta^u;Qf4>!N&bnM*QZxB1ukSkAMhrv!n?L?W+tZADo^z8A2kb15g};~* z4rQMFal1n*aRM`l?iAaHXr{iY>fN-sI2Co3p3KVzsb_=dpg%`n-t1wk%PmVuxDf}l zqhmUJBqFE-g#}&JvpDoG-(`JZ@XG52;mfAwO>1xI+qGWRnO$u;43X;OZ3(`on6Gl@ zU0Zf45$AGHDp4CSZGCkk?@4SsIp*Y;SYJzvqaLG>U2q9&Ky@g$v2LaNc*SJvhXzi^ zF}_i=e034??CzZ0L;)A-gNrncT4=BF!mWvY>flAbMrf8|{9ECu@rt1`t$*pEqL>0CaJ!DvdY zo~H5@knek&YaxBN(fo~{2S%}(LGPoRthN3=Q~P_pF$!4(Vt4u?9JL1+={3cmsmiud zC5jLHvGa)1j?^JM`hE>wWUr&AP#ven*`i9mEnRbx&5nxgKJ=(s0~B%^QD)1{Wf5ez zZQEI_2+aa0pKotuwM-L`v4OMblaFBjfq^ZHfqlB!*0&{m@azfNsbc_y$p!n}hlc}& zrFsS#mzB!PQ~`gf+FJM6g^@6JYDw|JwVTkNxokq1Z75lCZhC9tTm0_f#CruU<_tib zO|Bz|-F#i&X72S$h2E^dc82wuV>8+4(4VeDuX*pq!!tCc#*rIAFzh#Cqu(!! z{92D>b@D3;Ox4umGH9o{A~$HTegQ1bUbf=HP<@+g7uAoKF5pK$_8(~U1J!659M5j05oV_|y%fGCGiQ_I;?eq%DoZR6YfGNgUTaX18y z?5O1BO1C{^pZkPYwroDZ%SG{JQ79f_A&xO3~S&iy?3lF`YZ;>_6s@|*Lm ztjnOjLH8{1oGtt|Az6~YSk^ANxf0Eetd7#}hxyqBZdqx_W#usI9Keo@Uhg!`jzfG9 zTV)aZ4fxU6(HSB2sXtD#C&s!+m~rqjb&m&1Ia$wUdg8Y0<~Pa+Pn(41JHOj|PNT!u zC1ibL()r;y;VX1zAG8zW;%pPU_8HrwJLjhPy))_Qu>x>l4!~o;+nuc~g1IcQC60~Z zDL6494Vfp;KYFnC*lT}K?U+nHHfPR%dh{2_*`RFl#Bo>Y_X)8pbT6og;`M)JE$Yr) zz@^AWxlRS*XpF@zl-YY{?5sVyAOqsy>YY<_v^G6*+j4kQJ&FDVWT|Eg`Pm!I7cJJacVdl`X9;Az_4?d;|0Ps~?ZKESY- zr3pvDt=6j=JrOxUGKrzKp)n686)cogetL!BgR$<>IIzPMd!C9EZd>}S9cLBKQEy!ne^&E6HX4AK{j zh@F>8YD_xpIOYlM%tV3yz(tGZ(^_#UaMsiiXwjt&fgq5Gqx&M8yS15YV#T9=xEIBb z&yl(Is+xDjoqW$wQ6)|zY&B>PC2%(VEOldrFAIi z-3_(nBv)j%%#SX&{)Lb6PVXGVYMQDB+SQ8=YGSiQKvA37wMInRY6>{5ZpR(?{HLk7 zpLIqaf}cQ;WAA=S@O+z-gJwkxuN>mZl)er|9UiV*tC?G=gf;w%yeK17my)2VcAvBL z?xja0hy;e%cG?W9vdI8#6+PJts*z;U<o3vUF0Y zk;_pY_(}5evaQ&YAp+#f+iG&0dc-;tZf!J35HEDmnE7(yr9%dE>D%OJUK3FH1Q%-% zFCMgw+6cs`;Pmwiq@0BjhO45;wwhF@9_8>J6R&4;_u!6C{>{8i6Z+4^C#%|h{U2%W zc5mn!2oHL}R-~=6v`sT-G?~Vw}vb{aSkz0CCh#s(KjH{xl_odO( z*~|iqVuwHt`4lB9!GR^9_6rT}b*g1$3KF5dv%8cS4?#8jhFypZG*mR;Dv5=MTdqju zwBnKzrk!Rezy$RUZejpMOCn-IKJtJ*gub0`P7(g`{HC?1sit!s*!Fc*(0>;f5{~re zD76h^dIz_OQQKj5fC7ul^H5`}R@cwsD z=y!B3LJIGB=DuwPU7w}rBIg@R`;-q`t(+A@K(D;QoE4mZA7jkCvuBn)3v`%KDvJq0 zm3J+uUsgLX`~utWy?Pg0X-y?`0X@Cf>PGkeR3{O$k&32a(izr$E5E1nnZNNfYf*lN&cLT~@>{XV4}a=rwMlY7`j4zW@!=`?g zFpW6EkWcG3%J5QopKKMp+r)^!@Lp$b+R=8#1YNOuoejY8fj5SR;d@C_FHbi*F&^;b zX8&;YJ}oFwSbkS9aP*ERH7T#x9-*$0aQxJE{O$E+pUJa${mg9s_F#O8XtQQVC--KR z-a{&sqyRR!gYNKthok(dC2(Cnrywcb3Cj`Vqo|X={6BBBYE^ z<7&Jmh8eA77hc+p2fd+A%(zr>C4H1+jB7dM`TO2bP%m)l!@C7~JEiR=s|5D+^pf1w zPfgNO5I6mUI~=L@zI@=}GMA&f_Lrun|BEo3OdT3JtnM5fCA>j>tbHhP;;%&)^vb4k zYO?0o%oM|$ZwO@f=g;9h8j+=VvJFlven%5Tq@6JsX?Jv&NB7&wC-6i*{2b=zEb{xc zx4xnQD5~2POrqBmVV|#7gcf%1|J9+WKhW1g>ihx0oSVO!i!R!g0X->jy0}RD;>*_= z%9;nh@e!j;dzV~prt;7|t%oy`|XWgPZNK;8jp z9YC{6KDzxSCm7h}3u;Z>k5Rg(CWC@`j&(VMcf6hMUe^oL^v$VpwfGE+s6w|KtR;9PN;Nb6f+@S5bw>r_!MPp7@g7{Y19(X?qm)6D}{w&ap z0Z)aT$Zu(cLbE`2@GO>IOEtHT(Ia*f%3`C-#Y>X&4`NUS=t8$FNtFOIOWw{kI+xTM z>>qqr+q;ejh&Rp(MRH4%vr)b3_jyQpvrhH$F1-GYGq>Hnm014YbfPfY^qmHG z`Lg!(8DrR7KCoMb#8Y?HPC>wjzp2aojG@hPh%3QiOYA5J!H4=Y-GV3GQM_zjXDk*r z`-k>qz3P0Z!Fz~9ObEDT=ZQvt7i%;M-w4u9@RB7iRAmkjA5ma|2*nkzIK;vHGs`j! zw3Q}z8-Nj~ETi~w;M-%>zD+?T!grnYd)ZE1n?VFT&8XZ7if0=1w)xbk)4+$Jvh&eu`!MS@eQ^>R1kW{ z<~mB(!zp+*uQ1&dkoSa#@i%cOutxMQSg&0sQZM267Kj+)Pn z^D(7-wJ*Q>6AiclmW3uoyiVJ4R>s-+KB3>#^HE}jEOGAA;~#?UBvc>@keFCw5jai` z{781WA}3>O(aC(W3T5Sy{ZUqzV_R-sILSjW%u;j*tBmV8 zb8oU}${n0%?W8`z+LX3B7`+Fj&+vIyK< z;r&nhCRPNqpPc;$y`W$L;f47@)ylF*hgOw~MHAAFMYCXSS|~}kIbwKG*Etx10}OJp z3UpZzbldh@ub>a*kh6M1FO)R&(nv0-mv0|LD)QCp3FTPN@Q8RG%-&)c$H}pNJ-m^=B|qYdcf;I3v!)r&>ruE=$ikx z)*Ed-C}<2u-V&kzAZh3?8|KbNd(ZawH!W;u;%>Ej;Bjp`@4P?VfC=Axp=PfT^NMXR z)khn{7Rwb7q|G`9x;3UlUX#$eBikO5)ctBF;da0+d&C^Q&K6yDDIAC?SYT;_zd%hr zsf=v;U_1grYNq1qJ55p5c-Q7AHW}t|KkJ6p8}p7q&SYy2;tbxK_7uH>Hk{I@R3U8k z2-uf{(?Z}Hb26B11oIuz&HHLrR%rCo%7XzXEA{s5>unJ~h<@E)Dz_QBzqce@XyT21 z`YiJrf)4?nK-+NgZ+KuLpLH7f#EFb{{Hu4-!ug5dOJ^3PP-dYH|4}9gs<9a}3y-|{ zD=Ml?i+8mn(&XJ86Y6iPJnBQI8mn13L^u7}W*KDcjb|NgcjJiBvlr%}S^i+yL)`{j zq*Ll(r<5(QRBKeG%UQ~sM}xrY-$CSEdV?m8MJ!L%cs9fsIZBN<5nC2JV2NMq**&SP zT~|=Hpm%$K;oueTW5I>jICQ8}vn^hngOSmAilKa%j?j{Ph0OgBf0{e*$$+*=Yvl6r z@p~BRre*A`d!c!>TvMxdn;6|@z9e7ThU#ed#Lv5l$-AKxHO#Oe2xP|rE6J|m@7%&6 zxUuDXf_;Gtq7{#ggbzHSI1gsFl^D;31%kA&2vA=iKV54L+_H$AO z$O%3WwA0fZ$fd84msaF;z&TYj<=`z|xFzC3ZCO3A4ut)N(T(oAeg1X$x!FXHeQ&o_ z^!AdYx%e&jZ?o6LY+qHX<2tpg3HUqlKZce*;6@MPmOa~72wu3(CA4X*=c<_Rm9gqn zMJ3&y-#^-jAyNpdi5daH?o!$lB>nZbiB)F%ZlGQvk7aooI2&hKm=Hd`dXWt22?l-7 zyyW=P%>=~mSU=3VfJk^2H8aDDvi97K5Yb6q*dE|m_ZNyGEe z)^#YXC|A)%wm2i|1zjo{_WR~4 zlVdg_#M5s;M3ope0tSRGw^zkGM>Fi4LHv<+XZwwZvkj#cX+FXr31^SZ{gK;JRA=h1 zcFHQsgxDFUxhmk=I8cwL$lSGw&p=_~Je1Qw*1k0CEcqI>U+_+-1+YsiI3BKXlIK7% z0aCOXbe4|K1i?C7y$OywGL^h*tuO6UddQGowYq_NJ~~~aFYJ3AD}rI@k>72~ni}A_ zahGBTU+sKe(@p*c5i~_pRja<0VUhS`2eK`QP)%NcP2zK9nn4E0#U5rFo_}oI5HxIw z?LjlYe9z zuQ+D$T6;HvE5lb087pul0M9IpUa4ep+=_)12Lp^OdX7Z+%o9_7tNxYY{HEO8e3rZE zx#5ww-y*sSNqGmw>O{qJaz5<*L%!kU(WF;dMYZT+cT0~9JDt3!;h}7-s;Me0v1c=c z|I#bF?c=gsH6FrJnaVztP-7CwQca(|t9Rduc~TiX281XkFdV`12FT?Q>hrI}rT~j& zy}QD68CsG?h_Sa~SiSq|+J?*!BI~?Tk^^*mPjA7pUt}n^@SSl1V(cYdL$PU_*5^$83rn(Hf3m@6wxx?YqdS-0%tU0U2DHMkW0%jRO#ae}H_xgxDqpmO z@K)bj3jl-eN)m(;gixx0hI(WXrE1`J70eFkcK2-$YKR8s14#x^#%3^GU9UoEN^vDft$M~gfSJa zBlx6!nR3qtJIEBp%z&RY%>x0m6OW0DrMa}QVwJCsdx>__jNhVTUvfqy7={w(=y!0H z;GgD?V-m}P9LZ!QNh9pc`VXK$^^k_c%w%E;3UjbI+5I^=bb^t;BPS3HwWmsLXJON9>Jo{*nVF#5BApRKjWU|p*A!=s66lAJ&^GJJ~@{5Va6y#L~#7;vk z3yBB~G1d*C&#E?ZHCKd%1|7mq`JX=$QvH<&&F?<^RvQk1{x4R!0|g+YyhH&Z%d_== z8|dQw*ZnB}e$300yyGe)zQ+Gf5e~k!>H2)5J>TF}-^#&@yrRtIT?@RXamc(b_;xT2 zUDjQ#gx-hd8bG<>&OlQjn8^j9b3_0siLheLPt)9TXDrqVRjY2foJ|LYl?l{Kb{|hN zW_c^E=>zUW>&_!O|8sQTih=xJ3l5GZ z&_LtY<_gh@=3r5NOz1jOxtn*mgev*MoP2%cUA~1XI5l;1)Gru9y1JP4v)R22WW8+F zvx_pmF{%J14AJTxN-wVoZg1Fmt(&QVep015ji_amnUq^_gDzO$%6dl5Q=cc}keQpk z8=c(qVm|V^yglYijX<*ys0|<^%wpCRYynQwkhY~xh@v`p3!KZGvc|S06bxOb(r&Mh zC>!B|*CD)>z9Sn#m-%FDniHN2c@9DX%S~~Zg%ICSE7Z&KvJ5$UDV3ho2lWVG;>6!x z$P@sDdT6Ub0|Jend2)pz^#ROTZwzyi4TQ!69e;(^Qz=nhcFO0t`AAmA{DSJ`F(j`W z`okhTUkC3ER$8bzaK}s~D1igMs^Fd|XW6C^7Z3DP-Q3E&9ItI<^=|J}{o0S6*9T!9 zrF><0zAw@n5p_JX99|Z|QXgP8pDZ|0j5!rTUrTDNLe_~kEXewxxNFTspH`B$u*N{` z3Qt}nQ~xug)<3An#4FB7#AU!rd#RXP9lYRG8$C5t&SVLR$g3os^J`%qINkN4{YYYd zo0RxtauVS#jvgMc9|wwP^qw1AZVruLf2dAiZ|H=;)9}Xg1oXrwgVJC&@S-aqFp$;d zeKtJCev&?DeD3`0ChV9^ddvKUL<}o%HE(rGtGpqs3L2f~=I%_G1mdcU>=qSN2HzQM z-BZ*n!la#-Zu;wa=eEjsg5VA)69sw>${9{P4sWuM?k~4o#*}NTNF`R}XpAT)IMAAX zm=FjrV|@M1P(SR&6f%jFHxd^B@uYPL71z3K59Oyq;zpA13*evwpVX1X7eyDBTX`1} zTa_**=H6&m=og_A+qG3VkLTla0T=$ZVD9s08eGi%FO3$~Al1kd#Fjyn@gXFZNFR##Ta!c<^Pdk@ za6vsNf>BnZ!Udj|p^aHr^sOCoEgWuGNIL?bf*S$Hy2$1lh~Ww+@RBIJeCtW}kf~-? z+*LuLx}$4-NF;sEBq+;X%u6h#WxqKbxM1<1^Rn2v zrB+P3K=jU+sny>9;KKqYOtkPR(t?y%2D}M`&5Ir$6>d~Eji41|f*`LB=v#uftxj|5 zQR6`64q(}Bn%V5UBpSbb?&9T~rBhpQ41MDzk@02Ewu6^*^He5n=u9s8$0gXtol+~-sgg&RabVa6FvQ+~&Z%(T)C zr^wP#$jMEVrX$~h&V&oNovj^Ki*0{lmB}M-V&Y!-)U?`YU)HPhPSb}E^R;m(`U(iu zUmjrqy`B~>5N(c`4OrJR@WMS=>pDH{URg(7Ga`OR5`Y5X;subR*yAP=sAIi#-DB(9 z)>mOnA(xy8peKYg757@tvzvcHF0S~eeY$DMUp6`OF5h@;dWu0N)gX+m_rTM7C~DT} zz27=UnV9-mMca4~%2K-82zNH^#7`giF;A|(r-6Cu`5|-4X9IFgke2IxmZN|T?VZa< z$EyrzH=19|Q?uRuHRoy>^~vCpO!AAhnBA!oeU1Y`TNcfxiuWEJx1|f#YB$ZqaBXHD zCsCZn!5)HO6kH1+HZmoI?GzLI6KkydT3<1M=>c)Bu9ADI9Om35ITN2Yg02O5hrf6S zF~y&m3)k}Xe;rP*;5%qY{~$vmhc@Ni6OkcF&kG^sz=~;LV0^8FOyhF-mc+hV<(|;h z*2a*IHWaaGFsKT}^LL((hN#S<%l;VP0%DWyD9H?e9eu8K_dtyPzmLz+;U@om3j>h* z?{CnNYya& From c47d459221b3a7a09df5e70a8ecd493859b2daa7 Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Fri, 7 Feb 2020 15:46:51 -0500 Subject: [PATCH 068/168] fix autogen/configure errors Removes extra brackets that were surrounding uses of AC_MSG_ERROR in configure.ac. Silences a libtoolize warning by uncommenting the line in Makefile.am that set ACLOCAL_FLAGS. Removes use of AX_PTHREAD configure.ac, since the m4 definition is not always available (which leads to a weird configure failure), and we were not really using it as intended anyway. --- Makefile.am | 2 +- configure.ac | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Makefile.am b/Makefile.am index 76addc56e..5c402c5d0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -2,7 +2,7 @@ SUBDIRS = common meta server client examples extras t util CONFIG = ordered -#ACLOCAL_AMFLAGS = -I m4 +ACLOCAL_AMFLAGS = -I m4 pkgconfigdir = @pkgconfigdir@ pkgconfig_DATA = client/unifyfs.pc diff --git a/configure.ac b/configure.ac index 9bcb9b151..6e81d34d5 100755 --- a/configure.ac +++ b/configure.ac @@ -140,9 +140,10 @@ AS_IF([test "x$enable_fortran" = "xyes"],[ AC_LANG_POP ],[]) -AS_IF([test "$have_C_mpi" != "yes"],[ - AC_MSG_ERROR(["Couldn't find MPI"]) -],[]) +AS_IF([test "$have_C_mpi" != "yes"], + AC_MSG_ERROR(["Couldn't find MPI"]), + [] +) # look for leveldb library, sets LEVELDB_CFLAGS/LDFLAGS/LIBS UNIFYFS_AC_LEVELDB @@ -194,9 +195,6 @@ AC_CHECK_HEADERS(mntent.h sys/mount.h) AX_LIB_HDF5 AM_CONDITIONAL([HAVE_HDF5], [test x$with_hdf5 = xyes]) -# Look for pthreads -AX_PTHREAD([],[AC_MSG_ERROR([pthreads are required])]) - # libc functions wrapped by unifyfs CP_WRAPPERS+="-Wl,-wrap,access" From bdf8a88ab6be96ab1761ab21b76f156d0f4a773b Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Mon, 10 Feb 2020 11:03:31 -0500 Subject: [PATCH 069/168] add unifyfs_set_log_level() Use the set method in client/server rather than directly setting the global variable. --- client/src/unifyfs.c | 2 +- common/src/unifyfs_log.c | 47 ++++++++++++++++++++++++--------------- common/src/unifyfs_log.h | 6 ++++- server/src/unifyfs_init.c | 2 +- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 5aba007d6..6a2aa54bf 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -1747,7 +1747,7 @@ static int unifyfs_init(int rank) if (cfgval != NULL) { rc = configurator_int_val(cfgval, &l); if (rc == 0) { - unifyfs_log_level = (int)l; + unifyfs_set_log_level((unifyfs_log_level_t)l); } } diff --git a/common/src/unifyfs_log.c b/common/src/unifyfs_log.c index 6c6d68b47..bc087a11a 100644 --- a/common/src/unifyfs_log.c +++ b/common/src/unifyfs_log.c @@ -48,6 +48,7 @@ size_t unifyfs_log_source_base_len; // = 0 static const char* this_file = __FILE__; /* open specified file as log file stream, + * or stderr if no file given. * returns UNIFYFS_SUCCESS on success */ int unifyfs_log_open(const char* file) { @@ -59,30 +60,40 @@ int unifyfs_log_open(const char* file) } } - FILE* logf = fopen(file, "a"); - if (logf == NULL) { - /* failed to open file name, fall back to stderr */ - unifyfs_log_stream = stderr; - return (int)UNIFYFS_ERROR_DBG; - } else { - unifyfs_log_stream = logf; - return UNIFYFS_SUCCESS; + /* stderr is our default log file stream */ + unifyfs_log_stream = stderr; + + if (NULL != file) { + FILE* logf = fopen(file, "a"); + if (NULL != logf) { + unifyfs_log_stream = logf; + } else { + return EINVAL; + } } + return (int)UNIFYFS_SUCCESS; } -/* close our log file stream, +/* close our log file stream. * returns UNIFYFS_SUCCESS on success */ int unifyfs_log_close(void) { - if (unifyfs_log_stream == NULL) { - /* nothing to close */ - return (int)UNIFYFS_ERROR_DBG; - } else { - /* if stream is open, and its not stderr, close it */ - if (unifyfs_log_stream != stderr && - fclose(unifyfs_log_stream) == 0) { - return UNIFYFS_SUCCESS; + /* if stream is open, and its not stderr, close it */ + if (NULL != unifyfs_log_stream) { + if (unifyfs_log_stream != stderr) { + fclose(unifyfs_log_stream); + unifyfs_log_stream = stderr; } - return (int)UNIFYFS_ERROR_DBG; + } + return (int)UNIFYFS_SUCCESS; +} + +/* set log level */ +void unifyfs_set_log_level(unifyfs_log_level_t lvl) +{ + if (lvl < LOG_LEVEL_MAX) { + unifyfs_log_level = lvl; + } else { + LOGERR("invalid log level %d", (int)lvl); } } diff --git a/common/src/unifyfs_log.h b/common/src/unifyfs_log.h index 391d0f1f7..d83c4d5ac 100644 --- a/common/src/unifyfs_log.h +++ b/common/src/unifyfs_log.h @@ -45,7 +45,8 @@ typedef enum { LOG_ERR = 2, LOG_WARN = 3, LOG_INFO = 4, - LOG_DBG = 5 + LOG_DBG = 5, + LOG_LEVEL_MAX } unifyfs_log_level_t; extern unifyfs_log_level_t unifyfs_log_level; @@ -93,6 +94,9 @@ int unifyfs_log_open(const char* file); * returns UNIFYFS_SUCCESS on success */ int unifyfs_log_close(void); +/* set log level */ +void unifyfs_set_log_level(unifyfs_log_level_t lvl); + #ifdef __cplusplus } // extern "C" #endif diff --git a/server/src/unifyfs_init.c b/server/src/unifyfs_init.c index 2d2df5e69..c55993ea9 100644 --- a/server/src/unifyfs_init.c +++ b/server/src/unifyfs_init.c @@ -282,7 +282,7 @@ int main(int argc, char* argv[]) long l; rc = configurator_int_val(server_cfg.log_verbosity, &l); if (0 == rc) { - unifyfs_log_level = (int)l; + unifyfs_set_log_level((unifyfs_log_level_t)l); } } From 9144d777f8dd9b1ea17aa1a51978a2f9864c75a6 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Sat, 11 Jan 2020 16:06:29 -0800 Subject: [PATCH 070/168] Segment tree style edits and comments This is a restructuring of @adammoody's #430 PR. It makes the following changes to the seg_tree code: - Add comments - Correctly free memory on errors - Add seg_tree_find() - Update seg_tree_test.c with seg_tree_find() tests. --- common/src/seg_tree.c | 198 ++++++++++++++++++++++++++++-------------- common/src/seg_tree.h | 77 +++++++++++++++- t/seg_tree_test.c | 18 ++++ 3 files changed, 225 insertions(+), 68 deletions(-) diff --git a/common/src/seg_tree.c b/common/src/seg_tree.c index 2c11f138c..423860be0 100644 --- a/common/src/seg_tree.c +++ b/common/src/seg_tree.c @@ -72,19 +72,22 @@ void seg_tree_destroy(struct seg_tree* seg_tree) seg_tree_clear(seg_tree); }; -/* Allocate a range tree. Free it with free() when finished */ +/* Allocate a node for the range tree. Free node with free() when finished */ static struct seg_tree_node* seg_tree_node_alloc(unsigned long start, unsigned long end, unsigned long ptr) { + /* allocate a new node structure */ struct seg_tree_node* node; node = calloc(1, sizeof(*node)); if (!node) { return NULL; } + /* record logical range and physical offset */ node->start = start; node->end = end; node->ptr = ptr; + return node; } @@ -95,39 +98,50 @@ seg_tree_node_alloc(unsigned long start, unsigned long end, unsigned long ptr) * return 1 from this function, else return 0. If there are two * non-overlapping ranges, return the first one in new_start/new_end. */ -static int -get_non_overlapping_range(unsigned long start1, unsigned long end1, - long start2, long end2, long* new_start, long* new_end) +static int get_non_overlapping_range( + unsigned long start1, unsigned long end1, + long start2, long end2, + long* new_start, long* new_end) { - if (start1 >= start2 && end1 <= end2) { - /* Completely overlapping */ - return 1; - } else if (start1 < start2) { + /* + * This function is only called when we know that segment 1 and segment 2 + * overlap with each other. Find first portion of segment 1 that does not + * overlap with segment 2, if any. + */ + if (start1 < start2) { /* - * s1 ------- e1 - * s2--------e2 - * ---- non-overlap - * - * also: + * Segment 1 includes a portion before segment 2 starts return start/end + * of that leading portion of segment 1. * - * s1 -------------------e1 - * s2--------e2 - * ---- non-overlap + * s1-------e1 + * s2--------e2 + * ---- non-overlap */ *new_start = start1; - *new_end = MIN(end1, start2 - 1); - } else if (start1 > start2 && end1 > end2) { + *new_end = start2 - 1; + return 0; + } else if (end1 > end2) { /* - * s1 ----- e1 - * s2------- e2 + * Segment 1 does not start before segment 2, but segment 1 extends past + * end of segment 2. return start/end of trailing portion of segment 1. + * + * s1-----e1 + * s2-------e2 + * --- non-overlap */ - *new_start = MAX(start1, end2 + 1); - *new_end = end1; - } else if (start1 == start2 && end1 > end2) { *new_start = end2 + 1; *new_end = end1; + return 0; } - return 0; + + /* + * Segment 2 completely envelops segment 1 so nothing left of segment 1 to + * return, so return 1 to indicate this case. + * + * s1-------e1 + * s2-------------e2 + */ + return 1; } /* @@ -136,12 +150,15 @@ get_non_overlapping_range(unsigned long start1, unsigned long end1, int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, unsigned long end, unsigned long ptr) { + /* Assume we'll succeed */ + int rc = 0; struct seg_tree_node* node; - struct seg_tree_node* overlap = NULL; - struct seg_tree_node* resized; struct seg_tree_node* remaining; - long new_start = 0, new_end = 0; - int rc; + struct seg_tree_node* resized; + struct seg_tree_node* overlap; + long new_start; + long new_end; + int ret; /* Create our range */ node = seg_tree_node_alloc(start, end, ptr); @@ -149,13 +166,16 @@ int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, return ENOMEM; } + /* Lock the tree so we can modify it */ seg_tree_wrlock(seg_tree); + /* * Try to insert our range into the RB tree. If it overlaps with any other * range, then it is not inserted, and the overlapping range node is * returned in 'overlap'. If 'overlap' is NULL, then there were no * overlaps, and our range was successfully inserted. */ + overlap = NULL; while ((overlap = RB_INSERT(inttree, &seg_tree->head, node))) { /* * Our range overlaps with another range (in 'overlap'). Is there any @@ -163,13 +183,17 @@ int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, * delete the old 'overlap' and insert the smaller, non-overlapping * range. */ - rc = get_non_overlapping_range(overlap->start, overlap->end, start, end, - &new_start, &new_end); - if (rc) { - /* We can't find a non-overlapping range. Delete the old range. */ + ret = get_non_overlapping_range(overlap->start, overlap->end, start, + end, &new_start, &new_end); + if (ret) { + /* + * The new range we are adding completely covers the existing + * range in the tree defined in overlap. We can't find a + * non-overlapping range. Delete the existing range. + */ RB_REMOVE(inttree, &seg_tree->head, overlap); - seg_tree->count--; free(overlap); + seg_tree->count--; } else { /* * Part of the old range was non-overlapping. Split the old range @@ -181,47 +205,78 @@ int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, resized = seg_tree_node_alloc(new_start, new_end, overlap->ptr + (new_start - overlap->start)); if (!resized) { - return ENOMEM; + free(node); + rc = ENOMEM; + goto release_add; } - /* Remove our old range */ - RB_REMOVE(inttree, &seg_tree->head, overlap); - /* Insert the non-overlapping part of the new range */ - RB_INSERT(inttree, &seg_tree->head, resized); - - if (resized->end + 1 >= overlap->start && - resized->end +1 <= overlap->end) { + /* + * If the non-overlapping part came from the front portion of the + * existing range, then there is a trailing portion of the + * existing range to add back to be considered again in the next + * loop iteration. + */ + remaining = NULL; + if (resized->end < overlap->end) { /* * There's still a remaining section after the non-overlapping * part. Add it in. */ - remaining = seg_tree_node_alloc(resized->end + 1, overlap->end, + remaining = seg_tree_node_alloc( + resized->end + 1, overlap->end, overlap->ptr + (resized->end + 1 - overlap->start)); - if (!resized) { - free(overlap); - return ENOMEM; + if (!remaining) { + free(node); + free(resized); + rc = ENOMEM; + goto release_add; } + } + + /* Remove our old range */ + RB_REMOVE(inttree, &seg_tree->head, overlap); + free(overlap); + seg_tree->count--; + + /* Insert the non-overlapping part of the new range */ + RB_INSERT(inttree, &seg_tree->head, resized); + seg_tree->count++; + + /* + * If we have a trailing portion, insert range for that, and + * increase our extent count since we just turned one range entry + * into two + */ + if (remaining != NULL) { RB_INSERT(inttree, &seg_tree->head, remaining); seg_tree->count++; } - free(overlap); } } - if (!overlap) { - seg_tree->count++; - } + /* Increment segment count in the tree for the range we just added */ + seg_tree->count++; + + /* + * Update max ending offset if end of new range we just inserted + * is larger. + */ seg_tree->max = MAX(seg_tree->max, end); + +release_add: + seg_tree_unlock(seg_tree); - return 0; + return rc; } -/* Search tree for an entry that overlaps with given range of - * [start, end]. Returns the first overlapping entry if found, - * which is the overlapping entry having the lowest starting - * offset, and returns NULL otherwise. Assumes caller has lock - * on tree. */ +/* + * Search tree for an entry that overlaps with given range of [start, end]. + * Returns the first overlapping entry if found, which is the overlapping entry + * having the lowest starting offset, and returns NULL otherwise. + * + * This function assumes you've already locked the seg_tree. + */ struct seg_tree_node* seg_tree_find_nolock( struct seg_tree* seg_tree, unsigned long start, @@ -253,6 +308,25 @@ struct seg_tree_node* seg_tree_find_nolock( return NULL; } +/* + * Search tree for an entry that overlaps with given range of [start, end]. + * Returns the first overlapping entry if found, which is the overlapping entry + * having the lowest starting offset, and returns NULL otherwise. + */ +struct seg_tree_node* seg_tree_find( + struct seg_tree* seg_tree, + unsigned long start, + unsigned long end) +{ + struct seg_tree_node* node; + + seg_tree_rdlock(seg_tree); + node = seg_tree_find_nolock(seg_tree, start, end); + seg_tree_unlock(seg_tree); + + return node; +} + /* * Given a range tree and a starting node, iterate though all the nodes * in the tree, returning the next one each time. If start is NULL, then @@ -370,21 +444,17 @@ void seg_tree_clear(struct seg_tree* seg_tree) /* Return the number of segments in the segment tree */ unsigned long seg_tree_count(struct seg_tree* seg_tree) { - unsigned long count; - - seg_tree_wrlock(seg_tree); - count = seg_tree->count; + seg_tree_rdlock(seg_tree); + unsigned long count = seg_tree->count; seg_tree_unlock(seg_tree); return count; } -/* Return the maximum segment value in the tree */ +/* Return the maximum ending logical offset in the tree */ unsigned long seg_tree_max(struct seg_tree* seg_tree) { - unsigned long max; - - seg_tree_wrlock(seg_tree); - max = seg_tree->max; + seg_tree_rdlock(seg_tree); + unsigned long max = seg_tree->max; seg_tree_unlock(seg_tree); return max; } diff --git a/common/src/seg_tree.h b/common/src/seg_tree.h index d392e8916..20507eae7 100644 --- a/common/src/seg_tree.h +++ b/common/src/seg_tree.h @@ -5,33 +5,84 @@ struct seg_tree_node { RB_ENTRY(seg_tree_node) entry; - unsigned long start, end; /* our range */ - unsigned long ptr; /* pointer to our data buffer */ + unsigned long start; /* starting logical offset of range */ + unsigned long end; /* ending logical offset of range */ + unsigned long ptr; /* physical offset of data in log */ }; struct seg_tree { RB_HEAD(inttree, seg_tree_node) head; pthread_rwlock_t rwlock; - unsigned long count; /* number of segments */ - long max; /* maximum segment value in the tree */ + unsigned long count; /* number of segments stored in tree */ + unsigned long max; /* maximum logical offset value in the tree */ }; +/* Returns 0 on success, positive non-zero error code otherwise */ int seg_tree_init(struct seg_tree* seg_tree); + +/* + * Remove all nodes in seg_tree, but keep it initialized so you can + * seg_tree_add() to it. + */ void seg_tree_clear(struct seg_tree* seg_tree); + +/* + * Remove and free all nodes in the seg_tree. + */ void seg_tree_destroy(struct seg_tree* seg_tree); + +/* + * Add an entry to the range tree. Returns 0 on success, nonzero otherwise. + */ int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, unsigned long end, unsigned long ptr); +/* + * Find the first seg_tree_node that falls in a [start, end] range. + */ +struct seg_tree_node* seg_tree_find( + struct seg_tree* seg_tree, + unsigned long start, + unsigned long end +); + +/* + * Find the first seg_tree_node that falls in a [start, end] range. + * Assumes you've already locked the tree. + */ struct seg_tree_node* seg_tree_find_nolock( struct seg_tree* seg_tree, unsigned long start, unsigned long end ); +/* + * Given a range tree and a starting node, iterate though all the nodes + * in the tree, returning the next one each time. If start is NULL, then + * start with the first node in the tree. + * + * This is meant to be called in a loop, like: + * + * seg_tree_rdlock(seg_tree); + * + * struct seg_tree_node *node = NULL; + * while ((node = seg_tree_iter(seg_tree, node))) { + * printf("[%d-%d]", node->start, node->end); + * } + * + * seg_tree_unlock(seg_tree); + * + * Note: this function does no locking, and assumes you're properly locking + * and unlocking the seg_tree before doing the iteration (see + * seg_tree_rdlock()/seg_tree_wrlock()/seg_tree_unlock()). + */ struct seg_tree_node* seg_tree_iter(struct seg_tree* seg_tree, struct seg_tree_node* start); +/* Return the number of segments in the segment tree */ unsigned long seg_tree_count(struct seg_tree* seg_tree); + +/* Return the maximum ending logical offset in the tree */ unsigned long seg_tree_max(struct seg_tree* seg_tree); /* @@ -47,8 +98,26 @@ unsigned long seg_tree_max(struct seg_tree* seg_tree); * * seg_tree_unlock(&seg_tree); */ + +/* + * Lock a seg_tree for reading. This should only be used for calling + * seg_tree_iter(). All the other seg_tree functions provide their + * own locking. + */ void seg_tree_rdlock(struct seg_tree* seg_tree); + +/* + * Lock a seg_tree for read/write. This should only be used for calling + * seg_tree_iter(). All the other seg_tree functions provide their + * own locking. + */ void seg_tree_wrlock(struct seg_tree* seg_tree); + +/* + * Unlock a seg_tree for read/write. This should only be used for calling + * seg_tree_iter(). All the other seg_tree functions provide their + * own locking. + */ void seg_tree_unlock(struct seg_tree* seg_tree); #endif diff --git a/t/seg_tree_test.c b/t/seg_tree_test.c index b6ed850e7..7b7aaa8c8 100644 --- a/t/seg_tree_test.c +++ b/t/seg_tree_test.c @@ -48,6 +48,7 @@ int main(int argc, char** argv) char tmp[255]; int i; unsigned long max, count; + struct seg_tree_node* node; plan(NO_PLAN); @@ -128,6 +129,23 @@ int main(int argc, char** argv) ok(max == 50, "max 50 (got %lu)", max); ok(count == 8, "count is 8 (got %lu)", count); + /* + * Test seg_tree_find(). Find between a range that multiple segments. It + * should return the first one. + */ + node = seg_tree_find(&seg_tree, 2, 7); + ok(node->start == 2 && node->end == 2, "seg_tree_find found correct node"); + + /* Test finding a segment that partially overlaps our range */ + seg_tree_add(&seg_tree, 100, 200, (unsigned long) (buf + 100)); + node = seg_tree_find(&seg_tree, 90, 120); + ok(node->start == 100 && node->end == 200, + "seg_tree_find found partial overlapping node"); + + /* Look for a range that doesn't exist. Should return NULL. */ + node = seg_tree_find(&seg_tree, 2000, 3000); + ok(node == NULL, "seg_tree_find correctly returned NULL"); + /* * Write a range, then completely overwrite it with the * same range. Use a different buf value to verify it changed. From ec593f35f36a59ed5d3aeb84dbf053f50674172e Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Wed, 5 Feb 2020 17:16:23 -0600 Subject: [PATCH 071/168] rename unifyfs_error_e to unifyfs_rc This commit also removes all of the UnifyFS error codes that mirrored existing errno values, and converts the places we used those codes to just use the errno names. This works because the error enumerator was modified a while back to allow for errno values to not conflict with new error codes we define. There is also a new unifyfs_rc_errno() method to return a valid errno value from any of the unifyfs_rc values. This also removes the long dead domain socket code, which enabled removal of a bunch of UNIFYFS_ERROR_SOCK_XXXX codes. SKIP NOTE: checkpatch doesn't like the enumerator macros TEST_CHECKPATCH_SKIP_FILES=common/src/unifyfs_rc.h TEST_CHECKPATCH_SKIP_FILES+=,common/src/unifyfs_rc.c --- client/src/pmpi_wrappers.c | 4 +- client/src/unifyfs-fixed.c | 36 +-- client/src/unifyfs-internal.h | 6 - client/src/unifyfs-stdio.c | 66 +++--- client/src/unifyfs-sysio.c | 56 ++--- client/src/unifyfs.c | 207 ++-------------- common/src/Makefile.am | 6 +- common/src/err_enumerator.h | 140 ----------- common/src/unifyfs_configurator.c | 30 +-- common/src/unifyfs_const.h | 5 +- common/src/unifyfs_keyval.c | 96 ++++---- common/src/unifyfs_log.c | 19 +- common/src/{err_enumerator.c => unifyfs_rc.c} | 79 ++++--- common/src/unifyfs_rc.h | 104 +++++++++ common/src/unifyfs_runstate.c | 14 +- common/src/unifyfs_shm.c | 30 +-- server/src/Makefile.am | 4 +- server/src/unifyfs_cmd_handler.c | 4 +- server/src/unifyfs_global.h | 3 - server/src/unifyfs_init.c | 56 +---- server/src/unifyfs_request_manager.c | 36 +-- server/src/unifyfs_service_manager.c | 14 +- server/src/unifyfs_sock.c | 220 ------------------ server/src/unifyfs_sock.h | 58 ----- t/server/metadata_suite.c | 2 +- 25 files changed, 391 insertions(+), 904 deletions(-) delete mode 100644 common/src/err_enumerator.h rename common/src/{err_enumerator.c => unifyfs_rc.c} (56%) create mode 100644 common/src/unifyfs_rc.h delete mode 100644 server/src/unifyfs_sock.c delete mode 100644 server/src/unifyfs_sock.h diff --git a/client/src/pmpi_wrappers.c b/client/src/pmpi_wrappers.c index ac86d39ba..e4bad9f43 100644 --- a/client/src/pmpi_wrappers.c +++ b/client/src/pmpi_wrappers.c @@ -37,7 +37,7 @@ int unifyfs_mpi_init(int* argc, char*** argv) rc = unifyfs_mount("/unifyfs", rank, (size_t)world_sz, app_id); if (UNIFYFS_SUCCESS != rc) { fprintf(stderr, "UNIFYFS ERROR: unifyfs_mount() failed with '%s'\n", - unifyfs_error_enum_description((unifyfs_error_e)rc)); + unifyfs_rc_enum_description((unifyfs_rc)rc)); } return ret; @@ -66,7 +66,7 @@ int unifyfs_mpi_finalize(void) rc = unifyfs_unmount(); if (UNIFYFS_SUCCESS != rc) { fprintf(stderr, "UNIFYFS ERROR: unifyfs_unmount() failed with '%s'\n", - unifyfs_error_enum_description((unifyfs_error_e)rc)); + unifyfs_rc_enum_description((unifyfs_rc)rc)); } //fprintf(stderr, "DEBUG: %s - before PMPI_Finalize()\n", __func__); diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 6be3d3d93..869898cf8 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -164,7 +164,7 @@ static int unifyfs_chunk_alloc(int fid, unifyfs_filemeta_t* meta, int chunk_id) unifyfs_stack_unlock(); if (id < unifyfs_max_chunks) { LOGERR("spill-over device out of space (%d)", id); - return UNIFYFS_ERROR_NOSPC; + return ENOSPC; } /* got one from spill over */ @@ -173,7 +173,7 @@ static int unifyfs_chunk_alloc(int fid, unifyfs_filemeta_t* meta, int chunk_id) } else { /* spill over isn't available, so we're out of space */ LOGERR("memfs out of space (%d)", id); - return UNIFYFS_ERROR_NOSPC; + return ENOSPC; } } else if (unifyfs_use_spillover) { /* memory file system is not enabled, but spill over is */ @@ -188,7 +188,7 @@ static int unifyfs_chunk_alloc(int fid, unifyfs_filemeta_t* meta, int chunk_id) unifyfs_stack_unlock(); if (id < unifyfs_max_chunks) { LOGERR("spill-over device out of space (%d)", id); - return UNIFYFS_ERROR_NOSPC; + return ENOSPC; } /* got one from spill over */ @@ -197,7 +197,7 @@ static int unifyfs_chunk_alloc(int fid, unifyfs_filemeta_t* meta, int chunk_id) } else { /* don't know how to allocate chunk */ chunk_meta->location = CHUNK_LOCATION_NULL; - return UNIFYFS_ERROR_IO; + return EIO; } return UNIFYFS_SUCCESS; @@ -222,7 +222,7 @@ static int unifyfs_chunk_free(int fid, unifyfs_filemeta_t* meta, int chunk_id) } else { /* unkwown chunk location */ LOGERR("unknown chunk location %d", chunk_meta->location); - return UNIFYFS_ERROR_IO; + return EIO; } /* update location of chunk */ @@ -255,12 +255,12 @@ static int unifyfs_chunk_read( off_t spill_offset = unifyfs_compute_spill_offset(meta, chunk_id, chunk_offset); ssize_t rc = pread(unifyfs_spilloverblock, buf, count, spill_offset); if (rc < 0) { - return unifyfs_errno_map_to_err(rc); + return errno; } } else { /* unknown chunk type */ LOGERR("unknown chunk type"); - return UNIFYFS_ERROR_IO; + return EIO; } /* assume read was successful if we get to here */ @@ -373,7 +373,7 @@ int unifyfs_sync(int gfid) * we called the real fsync which should * have already set errno to something reasonable */ LOGERR("failed to flush data to spill over file"); - return UNIFYFS_ERROR_IO; + return EIO; } } @@ -382,7 +382,7 @@ int unifyfs_sync(int gfid) if (ret != UNIFYFS_SUCCESS) { /* something went wrong when trying to flush key/values */ LOGERR("failed to flush key/value index to server"); - return UNIFYFS_ERROR_IO; + return EIO; } /* flushed, clear buffer and refresh number of entries @@ -488,7 +488,7 @@ static int unifyfs_logio_add_write_meta_to_index(unifyfs_filemeta_t* meta, if (ret != UNIFYFS_SUCCESS) { /* something went wrong when trying to flush key/values */ LOGERR("failed to flush key/value index to server"); - return UNIFYFS_ERROR_IO; + return EIO; } } @@ -527,7 +527,7 @@ static int unifyfs_logio_chunk_write( chunk_meta->location != CHUNK_LOCATION_SPILLOVER) { /* unknown chunk type */ LOGERR("unknown chunk type"); - return UNIFYFS_ERROR_IO; + return EIO; } /* copy data into chunk and record its starting offset within the log */ @@ -564,7 +564,7 @@ static int unifyfs_logio_chunk_write( bufpos, remaining, filepos); if (rc < 0) { LOGERR("pwrite failed: errno=%d (%s)", errno, strerror(errno)); - return UNIFYFS_ERROR_IO; + return EIO; } /* wrote without an error, total up bytes written so far */ @@ -679,7 +679,7 @@ static int unifyfs_chunk_write( } else { /* unknown chunk type */ LOGERR("unknown chunk type"); - return UNIFYFS_ERROR_IO; + return EIO; } /* assume write was successful if we get to here */ @@ -705,7 +705,7 @@ int unifyfs_fid_store_fixed_extend(int fid, unifyfs_filemeta_t* meta, if (rc != UNIFYFS_SUCCESS) { /* ran out of space to store data */ LOGERR("failed to allocate chunk"); - return UNIFYFS_ERROR_NOSPC; + return ENOSPC; } /* increase chunk count and subtract bytes from the number we need */ @@ -796,7 +796,7 @@ int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, chunk_id = meta->log_size >> unifyfs_chunk_bits; chunk_offset = meta->log_size & unifyfs_chunk_mask; } else { - return UNIFYFS_ERROR_IO; + return EIO; } /* determine how many bytes remain in the current chunk */ @@ -807,7 +807,7 @@ int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, chunk_offset, buf, count); } else { - return UNIFYFS_ERROR_IO; + return EIO; } } else { /* otherwise, fill up the remainder of the current chunk */ @@ -816,7 +816,7 @@ int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, chunk_offset, (void*)ptr, remaining); } else { - return UNIFYFS_ERROR_IO; + return EIO; } ptr += remaining; @@ -840,7 +840,7 @@ int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, 0, (void*)ptr, num); } else { - return UNIFYFS_ERROR_IO; + return EIO; } ptr += num; pos += num; diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 511756e20..99b633890 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -490,12 +490,6 @@ int unifyfs_gfid_from_fid(const int fid); * returns -1 otherwise */ int unifyfs_fid_from_gfid(const int gfid); -/* given an UNIFYFS error code, return corresponding errno code */ -int unifyfs_err_map_to_errno(int rc); - -/* given an errno error code, return corresponding UnifyFS error code */ -int unifyfs_errno_map_to_err(int rc); - /* checks to see if fid is a directory * returns 1 for yes * returns 0 for no */ diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index be9fbe3b0..bd4e49940 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -152,7 +152,7 @@ int unifyfs_stream_set_pointers(unifyfs_stream_t* s) /* ERROR: invalid file descriptor */ s->err = 1; errno = EBADF; - return UNIFYFS_ERROR_BADF; + return EBADF; } /* if we have anything on the push back buffer, that must be @@ -189,7 +189,7 @@ int unifyfs_stream_set_pointers(unifyfs_stream_t* s) /* given a mode like "r", "wb+", or "a+" return flags read, write, * append, and plus to indicate which were set, - * returns UNIFYFS_ERROR_INVAL if invalid character is found + * returns EINVAL if invalid character is found */ static int unifyfs_fopen_parse_mode( const char* mode, @@ -206,13 +206,13 @@ static int unifyfs_fopen_parse_mode( /* ensure that user specified an input mode */ if (mode == NULL) { - return UNIFYFS_ERROR_INVAL; + return EINVAL; } /* get number of characters in mode */ size_t len = strlen(mode); if (len <= 0 || len > 3) { - return UNIFYFS_ERROR_INVAL; + return EINVAL; } /* first character must either be r, w, or a */ @@ -228,7 +228,7 @@ static int unifyfs_fopen_parse_mode( *append = 1; break; default: - return UNIFYFS_ERROR_INVAL; + return EINVAL; } /* optional second character may either be + or b */ @@ -243,7 +243,7 @@ static int unifyfs_fopen_parse_mode( char third = mode[2]; if (third != 'b') { /* third character something other than + or b */ - return UNIFYFS_ERROR_INVAL; + return EINVAL; } } } else if (second == 'b') { @@ -254,12 +254,12 @@ static int unifyfs_fopen_parse_mode( *plus = 1; } else { /* third character something other than + or b */ - return UNIFYFS_ERROR_INVAL; + return EINVAL; } } } else { /* second character something other than + or b */ - return UNIFYFS_ERROR_INVAL; + return EINVAL; } } @@ -295,7 +295,7 @@ static int unifyfs_fopen( off_t pos; if (read) { /* read shall fail if file does not already exist, unifyfs_fid_open - * returns UNIFYFS_ERROR_NOENT if file does not exist w/o O_CREAT + * returns ENOENT if file does not exist w/o O_CREAT */ if (plus) { /* r+ ==> open file for update (reading and writing) */ @@ -342,7 +342,7 @@ static int unifyfs_fopen( * process has hit file stream limit, not the OS */ /* exhausted our file streams */ - return UNIFYFS_ERROR_NFILE; + return ENFILE; } /* get stream structure corresponding to stream id */ @@ -358,7 +358,7 @@ static int unifyfs_fopen( unifyfs_stack_push(unifyfs_stream_stack, sid); /* exhausted our file descriptors */ - return UNIFYFS_ERROR_NFILE; + return ENFILE; } /* set file pointer and read/write mode in file descriptor */ @@ -422,19 +422,19 @@ static int unifyfs_setvbuf( /* check whether we've already associated a buffer */ if (s->buf != NULL) { /* ERROR: stream already has buffer */ - return UNIFYFS_ERROR_BADF; + return EBADF; } /* check that the type argument is valid */ if (type != _IOFBF && type != _IOLBF && type != _IONBF) { /* ERROR: invalid type argument */ - return UNIFYFS_ERROR_INVAL; + return EINVAL; } /* check that size is valid */ if (size <= 0) { /* ERROR: invalid size argument */ - return UNIFYFS_ERROR_INVAL; + return EINVAL; } /* associate buffer with stream */ @@ -443,7 +443,7 @@ static int unifyfs_setvbuf( s->buf = malloc(size); if (s->buf == NULL) { /* ERROR: no memory */ - return UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* remember that we need to free the buffer at the end */ s->buffree = 1; @@ -479,7 +479,7 @@ static int unifyfs_stream_flush(FILE* stream) int write_rc = unifyfs_fd_write(s->fd, s->bufpos, s->buf, s->buflen); if (write_rc != UNIFYFS_SUCCESS) { s->err = 1; - errno = unifyfs_err_map_to_errno(write_rc); + errno = unifyfs_rc_errno(write_rc); return write_rc; } @@ -514,7 +514,7 @@ static int unifyfs_stream_read( s->err = 1; errno = EBADF; LOGDBG("Invalid file descriptor"); - return UNIFYFS_ERROR_BADF; + return EBADF; } /* bail with error if stream not open for reading */ @@ -523,7 +523,7 @@ static int unifyfs_stream_read( errno = EBADF; LOGDBG("Stream not open for reading"); - return UNIFYFS_ERROR_BADF; + return EBADF; } /* associate buffer with stream if we need to */ @@ -533,7 +533,7 @@ static int unifyfs_stream_read( if (setvbuf_rc != UNIFYFS_SUCCESS) { /* ERROR: failed to associate buffer */ s->err = 1; - errno = unifyfs_err_map_to_errno(setvbuf_rc); + errno = unifyfs_rc_errno(setvbuf_rc); LOGDBG("Couldn't setvbuf"); return setvbuf_rc; } @@ -554,7 +554,7 @@ static int unifyfs_stream_read( if (unifyfs_would_overflow_offt(current, (off_t) count)) { s->err = 1; errno = EOVERFLOW; - return UNIFYFS_ERROR_OVERFLOW; + return EOVERFLOW; } /* take bytes from push back buffer if they exist */ @@ -607,7 +607,7 @@ static int unifyfs_stream_read( * by unifyfs_fd_read() */ s->err = 1; - return UNIFYFS_ERROR_IO; + return EIO; } /* record new buffer range within file */ @@ -680,7 +680,7 @@ static int unifyfs_stream_write( LOGDBG("Bad file descriptor"); s->err = 1; errno = EBADF; - return UNIFYFS_ERROR_BADF; + return EBADF; } /* bail with error if stream not open for writing */ @@ -688,7 +688,7 @@ static int unifyfs_stream_write( LOGDBG("Stream not open for writing"); s->err = 1; errno = EBADF; - return UNIFYFS_ERROR_BADF; + return EBADF; } /* TODO: Don't know what to do with push back bytes if write @@ -702,7 +702,7 @@ static int unifyfs_stream_write( if (fid < 0) { s->err = 1; errno = EBADF; - return UNIFYFS_ERROR_BADF; + return EBADF; } current = unifyfs_fid_logical_size(fid); @@ -726,7 +726,7 @@ static int unifyfs_stream_write( if (unifyfs_would_overflow_offt(current, (off_t) count)) { s->err = 1; errno = EFBIG; - return UNIFYFS_ERROR_FBIG; + return EFBIG; } /* associate buffer with stream if we need to */ @@ -736,7 +736,7 @@ static int unifyfs_stream_write( if (setvbuf_rc != UNIFYFS_SUCCESS) { /* ERROR: failed to associate buffer */ s->err = 1; - errno = unifyfs_err_map_to_errno(setvbuf_rc); + errno = unifyfs_rc_errno(setvbuf_rc); return setvbuf_rc; } } @@ -748,7 +748,7 @@ static int unifyfs_stream_write( if (write_rc != UNIFYFS_SUCCESS) { /* ERROR: write error, set error indicator and errno */ s->err = 1; - errno = unifyfs_err_map_to_errno(write_rc); + errno = unifyfs_rc_errno(write_rc); return write_rc; } @@ -798,7 +798,7 @@ static int unifyfs_stream_write( /* ERROR: write error, set error indicator and errno */ s->err = 1; errno = ENOMEM; - return UNIFYFS_ERROR_NOMEM; + return ENOMEM; } } else { /* fully buffered, write until we hit the buffer limit */ @@ -943,7 +943,7 @@ FILE* UNIFYFS_WRAP(fopen)(const char* path, const char* mode) FILE* stream; int rc = unifyfs_fopen(path, mode, &stream); if (rc != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(rc); + errno = unifyfs_rc_errno(rc); return NULL; } return stream; @@ -980,7 +980,7 @@ int UNIFYFS_WRAP(setvbuf)(FILE* stream, char* buf, int type, size_t size) if (unifyfs_intercept_stream(stream)) { int rc = unifyfs_setvbuf(stream, buf, type, size); if (rc != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(rc); + errno = unifyfs_rc_errno(rc); return 1; } return 0; @@ -1565,7 +1565,7 @@ void UNIFYFS_WRAP(rewind)(FILE* stream) int rc = unifyfs_fseek(stream, (off_t) 0L, SEEK_SET); /* set errno */ - errno = unifyfs_err_map_to_errno(rc); + errno = unifyfs_rc_errno(rc); /* clear error indicator if seek successful */ if (rc == 0) { @@ -1836,7 +1836,7 @@ int UNIFYFS_WRAP(fclose)(FILE* stream) /* close the file */ int close_rc = unifyfs_fid_close(fid); if (close_rc != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(close_rc); + errno = unifyfs_rc_errno(close_rc); return EOF; } @@ -2240,7 +2240,7 @@ static int __srefill(unifyfs_stream_t* stream) if (setvbuf_rc != UNIFYFS_SUCCESS) { /* ERROR: failed to associate buffer */ s->err = 1; - errno = unifyfs_err_map_to_errno(setvbuf_rc); + errno = unifyfs_rc_errno(setvbuf_rc); return 1; } } diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 9c37bce83..57da3ec90 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -103,7 +103,7 @@ int UNIFYFS_WRAP(mkdir)(const char* path, mode_t mode) if (ret != UNIFYFS_SUCCESS) { /* failed to create the directory, * set errno and return */ - errno = unifyfs_err_map_to_errno(ret); + errno = unifyfs_rc_errno(ret); return -1; } @@ -150,7 +150,7 @@ int UNIFYFS_WRAP(rmdir)(const char* path) if (ret != UNIFYFS_SUCCESS) { /* failed to remove the directory, * set errno and return */ - errno = unifyfs_err_map_to_errno(ret); + errno = unifyfs_rc_errno(ret); return -1; } @@ -241,7 +241,7 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) int ret = unifyfs_sync(gfid); if (ret != UNIFYFS_SUCCESS) { /* sync failed for some reason, set errno and return error */ - errno = unifyfs_err_map_to_errno(ret); + errno = unifyfs_rc_errno(ret); return -1; } @@ -297,7 +297,7 @@ int UNIFYFS_WRAP(unlink)(const char* path) /* delete the file */ int ret = unifyfs_fid_unlink(fid); if (ret != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(ret); + errno = unifyfs_rc_errno(ret); return -1; } @@ -336,7 +336,7 @@ int UNIFYFS_WRAP(remove)(const char* path) /* delete the file */ int ret = unifyfs_fid_unlink(fid); if (ret != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(ret); + errno = unifyfs_rc_errno(ret); return -1; } @@ -627,25 +627,25 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { - return UNIFYFS_ERROR_BADF; + return EBADF; } /* it's an error to write to a directory */ if (unifyfs_fid_is_dir(fid)) { - return UNIFYFS_ERROR_INVAL; + return EINVAL; } /* check that file descriptor is open for write */ unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); if (!filedesc->write) { - return UNIFYFS_ERROR_BADF; + return EBADF; } /* TODO: is it safe to assume that off_t is bigger than size_t? */ /* check that our write won't overflow the length */ if (unifyfs_would_overflow_offt(pos, (off_t) count)) { /* TODO: want to return EFBIG here for streams */ - return UNIFYFS_ERROR_OVERFLOW; + return EOVERFLOW; } /* get current log size before extending the log */ @@ -684,7 +684,7 @@ int UNIFYFS_WRAP(creat)(const char* path, mode_t mode) off_t pos; int rc = unifyfs_fid_open(path, O_WRONLY | O_CREAT | O_TRUNC, mode, &fid, &pos); if (rc != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(rc); + errno = unifyfs_rc_errno(rc); return -1; } @@ -751,7 +751,7 @@ int UNIFYFS_WRAP(open)(const char* path, int flags, ...) off_t pos; int rc = unifyfs_fid_open(path, flags, mode, &fid, &pos); if (rc != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(rc); + errno = unifyfs_rc_errno(rc); return -1; } @@ -1055,7 +1055,7 @@ ssize_t UNIFYFS_WRAP(write)(int fd, const void* buf, size_t count) /* write data to file */ int write_rc = unifyfs_fd_write(fd, pos, buf, count); if (write_rc != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(write_rc); + errno = unifyfs_rc_errno(write_rc); return (ssize_t)(-1); } ret = count; @@ -1290,28 +1290,6 @@ static int delegator_wait(void) { int rc = (int)UNIFYFS_SUCCESS; -#if defined(UNIFYFS_USE_DOMAIN_SOCKET) - /* wait for signal on socket */ - cmd_fd.events = POLLIN | POLLPRI; - cmd_fd.revents = 0; - rc = poll(&cmd_fd, 1, -1); - - /* check that we got something good */ - if (rc == 0) { - if (cmd_fd.revents != 0) { - if (cmd_fd.revents == POLLIN) { - return UNIFYFS_SUCCESS; - } else { - printf("poll returned %d; error: %s\n", rc, strerror(errno)); - } - } else { - printf("poll returned %d; error: %s\n", rc, strerror(errno)); - } - } else { - printf("poll returned %d; error: %s\n", rc, strerror(errno)); - } -#endif - /* specify time to sleep between checking flag in shared * memory indicating server has produced */ struct timespec shm_wait_tm; @@ -1674,7 +1652,7 @@ static void service_local_reqs( if (rc != length) { /* had a problem reading, * set the request error code */ - req->errcode = UNIFYFS_ERROR_IO; + req->errcode = EIO; } } @@ -1746,7 +1724,7 @@ int unifyfs_fd_logreadlist(read_req_t* in_reqs, int in_count) size_t reqs_size = 2 * in_count * sizeof(read_req_t); reqs = (read_req_t*) malloc(reqs_size); if (reqs == NULL) { - return UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* define pointers to space where we can build our list @@ -2036,7 +2014,7 @@ ssize_t UNIFYFS_WRAP(pwrite)(int fd, const void* buf, size_t count, /* write data to file */ int write_rc = unifyfs_fd_write(fd, offset, buf, count); if (write_rc != UNIFYFS_SUCCESS) { - errno = unifyfs_err_map_to_errno(write_rc); + errno = unifyfs_rc_errno(write_rc); return (ssize_t)(-1); } @@ -2089,7 +2067,7 @@ int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) int ret = unifyfs_sync(gfid); if (ret != UNIFYFS_SUCCESS) { /* sync failed for some reason, set errno and return error */ - errno = unifyfs_err_map_to_errno(ret); + errno = unifyfs_rc_errno(ret); return -1; } @@ -2131,7 +2109,7 @@ int UNIFYFS_WRAP(fsync)(int fd) int ret = unifyfs_sync(gfid); if (ret != UNIFYFS_SUCCESS) { /* sync failed for some reason, set errno and return error */ - errno = unifyfs_err_map_to_errno(ret); + errno = unifyfs_rc_errno(ret); return -1; } diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 6a2aa54bf..d31500e08 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -236,82 +236,6 @@ int unifyfs_unsupported( return rc; } -/* given an UNIFYFS error code, return corresponding errno code */ -int unifyfs_err_map_to_errno(int rc) -{ - unifyfs_error_e ec = (unifyfs_error_e)rc; - - switch (ec) { - case UNIFYFS_SUCCESS: - return 0; - case UNIFYFS_ERROR_BADF: - return EBADF; - case UNIFYFS_ERROR_EXIST: - return EEXIST; - case UNIFYFS_ERROR_FBIG: - return EFBIG; - case UNIFYFS_ERROR_INVAL: - return EINVAL; - case UNIFYFS_ERROR_ISDIR: - return EISDIR; - case UNIFYFS_ERROR_NAMETOOLONG: - return ENAMETOOLONG; - case UNIFYFS_ERROR_NFILE: - return ENFILE; - case UNIFYFS_ERROR_NOENT: - return ENOENT; - case UNIFYFS_ERROR_NOMEM: - return ENOMEM; - case UNIFYFS_ERROR_NOSPC: - return ENOSPC; - case UNIFYFS_ERROR_NOTDIR: - return ENOTDIR; - case UNIFYFS_ERROR_OVERFLOW: - return EOVERFLOW; - default: - break; - } - return ec; -} - -/* given an errno error code, return corresponding UnifyFS error code */ -int unifyfs_errno_map_to_err(int rc) -{ - switch (rc) { - case 0: - return (int)UNIFYFS_SUCCESS; - case EBADF: - return (int)UNIFYFS_ERROR_BADF; - case EEXIST: - return (int)UNIFYFS_ERROR_EXIST; - case EFBIG: - return (int)UNIFYFS_ERROR_FBIG; - case EINVAL: - return (int)UNIFYFS_ERROR_INVAL; - case EIO: - return (int)UNIFYFS_ERROR_IO; - case EISDIR: - return (int)UNIFYFS_ERROR_ISDIR; - case ENAMETOOLONG: - return (int)UNIFYFS_ERROR_NAMETOOLONG; - case ENFILE: - return (int)UNIFYFS_ERROR_NFILE; - case ENOENT: - return (int)UNIFYFS_ERROR_NOENT; - case ENOMEM: - return (int)UNIFYFS_ERROR_NOMEM; - case ENOSPC: - return (int)UNIFYFS_ERROR_NOSPC; - case ENOTDIR: - return (int)UNIFYFS_ERROR_NOTDIR; - case EOVERFLOW: - return (int)UNIFYFS_ERROR_OVERFLOW; - default: - break; - } - return (int)UNIFYFS_FAILURE; -} - /* returns 1 if two input parameters will overflow their type when * added together */ inline int unifyfs_would_overflow_offt(off_t a, off_t b) @@ -964,7 +888,7 @@ int unifyfs_fid_create_file(const char* path) /* check that pathname is within bounds */ size_t pathlen = strlen(path) + 1; if (pathlen > UNIFYFS_MAX_FILENAME) { - return (int) UNIFYFS_ERROR_NAMETOOLONG; + return ENAMETOOLONG; } /* allocate an id for this file */ @@ -1026,7 +950,7 @@ int unifyfs_fid_create_directory(const char* path) /* check that pathname is within bounds */ size_t pathlen = strlen(path) + 1; if (pathlen > UNIFYFS_MAX_FILENAME) { - return (int) UNIFYFS_ERROR_NAMETOOLONG; + return (int) ENAMETOOLONG; } /* get local and global file ids */ @@ -1045,7 +969,7 @@ int unifyfs_fid_create_directory(const char* path) /* can't create if it already exists */ if (found_global) { - return (int) UNIFYFS_ERROR_EXIST; + return (int) EEXIST; } if (found_local) { @@ -1065,14 +989,14 @@ int unifyfs_fid_create_directory(const char* path) * we currently return EIO, and this needs to be addressed according to * a consistency model this fs intance assumes. */ - return (int) UNIFYFS_ERROR_IO; + return EIO; } /* now, we need to create a new directory. */ fid = unifyfs_fid_create_file(path); if (fid < 0) { /* FIXME: ENOSPC or EIO? */ - return (int) UNIFYFS_ERROR_IO; + return EIO; } /* Set as directory */ @@ -1084,7 +1008,7 @@ int unifyfs_fid_create_directory(const char* path) if (ret != UNIFYFS_SUCCESS) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); - return (int) UNIFYFS_ERROR_IO; + return EIO; } return UNIFYFS_SUCCESS; @@ -1114,7 +1038,7 @@ int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count) rc = unifyfs_fid_store_fixed_write(fid, meta, pos, buf, count); } else { /* unknown storage type */ - rc = (int)UNIFYFS_ERROR_IO; + rc = EIO; } return rc; @@ -1130,7 +1054,7 @@ int unifyfs_fid_write_zero(int fid, off_t pos, off_t count) size_t buf_size = 1024 * 1024; void* buf = (void*) malloc(buf_size); if (buf == NULL) { - return (int)UNIFYFS_ERROR_IO; + return EIO; } /* set values in this buffer to zero */ @@ -1150,7 +1074,7 @@ int unifyfs_fid_write_zero(int fid, off_t pos, off_t count) /* write data to file */ int write_rc = unifyfs_fid_write(fid, curpos, buf, num); if (write_rc != UNIFYFS_SUCCESS) { - rc = (int)UNIFYFS_ERROR_IO; + rc = EIO; break; } @@ -1181,7 +1105,7 @@ int unifyfs_fid_extend(int fid, off_t length) rc = unifyfs_fid_store_fixed_extend(fid, meta, length); } else { /* unknown storage type */ - rc = (int)UNIFYFS_ERROR_IO; + rc = EIO; } return rc; @@ -1192,7 +1116,7 @@ int unifyfs_fid_shrink(int fid, off_t length) { /* TODO: implement this function */ - return UNIFYFS_ERROR_IO; + return EIO; } /* truncate file id to given length, frees resources if length is @@ -1204,7 +1128,7 @@ int unifyfs_fid_truncate(int fid, off_t length) unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); if (meta->is_laminated) { /* Can't truncate a laminated file */ - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } /* determine file storage type */ @@ -1222,7 +1146,7 @@ int unifyfs_fid_truncate(int fid, off_t length) meta->local_size = length; } else { /* unknown storage type */ - return (int)UNIFYFS_ERROR_IO; + return EIO; } return UNIFYFS_SUCCESS; @@ -1243,7 +1167,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, /* check that pathname is within bounds */ size_t pathlen = strlen(path) + 1; if (pathlen > UNIFYFS_MAX_FILENAME) { - return (int) UNIFYFS_ERROR_NAMETOOLONG; + return ENAMETOOLONG; } /* check whether this file already exists */ @@ -1290,7 +1214,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, LOGDBG("file found locally, but seems to be deleted globally. " "invalidating the local cache."); unifyfs_fid_unlink(fid); - return (int) UNIFYFS_ERROR_NOENT; + return ENOENT; } /* for all other three cases below, we need to open the file and allocate a @@ -1305,9 +1229,9 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, /* initialize local metadata for this file */ fid = unifyfs_fid_create_file(path); if (fid < 0) { - /* FIXME: UNIFYFS_ERROR_NFILE or UNIFYFS_ERROR_IO ? */ + /* FIXME: ENFILE or EIO ? */ LOGERR("failed to create a new file %s", path); - return (int) UNIFYFS_ERROR_IO; + return EIO; } /* initialize local storage for this file */ @@ -1315,7 +1239,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, if (ret != UNIFYFS_SUCCESS) { LOGERR("failed to allocate storage space for file %s (fid=%d)", path, fid); - return (int) UNIFYFS_ERROR_IO; + return EIO; } /* initialize global size of file from global metadata */ @@ -1323,15 +1247,15 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } else if (found_local && found_global) { /* file exists and is valid. */ if ((flags & O_CREAT) && (flags & O_EXCL)) { - return (int)UNIFYFS_ERROR_EXIST; + return EEXIST; } if ((flags & O_DIRECTORY) && !unifyfs_fid_is_dir(fid)) { - return (int)UNIFYFS_ERROR_NOTDIR; + return ENOTDIR; } if (!(flags & O_DIRECTORY) && unifyfs_fid_is_dir(fid)) { - return (int)UNIFYFS_ERROR_NOTDIR; + return ENOTDIR; } /* update local metadata from global metadata */ @@ -1352,7 +1276,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, */ if (!(flags & O_CREAT)) { LOGERR("%s does not exist (O_CREAT not given).", path); - return (int) UNIFYFS_ERROR_NOENT; + return ENOENT; } LOGDBG("Creating a new entry for %s.", path); @@ -1365,14 +1289,14 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, fid = unifyfs_fid_create_file(path); if (fid < 0) { LOGERR("Failed to create new file %s", path); - return (int) UNIFYFS_ERROR_NFILE; + return ENFILE; } /* initialize the storage for the file */ int store_rc = unifyfs_fid_store_alloc(fid); if (store_rc != UNIFYFS_SUCCESS) { LOGERR("Failed to create storage for file %s", path); - return (int) UNIFYFS_ERROR_IO; + return EIO; } /* insert file attribute for file in key-value store */ @@ -1380,7 +1304,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, if (ret != UNIFYFS_SUCCESS) { LOGERR("Failed to populate the global meta entry for %s (fid:%d)", path, fid); - return (int) UNIFYFS_ERROR_IO; + return EIO; } } @@ -2094,77 +2018,6 @@ static int unifyfs_init_req_shm(int local_rank_idx, int app_id) return UNIFYFS_SUCCESS; } - -#if defined(UNIFYFS_USE_DOMAIN_SOCKET) -/** - * initialize the client-side socket - * used to communicate with the server-side - * delegators. Each client is serviced by - * one delegator. - * @param proc_id: local process id - * @param l_num_procs_per_node: number - * of ranks on each compute node - * @param l_num_del_per_node: number of server-side - * delegators on the same node - * @return success/error code - */ -static int unifyfs_init_socket(int proc_id, int l_num_procs_per_node, - int l_num_del_per_node) -{ - int rc = -1; - int nprocs_per_del; - int len; - int result; - int flag; - struct sockaddr_un serv_addr; - char tmp_path[UNIFYFS_MAX_FILENAME] = {0}; - char* pmi_path = NULL; - - client_sockfd = socket(AF_UNIX, SOCK_STREAM, 0); - if (client_sockfd < 0) { - LOGERR("socket create failed"); - return -1; - } - - /* calculate delegator assignment */ - nprocs_per_del = l_num_procs_per_node / l_num_del_per_node; - if ((l_num_procs_per_node % l_num_del_per_node) != 0) { - nprocs_per_del++; - } - snprintf(tmp_path, sizeof(tmp_path), "%s.%d.%d", - SOCKET_PATH, getuid(), (proc_id / nprocs_per_del)); - - // lookup domain socket path in key-val store - if (unifyfs_keyval_lookup_local(key_unifyfsd_socket, &pmi_path) == 0) { - memset(tmp_path, 0, sizeof(tmp_path)); - snprintf(tmp_path, sizeof(tmp_path), "%s", pmi_path); - free(pmi_path); - } - - memset(&serv_addr, 0, sizeof(serv_addr)); - serv_addr.sun_family = AF_UNIX; - strcpy(serv_addr.sun_path, tmp_path); - len = sizeof(serv_addr); - result = connect(client_sockfd, (struct sockaddr*)&serv_addr, len); - - /* exit with error if connection is not successful */ - if (result == -1) { - rc = -1; - LOGERR("socket connect failed"); - return rc; - } - - flag = fcntl(client_sockfd, F_GETFL); - fcntl(client_sockfd, F_SETFL, flag | O_NONBLOCK); - - cmd_fd.fd = client_sockfd; - cmd_fd.events = POLLIN | POLLHUP; - cmd_fd.revents = 0; - - return 0; -} -#endif // UNIFYFS_USE_DOMAIN_SOCKET - static int compare_int(const void* a, const void* b) { const int* ptr_a = a; @@ -2521,16 +2374,6 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, unifyfs_shm_unlink(shm_req_name); unifyfs_shm_unlink(shm_recv_name); -#if defined(UNIFYFS_USE_DOMAIN_SOCKET) - /* open a socket to the server */ - rc = unifyfs_init_socket(local_rank_idx, local_rank_cnt, - local_del_cnt); - if (rc < 0) { - LOGERR("failed to initialize socket, rc == %d", rc); - return UNIFYFS_FAILURE; - } -#endif - /* add mount point as a new directory in the file list */ if (unifyfs_get_fid_from_path(prefix) < 0) { /* no entry exists for mount point, so create one */ diff --git a/common/src/Makefile.am b/common/src/Makefile.am index 92c0a6a38..725c509e0 100644 --- a/common/src/Makefile.am +++ b/common/src/Makefile.am @@ -1,14 +1,12 @@ lib_LTLIBRARIES = libunifyfs_common.la -include_HEADERS = unifyfs_const.h err_enumerator.h +include_HEADERS = unifyfs_const.h unifyfs_rc.h libunifyfs_commondir = $(includedir) BASE_SRCS = \ ini.h \ ini.c \ - err_enumerator.h \ - err_enumerator.c \ cm_enumerator.h \ cm_enumerator.c \ rm_enumerator.h \ @@ -31,6 +29,8 @@ BASE_SRCS = \ unifyfs_rpc_util.c \ unifyfs_client_rpcs.h \ unifyfs_server_rpcs.h \ + unifyfs_rc.h \ + unifyfs_rc.c \ unifyfs_runstate.h \ unifyfs_runstate.c \ unifyfs_shm.h \ diff --git a/common/src/err_enumerator.h b/common/src/err_enumerator.h deleted file mode 100644 index 021262ebd..000000000 --- a/common/src/err_enumerator.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2019, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2019, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyFS. - * For details, see https://github.com/LLNL/UnifyFS. - * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. - */ - -/* Copyright (c) 2018 - Michael J. Brim - * - * Enumerator is part of https://github.com/MichaelBrim/tedium - * - * MIT License - See LICENSE.tedium - */ - -#ifndef _UNIFYFS_ERROR_ENUMERATOR_H_ -#define _UNIFYFS_ERROR_ENUMERATOR_H_ -#include - -/** - * @brief enumerator list expanded many times with varied ENUMITEM() definitions - * - * @param item name - * @param item short description - */ -#define UNIFYFS_ERROR_ENUMERATOR \ - ENUMITEM(ACCEPT, "Failed to accept RDMA connection.") \ - ENUMITEM(ADDR, "Failed to parse IP address and port.") \ - ENUMITEM(APPCONFIG, "Failed to initialize application config.") \ - ENUMITEM(ARRAY_BOUNDS, "Array access out of bounds.") \ - ENUMITEM(BADF, "Bad file descriptor.") \ - ENUMITEM(CHANNEL, "Error creating completion channel.") \ - ENUMITEM(CONNECT, "Error in RDMA connect or disconnect.") \ - ENUMITEM(CONTEXT, "Wrong connection context.") \ - ENUMITEM(CQ, "Error creating or polling completion queue.") \ - ENUMITEM(DBG, "Failed to open/close debug file.") \ - ENUMITEM(EVENT_UNKNOWN, "Unknown event detected.") \ - ENUMITEM(EXIST, "File or directory exists.") \ - ENUMITEM(EXIT, "Error - remote peer exited.") \ - ENUMITEM(FAILURE, "General failure.") \ - ENUMITEM(FBIG, "File too large.") \ - ENUMITEM(FILE, "File operation error.") \ - ENUMITEM(GENERAL, "General system call error.") \ - ENUMITEM(INVAL, "Invalid argument.") \ - ENUMITEM(IO, "Generic I/O error.") \ - ENUMITEM(ISDIR, "Invalid operation for directory.") \ - ENUMITEM(MARGO, "Mercury/Argobots operation error.") \ - ENUMITEM(MDHIM, "MDHIM operation error.") \ - ENUMITEM(MDINIT, "MDHIM initialization error.") \ - ENUMITEM(NAMETOOLONG, "Filename is too long.") \ - ENUMITEM(NFILE, "Too many open files.") \ - ENUMITEM(NOENT, "No such file or directory.") \ - ENUMITEM(NOENV, "Environment variable is not defined.") \ - ENUMITEM(NOMEM, "Error in memory allocation/free.") \ - ENUMITEM(NOSPC, "No space left on device.") \ - ENUMITEM(NOTDIR, "Not a directory.") \ - ENUMITEM(OVERFLOW, "Value too large for data type.") \ - ENUMITEM(PD, "Error creating PD.") \ - ENUMITEM(PIPE, "Pipe error.") \ - ENUMITEM(PMIX, "PMIx error.") \ - ENUMITEM(POLL, "Error on poll.") \ - ENUMITEM(POSTRECV, "Failed to post receive operation.") \ - ENUMITEM(POSTSEND, "Failed to post send operation.") \ - ENUMITEM(QP, "Error creating or destroying QP.") \ - ENUMITEM(READ, "Read error.") \ - ENUMITEM(RECV, "Receive error.") \ - ENUMITEM(REGMEM, "Memory [de]registration failure.") \ - ENUMITEM(RM_INIT, "Failed to init request manager.") \ - ENUMITEM(RM_RECV, "Fail to receive data in request manager.") \ - ENUMITEM(ROUTE, "Failed to resolve route.") \ - ENUMITEM(SEND, "Send error.") \ - ENUMITEM(SHMEM, "Error on shared memory attach.") \ - ENUMITEM(SOCKET, "Error creating/open socket.") \ - ENUMITEM(SOCKET_FD_EXCEED, "Exceeded max number of connections.") \ - ENUMITEM(SOCK_CMD, "Unknown exception on the remote peer.") \ - ENUMITEM(SOCK_DISCONNECT, "Remote peer disconnected.") \ - ENUMITEM(SOCK_LISTEN, "Exception on listening socket.") \ - ENUMITEM(SOCK_OTHER, "Unknown socket error.") \ - ENUMITEM(THRDINIT, "Thread initialization failure.") \ - ENUMITEM(TIMEOUT, "Error - timed out.") \ - ENUMITEM(WC, "Write completion with error.") \ - ENUMITEM(WRITE, "Write error.") \ - - -#ifdef __cplusplus -extern "C" { -#endif - -/* #define __ELASTERROR if our errno.h doesn't define it for us */ -#ifndef __ELASTERROR -#define __ELASTERROR 2000 -#endif - -/** - * @brief enum for error codes - */ -typedef enum { - UNIFYFS_INVALID_ERROR = -2, - UNIFYFS_FAILURE = -1, - UNIFYFS_SUCCESS = 0, - /* Start our error numbers after the standard errno.h ones */ - UNIFRFS_START_OF_ERRORS = __ELASTERROR, -#define ENUMITEM(name, desc) \ - UNIFYFS_ERROR_ ## name, - UNIFYFS_ERROR_ENUMERATOR -#undef ENUMITEM - UNIFYFS_ERROR_MAX -} unifyfs_error_e; - -/** - * @brief get C-string for given error enum value - */ -const char *unifyfs_error_enum_str(unifyfs_error_e e); - -/** - * @brief get description for given error enum value - */ -const char *unifyfs_error_enum_description(unifyfs_error_e e); - -/** - * @brief check validity of given error enum value - */ -int check_valid_unifyfs_error_enum(unifyfs_error_e e); - -/** - * @brief get enum value for given error C-string - */ -unifyfs_error_e unifyfs_error_enum_from_str(const char *s); - -#ifdef __cplusplus -} /* extern C */ -#endif - -#endif /* UNIFYFS_ERROR_ENUMERATOR_H */ diff --git a/common/src/unifyfs_configurator.c b/common/src/unifyfs_configurator.c index 56e1bba34..3b7dbb1c0 100644 --- a/common/src/unifyfs_configurator.c +++ b/common/src/unifyfs_configurator.c @@ -58,7 +58,7 @@ int unifyfs_config_init(unifyfs_cfg_t *cfg, char *syscfg = NULL; if (cfg == NULL) - return -1; + return EINVAL; memset((void *)cfg, 0, sizeof(unifyfs_cfg_t)); @@ -101,14 +101,14 @@ int unifyfs_config_init(unifyfs_cfg_t *cfg, if (rc) return rc; - return 0; + return (int)UNIFYFS_SUCCESS; } // cleanup allocated state int unifyfs_config_fini(unifyfs_cfg_t *cfg) { if (cfg == NULL) - return -1; + return EINVAL; #define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ if (cfg->sec##_##key != NULL) { \ @@ -146,7 +146,7 @@ int unifyfs_config_fini(unifyfs_cfg_t *cfg) #undef UNIFYFS_CFG_MULTI #undef UNIFYFS_CFG_MULTI_CLI - return 0; + return (int)UNIFYFS_SUCCESS; } // print configuration to specified file (or stderr) @@ -266,7 +266,7 @@ int unifyfs_config_set_defaults(unifyfs_cfg_t *cfg) char *val; if (cfg == NULL) - return -1; + return EINVAL; #define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ val = stringify(dv); \ @@ -292,7 +292,7 @@ int unifyfs_config_set_defaults(unifyfs_cfg_t *cfg) #undef UNIFYFS_CFG_MULTI #undef UNIFYFS_CFG_MULTI_CLI - return 0; + return (int)UNIFYFS_SUCCESS; } @@ -363,7 +363,7 @@ int unifyfs_config_process_cli_args(unifyfs_cfg_t *cfg, extern int optind, optopt; if (cfg == NULL) - return -1; + return EINVAL; // setup short_opts and cli_options memset((void *)short_opts, 0, sizeof(short_opts)); @@ -461,9 +461,9 @@ int unifyfs_config_process_cli_args(unifyfs_cfg_t *cfg, } if (!usage_err) - rc = 0; + rc = (int)UNIFYFS_SUCCESS; else { - rc = -1; + rc = (int)UNIFYFS_FAILURE; unifyfs_config_cli_usage_error(argv[0], errmsg); } @@ -512,7 +512,7 @@ int unifyfs_config_process_environ(unifyfs_cfg_t *cfg) char *envval; if (cfg == NULL) - return -1; + return EINVAL; #define UNIFYFS_CFG(sec, key, typ, dv, desc, vfn) \ @@ -559,7 +559,7 @@ int unifyfs_config_process_environ(unifyfs_cfg_t *cfg) #undef UNIFYFS_CFG_MULTI #undef UNIFYFS_CFG_MULTI_CLI - return 0; + return (int)UNIFYFS_SUCCESS; } // inih callback handler @@ -611,7 +611,7 @@ int inih_config_handler(void *user, cfg->sec##_##key[cfg->n_##sec##_##key++] = strdup(val); \ } - UNIFYFS_CONFIGS; +UNIFYFS_CONFIGS #undef UNIFYFS_CFG #undef UNIFYFS_CFG_CLI #undef UNIFYFS_CFG_MULTI @@ -636,7 +636,7 @@ int unifyfs_config_process_ini_file(unifyfs_cfg_t *cfg, inih_rc = ini_parse(file, inih_config_handler, cfg); switch (inih_rc) { case 0: - rc = 0; + rc = (int)UNIFYFS_SUCCESS; break; case -1: snprintf(errmsg, sizeof(errmsg), @@ -662,7 +662,7 @@ int unifyfs_config_process_ini_file(unifyfs_cfg_t *cfg, snprintf(errmsg, sizeof(errmsg), "failed to parse config file %s", file); - rc = EINVAL; + rc = (int)UNIFYFS_ERROR_BADCONFIG; fprintf(stderr, "UNIFYFS CONFIG ERROR: %s\n", errmsg); break; } @@ -697,7 +697,7 @@ int validate_value(const char *section, // validate configuration int unifyfs_config_validate(unifyfs_cfg_t *cfg) { - int rc = 0; + int rc = (int)UNIFYFS_SUCCESS; int vrc; char *new_val = NULL; diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h index cdca51240..d11301045 100644 --- a/common/src/unifyfs_const.h +++ b/common/src/unifyfs_const.h @@ -30,9 +30,8 @@ #ifndef UNIFYFS_CONST_H #define UNIFYFS_CONST_H -/* ********************** ERROR CODES ************************ */ -#include "err_enumerator.h" -#define ULFS_SUCCESS ((int)UNIFYFS_SUCCESS) +/* ********************** RETURN CODES ************************ */ +#include "unifyfs_rc.h" /* ********************** STRING CONSTANTS ************************ */ #define DEFAULT_INTERFACE "ib0" diff --git a/common/src/unifyfs_keyval.c b/common/src/unifyfs_keyval.c index 8bdcc02d6..caaeda29c 100644 --- a/common/src/unifyfs_keyval.c +++ b/common/src/unifyfs_keyval.c @@ -143,7 +143,7 @@ static int unifyfs_pmi2_init(void) if (rc != PMI2_SUCCESS) { unifyfs_pmi2_errstr(rc); LOGERR("PMI2_Init() failed: %s", pmi2_errstr); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } pmi_world_rank = rank; pmi_world_nprocs = nprocs; @@ -155,7 +155,7 @@ static int unifyfs_pmi2_init(void) if (rc != PMI2_SUCCESS) { unifyfs_pmi2_errstr(rc); LOGERR("PMI2_Job_GetRank() failed: %s", pmi2_errstr); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } else { pmi_world_rank = rank; } @@ -185,7 +185,7 @@ static int unifyfs_pmi2_init(void) if (rc != PMI2_SUCCESS) { unifyfs_pmi2_errstr(rc); LOGERR("PMI2_Job_GetId() failed: %s", pmi2_errstr); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } kv_myrank = pmi_world_rank; @@ -209,7 +209,7 @@ static int unifyfs_pmi2_fini(void) if (rc != PMI2_SUCCESS) { unifyfs_pmi2_errstr(rc); LOGERR("PMI2_Finalize() failed: %s", pmi2_errstr); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } pmi2_need_finalize = 0; pmi2_initialized = 0; @@ -227,7 +227,7 @@ static int unifyfs_pmi2_lookup(const char* key, char pmi2_val[PMI2_MAX_VALLEN] = {0}; if (!pmi2_initialized) { - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } strncpy(pmi2_key, key, sizeof(pmi2_key)); @@ -237,7 +237,7 @@ static int unifyfs_pmi2_lookup(const char* key, if (rc != PMI2_SUCCESS) { unifyfs_pmi2_errstr(rc); LOGERR("PMI2_KVS_Get(%s) failed: %s", key, pmi2_errstr); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } *oval = strdup(pmi2_val); return (int)UNIFYFS_SUCCESS; @@ -252,7 +252,7 @@ static int unifyfs_pmi2_publish(const char* key, char pmi2_val[PMI2_MAX_VALLEN] = {0}; if (!pmi2_initialized) { - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } strncpy(pmi2_key, key, sizeof(pmi2_key)); @@ -261,7 +261,7 @@ static int unifyfs_pmi2_publish(const char* key, if (rc != PMI2_SUCCESS) { unifyfs_pmi2_errstr(rc); LOGERR("PMI2_KVS_Put(%s) failed: %s", key, pmi2_errstr); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } return (int)UNIFYFS_SUCCESS; } @@ -274,7 +274,7 @@ static int unifyfs_pmi2_fence(void) if (rc != PMI2_SUCCESS) { unifyfs_pmi2_errstr(rc); LOGERR("PMI2_KVS_Fence() failed: %s", pmi2_errstr); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } return (int)UNIFYFS_SUCCESS; } @@ -307,7 +307,7 @@ static int unifyfs_pmix_init(void) rc = PMIx_Init(&pmix_myproc, NULL, 0); if (rc != PMIX_SUCCESS) { LOGERR("PMIx_Init() failed: %s", PMIx_Error_string(rc)); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } kv_max_keylen = PMIX_MAX_KEYLEN; @@ -321,7 +321,7 @@ static int unifyfs_pmix_init(void) if (rc != PMIX_SUCCESS) { LOGERR("PMIx rank %d: PMIx_Get(UNIV_SIZE) failed: %s", pmix_myproc.rank, PMIx_Error_string(rc)); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } pmix_univ_nprocs = (size_t) valp->data.uint32; @@ -365,7 +365,7 @@ static int unifyfs_pmix_fini(void) if (rc != PMIX_SUCCESS) { LOGERR("PMIx rank %d: PMIx_Finalize() failed: %s", pmix_myproc.rank, PMIx_Error_string(rc)); - rc = (int) UNIFYFS_FAILURE; + rc = (int) UNIFYFS_ERROR_PMI; } else { PMIX_PROC_DESTRUCT(&pmix_myproc); pmix_initialized = 0; @@ -385,7 +385,7 @@ static int unifyfs_pmix_lookup(const char* key, char pmix_key[PMIX_MAX_KEYLEN+1]; if (!pmix_initialized) { - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } /* set key to lookup */ @@ -407,7 +407,7 @@ static int unifyfs_pmix_lookup(const char* key, LOGERR("PMIx rank %d: PMIx_Lookup(%s) failed: %s", pmix_myproc.rank, pmix_key, PMIx_Error_string(rc)); *oval = NULL; - rc = (int)UNIFYFS_FAILURE; + rc = (int)UNIFYFS_ERROR_PMI; } else { if (pdata[0].value.data.string != NULL) { *oval = strdup(pdata[0].value.data.string); @@ -416,7 +416,7 @@ static int unifyfs_pmix_lookup(const char* key, LOGERR("PMIx rank %d: PMIx_Lookup(%s) returned NULL string", pmix_myproc.rank, pmix_key); *oval = NULL; - rc = (int)UNIFYFS_FAILURE; + rc = (int)UNIFYFS_ERROR_PMI; } } /* cleanup */ @@ -436,7 +436,7 @@ static int unifyfs_pmix_publish(const char* key, char pmix_key[PMIX_MAX_KEYLEN+1]; if (!pmix_initialized) { - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_PMI; } /* set key-val and modify publish behavior */ @@ -452,7 +452,7 @@ static int unifyfs_pmix_publish(const char* key, if (rc != PMIX_SUCCESS) { LOGERR("PMIx rank %d: PMIx_Publish failed: %s", pmix_myproc.rank, PMIx_Error_string(rc)); - rc = (int)UNIFYFS_FAILURE; + rc = (int)UNIFYFS_ERROR_PMI; } else { rc = (int)UNIFYFS_SUCCESS; } @@ -468,7 +468,7 @@ static int unifyfs_pmix_fence(void) if (rc != PMIX_SUCCESS) { LOGERR("PMIx rank %d: PMIx_Fence failed: %s", pmix_myproc.rank, PMIx_Error_string(rc)); - rc = (int)UNIFYFS_FAILURE; + rc = (int)UNIFYFS_ERROR_PMI; } else { rc = (int)UNIFYFS_SUCCESS; } @@ -493,7 +493,7 @@ static int unifyfs_fskv_init(unifyfs_cfg_t* cfg) if (NULL == cfg) { LOGERR("NULL config"); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } memset(localfs_kvdir, 0, sizeof(localfs_kvdir)); @@ -503,7 +503,7 @@ static int unifyfs_fskv_init(unifyfs_cfg_t* cfg) // find or create local kvstore directory if (NULL == cfg->runstate_dir) { LOGERR("local file system k-v store requires cfg.runstate_dir"); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_BADCONFIG; } snprintf(localfs_kvdir, sizeof(localfs_kvdir), "%s/kvstore", cfg->runstate_dir); @@ -517,11 +517,11 @@ static int unifyfs_fskv_init(unifyfs_cfg_t* cfg) if ((rc != 0) && (err != EEXIST)) { LOGERR("failed to create local kvstore directory %s - %s", localfs_kvdir, strerror(err)); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } } else { LOGERR("missing local kvstore directory %s", localfs_kvdir); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } } @@ -538,7 +538,7 @@ static int unifyfs_fskv_init(unifyfs_cfg_t* cfg) if ((rc != 0) && (err != EEXIST)) { LOGERR("failed to create kvstore directory %s - %s", sharedfs_kvdir, strerror(err)); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } } @@ -554,7 +554,7 @@ static int unifyfs_fskv_init(unifyfs_cfg_t* cfg) if ((rc != 0) && (err != EEXIST)) { LOGERR("failed to create rank kvstore directory %s - %s", sharedfs_rank_kvdir, strerror(err)); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } } have_sharedfs_kvstore = 1; @@ -583,7 +583,7 @@ static int unifyfs_fskv_fini(void) DIR* lkv = opendir(localfs_kvdir); if (NULL == lkv) { LOGERR("failed to opendir(%s)", localfs_kvdir); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } while (NULL != (de = readdir(lkv))) { if ((0 == strcmp(".", de->d_name)) || @@ -606,7 +606,7 @@ static int unifyfs_fskv_fini(void) if (rc != 0) { LOGERR("failed to remove local kvstore dir %s", sharedfs_rank_kvdir); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } } @@ -620,7 +620,7 @@ static int unifyfs_fskv_fini(void) DIR* rkv = opendir(sharedfs_rank_kvdir); if (NULL == rkv) { LOGERR("failed to opendir(%s)", sharedfs_rank_kvdir); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } while (NULL != (de = readdir(rkv))) { if ((0 == strcmp(".", de->d_name)) || @@ -643,7 +643,7 @@ static int unifyfs_fskv_fini(void) if (rc != 0) { LOGERR("failed to remove rank-specific kvstore dir %s", sharedfs_rank_kvdir); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } } @@ -656,7 +656,7 @@ static int unifyfs_fskv_fini(void) DIR* skv = opendir(sharedfs_kvdir); if (NULL == skv) { LOGERR("failed to opendir(%s)", sharedfs_kvdir); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } while (NULL != (de = readdir(skv))) { if ((0 == strcmp(".", de->d_name)) || @@ -672,7 +672,7 @@ static int unifyfs_fskv_fini(void) if (rc != 0) { LOGERR("failed to remove sharedfs kvstore dir %s", sharedfs_kvdir); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } } } @@ -694,7 +694,7 @@ static int unifyfs_fskv_lookup_local(const char* key, kvf = fopen(kvfile, "r"); if (NULL == kvf) { LOGERR("failed to open kvstore entry %s", kvfile); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } memset(kvalue, 0, sizeof(kvalue)); fscanf(kvf, "%s\n", kvalue); @@ -713,7 +713,7 @@ static int unifyfs_fskv_lookup_remote(int rank, char kvalue[kv_max_vallen]; if (!have_sharedfs_kvstore) { - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } snprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%d/%s", @@ -721,7 +721,7 @@ static int unifyfs_fskv_lookup_remote(int rank, kvf = fopen(rank_kvfile, "r"); if (NULL == kvf) { LOGERR("failed to open kvstore entry %s", rank_kvfile); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } memset(kvalue, 0, sizeof(kvalue)); fscanf(kvf, "%s\n", kvalue); @@ -743,7 +743,7 @@ static int unifyfs_fskv_publish_local(const char* key, kvf = fopen(kvfile, "w"); if (NULL == kvf) { LOGERR("failed to create kvstore entry %s", kvfile); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } fprintf(kvf, "%s\n", val); fclose(kvf); @@ -758,7 +758,7 @@ static int unifyfs_fskv_publish_remote(const char* key, char rank_kvfile[UNIFYFS_MAX_FILENAME]; if (!have_sharedfs_kvstore) { - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } snprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%s", @@ -766,7 +766,7 @@ static int unifyfs_fskv_publish_remote(const char* key, kvf = fopen(rank_kvfile, "w"); if (NULL == kvf) { LOGERR("failed to create kvstore entry %s", rank_kvfile); - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } fprintf(kvf, "%s\n", val); fclose(kvf); @@ -777,7 +777,7 @@ static int unifyfs_fskv_publish_remote(const char* key, static int unifyfs_fskv_fence(void) { if (!have_sharedfs_kvstore) { - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } if (1 == kv_nranks) { @@ -877,7 +877,7 @@ int unifyfs_keyval_lookup_local(const char* key, if ((NULL == key) || (NULL == oval)) { LOGERR("NULL parameter"); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } if (!kv_initialized) { @@ -891,7 +891,7 @@ int unifyfs_keyval_lookup_local(const char* key, if (len >= (kv_max_keylen - 1)) { LOGERR("length of key (%zd) exceeds max %zd", len, kv_max_keylen); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } // do the lookup @@ -911,7 +911,7 @@ int unifyfs_keyval_lookup_remote(int rank, if ((NULL == key) || (NULL == oval)) { LOGERR("NULL parameter"); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } if (!kv_initialized) { @@ -926,7 +926,7 @@ int unifyfs_keyval_lookup_remote(int rank, if (len >= (kv_max_keylen - 1)) { LOGERR("length of key (%zd) exceeds max %zd", len, kv_max_keylen); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } // generate full key, which includes remote host @@ -958,26 +958,26 @@ int unifyfs_keyval_publish_local(const char* key, int rc; if (!kv_initialized) { - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } if ((key == NULL) || (val == NULL)) { LOGERR("NULL key or value"); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } size_t len = strlen(key); if (len >= (kv_max_keylen - 1)) { LOGERR("length of key (%zd) exceeds max %zd", len, kv_max_keylen); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } len = strlen(val); if (len >= kv_max_vallen) { LOGERR("length of val (%zd) exceeds max %zd", len, kv_max_vallen); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } // publish it @@ -997,12 +997,12 @@ int unifyfs_keyval_publish_remote(const char* key, int rc; if (!kv_initialized) { - return (int)UNIFYFS_FAILURE; + return (int)UNIFYFS_ERROR_KEYVAL; } if ((key == NULL) || (val == NULL)) { LOGERR("NULL key or value"); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } // NOTE: assumes rank value fits in 10 characters @@ -1010,7 +1010,7 @@ int unifyfs_keyval_publish_remote(const char* key, if (len >= (kv_max_keylen - 1)) { LOGERR("length of key (%zd) exceeds max %zd", len, kv_max_keylen); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } // generate full key, which includes remote host diff --git a/common/src/unifyfs_log.c b/common/src/unifyfs_log.c index bc087a11a..01d5ecf93 100644 --- a/common/src/unifyfs_log.c +++ b/common/src/unifyfs_log.c @@ -43,7 +43,7 @@ time_t unifyfs_log_time; struct tm* unifyfs_log_ltime; char unifyfs_log_timestamp[256]; -/* used to reduce source file name length */ +/* used to reduce source file pathname length */ size_t unifyfs_log_source_base_len; // = 0 static const char* this_file = __FILE__; @@ -60,17 +60,20 @@ int unifyfs_log_open(const char* file) } } - /* stderr is our default log file stream */ - unifyfs_log_stream = stderr; + if (NULL == unifyfs_log_stream) { + /* stderr is the default log stream */ + unifyfs_log_stream = stderr; + } if (NULL != file) { FILE* logf = fopen(file, "a"); - if (NULL != logf) { - unifyfs_log_stream = logf; + if (logf == NULL) { + return ENOENT; } else { - return EINVAL; + unifyfs_log_stream = logf; } } + return (int)UNIFYFS_SUCCESS; } @@ -82,6 +85,8 @@ int unifyfs_log_close(void) if (NULL != unifyfs_log_stream) { if (unifyfs_log_stream != stderr) { fclose(unifyfs_log_stream); + + /* revert to stderr for any future log messages */ unifyfs_log_stream = stderr; } } @@ -93,7 +98,5 @@ void unifyfs_set_log_level(unifyfs_log_level_t lvl) { if (lvl < LOG_LEVEL_MAX) { unifyfs_log_level = lvl; - } else { - LOGERR("invalid log level %d", (int)lvl); } } diff --git a/common/src/err_enumerator.c b/common/src/unifyfs_rc.c similarity index 56% rename from common/src/err_enumerator.c rename to common/src/unifyfs_rc.c index 8f978f175..83a92d868 100644 --- a/common/src/err_enumerator.c +++ b/common/src/unifyfs_rc.c @@ -34,21 +34,24 @@ * MIT License - See LICENSE.tedium */ -#include "err_enumerator.h" -#include +#include #include +#include #include +#include "unifyfs_rc.h" + + /* c-strings for enum names */ -#define ENUMITEM(name, desc) \ - const char *UNIFYFS_ERROR_ ## name ## _NAME_STR = #name; +#define ENUMITEM(name, desc) \ + const char* UNIFYFS_ERROR_ ## name ## _NAME_STR = #name; UNIFYFS_ERROR_ENUMERATOR #undef ENUMITEM -const char *unifyfs_error_enum_str(unifyfs_error_e e) +const char* unifyfs_rc_enum_str(unifyfs_rc rc) { - switch (e) { + switch (rc) { case UNIFYFS_FAILURE: return "UNIFYFS_FAILURE"; case UNIFYFS_SUCCESS: @@ -56,7 +59,7 @@ const char *unifyfs_error_enum_str(unifyfs_error_e e) #define ENUMITEM(name, desc) \ case UNIFYFS_ERROR_ ## name: \ return UNIFYFS_ERROR_ ## name ## _NAME_STR; - UNIFYFS_ERROR_ENUMERATOR +UNIFYFS_ERROR_ENUMERATOR #undef ENUMITEM default : break; @@ -66,14 +69,16 @@ const char *unifyfs_error_enum_str(unifyfs_error_e e) /* c-strings for enum descriptions */ -#define ENUMITEM(name, desc) \ - const char *UNIFYFS_ERROR_ ## name ## _DESC_STR = #desc; +#define ENUMITEM(name, desc) \ + const char* UNIFYFS_ERROR_ ## name ## _DESC_STR = #desc; UNIFYFS_ERROR_ENUMERATOR #undef ENUMITEM -const char *unifyfs_error_enum_description(unifyfs_error_e e) +char posix_errstr[1024]; + +const char* unifyfs_rc_enum_description(unifyfs_rc rc) { - switch (e) { + switch (rc) { case UNIFYFS_FAILURE: return "Failure"; case UNIFYFS_SUCCESS: @@ -81,33 +86,55 @@ const char *unifyfs_error_enum_description(unifyfs_error_e e) #define ENUMITEM(name, desc) \ case UNIFYFS_ERROR_ ## name: \ return UNIFYFS_ERROR_ ## name ## _DESC_STR; - UNIFYFS_ERROR_ENUMERATOR +UNIFYFS_ERROR_ENUMERATOR #undef ENUMITEM - default : - break; + default: + /* assume it's a POSIX errno value */ + snprintf(posix_errstr, sizeof(posix_errstr), "%s", + strerror((int)rc)); + return (const char*)posix_errstr; } return NULL; } -unifyfs_error_e unifyfs_error_enum_from_str(const char *s) +unifyfs_rc unifyfs_rc_enum_from_str(const char* s) { - if (0) - ; + if (strcmp(s, "Success") == 0) { + return UNIFYFS_SUCCESS; + } else if (strcmp(s, "Failure") == 0) { + return UNIFYFS_FAILURE; + } #define ENUMITEM(name, desc) \ - else if (strcmp(s, #name) == 0) \ - return UNIFYFS_ERROR_ ## name; - UNIFYFS_ERROR_ENUMERATOR; + else if (strcmp(s, #name) == 0) { \ + return UNIFYFS_ERROR_ ## name; \ + } +UNIFYFS_ERROR_ENUMERATOR #undef ENUMITEM - return UNIFYFS_INVALID_ERROR; + return UNIFYFS_INVALID_RC; } /* validity check */ - -int check_valid_unifyfs_error_enum(unifyfs_error_e e) +int check_valid_unifyfs_rc_enum(unifyfs_rc rc) { - return ((e > UNIFYFS_INVALID_ERROR) && - (e < UNIFYFS_ERROR_MAX) && - (unifyfs_error_enum_str(e) != NULL)); + return ((rc > UNIFYFS_INVALID_RC) && + (rc < UNIFYFS_END_ERRORS) && + (unifyfs_rc_enum_str(rc) != NULL)); } +/* convert to an errno value */ +int unifyfs_rc_errno(unifyfs_rc rc) +{ + if (rc == UNIFYFS_SUCCESS) { + return 0; + } else if (rc == UNIFYFS_INVALID_RC) { + return EINVAL; + } else if ((rc == UNIFYFS_FAILURE) || + ((rc > UNIFYFS_BEGIN_ERRORS) && (rc < UNIFYFS_END_ERRORS))) { + /* none of our custom errors have good errno counterparts, use EIO */ + return EIO; + } else { + /* should be a normal errno value already */ + return (int)rc; + } +} diff --git a/common/src/unifyfs_rc.h b/common/src/unifyfs_rc.h new file mode 100644 index 000000000..fb96635c4 --- /dev/null +++ b/common/src/unifyfs_rc.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* Copyright (c) 2018 - Michael J. Brim + * + * Enumerator is part of https://github.com/MichaelBrim/tedium + * + * MIT License - See LICENSE.tedium + */ + +#ifndef _UNIFYFS_RC_ENUMERATOR_H_ +#define _UNIFYFS_RC_ENUMERATOR_H_ + +#include + +/* #define __ELASTERROR if our errno.h doesn't define it for us */ +#ifndef __ELASTERROR +#define __ELASTERROR 1000 +#endif + +/* NOTE: If POSIX errno.h defines an error code that we can use sensibly, + * don't create a duplicate one for UnifyFS */ + +/** + * @brief enumerator list expanded many times with varied ENUMITEM() definitions + * + * @param item name + * @param item short description + */ +#define UNIFYFS_ERROR_ENUMERATOR \ + ENUMITEM(BADCONFIG, "Configuration has invalid setting") \ + ENUMITEM(KEYVAL, "Key-value store operation error") \ + ENUMITEM(MARGO, "Mercury/Argobots operation error") \ + ENUMITEM(MDHIM, "MDHIM operation error") \ + ENUMITEM(META, "Metadata store operation error") \ + ENUMITEM(NYI, "Not yet implemented") \ + ENUMITEM(PMI, "PMI2/PMIx error") \ + ENUMITEM(SHMEM, "Shared memory region init/access error") \ + ENUMITEM(THRDINIT, "Thread initialization failed") \ + ENUMITEM(TIMEOUT, "Timed out") \ + + +#ifdef __cplusplus +extern "C" { +#endif + + +/** + * @brief enum for UnifyFS return codes + */ +typedef enum { + UNIFYFS_INVALID_RC = -2, + UNIFYFS_FAILURE = -1, + UNIFYFS_SUCCESS = 0, + /* Start our error numbers after the standard errno.h ones */ + UNIFYFS_BEGIN_ERRORS = __ELASTERROR, +#define ENUMITEM(name, desc) \ + UNIFYFS_ERROR_ ## name, +UNIFYFS_ERROR_ENUMERATOR +#undef ENUMITEM + UNIFYFS_END_ERRORS +} unifyfs_rc; + +/** + * @brief get C-string for given error enum value + */ +const char* unifyfs_rc_enum_str(unifyfs_rc rc); + +/** + * @brief get description for given error enum value + */ +const char* unifyfs_rc_enum_description(unifyfs_rc rc); + +/** + * @brief check validity of given error enum value + */ +int check_valid_unifyfs_rc_enum(unifyfs_rc rc); + +/** + * @brief get enum value for given error C-string + */ +unifyfs_rc unifyfs_rc_enum_from_str(const char* s); + +/** + * @brief convert a UnifyFS error to an errno value + */ +int unifyfs_rc_errno(unifyfs_rc rc); + +#ifdef __cplusplus +} /* extern C */ +#endif + +#endif /* UNIFYFS_RC_ENUMERATOR_H */ diff --git a/common/src/unifyfs_runstate.c b/common/src/unifyfs_runstate.c index 660b7e693..db9a75b41 100644 --- a/common/src/unifyfs_runstate.c +++ b/common/src/unifyfs_runstate.c @@ -19,13 +19,13 @@ int unifyfs_read_runstate(unifyfs_cfg_t* cfg, if (cfg == NULL) { LOGERR("NULL config"); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } if (runstate_path == NULL) { if (cfg->runstate_dir == NULL) { LOGERR("bad runstate dir config setting"); - return (int)UNIFYFS_ERROR_APPCONFIG; + return (int)UNIFYFS_ERROR_BADCONFIG; } snprintf(runstate_fname, sizeof(runstate_fname), "%s/%s.%d", cfg->runstate_dir, runstate_file, uid); @@ -36,7 +36,7 @@ int unifyfs_read_runstate(unifyfs_cfg_t* cfg, if (unifyfs_config_process_ini_file(cfg, runstate_fname) != 0) { LOGERR("failed to process runstate file %s", runstate_fname); - rc = (int)UNIFYFS_ERROR_APPCONFIG; + rc = (int)UNIFYFS_ERROR_BADCONFIG; } return rc; @@ -51,7 +51,7 @@ int unifyfs_write_runstate(unifyfs_cfg_t* cfg) if (cfg == NULL) { LOGERR("NULL config"); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } snprintf(runstate_fname, sizeof(runstate_fname), @@ -59,8 +59,8 @@ int unifyfs_write_runstate(unifyfs_cfg_t* cfg) runstate_fp = fopen(runstate_fname, "w"); if (runstate_fp == NULL) { + rc = errno; LOGERR("failed to create file %s", runstate_fname); - rc = (int)UNIFYFS_ERROR_FILE; } else { if ((unifyfs_log_stream != NULL) && (unifyfs_log_level >= LOG_INFO)) { @@ -81,7 +81,7 @@ int unifyfs_clean_runstate(unifyfs_cfg_t* cfg) if (cfg == NULL) { LOGERR("invalid config arg"); - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } snprintf(runstate_fname, sizeof(runstate_fname), @@ -89,8 +89,8 @@ int unifyfs_clean_runstate(unifyfs_cfg_t* cfg) rc = unlink(runstate_fname); if (rc != 0) { + rc = errno; LOGERR("failed to remove file %s", runstate_fname); - rc = (int)UNIFYFS_ERROR_FILE; } return rc; diff --git a/common/src/unifyfs_shm.c b/common/src/unifyfs_shm.c index 8ab912ac9..b5ddceb1a 100644 --- a/common/src/unifyfs_shm.c +++ b/common/src/unifyfs_shm.c @@ -37,8 +37,8 @@ void* unifyfs_shm_alloc(const char* name, size_t size) int fd = shm_open(name, O_RDWR | O_CREAT, 0770); if (fd == -1) { /* failed to open shared memory */ - LOGERR("Failed to open shared memory %s errno=%d (%s)", - name, errno, strerror(errno)); + LOGERR("Failed to open shared memory %s (%s)", + name, strerror(errno)); return NULL; } @@ -48,8 +48,8 @@ void* unifyfs_shm_alloc(const char* name, size_t size) if (ret != 0) { /* failed to set size shared memory */ errno = ret; - LOGERR("posix_fallocate failed for %s errno=%d (%s)", - name, errno, strerror(errno)); + LOGERR("posix_fallocate failed for %s (%s)", + name, strerror(errno)); close(fd); return NULL; } @@ -58,8 +58,8 @@ void* unifyfs_shm_alloc(const char* name, size_t size) ret = ftruncate(fd, size); if (ret == -1) { /* failed to set size of shared memory */ - LOGERR("ftruncate failed for %s errno=%d (%s)", - name, errno, strerror(errno)); + LOGERR("ftruncate failed for %s (%s)", + name, strerror(errno)); close(fd); return NULL; } @@ -71,8 +71,8 @@ void* unifyfs_shm_alloc(const char* name, size_t size) fd, 0); if (addr == MAP_FAILED) { /* failed to open shared memory */ - LOGERR("Failed to mmap shared memory %s errno=%d (%s)", - name, errno, strerror(errno)); + LOGERR("Failed to mmap shared memory %s (%s)", + name, strerror(errno)); close(fd); return NULL; } @@ -82,8 +82,8 @@ void* unifyfs_shm_alloc(const char* name, size_t size) ret = close(fd); if (ret == -1) { /* failed to open shared memory */ - LOGERR("Failed to mmap shared memory %s errno=%d (%s)", - name, errno, strerror(errno)); + LOGERR("Failed to mmap shared memory %s (%s)", + name, strerror(errno)); /* not fatal, so keep going */ } @@ -100,7 +100,7 @@ int unifyfs_shm_free(const char* name, size_t size, void** paddr) { /* check that we got an address (to something) */ if (paddr == NULL) { - return UNIFYFS_FAILURE; + return EINVAL; } /* get address of shared memory region */ @@ -114,8 +114,8 @@ int unifyfs_shm_free(const char* name, size_t size, void** paddr) int rc = munmap(addr, size); if (rc == -1) { /* failed to unmap shared memory */ - LOGERR("Failed to unmap shared memory %s errno=%d (%s)", - name, errno, strerror(errno)); + LOGERR("Failed to unmap shared memory %s (%s)", + name, strerror(errno)); /* not fatal, so keep going */ } @@ -139,8 +139,8 @@ int unifyfs_shm_unlink(const char* name) int err = errno; if (ENOENT != err) { /* failed to remove shared memory */ - LOGERR("Failed to unlink shared memory %s errno=%d (%s)", - name, err, strerror(err)); + LOGERR("Failed to unlink shared memory %s (%s)", + name, strerror(err)); } /* not fatal, so keep going */ } diff --git a/server/src/Makefile.am b/server/src/Makefile.am index 89338dbf6..31c0efae6 100644 --- a/server/src/Makefile.am +++ b/server/src/Makefile.am @@ -12,9 +12,7 @@ libunifyfsd_a_SOURCES = \ unifyfs_request_manager.c \ unifyfs_request_manager.h \ unifyfs_service_manager.c \ - unifyfs_service_manager.h \ - unifyfs_sock.c \ - unifyfs_sock.h + unifyfs_service_manager.h bin_PROGRAMS = unifyfsd diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index 7449409ef..e0eecbcff 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -142,7 +142,7 @@ static int open_log_file(app_config_t* app_config, app_config->spill_log_fds[client_side_id] = open(path, O_RDONLY, 0666); if (app_config->spill_log_fds[client_side_id] < 0) { LOGERR("failed to open spill file %s", path); - return (int)UNIFYFS_ERROR_FILE; + return ENOENT; } /* build name of spill over index file, @@ -159,7 +159,7 @@ static int open_log_file(app_config_t* app_config, open(path, O_RDONLY, 0666); if (app_config->spill_index_log_fds[client_side_id] < 0) { LOGERR("failed to open spill index file %s", path); - return (int)UNIFYFS_ERROR_FILE; + return ENOENT; } return UNIFYFS_SUCCESS; diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index 23a60d99f..2476b552c 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -47,7 +47,6 @@ #include "unifyfs_log.h" #include "unifyfs_meta.h" #include "unifyfs_shm.h" -#include "unifyfs_sock.h" #include #include @@ -170,8 +169,6 @@ typedef struct { typedef int fattr_key_t; -int invert_sock_ids[MAX_NUM_CLIENTS]; - typedef struct { //char* hostname; char* margo_svr_addr_str; diff --git a/server/src/unifyfs_init.c b/server/src/unifyfs_init.c index c55993ea9..ed920b6fc 100644 --- a/server/src/unifyfs_init.c +++ b/server/src/unifyfs_init.c @@ -56,8 +56,6 @@ server_info_t* glb_servers; // array of server_info_t arraylist_t* app_config_list; -int invert_sock_ids[MAX_NUM_CLIENTS]; /*records app_id for each sock_id*/ - unifyfs_cfg_t server_cfg; static int unifyfs_exit(void); @@ -192,7 +190,7 @@ static int allocate_servers(size_t n_servers) glb_servers = (server_info_t*) calloc(n_servers, sizeof(server_info_t)); if (NULL == glb_servers) { LOGERR("failed to allocate server_info array"); - return (int)UNIFYFS_ERROR_NOMEM; + return ENOMEM; } return (int)UNIFYFS_SUCCESS; } @@ -205,7 +203,7 @@ static int process_servers_hostfile(const char* hostfile) char hostbuf[UNIFYFS_MAX_HOSTNAME+1]; if (NULL == hostfile) { - return (int)UNIFYFS_ERROR_INVAL; + return EINVAL; } fp = fopen(hostfile, "r"); if (!fp) { @@ -296,13 +294,13 @@ int main(int argc, char* argv[]) app_config_list = arraylist_create(); if (app_config_list == NULL) { - LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_NOMEM)); + LOGERR("%s", unifyfs_rc_enum_description(ENOMEM)); exit(1); } rm_thrd_list = arraylist_create(); if (rm_thrd_list == NULL) { - LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_NOMEM)); + LOGERR("%s", unifyfs_rc_enum_description(ENOMEM)); exit(1); } @@ -316,7 +314,7 @@ int main(int argc, char* argv[]) server_cfg.log_dir, server_cfg.log_file, glb_host); rc = unifyfs_log_open(dbg_fname); if (rc != UNIFYFS_SUCCESS) { - LOGERR("%s", unifyfs_error_enum_description((unifyfs_error_e)rc)); + LOGERR("%s", unifyfs_rc_enum_description((unifyfs_rc)rc)); } if (NULL != server_cfg.server_hostfile) { @@ -364,66 +362,36 @@ int main(int argc, char* argv[]) rc = configurator_bool_val(server_cfg.margo_tcp, &margo_use_tcp); rc = margo_server_rpc_init(); if (rc != UNIFYFS_SUCCESS) { - LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_MARGO)); + LOGERR("%s", unifyfs_rc_enum_description(rc)); exit(1); } LOGDBG("connecting rpc servers"); rc = margo_connect_servers(); if (rc != UNIFYFS_SUCCESS) { - LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_MARGO)); - exit(1); - } - -#if defined(UNIFYFS_USE_DOMAIN_SOCKET) - int srvr_rank_idx = 0; -#if defined(UNIFYFS_MULTIPLE_DELEGATORS) - rc = CountTasksPerNode(glb_pmi_rank, glb_pmi_size); - if (rc < 0) { - exit(1); - } - srvr_rank_idx = find_rank_idx(glb_pmi_rank); -#endif // UNIFYFS_MULTIPLE_DELEGATORS - LOGDBG("creating server domain socket"); - rc = sock_init_server(srvr_rank_idx); - if (rc != 0) { - LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_SOCKET)); + LOGERR("%s", unifyfs_rc_enum_description(rc)); exit(1); } -#endif // UNIFYFS_USE_DOMAIN_SOCKET /* launch the service manager */ LOGDBG("launching service manager thread"); rc = svcmgr_init(); if (rc != (int)UNIFYFS_SUCCESS) { - LOGERR("launch failed - %s", unifyfs_error_enum_description(rc)); + LOGERR("launch failed - %s", unifyfs_rc_enum_description(rc)); exit(1); } LOGDBG("initializing metadata store"); rc = meta_init_store(&server_cfg); if (rc != 0) { - LOGERR("%s", unifyfs_error_enum_description(UNIFYFS_ERROR_MDINIT)); + LOGERR("%s", unifyfs_rc_enum_description(rc)); exit(1); } LOGDBG("finished service initialization"); while (1) { -#if defined(UNIFYFS_USE_DOMAIN_SOCKET) - int timeout_ms = 2000; /* in milliseconds */ - rc = sock_wait_cmd(timeout_ms); - if (rc != UNIFYFS_SUCCESS) { - // we ignore disconnects, they are expected - if (rc != UNIFYFS_ERROR_SOCK_DISCONNECT) { - LOGDBG("domain socket error %s", - unifyfs_error_enum_description((unifyfs_error_e)rc)); - time_to_exit = 1; - } - } -#else sleep(1); -#endif // UNIFYFS_USE_DOMAIN_SOCKET if (time_to_exit) { LOGDBG("starting service shutdown"); break; @@ -621,12 +589,6 @@ static int unifyfs_exit(void) LOGDBG("stopping rpc service"); margo_server_rpc_finalize(); -#if defined(UNIFYFS_USE_DOMAIN_SOCKET) - /* close remaining sockets */ - LOGDBG("closing sockets"); - sock_sanitize(); -#endif - /* finalize kvstore service*/ LOGDBG("finalizing kvstore service"); unifyfs_keyval_fini(); diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index eb365a578..86163376d 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -304,7 +304,7 @@ static int release_read_req(reqmgr_thrd_t* thrd_ctrl, thrd_ctrl->num_read_reqs, thrd_ctrl->next_rdreq_ndx); debug_print_read_req(rdreq); } else { - rc = UNIFYFS_ERROR_INVAL; + rc = EINVAL; LOGERR("NULL read_req"); } RM_UNLOCK(thrd_ctrl); @@ -360,7 +360,7 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, calloc((size_t)num_vals, sizeof(chunk_read_req_t)); if (NULL == all_chunk_reads) { LOGERR("failed to allocate chunk-reads array"); - return UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* wait on lock before we attach new array to global variable */ @@ -408,7 +408,7 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, if (NULL == rdreq->remote_reads) { LOGERR("failed to allocate remote-reads array"); RM_UNLOCK(thrd_ctrl); - return UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* get pointer to start of chunk read request array */ @@ -660,7 +660,7 @@ static int truncate_delete_keys( (NULL == unifyfs_key_lens) || (NULL == unifyfs_val_lens)) { LOGERR("failed to allocate memory for file extents"); - ret = (int)UNIFYFS_ERROR_NOMEM; + ret = ENOMEM; goto truncate_delete_exit; } @@ -758,7 +758,7 @@ static int truncate_rewrite_keys( (NULL == unifyfs_key_lens) || (NULL == unifyfs_val_lens)) { LOGERR("failed to allocate memory for file extents"); - ret = (int)UNIFYFS_ERROR_NOMEM; + ret = ENOMEM; goto truncate_rewrite_exit; } @@ -1024,7 +1024,7 @@ int rm_cmd_laminate( mode_t mode = (mode_t) attr.mode; if ((mode & S_IFMT) != S_IFREG) { /* item is not a regular file */ - return UNIFYFS_ERROR_INVAL; + return EINVAL; } /* lookup current file size */ @@ -1064,7 +1064,7 @@ int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, free(keyvals); keyvals = NULL; } - return UNIFYFS_ERROR_NOMEM; + return ENOMEM; } if (UNIFYFS_SUCCESS != rc) { @@ -1286,7 +1286,7 @@ int rm_cmd_read( size_t slices = num_slices(offset, length); if (slices >= UNIFYFS_MAX_SPLIT_CNT) { LOGERR("Error allocating buffers"); - return (int)UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* allocate key storage */ @@ -1298,7 +1298,7 @@ int rm_cmd_read( // this is a fatal error // TODO: we need better error handling LOGERR("Error allocating buffers"); - return (int)UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* split range of read request at boundaries used for @@ -1364,7 +1364,7 @@ int rm_cmd_mread( } if (slices >= UNIFYFS_MAX_SPLIT_CNT) { LOGERR("Error allocating buffers"); - return (int)UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* allocate key storage */ @@ -1376,7 +1376,7 @@ int rm_cmd_mread( // this is a fatal error // TODO: we need better error handling LOGERR("Error allocating buffers"); - return (int)UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* get chunks corresponding to requested client read extents */ @@ -1540,7 +1540,7 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) } if (slices >= UNIFYFS_MAX_SPLIT_CNT) { LOGERR("Error allocating buffers"); - return (int)UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* pointers to memory we'll dynamically allocate for file extents */ @@ -1560,7 +1560,7 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) (NULL == key_lens) || (NULL == val_lens)) { LOGERR("failed to allocate memory for file extents"); - ret = (int)UNIFYFS_ERROR_NOMEM; + ret = ENOMEM; goto rm_cmd_fsync_exit; } @@ -1710,7 +1710,7 @@ static int rm_request_remote_chunks(reqmgr_thrd_t* thrd_ctrl) ret = rc; LOGERR("server request rpc to %d failed - %s", del_rank, - unifyfs_error_enum_str((unifyfs_error_e)rc)); + unifyfs_rc_enum_str((unifyfs_rc)rc)); } } } else { @@ -1924,10 +1924,10 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, if (del_reads->status != READREQ_STARTED) { LOGERR("chunk read response for non-started req @ index=%d", rdreq->req_ndx); - ret = (int32_t)UNIFYFS_ERROR_INVAL; + ret = (int32_t)EINVAL; } else if (0 == del_reads->total_sz) { LOGERR("empty chunk read response for gfid=%d", gfid); - ret = (int32_t)UNIFYFS_ERROR_INVAL; + ret = (int32_t)EINVAL; } else { LOGDBG("handling chunk read responses from server %d: " "gfid=%d num_chunks=%d buf_size=%zu", @@ -2283,14 +2283,14 @@ static void chunk_read_response_rpc(hg_handle_t handle) * don't think that should happen unless maybe * we had sent a read request list that was empty? */ LOGERR("empty response buffer"); - ret = (int32_t)UNIFYFS_ERROR_INVAL; + ret = (int32_t)EINVAL; } else { /* allocate a buffer to hold the incoming data */ char* resp_buf = (char*) malloc(bulk_sz); if (NULL == resp_buf) { /* allocation failed, that's bad */ LOGERR("failed to allocate chunk read responses buffer"); - ret = (int32_t)UNIFYFS_ERROR_NOMEM; + ret = (int32_t)ENOMEM; } else { /* got a buffer, now pull response data */ ret = (int32_t)UNIFYFS_SUCCESS; diff --git a/server/src/unifyfs_service_manager.c b/server/src/unifyfs_service_manager.c index a1c7ea9db..a1eb3dff7 100644 --- a/server/src/unifyfs_service_manager.c +++ b/server/src/unifyfs_service_manager.c @@ -124,7 +124,7 @@ int sm_issue_chunk_reads(int src_rank, char* crbuf = (char*) calloc(1, buf_sz); if (NULL == crbuf) { LOGERR("failed to allocate chunk_read_reqs"); - return UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* the chunk read response array starts as the first @@ -138,7 +138,7 @@ int sm_issue_chunk_reads(int src_rank, calloc(1, sizeof(remote_chunk_reads_t)); if (NULL == rcr) { LOGERR("failed to allocate remote_chunk_reads"); - return UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* fill in chunk read request */ @@ -302,7 +302,7 @@ int svcmgr_init(void) sm = (svcmgr_state_t*)calloc(1, sizeof(svcmgr_state_t)); if (NULL == sm) { LOGERR("failed to allocate service manager state!"); - return (int)UNIFYFS_ERROR_NOMEM; + return ENOMEM; } /* tracks how much data we process in each burst */ @@ -313,7 +313,7 @@ int svcmgr_init(void) if (sm->chunk_reads == NULL) { LOGERR("failed to allocate service manager chunk_reads!"); svcmgr_fini(); - return (int)UNIFYFS_ERROR_NOMEM; + return ENOMEM; } int rc = pthread_mutex_init(&(sm->sync), NULL); @@ -615,7 +615,7 @@ static void server_request_rpc(hg_handle_t handle) /* allocate and register local target buffer for bulk access */ reqbuf = malloc(bulk_sz); if (NULL == reqbuf) { - ret = (int32_t)UNIFYFS_ERROR_NOMEM; + ret = (int32_t)ENOMEM; goto request_out; } hret = margo_bulk_create(mid, 1, &reqbuf, &in.bulk_size, @@ -632,7 +632,7 @@ static void server_request_rpc(hg_handle_t handle) switch (tag) { default: { LOGERR("invalid request tag %d", tag); - ret = (int32_t)UNIFYFS_ERROR_INVAL; + ret = (int32_t)EINVAL; break; } } @@ -721,7 +721,7 @@ static void chunk_read_request_rpc(hg_handle_t handle) } else { LOGERR("invalid chunk read request command %d from server %d", reqcmd, src_rank); - ret = (int32_t)UNIFYFS_ERROR_INVAL; + ret = (int32_t)EINVAL; } /* fill output structure */ diff --git a/server/src/unifyfs_sock.c b/server/src/unifyfs_sock.c deleted file mode 100644 index 37dca20dd..000000000 --- a/server/src/unifyfs_sock.c +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyFS. - * For details, see https://github.com/LLNL/UnifyFS. - * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "arraylist.h" -#include "unifyfs_const.h" -#include "unifyfs_global.h" -#include "unifyfs_keyval.h" -#include "unifyfs_log.h" -#include "unifyfs_sock.h" - -char sock_path[UNIFYFS_MAX_FILENAME]; -int server_sockfd = -1; -int num_fds; -struct pollfd poll_set[MAX_NUM_CLIENTS]; -struct sockaddr_un server_address; - -int detached_sock_idx = -1; -int cur_sock_idx = -1; - -/* initialize the listening socket on this delegator - * @return success/error code */ -int sock_init_server(int srvr_id) -{ - int i, rc; - - for (i = 0; i < MAX_NUM_CLIENTS; i++) { - poll_set[i].fd = -1; - } - - num_fds = 0; - - snprintf(sock_path, sizeof(sock_path), "%s.%d.%d", - SOCKET_PATH, getuid(), srvr_id); - LOGDBG("domain socket path is %s", sock_path); - unlink(sock_path); // remove domain socket leftover from prior run - - server_sockfd = socket(AF_UNIX, SOCK_STREAM, 0); - - memset(&server_address, 0, sizeof(server_address)); - server_address.sun_family = AF_UNIX; - strcpy(server_address.sun_path, sock_path); - rc = bind(server_sockfd, (struct sockaddr*)&server_address, - (socklen_t)sizeof(server_address)); - if (rc != 0) { - close(server_sockfd); - return -1; - } - - rc = listen(server_sockfd, MAX_NUM_CLIENTS); - if (rc != 0) { - close(server_sockfd); - return -1; - } - - sock_add(server_sockfd); // puts server fd at index 0 of poll_set - LOGDBG("completed sock init server"); - - // publish domain socket path - unifyfs_keyval_publish_local(key_unifyfsd_socket, sock_path); - - return 0; -} - -void sock_sanitize_client(int client_idx) -{ - /* close socket for this client id - * and set fd back to -1 */ - if (poll_set[client_idx].fd != -1) { - close(poll_set[client_idx].fd); - poll_set[client_idx].fd = -1; - } -} - -int sock_sanitize(void) -{ - int i; - for (i = 0; i < num_fds; i++) { - sock_sanitize_client(i); - } - - if (server_sockfd != -1) { - server_sockfd = -1; - unlink(sock_path); - } - - return 0; -} - -int sock_add(int fd) -{ - if (num_fds == MAX_NUM_CLIENTS) { - LOGERR("exceeded MAX_NUM_CLIENTS"); - return -1; - } - - int flag = fcntl(fd, F_GETFL); - fcntl(fd, F_SETFL, flag | O_NONBLOCK); - - LOGDBG("sock_adding fd: %d", fd); - poll_set[num_fds].fd = fd; - poll_set[num_fds].events = POLLIN | POLLHUP; - poll_set[num_fds].revents = 0; - num_fds++; - return 0; -} - -void sock_reset(void) -{ - int i; - cur_sock_idx = -1; - detached_sock_idx = -1; - for (i = 0; i < num_fds; i++) { - poll_set[i].events = POLLIN | POLLHUP; - poll_set[i].revents = 0; - } -} - -int sock_remove(int client_idx) -{ - /* in this case, we simply disable the disconnected - * file descriptor. */ - poll_set[client_idx].fd = -1; - return 0; -} - -/* - * wait for the client-side command - * */ - -int sock_wait_cmd(int poll_timeout) -{ - int rc, i, client_fd; - - sock_reset(); - rc = poll(poll_set, num_fds, poll_timeout); - if (rc < 0) { - return (int)UNIFYFS_ERROR_POLL; - } else if (rc == 0) { // timeout - return (int)UNIFYFS_SUCCESS; - } else { - LOGDBG("poll detected socket activity"); - for (i = 0; i < num_fds; i++) { - if (poll_set[i].fd == -1) { - continue; - } - if (i == 0) { // listening socket - if (poll_set[i].revents & POLLIN) { - int client_len = sizeof(struct sockaddr_un); - struct sockaddr_un client_address; - client_fd = accept(server_sockfd, - (struct sockaddr*)&client_address, - (socklen_t*)&client_len); - LOGDBG("accepted client on socket %d", client_fd); - rc = sock_add(client_fd); - if (rc < 0) { - return (int)UNIFYFS_ERROR_SOCKET_FD_EXCEED; - } - } else if (poll_set[i].revents & POLLERR) { - // unknown error on listening socket - return (int)UNIFYFS_ERROR_SOCK_LISTEN; - } - } else { // (i != 0) client sockets - rc = 0; - if (poll_set[i].revents & POLLIN) { - assert(!"This is dead code"); - } else if (poll_set[i].revents & POLLHUP) { - rc = (int)UNIFYFS_ERROR_SOCK_DISCONNECT; - } else if (poll_set[i].revents & POLLERR) { - // unknown error on client socket - rc = (int)UNIFYFS_ERROR_SOCK_OTHER; - } - if (rc) { - if (rc == (int)UNIFYFS_ERROR_SOCK_DISCONNECT) { - sock_remove(i); - detached_sock_idx = i; - } - return rc; - } - } - } - } - - return UNIFYFS_SUCCESS; -} - diff --git a/server/src/unifyfs_sock.h b/server/src/unifyfs_sock.h deleted file mode 100644 index 794f02e9f..000000000 --- a/server/src/unifyfs_sock.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * - * Copyright 2017, UT-Battelle, LLC. - * - * LLNL-CODE-741539 - * All rights reserved. - * - * This is the license for UnifyFS. - * For details, see https://github.com/LLNL/UnifyFS. - * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. - */ - -/* - * Copyright (c) 2017, Lawrence Livermore National Security, LLC. - * Produced at the Lawrence Livermore National Laboratory. - * Copyright (c) 2017, Florida State University. Contributions from - * the Computer Architecture and Systems Research Laboratory (CASTL) - * at the Department of Computer Science. - * - * Written by: Teng Wang, Adam Moody, Weikuan Yu, Kento Sato, Kathryn Mohror - * LLNL-CODE-728877. All rights reserved. - * - * This file is part of burstfs. - * For details, see https://github.com/llnl/burstfs - * Please read https://github.com/llnl/burstfs/LICENSE for full license text. - */ - -#ifndef UNIFYFS_SOCK_H -#define UNIFYFS_SOCK_H - -#include -#include "unifyfs_const.h" - -extern int server_sockfd; -extern int client_sockfd; -extern struct pollfd poll_set[MAX_NUM_CLIENTS]; - -int sock_init_server(int srvr_id); -void sock_sanitize_client(int client_idx); -int sock_sanitize(void); -int sock_add(int fd); -int sock_remove(int client_idx); -void sock_reset(void); -int sock_wait_cmd(int poll_timeout); - -#if 0 // DEPRECATED DUE TO MARGO -int sock_handle_error(int sock_error_no); -int sock_get_id(void); -int sock_get_error_id(void); -char* sock_get_cmd_buf(int client_idx); -char* sock_get_ack_buf(int client_idx); -int sock_ack_client(int client_idx, int ret_sz); -int sock_notify_client(int client_idx, int cmd); -#endif // DEPRECATED DUE TO MARGO - -#endif diff --git a/t/server/metadata_suite.c b/t/server/metadata_suite.c index db91f57fc..4ba5fc053 100644 --- a/t/server/metadata_suite.c +++ b/t/server/metadata_suite.c @@ -19,7 +19,7 @@ int main(int argc, char* argv[]) rc = meta_init_store(&server_cfg); if (rc != 0) { LOG(LOG_ERR, "%s", - unifyfs_error_enum_description(UNIFYFS_ERROR_MDINIT)); + unifyfs_rc_enum_description(UNIFYFS_ERROR_META)); exit(1); } From c044ce5b3fcfac43809e94240bec075a503d937b Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Mon, 10 Feb 2020 13:46:23 -0500 Subject: [PATCH 072/168] move unifyfs_generate_gfid() to common In addition to the move to common, there are five important changes in this commit. First, it fixes a bug in the hash computation, where abs() was used on the hash value to enforce a positive integer. This had the potential to result in conflicts between the positive and negative values of any number (i.e., x and -x). Now, we treat the hash value as unsigned, and just cast to signed, which doesn't change its value. Second, the hash value will now be computed identically regardless of machine architecture endianness. Third, the hash calculation now generates a uint64_t, which enables a switch to 64-bit gfids in the future. For now, to convert to an integer gfid, we use the most significant 32 bits and cast to int. Fourth, this fixes a bug in MDHIM code where all UnifyFS extent keys were assigned to be stored on server 0. It now distributes them among servers based on file slice offset. Fifth, on IBM LSF-CSM systems, we now capture the stdout/err for the servers in files. The server stderr is where MDHIM error messages get printed. --- client/src/Makefile.am | 2 +- client/src/unifyfs-internal.h | 2 -- client/src/unifyfs.c | 17 ----------- common/src/Makefile.am | 3 +- common/src/unifyfs_meta.c | 37 +++++++++++++++++++++++ common/src/unifyfs_meta.h | 44 ++++++++++++++++++--------- meta/src/indexes.c | 4 +-- meta/src/mdhim_private.c | 1 - meta/src/partitioner.c | 33 +++++++++----------- meta/src/partitioner.h | 4 ++- server/src/unifyfs_cmd_handler.c | 2 ++ server/src/unifyfs_metadata.c | 45 ++++++++++++++++++---------- server/src/unifyfs_request_manager.c | 7 +++++ util/unifyfs/src/unifyfs-rm.c | 16 ++++++---- 14 files changed, 137 insertions(+), 80 deletions(-) create mode 100644 common/src/unifyfs_meta.c diff --git a/client/src/Makefile.am b/client/src/Makefile.am index d7679af31..6186d1b77 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -39,7 +39,7 @@ CLIENT_COMMON_LIBADD = \ $(top_builddir)/common/src/libunifyfs_common.la \ $(MARGO_LIBS) \ $(FLATCC_LIBS) \ - -lcrypto -lrt -lpthread + -lrt -lpthread CLIENT_COMMON_SOURCES = \ margo_client.c \ diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 99b633890..30861856c 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -574,8 +574,6 @@ int unifyfs_fid_unlink(int fid); /* functions used in UnifyFS */ -int unifyfs_generate_gfid(const char* path); - int unifyfs_set_global_file_meta_from_fid( int fid, int create); diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index d31500e08..687696ea9 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -46,7 +46,6 @@ #include #include -#include #ifdef UNIFYFS_GOTCHA #include "gotcha/gotcha_types.h" @@ -596,22 +595,6 @@ int unifyfs_fid_is_dir(int fid) } } -/* - * hash a path to gfid - * @param path: file path - * return: gfid - */ -int unifyfs_generate_gfid(const char* path) -{ - unsigned char digested[16] = { 0, }; - unsigned long len = strlen(path); - int* ival = (int*) digested; - - MD5((const unsigned char*) path, len, digested); - - return abs(ival[0]); -} - int unifyfs_gfid_from_fid(const int fid) { /* check that local file id is in range */ diff --git a/common/src/Makefile.am b/common/src/Makefile.am index 725c509e0..4997a83d8 100644 --- a/common/src/Makefile.am +++ b/common/src/Makefile.am @@ -25,6 +25,7 @@ BASE_SRCS = \ unifyfs_log.h \ unifyfs_log.c \ unifyfs_meta.h \ + unifyfs_meta.c \ unifyfs_rpc_util.h \ unifyfs_rpc_util.c \ unifyfs_client_rpcs.h \ @@ -68,6 +69,6 @@ libunifyfs_common_la_LDFLAGS = \ -version-info $(LIBUNIFYFS_LT_VERSION) libunifyfs_common_la_LIBADD = \ - $(OPT_LIBS) -lm -lrt + $(OPT_LIBS) -lm -lrt -lcrypto AM_CFLAGS = -Wall -Wno-strict-aliasing diff --git a/common/src/unifyfs_meta.c b/common/src/unifyfs_meta.c new file mode 100644 index 000000000..227745343 --- /dev/null +++ b/common/src/unifyfs_meta.c @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include +#include +#include + +#include "unifyfs_meta.h" + +/** + * Hash a file path to a 64-bit unsigned integer using MD5 + * @param path absolute file path + * @return hash value + */ +uint64_t compute_path_md5(const char* path) +{ + unsigned long len; + unsigned char digested[16] = {0}; + + len = strlen(path); + MD5((const unsigned char*) path, len, digested); + + /* construct uint64_t hash from first 8 digest bytes */ + uint64_t hash = be64toh(*((uint64_t*)digested)); + return hash; +} diff --git a/common/src/unifyfs_meta.h b/common/src/unifyfs_meta.h index ce909ea49..ee24111e5 100644 --- a/common/src/unifyfs_meta.h +++ b/common/src/unifyfs_meta.h @@ -27,20 +27,6 @@ extern "C" { #endif -/** - * Server commands - */ -typedef enum { - COMM_MOUNT, - COMM_META_FSYNC, - COMM_META_GET, - COMM_META_SET, - COMM_READ, - COMM_UNMOUNT, - COMM_DIGEST, - COMM_SYNC_DEL, -} cmd_lst_t; - typedef struct { int gfid; char filename[UNIFYFS_MAX_FILENAME]; @@ -102,6 +88,36 @@ typedef struct { int gfid; /* global file id */ } unifyfs_index_t; +/* + * Hash a file path to a uint64_t using MD5 + * @param path absolute file path + * @return hash value + */ +uint64_t compute_path_md5(const char* path); + +/* + * Hash a file path to an integer gfid + * @param path absolute file path + * @return gfid + */ +static inline +int unifyfs_generate_gfid(const char* path) +{ + /* until we support 64-bit gfids, use top 32 bits */ + uint64_t hash64 = compute_path_md5(path); + uint32_t hash32 = (uint32_t)(hash64 >> 32); + + /* TODO: Remove next statement once we get rid of MDHIM. + * + * MDHIM requires positive values for integer keys, due to the way + * slice servers are calculated. We use an integer key for the + * gfid -> file attributes index. To guarantee a positive value, we + * shift right one bit to make sure the top bit is zero. */ + hash32 = hash32 >> 1; + + return (int)hash32; +} + /* Header for read request reply in client shared memory region. * The associated data payload immediately follows the header in * the shmem region. diff --git a/meta/src/indexes.c b/meta/src/indexes.c index 359179d9a..0721a3cf8 100644 --- a/meta/src/indexes.c +++ b/meta/src/indexes.c @@ -287,8 +287,8 @@ int update_stat(struct mdhim_t *md, struct index_t *index, void *key, uint32_t k *(unsigned long *)val1 = get_byte_num(key, key_len); *(unsigned long *)val2 = *(unsigned long *)val1; } else if (index->key_type == MDHIM_UNIFYFS_KEY) { - val1 = get_meta_pair(key, key_len); - val2 = get_meta_pair(key, key_len); + val1 = copy_unifyfs_key(key, key_len); + val2 = copy_unifyfs_key(key, key_len); } gettimeofday(&metaend, NULL); metatime+=1000000*(metaend.tv_sec-metastart.tv_sec)+metaend.tv_usec-metastart.tv_usec; diff --git a/meta/src/mdhim_private.c b/meta/src/mdhim_private.c index dc499c5ef..399ab0e0f 100644 --- a/meta/src/mdhim_private.c +++ b/meta/src/mdhim_private.c @@ -357,7 +357,6 @@ struct mdhim_bgetrm_t *_bget_records(struct mdhim_t *md, struct index_t *index, if ((op == MDHIM_GET_EQ || op == MDHIM_GET_PRIMARY_EQ || op == MDHIM_RANGE_BGET) && index->type != LOCAL_INDEX && (rl = get_range_servers(md, index, keys[i], key_lens[i])) == NULL) { - printf("here\n"); fflush(stdout); mlog(MDHIM_CLIENT_CRIT, "MDHIM Rank: %d - " "Error while determining range server in mdhimBget", md->mdhim_rank); diff --git a/meta/src/partitioner.c b/meta/src/partitioner.c index 37c171bcb..bf637d4dd 100644 --- a/meta/src/partitioner.c +++ b/meta/src/partitioner.c @@ -34,12 +34,15 @@ * */ +#include #include #include #include #include #include "partitioner.h" +#include "unifyfs_metadata.h" + struct timeval calslicestart, calsliceend; double calslicetime = 0; struct timeval rangehashstart, rangehashend; @@ -85,13 +88,12 @@ long double get_str_num(void *key, uint32_t key_len) { return str_num; } -unsigned long * get_meta_pair(void *key, uint32_t key_len) { - ulong *meta_pair = (ulong *)malloc (2*sizeof(ulong)); - memset(meta_pair, 0, 2*sizeof(unsigned long)); - meta_pair[0] = *((unsigned long *)(((char *)key))); - meta_pair[1] = *((unsigned long *)(((char *)key)+sizeof(unsigned long))); - // printf("meta_pair[1] is %ld\n", meta_pair[1]); - return meta_pair; +/* Allocate a copy of a key and return it. The returned key must be freed. */ +void* copy_unifyfs_key(void* key, uint32_t key_len) +{ + void* key_copy = malloc((size_t)key_len); + memcpy(key_copy, key, (size_t)key_len); + return key_copy; } uint64_t get_byte_num(void *key, uint32_t key_len) { @@ -386,26 +388,19 @@ int get_slice_num(struct mdhim_t *md, struct index_t *index, void *key, int key_ key_num = floorl(map_num * total_keys); break; + case MDHIM_UNIFYFS_KEY: + /* Use only the gfid portion of the key, which ensures all extents + * for the same file hash to the same server */ + key_num = (uint64_t) UNIFYFS_KEY_FID(key); + break; default: return 0; - break; } /* Convert the key to a slice number */ slice_num = key_num/index->mdhim_max_recs_per_slice; - if (key_type == MDHIM_UNIFYFS_KEY) { - unsigned long *meta_pair = get_meta_pair(key, key_len); - unsigned long surplus = meta_pair[1]; - unsigned long highval = (meta_pair[0] << 1); - unsigned long multiply = (unsigned long)1 << (sizeof(unsigned long)*8 - 1); -/* slice_num = (surplus + highval * multiply%index->mdhim_max_recs_per_slice) % \ - index->mdhim_max_recs_per_slice; -*/ - slice_num = highval * (multiply/index->mdhim_max_recs_per_slice) + surplus/index->mdhim_max_recs_per_slice; - free(meta_pair); - } //Return the slice number return slice_num; } diff --git a/meta/src/partitioner.h b/meta/src/partitioner.h index 9707af159..330d65528 100644 --- a/meta/src/partitioner.h +++ b/meta/src/partitioner.h @@ -102,12 +102,14 @@ long double get_str_num(void *key, uint32_t key_len); uint64_t get_byte_num(void *key, uint32_t key_len); int get_slice_num(struct mdhim_t *md, struct index_t *index, void *key, int key_len); int is_float_key(int type); -unsigned long * get_meta_pair(void *key, uint32_t key_len); + rangesrv_list *get_range_servers_from_stats(struct mdhim_t *md, struct index_t *index, void *key, int key_len, int op); rangesrv_list *get_range_servers_from_range(struct mdhim_t *md, struct index_t *index, void *start_key, void *end_key, int key_len); +void* copy_unifyfs_key(void* key, uint32_t key_len); + #ifdef __cplusplus } #endif diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index e0eecbcff..daafd4cfd 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -201,6 +201,8 @@ static void unifyfs_mount_rpc(hg_handle_t handle) /* fill in and insert a new entry for this app_id * if we don't already have one */ if (tmp_config == NULL) { + LOGDBG("creating app_config for app_id=%d", app_id); + /* don't have an app_config for this app_id, * so allocate and fill one in */ tmp_config = (app_config_t*)malloc(sizeof(app_config_t)); diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index 2715c6a5f..82a475e93 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -255,7 +255,6 @@ int unifyfs_set_file_attribute( NULL, NULL); if (!brm || brm->error) { - LOGERR("Error inserting file attribute into MDHIM"); rc = (int)UNIFYFS_ERROR_MDHIM; } @@ -263,6 +262,9 @@ int unifyfs_set_file_attribute( mdhim_full_release_msg(brm); } + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to insert attributes for gfid=%d", gfid); + } return rc; } @@ -286,7 +288,6 @@ int unifyfs_set_file_attributes(int num_entries, /* check for errors and free resources */ if (!brm) { - LOGERR("Error inserting file attributes into MDHIM"); rc = (int)UNIFYFS_ERROR_MDHIM; } else { /* step through linked list of messages, @@ -294,8 +295,8 @@ int unifyfs_set_file_attributes(int num_entries, struct mdhim_brm_t* brmp = brm; while (brmp) { /* check current item for error */ - if (brmp->error < 0) { - LOGERR("Error inserting file attributes into MDHIM"); + if (brmp->error) { + LOGERR("MDHIM bulk put error=%d", brmp->error); rc = (int)UNIFYFS_ERROR_MDHIM; } @@ -308,6 +309,9 @@ int unifyfs_set_file_attributes(int num_entries, } } + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to bulk insert file attributes"); + } return rc; } @@ -338,6 +342,9 @@ int unifyfs_get_file_attribute( mdhim_full_release_msg(bgrm); } + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to retrieve attributes for gfid=%d", gfid); + } return rc; } @@ -355,7 +362,6 @@ int unifyfs_delete_file_attribute( /* check for errors and free resources */ if (!brm) { - LOGERR("Error deleting file attributes from MDHIM"); rc = (int)UNIFYFS_ERROR_MDHIM; } else { /* step through linked list of messages, @@ -363,8 +369,8 @@ int unifyfs_delete_file_attribute( struct mdhim_brm_t* brmp = brm; while (brmp) { /* check current item for error */ - if (brmp->error < 0) { - LOGERR("Error deleting file attributes from MDHIM"); + if (brmp->error) { + LOGERR("MDHIM delete error=%d", brmp->error); rc = (int)UNIFYFS_ERROR_MDHIM; } @@ -377,6 +383,9 @@ int unifyfs_delete_file_attribute( } } + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to delete attributes for gfid=%d", gfid); + } return rc; } @@ -411,9 +420,9 @@ int unifyfs_get_file_extents(int num_keys, unifyfs_key_t** keys, struct mdhim_bgetrm_t* ptr = bkvlist; while (ptr) { /* check that we don't have an error condition */ - if (ptr->error < 0) { + if (ptr->error) { /* hit an error */ - LOGERR("MDHIM Range Query error"); + LOGERR("MDHIM range query error=%d", ptr->error); return (int)UNIFYFS_ERROR_MDHIM; } @@ -429,7 +438,7 @@ int unifyfs_get_file_extents(int num_keys, unifyfs_key_t** keys, tot_num, sizeof(unifyfs_keyval_t)); if (NULL == kvs) { LOGERR("failed to allocate keyvals"); - return (int)UNIFYFS_ERROR_MDHIM; + return ENOMEM; } /* iterate over list and copy each key/value into output array */ @@ -486,7 +495,6 @@ int unifyfs_set_file_extents(int num_entries, /* check for errors and free resources */ if (!brm) { - LOGERR("Error inserting file extents into MDHIM"); rc = (int)UNIFYFS_ERROR_MDHIM; } else { /* step through linked list of messages, @@ -494,8 +502,8 @@ int unifyfs_set_file_extents(int num_entries, struct mdhim_brm_t* brmp = brm; while (brmp) { /* check current item for error */ - if (brmp->error < 0) { - LOGERR("Error inserting file extents into MDHIM"); + if (brmp->error) { + LOGERR("MDHIM bulk put error=%d", brmp->error); rc = (int)UNIFYFS_ERROR_MDHIM; } @@ -508,6 +516,9 @@ int unifyfs_set_file_extents(int num_entries, } } + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to bulk insert file extents"); + } return rc; } @@ -529,7 +540,6 @@ int unifyfs_delete_file_extents( /* check for errors and free resources */ if (!brm) { - LOGERR("Error deleting file extents from MDHIM"); rc = (int)UNIFYFS_ERROR_MDHIM; } else { /* step through linked list of messages, @@ -537,8 +547,8 @@ int unifyfs_delete_file_extents( struct mdhim_brm_t* brmp = brm; while (brmp) { /* check current item for error */ - if (brmp->error < 0) { - LOGERR("Error deleting file extents from MDHIM"); + if (brmp->error) { + LOGERR("MDHIM bulk delete error=%d", brmp->error); rc = (int)UNIFYFS_ERROR_MDHIM; } @@ -551,5 +561,8 @@ int unifyfs_delete_file_extents( } } + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to bulk delete file extents"); + } return rc; } diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 86163376d..03100d023 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -580,6 +580,7 @@ int rm_cmd_filesize( &num_vals, &keyvals); if (UNIFYFS_SUCCESS != rc) { /* failed to look up extents, bail with error */ + LOGERR("failed to retrieve extent metadata for gfid=%d", gfid); return UNIFYFS_FAILURE; } @@ -620,6 +621,7 @@ int rm_cmd_filesize( filesize_meta = fattr.size; } else { /* failed to find file attributes for this file */ + LOGERR("failed to retrieve attributes for gfid=%d", gfid); return UNIFYFS_FAILURE; } @@ -1024,6 +1026,7 @@ int rm_cmd_laminate( mode_t mode = (mode_t) attr.mode; if ((mode & S_IFMT) != S_IFREG) { /* item is not a regular file */ + LOGERR("ERROR: only regular files can be laminated (gfid=%d)", gfid); return EINVAL; } @@ -1032,6 +1035,7 @@ int rm_cmd_laminate( ret = rm_cmd_filesize(app_id, client_id, gfid, &filesize); if (ret != UNIFYFS_SUCCESS) { /* failed to get file size for file */ + LOGERR("lamination file size calculation failed (gfid=%d)", gfid); return ret; } @@ -1041,6 +1045,9 @@ int rm_cmd_laminate( /* update metadata, set size and laminate */ rc = unifyfs_set_file_attribute(1, 1, &attr); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("lamination metadata update failed (gfid=%d)", gfid); + } return rc; } diff --git a/util/unifyfs/src/unifyfs-rm.c b/util/unifyfs/src/unifyfs-rm.c index 46f030a3a..cd9744ccd 100644 --- a/util/unifyfs/src/unifyfs-rm.c +++ b/util/unifyfs/src/unifyfs-rm.c @@ -474,7 +474,7 @@ static int jsrun_launch(unifyfs_resource_t* resource, char n_nodes[16]; // full command: jsrun - jsrun_argc = 9; + jsrun_argc = 13; snprintf(n_nodes, sizeof(n_nodes), "%zu", resource->n_nodes); server_argc = construct_server_argv(args, NULL); @@ -486,11 +486,15 @@ static int jsrun_launch(unifyfs_resource_t* resource, argv[1] = strdup("--immediate"); argv[2] = strdup("-e"); argv[3] = strdup("individual"); - argv[4] = strdup("--nrs"); - argv[5] = strdup(n_nodes); - argv[6] = strdup("-r1"); - argv[7] = strdup("-c1"); - argv[8] = strdup("-a1"); + argv[4] = strdup("--stdio_stderr"); + argv[5] = strdup("unifyfsd.err.%h.%p"); + argv[6] = strdup("--stdio_stdout"); + argv[7] = strdup("unifyfsd.out.%h.%p"); + argv[8] = strdup("--nrs"); + argv[9] = strdup(n_nodes); + argv[10] = strdup("-r1"); + argv[11] = strdup("-c1"); + argv[12] = strdup("-a1"); construct_server_argv(args, argv + jsrun_argc); execvp(argv[0], argv); From 51f2f823af357ecd4c1fe56a0771f99f5eb339ed Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 28 Jan 2020 16:33:37 -0800 Subject: [PATCH 073/168] examples: add write+sync/read bandwidth output from global timers --- examples/src/testutil_rdwr.h | 15 ++++ examples/src/writeread.c | 129 ++++++++++++++++++++++++++--------- t/ci/100-writeread-tests.sh | 4 +- 3 files changed, 115 insertions(+), 33 deletions(-) diff --git a/examples/src/testutil_rdwr.h b/examples/src/testutil_rdwr.h index ff9297ff3..d2c0db012 100644 --- a/examples/src/testutil_rdwr.h +++ b/examples/src/testutil_rdwr.h @@ -239,6 +239,21 @@ int write_laminate(test_cfg* cfg, const char* filepath) return rc; } +static inline +int stat_file(test_cfg* cfg, const char* filepath) +{ + int rc = 0; + if (cfg->rank == 0 || cfg->io_pattern == IO_PATTERN_NN) { + struct stat s; + int stat_rc = stat(filepath, &s); + if (-1 == stat_rc) { + test_print(cfg, "ERROR - stat(%s) failed", filepath); + rc = -1; + } + } + return rc; +} + /* -------- Read Helper Methods -------- */ static inline diff --git a/examples/src/writeread.c b/examples/src/writeread.c index b9967947c..a101990a4 100644 --- a/examples/src/writeread.c +++ b/examples/src/writeread.c @@ -182,14 +182,20 @@ int main(int argc, char* argv[]) test_timer time_wr; test_timer time_rd; test_timer time_sync; + test_timer time_stat_pre; + test_timer time_stat_pre2; test_timer time_laminate; + test_timer time_stat_post; timer_init(&time_create2laminate, "create2laminate"); timer_init(&time_create, "create"); timer_init(&time_wr, "write"); timer_init(&time_rd, "read"); timer_init(&time_sync, "sync"); + timer_init(&time_stat_pre, "statpre"); + timer_init(&time_stat_pre2, "statpre2"); timer_init(&time_laminate, "laminate"); + timer_init(&time_stat_post, "statpost"); rc = test_init(argc, argv, cfg); if (rc) { @@ -257,6 +263,18 @@ int main(int argc, char* argv[]) // close file + // stat file pre-laminate + timer_start_barrier(cfg, &time_stat_pre); + stat_file(cfg, target_file); + timer_stop_barrier(cfg, &time_stat_pre); + test_print_verbose_once(cfg, "DEBUG: finished stat pre"); + + // stat file pre-laminate (again) + timer_start_barrier(cfg, &time_stat_pre2); + stat_file(cfg, target_file); + timer_stop_barrier(cfg, &time_stat_pre2); + test_print_verbose_once(cfg, "DEBUG: finished stat pre2"); + // laminate timer_start_barrier(cfg, &time_laminate); rc = write_laminate(cfg, target_file); @@ -266,6 +284,12 @@ int main(int argc, char* argv[]) timer_stop_barrier(cfg, &time_laminate); test_print_verbose_once(cfg, "DEBUG: finished laminate"); + // stat file post-laminate + timer_start_barrier(cfg, &time_stat_post); + stat_file(cfg, target_file); + timer_stop_barrier(cfg, &time_stat_post); + test_print_verbose_once(cfg, "DEBUG: finished stat post"); + // post-write cleanup free(wr_buf); free(reqs); @@ -310,41 +334,81 @@ int main(int argc, char* argv[]) // calculate achieved bandwidth rates size_t rank_bytes = test_config.n_blocks * test_config.block_sz; - double write_bw, read_bw; - double aggr_write_bw, aggr_read_bw; - double max_write_time, max_read_time; - double min_sync_time, max_sync_time; - - write_bw = bandwidth_mib(rank_bytes, time_wr.elapsed_sec); - aggr_write_bw = test_reduce_double_sum(cfg, write_bw); - max_write_time = test_reduce_double_max(cfg, time_wr.elapsed_sec); - - min_sync_time = test_reduce_double_min(cfg, time_sync.elapsed_sec); - max_sync_time = test_reduce_double_max(cfg, time_sync.elapsed_sec); - - read_bw = bandwidth_mib(rank_bytes, time_rd.elapsed_sec); - aggr_read_bw = test_reduce_double_sum(cfg, read_bw); - max_read_time = test_reduce_double_max(cfg, time_rd.elapsed_sec); + size_t total_bytes = rank_bytes * test_config.n_ranks; + + double min_local_write_time = test_reduce_double_min(cfg, + time_wr.elapsed_sec); + double max_local_write_time = test_reduce_double_max(cfg, + time_wr.elapsed_sec); + double max_global_write_time = test_reduce_double_max(cfg, + time_wr.elapsed_sec_all); + + double local_write_bw = bandwidth_mib(rank_bytes, time_wr.elapsed_sec); + double min_local_write_bw = test_reduce_double_min(cfg, local_write_bw); + double max_local_write_bw = test_reduce_double_max(cfg, local_write_bw); + double aggr_local_write_bw = test_reduce_double_sum(cfg, local_write_bw); + + double global_write_bw = bandwidth_mib(total_bytes, max_global_write_time); + + double min_local_sync_time = test_reduce_double_min(cfg, + time_sync.elapsed_sec); + double max_local_sync_time = test_reduce_double_max(cfg, + time_sync.elapsed_sec); + double max_global_sync_time = test_reduce_double_max(cfg, + time_sync.elapsed_sec_all); + + double global_write_sync_bw = bandwidth_mib(total_bytes, + max_global_write_time + max_global_sync_time); + + double min_local_read_time = test_reduce_double_min(cfg, + time_rd.elapsed_sec); + double max_local_read_time = test_reduce_double_max(cfg, + time_rd.elapsed_sec); + double max_global_read_time = test_reduce_double_max(cfg, + time_rd.elapsed_sec_all); + + double local_read_bw = bandwidth_mib(rank_bytes, time_rd.elapsed_sec); + double min_local_read_bw = test_reduce_double_min(cfg, local_read_bw); + double max_local_read_bw = test_reduce_double_max(cfg, local_read_bw); + double aggr_local_read_bw = test_reduce_double_sum(cfg, local_read_bw); + + double global_read_bw = bandwidth_mib(total_bytes, max_global_read_time); if (test_config.rank == 0) { - size_t total_bytes = rank_bytes * test_config.n_ranks; - double eff_write_bw, eff_read_bw; - eff_write_bw = bandwidth_mib(total_bytes, max_write_time); - eff_read_bw = bandwidth_mib(total_bytes, max_read_time); - - printf("File create time is %.3lf s\n", + printf("File Create Time is %.6lf s\n", time_create.elapsed_sec_all); - printf("Aggregate Write BW is %.3lf MiB/s\n" - "Effective Write BW is %.3lf MiB/s\n\n", - aggr_write_bw, eff_write_bw); - printf("Minimum Sync Time is %.6lf s\n" - "Maximum Sync Time is %.6lf s\n\n", - min_sync_time, max_sync_time); - printf("Aggregate Read BW is %.3lf MiB/s\n" - "Effective Read BW is %.3lf MiB/s\n\n", - aggr_read_bw, eff_read_bw); - printf("File laminate time is %.3lf s\n", + printf("Minimum Local Write BW is %.3lf MiB/s\n", + min_local_write_bw); + printf("Maximum Local Write BW is %.3lf MiB/s\n", + max_local_write_bw); + printf("Aggregate Local Write BW is %.3lf MiB/s\n", + aggr_local_write_bw); + printf("Global Write BW is %.3lf MiB/s\n", + global_write_bw); + printf("Minimum Local Sync Time is %.6lf s\n", + min_local_sync_time); + printf("Maximum Local Sync Time is %.6lf s\n", + max_local_sync_time); + printf("Global Sync Time is %.6lf s\n", + max_global_sync_time); + printf("Global Write+Sync BW is %.3lf MiB/s\n", + global_write_sync_bw); + printf("Stat Time Pre-Laminate is %.6lf s\n", + time_stat_pre.elapsed_sec_all); + printf("Stat Time Pre-Laminate2 is %.6lf s\n", + time_stat_pre.elapsed_sec_all); + printf("File Laminate Time is %.6lf s\n", time_laminate.elapsed_sec_all); + printf("Stat Time Post-Laminate is %.6lf s\n", + time_stat_post.elapsed_sec_all); + printf("Minimum Local Read BW is %.3lf MiB/s\n", + min_local_read_bw); + printf("Maximum Local Read BW is %.3lf MiB/s\n", + max_local_read_bw); + printf("Aggregate Local Read BW is %.3lf MiB/s\n", + aggr_local_read_bw); + printf("Global Read BW is %.3lf MiB/s\n", + global_read_bw); fflush(stdout); } @@ -356,7 +420,10 @@ int main(int argc, char* argv[]) timer_fini(&time_wr); timer_fini(&time_rd); timer_fini(&time_sync); + timer_fini(&time_stat_pre); + timer_fini(&time_stat_pre2); timer_fini(&time_laminate); + timer_fini(&time_stat_post); test_fini(cfg); diff --git a/t/ci/100-writeread-tests.sh b/t/ci/100-writeread-tests.sh index 32139e704..4d5ad4cf4 100755 --- a/t/ci/100-writeread-tests.sh +++ b/t/ci/100-writeread-tests.sh @@ -68,7 +68,7 @@ unify_test_writeread() { # Evaluate output test_expect_success "$app_name $app_args: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 11 + test $lcount = 17 ' } @@ -86,7 +86,7 @@ unify_test_writeread_posix() { # Evaluate output test_expect_success POSIX "$app_name $1: (line_count=${lcount}, rc=$rc)" ' test $rc = 0 && - test $lcount = 11 && + test $lcount = 17 && if [[ $io_pattern =~ (n1)$ ]]; then test_path_is_file ${CI_POSIX_MP}/$filename else From b69963f9e118ac96693505d104700a5d5ec9d1fc Mon Sep 17 00:00:00 2001 From: CamStan Date: Tue, 3 Mar 2020 13:22:02 -0800 Subject: [PATCH 074/168] Move mpi.h include to fix --enable-mpi-mount bug A bug was reported in #467 that the --enable-mpi-mount configure option would cause the build to fail. This moves the include for mpi.h from pmpi_wrappers.c to pmpi_wrappers.h. The build started passing as a result. Fixes: #467 --- client/src/pmpi_wrappers.c | 1 - client/src/pmpi_wrappers.h | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/client/src/pmpi_wrappers.c b/client/src/pmpi_wrappers.c index e4bad9f43..b94958709 100644 --- a/client/src/pmpi_wrappers.c +++ b/client/src/pmpi_wrappers.c @@ -14,7 +14,6 @@ #include "pmpi_wrappers.h" #include "unifyfs.h" -#include #include int unifyfs_mpi_init(int* argc, char*** argv) diff --git a/client/src/pmpi_wrappers.h b/client/src/pmpi_wrappers.h index b90057ce4..995e44bbe 100644 --- a/client/src/pmpi_wrappers.h +++ b/client/src/pmpi_wrappers.h @@ -15,6 +15,8 @@ #ifndef UNIFYFS_PMPI_WRAPPERS_H #define UNIFYFS_PMPI_WRAPPERS_H +#include + /* MPI_Init PMPI wrapper */ int unifyfs_mpi_init(int* argc, char*** argv); int MPI_Init(int* argc, char*** argv); From be26e612cd5b5dcecc4f92f495970535605aaf3e Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 28 Feb 2020 12:19:28 -0800 Subject: [PATCH 075/168] coalesce entries in segment tree This updates the segment tree to coalesce adjacent entries when a new entry is added. To be merged, two entries must be adjacent in both [start, end] logical offsets as well as the log position. It checks any extent that comes before or after the new extent. Existing test cases are modified to ensure entries are added correctly when they can't be combined, and new test cases are added to check that coalescing is performed correctly. --- common/src/seg_tree.c | 59 +++++++++++++++++++++++ t/seg_tree_test.c | 109 +++++++++++++++++++++++++++++------------- 2 files changed, 134 insertions(+), 34 deletions(-) diff --git a/common/src/seg_tree.c b/common/src/seg_tree.c index 423860be0..8c1e7dfbf 100644 --- a/common/src/seg_tree.c +++ b/common/src/seg_tree.c @@ -156,8 +156,12 @@ int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, struct seg_tree_node* remaining; struct seg_tree_node* resized; struct seg_tree_node* overlap; + struct seg_tree_node* target; + struct seg_tree_node* prev; + struct seg_tree_node* next; long new_start; long new_end; + unsigned long ptr_end; int ret; /* Create our range */ @@ -263,6 +267,61 @@ int seg_tree_add(struct seg_tree* seg_tree, unsigned long start, */ seg_tree->max = MAX(seg_tree->max, end); + /* Get temporary pointer to the node we just added. */ + target = node; + + /* Check whether we can coalesce new extent with any preceding extent. */ + prev = RB_PREV(inttree, &seg_tree->head, target); + if (prev != NULL && prev->end + 1 == target->start) { + /* + * We found a extent that ends just before the new extent starts. + * Check whether they are also contiguous in the log. + */ + ptr_end = prev->ptr + (prev->end - prev->start + 1); + if (ptr_end == target->ptr) { + /* + * The preceding extent describes a log position adjacent to + * the extent we just added, so we can merge them. + * Append entry to previous by extending end of previous. + */ + prev->end = target->end; + + /* Delete new extent from the tree and free it. */ + RB_REMOVE(inttree, &seg_tree->head, target); + free(target); + seg_tree->count--; + + /* + * Update target to point at previous extent since we just + * merged our new extent into it. + */ + target = prev; + } + } + + /* Check whether we can coalesce new extent with any trailing extent. */ + next = RB_NEXT(inttree, &seg_tree->head, target); + if (next != NULL && target->end + 1 == next->start) { + /* + * We found a extent that starts just after the new extent ends. + * Check whether they are also contiguous in the log. + */ + ptr_end = target->ptr + (target->end - target->start + 1); + if (ptr_end == next->ptr) { + /* + * The target extent describes a log position adjacent to + * the next extent, so we can merge them. + * Append entry to target by extending end of to cover next. + */ + target->end = next->end; + + /* Delete next extent from the tree and free it. */ + RB_REMOVE(inttree, &seg_tree->head, next); + free(next); + seg_tree->count--; + } + } + release_add: seg_tree_unlock(seg_tree); diff --git a/t/seg_tree_test.c b/t/seg_tree_test.c index 7b7aaa8c8..7176eda19 100644 --- a/t/seg_tree_test.c +++ b/t/seg_tree_test.c @@ -34,8 +34,8 @@ char* print_tree(char* dst, struct seg_tree* seg_tree) seg_tree_rdlock(seg_tree); while ((node = seg_tree_iter(seg_tree, node))) { - ptr += sprintf(&dst[ptr], "[%lu-%lu:%c]", node->start, node->end, - *((char*)node->ptr)); + ptr += sprintf(&dst[ptr], "[%lu-%lu:%lu]", node->start, node->end, + node->ptr); } seg_tree_unlock(seg_tree); return dst; @@ -44,47 +44,36 @@ char* print_tree(char* dst, struct seg_tree* seg_tree) int main(int argc, char** argv) { struct seg_tree seg_tree; - char buf[500]; char tmp[255]; - int i; unsigned long max, count; struct seg_tree_node* node; plan(NO_PLAN); - /* - * Initialize our buffer with the character for the buffer pos (0-9). - * We'll use this to make sure the node.ptr value is getting updated - * correctly by the seg_tree. - */ - for (i = 0; i < sizeof(buf); i++) { - buf[i] = (i % 10) + '0'; - } - seg_tree_init(&seg_tree); /* Initial insert */ - seg_tree_add(&seg_tree, 5, 10, (unsigned long) (buf + 5)); - is("[5-10:5]", print_tree(tmp, &seg_tree), "Initial insert works"); + seg_tree_add(&seg_tree, 5, 10, 0); + is("[5-10:0]", print_tree(tmp, &seg_tree), "Initial insert works"); /* Non-overlapping insert */ - seg_tree_add(&seg_tree, 100, 150, (unsigned long) (buf + 100)); - is("[5-10:5][100-150:0]", print_tree(tmp, &seg_tree), + seg_tree_add(&seg_tree, 100, 150, 100); + is("[5-10:0][100-150:100]", print_tree(tmp, &seg_tree), "Non-overlapping works"); /* Add range overlapping part of the left size */ - seg_tree_add(&seg_tree, 2, 7, (unsigned long) (buf + 2)); - is("[2-7:2][8-10:8][100-150:0]", print_tree(tmp, &seg_tree), + seg_tree_add(&seg_tree, 2, 7, 200); + is("[2-7:200][8-10:3][100-150:100]", print_tree(tmp, &seg_tree), "Left size overlap works"); /* Add range overlapping part of the right size */ - seg_tree_add(&seg_tree, 9, 12, (unsigned long) (buf + 9)); - is("[2-7:2][8-8:8][9-12:9][100-150:0]", print_tree(tmp, &seg_tree), + seg_tree_add(&seg_tree, 9, 12, 300); + is("[2-7:200][8-8:3][9-12:300][100-150:100]", print_tree(tmp, &seg_tree), "Right size overlap works"); /* Add range totally within another range */ - seg_tree_add(&seg_tree, 3, 4, (unsigned long) (buf + 3)); - is("[2-2:2][3-4:3][5-7:5][8-8:8][9-12:9][100-150:0]", + seg_tree_add(&seg_tree, 3, 4, 400); + is("[2-2:200][3-4:400][5-7:203][8-8:3][9-12:300][100-150:100]", print_tree(tmp, &seg_tree), "Inside range works"); /* Test counts */ @@ -94,8 +83,8 @@ int main(int argc, char** argv) ok(count == 6, "count is 6 (got %lu)", count); /* Add a range that blows away multiple ranges, and overlaps */ - seg_tree_add(&seg_tree, 4, 120, (unsigned long) (buf + 4)); - is("[2-2:2][3-3:3][4-120:4][121-150:1]", print_tree(tmp, &seg_tree), + seg_tree_add(&seg_tree, 4, 120, 500); + is("[2-2:200][3-3:400][4-120:500][121-150:121]", print_tree(tmp, &seg_tree), "Blow away multiple ranges works"); /* Test counts */ @@ -116,12 +105,12 @@ int main(int argc, char** argv) * Now let's write a long extent, and then sawtooth over it with 1 byte * extents. */ - seg_tree_add(&seg_tree, 0, 50, (unsigned long) (buf + 50)); - seg_tree_add(&seg_tree, 0, 0, (unsigned long) (buf + 0)); - seg_tree_add(&seg_tree, 2, 2, (unsigned long) (buf + 2)); - seg_tree_add(&seg_tree, 4, 4, (unsigned long) (buf + 4)); - seg_tree_add(&seg_tree, 6, 6, (unsigned long) (buf + 6)); - is("[0-0:0][1-1:1][2-2:2][3-3:3][4-4:4][5-5:5][6-6:6][7-50:7]", + seg_tree_add(&seg_tree, 0, 50, 50); + seg_tree_add(&seg_tree, 0, 0, 0); + seg_tree_add(&seg_tree, 2, 2, 2); + seg_tree_add(&seg_tree, 4, 4, 4); + seg_tree_add(&seg_tree, 6, 6, 6); + is("[0-0:0][1-1:51][2-2:2][3-3:53][4-4:4][5-5:55][6-6:6][7-50:57]", print_tree(tmp, &seg_tree), "Sawtooth extents works"); max = seg_tree_max(&seg_tree); @@ -137,7 +126,7 @@ int main(int argc, char** argv) ok(node->start == 2 && node->end == 2, "seg_tree_find found correct node"); /* Test finding a segment that partially overlaps our range */ - seg_tree_add(&seg_tree, 100, 200, (unsigned long) (buf + 100)); + seg_tree_add(&seg_tree, 100, 200, 100); node = seg_tree_find(&seg_tree, 90, 120); ok(node->start == 100 && node->end == 200, "seg_tree_find found partial overlapping node"); @@ -151,12 +140,64 @@ int main(int argc, char** argv) * same range. Use a different buf value to verify it changed. */ seg_tree_clear(&seg_tree); - seg_tree_add(&seg_tree, 20, 30, (unsigned long) (buf + 0)); + seg_tree_add(&seg_tree, 20, 30, 0); is("[20-30:0]", print_tree(tmp, &seg_tree), "Initial [20-30] write works"); - seg_tree_add(&seg_tree, 20, 30, (unsigned long) (buf + 8)); + seg_tree_add(&seg_tree, 20, 30, 8); is("[20-30:8]", print_tree(tmp, &seg_tree), "Same range overwrite works"); + /* Test coalescing */ + seg_tree_clear(&seg_tree); + seg_tree_add(&seg_tree, 5, 10, 105); + is("[5-10:105]", print_tree(tmp, &seg_tree), "Initial insert works"); + + /* Non-overlapping insert */ + seg_tree_add(&seg_tree, 100, 150, 200); + is("[5-10:105][100-150:200]", print_tree(tmp, &seg_tree), + "Non-overlapping works"); + + /* + * Add range overlapping part of the left size. + * Check that it coalesces + */ + seg_tree_add(&seg_tree, 2, 7, 102); + is("[2-10:102][100-150:200]", print_tree(tmp, &seg_tree), + "Left size overlap works"); + + /* + * Add range overlapping part of the right size. + * Check that is coalesces. + */ + seg_tree_add(&seg_tree, 9, 12, 109); + is("[2-12:102][100-150:200]", print_tree(tmp, &seg_tree), + "Right size overlap works"); + + /* + * Add range totally within another range. + * Check that it is consumed. + */ + seg_tree_add(&seg_tree, 3, 4, 103); + is("[2-12:102][100-150:200]", + print_tree(tmp, &seg_tree), "Inside range works"); + + /* Test counts */ + max = seg_tree_max(&seg_tree); + count = seg_tree_count(&seg_tree); + ok(max == 150, "max is 150 (got %lu)", max); + ok(count == 2, "count is 2 (got %lu)", count); + + /* Add a range that connects two other ranges. */ + seg_tree_add(&seg_tree, 4, 120, 104); + is("[2-150:102]", print_tree(tmp, &seg_tree), + "Connect two ranges works"); + + /* Test counts */ + max = seg_tree_max(&seg_tree); + count = seg_tree_count(&seg_tree); + ok(max == 150, "max is 150 (got %lu)", max); + ok(count == 1, "count is 1 (got %lu)", count); + + seg_tree_clear(&seg_tree); seg_tree_destroy(&seg_tree); done_testing(); From 0e043d999ae1af192e30bca2dc6a9a040c97d689 Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Wed, 19 Feb 2020 17:01:01 -0500 Subject: [PATCH 076/168] implement and use common log-based I/O methods This implements common methods for client and server access to log-based I/O. A new logio_context structure contains all state for managing both the shared memory and spill file portions. The shared memory for data chunks is no longer allocated out of the client superblock, but instead uses a dedicated shmem region. Chunk management is now handled by a new slotmap structure that maintains a bitmap of free/used chunk slots. The chunk slotmap resides within the first page of the shmem region and spill file. This also includes the following code cleanup: * remove unused constants * remove unused read request shmem region * remove unused client socket state * libcommon needs to link against libpthread * compiler warning fixes TEST_CHECKPATCH_SKIP_FILES=common/src/unifyfs_configurator.h --- client/src/margo_client.c | 73 +- client/src/margo_client.h | 4 +- client/src/unifyfs-fixed.c | 621 ++------------ client/src/unifyfs-fixed.h | 43 +- client/src/unifyfs-internal.h | 52 +- client/src/unifyfs-sysio.c | 109 +-- client/src/unifyfs.c | 556 ++---------- client/src/unifyfs.h | 22 +- common/src/Makefile.am | 18 +- common/src/slotmap.c | 368 ++++++++ common/src/slotmap.h | 96 +++ common/src/unifyfs_client_rpcs.h | 42 +- common/src/unifyfs_configurator.h | 17 +- common/src/unifyfs_const.h | 20 +- common/src/unifyfs_logio.c | 805 ++++++++++++++++++ common/src/unifyfs_logio.h | 143 ++++ common/src/unifyfs_meta.h | 99 ++- common/src/unifyfs_shm.c | 80 +- common/src/unifyfs_shm.h | 89 +- docs/add-rpcs.rst | 2 +- docs/configuration.rst | 105 +-- examples/src/sysio-writeread.c | 2 +- server/src/margo_server.c | 6 +- server/src/unifyfs_cmd_handler.c | 207 ++--- server/src/unifyfs_global.h | 23 +- server/src/unifyfs_init.c | 56 +- server/src/unifyfs_request_manager.c | 84 +- server/src/unifyfs_request_manager.h | 10 +- server/src/unifyfs_service_manager.c | 76 +- t/9200-seg-tree-test.t | 8 + ...06-seg-tree-test.t => 9201-slotmap-test.t} | 2 +- t/Makefile.am | 55 +- t/{ => common}/seg_tree_test.c | 0 t/common/slotmap_test.c | 125 +++ t/server/unifyfs_meta_get_test.c | 4 +- t/sharness.d/01-unifyfs-settings.sh | 6 +- 36 files changed, 2247 insertions(+), 1781 deletions(-) create mode 100644 common/src/slotmap.c create mode 100644 common/src/slotmap.h create mode 100644 common/src/unifyfs_logio.c create mode 100644 common/src/unifyfs_logio.h create mode 100755 t/9200-seg-tree-test.t rename t/{9006-seg-tree-test.t => 9201-slotmap-test.t} (82%) rename t/{ => common}/seg_tree_test.c (100%) create mode 100644 t/common/slotmap_test.c diff --git a/client/src/margo_client.c b/client/src/margo_client.c index 58d9d36ad..f14570abd 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -55,9 +55,9 @@ static void register_client_rpcs(client_rpc_context_t* ctx) unifyfs_laminate_out_t, NULL); - ctx->rpcs.fsync_id = MARGO_REGISTER(mid, "unifyfs_fsync_rpc", - unifyfs_fsync_in_t, - unifyfs_fsync_out_t, + ctx->rpcs.sync_id = MARGO_REGISTER(mid, "unifyfs_sync_rpc", + unifyfs_sync_in_t, + unifyfs_sync_out_t, NULL); ctx->rpcs.read_id = MARGO_REGISTER(mid, "unifyfs_read_rpc", @@ -262,8 +262,8 @@ int invoke_client_unmount_rpc(void) /* fill in input struct */ unifyfs_unmount_in_t in; - in.app_id = app_id; - in.local_rank_idx = local_rank_idx; + in.app_id = (int32_t) app_id; + in.client_id = (int32_t) local_rank_idx; /* call rpc function */ LOGDBG("invoking the unmount rpc function in client"); @@ -307,7 +307,7 @@ int invoke_client_metaset_rpc(int create, unifyfs_file_attr_t* f_meta) /* fill in input struct */ unifyfs_metaset_in_t in; in.create = (int32_t) create; - in.gfid = f_meta->gfid; + in.gfid = (int32_t) f_meta->gfid; in.filename = f_meta->filename; in.mode = f_meta->mode; in.uid = f_meta->uid; @@ -397,9 +397,9 @@ int invoke_client_filesize_rpc(int gfid, size_t* outsize) /* fill in input struct */ unifyfs_filesize_in_t in; - in.app_id = (int32_t)app_id; - in.local_rank_idx = (int32_t)local_rank_idx; - in.gfid = (int32_t)gfid; + in.app_id = (int32_t) app_id; + in.client_id = (int32_t) local_rank_idx; + in.gfid = (int32_t) gfid; /* call rpc function */ LOGDBG("invoking the filesize rpc function in client"); @@ -435,10 +435,10 @@ int invoke_client_truncate_rpc(int gfid, size_t filesize) /* fill in input struct */ unifyfs_truncate_in_t in; - in.app_id = (int32_t)app_id; - in.local_rank_idx = (int32_t)local_rank_idx; - in.gfid = (int32_t)gfid; - in.filesize = (hg_size_t)filesize; + in.app_id = (int32_t) app_id; + in.client_id = (int32_t) local_rank_idx; + in.gfid = (int32_t) gfid; + in.filesize = (hg_size_t) filesize; /* call rpc function */ LOGDBG("invoking the truncate rpc function in client"); @@ -471,9 +471,9 @@ int invoke_client_unlink_rpc(int gfid) /* fill in input struct */ unifyfs_unlink_in_t in; - in.app_id = (int32_t)app_id; - in.local_rank_idx = (int32_t)local_rank_idx; - in.gfid = (int32_t)gfid; + in.app_id = (int32_t) app_id; + in.client_id = (int32_t) local_rank_idx; + in.gfid = (int32_t) gfid; /* call rpc function */ LOGDBG("invoking the unlink rpc function in client"); @@ -506,9 +506,9 @@ int invoke_client_laminate_rpc(int gfid) /* fill in input struct */ unifyfs_unlink_in_t in; - in.app_id = (int32_t)app_id; - in.local_rank_idx = (int32_t)local_rank_idx; - in.gfid = (int32_t)gfid; + in.app_id = (int32_t) app_id; + in.client_id = (int32_t) local_rank_idx; + in.gfid = (int32_t) gfid; /* call rpc function */ LOGDBG("invoking the laminate rpc function in client"); @@ -528,8 +528,8 @@ int invoke_client_laminate_rpc(int gfid) return (int)ret; } -/* invokes the client fsync rpc function */ -int invoke_client_fsync_rpc(int gfid) +/* invokes the client sync rpc function */ +int invoke_client_sync_rpc(void) { /* check that we have initialized margo */ if (NULL == client_rpc_context) { @@ -537,21 +537,20 @@ int invoke_client_fsync_rpc(int gfid) } /* get handle to rpc function */ - hg_handle_t handle = create_handle(client_rpc_context->rpcs.fsync_id); + hg_handle_t handle = create_handle(client_rpc_context->rpcs.sync_id); /* fill in input struct */ - unifyfs_fsync_in_t in; - in.app_id = (int32_t)app_id; - in.local_rank_idx = (int32_t)local_rank_idx; - in.gfid = (int32_t)gfid; + unifyfs_sync_in_t in; + in.app_id = (int32_t) app_id; + in.client_id = (int32_t) local_rank_idx; /* call rpc function */ - LOGDBG("invoking the fsync rpc function in client"); + LOGDBG("invoking the sync rpc function in client"); hg_return_t hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); /* decode response */ - unifyfs_fsync_out_t out; + unifyfs_sync_out_t out; hret = margo_get_output(handle, &out); assert(hret == HG_SUCCESS); int32_t ret = out.ret; @@ -576,11 +575,11 @@ int invoke_client_read_rpc(int gfid, size_t offset, size_t length) /* fill in input struct */ unifyfs_read_in_t in; - in.app_id = (int32_t)app_id; - in.local_rank_idx = (int32_t)local_rank_idx; - in.gfid = (int32_t)gfid; - in.offset = (hg_size_t)offset; - in.length = (hg_size_t)length; + in.app_id = (int32_t) app_id; + in.client_id = (int32_t) local_rank_idx; + in.gfid = (int32_t) gfid; + in.offset = (hg_size_t) offset; + in.length = (hg_size_t) length; /* call rpc function */ LOGDBG("invoking the read rpc function in client"); @@ -618,10 +617,10 @@ int invoke_client_mread_rpc(int read_count, size_t size, void* buffer) assert(hret == HG_SUCCESS); /* fill in input struct */ - in.app_id = (int32_t)app_id; - in.local_rank_idx = (int32_t)local_rank_idx; - in.read_count = (int32_t)read_count; - in.bulk_size = (hg_size_t)size; + in.app_id = (int32_t) app_id; + in.client_id = (int32_t) local_rank_idx; + in.read_count = (int32_t) read_count; + in.bulk_size = (hg_size_t) size; /* call rpc function */ LOGDBG("invoking the read rpc function in client"); diff --git a/client/src/margo_client.h b/client/src/margo_client.h index 694770640..3470b4c1a 100644 --- a/client/src/margo_client.h +++ b/client/src/margo_client.h @@ -18,7 +18,7 @@ typedef struct ClientRpcIds { hg_id_t truncate_id; hg_id_t unlink_id; hg_id_t laminate_id; - hg_id_t fsync_id; + hg_id_t sync_id; hg_id_t read_id; hg_id_t mread_id; } client_rpcs_t; @@ -54,7 +54,7 @@ int invoke_client_unlink_rpc(int gfid); int invoke_client_laminate_rpc(int gfid); -int invoke_client_fsync_rpc(int gfid); +int invoke_client_sync_rpc(void); int invoke_client_read_rpc(int gfid, size_t offset, size_t length); diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 869898cf8..4d7141b80 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -46,229 +46,12 @@ #include "margo_client.h" #include "seg_tree.h" -static inline -unifyfs_chunkmeta_t* filemeta_get_chunkmeta(const unifyfs_filemeta_t* meta, - int cid) -{ - unifyfs_chunkmeta_t* chunkmeta = NULL; - uint64_t limit = 0; - - if (unifyfs_use_memfs) { - limit += unifyfs_max_chunks; - } - - if (unifyfs_use_spillover) { - limit += unifyfs_spillover_max_chunks; - } - - if (meta && (cid >= 0 && cid < limit)) { - chunkmeta = &unifyfs_chunkmetas[meta->chunkmeta_idx + cid]; - } - - return chunkmeta; -} - -/* given a file id and logical chunk id, return pointer to meta data - * for specified chunk, return NULL if not found */ -static inline unifyfs_chunkmeta_t* unifyfs_get_chunkmeta(int fid, int cid) -{ - /* lookup file meta data for specified file id */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - - return filemeta_get_chunkmeta(meta, cid); -} - /* --------------------------------------- - * Operations on file chunks + * Operations on client write index * --------------------------------------- */ -/* given a logical chunk id and an offset within that chunk, return the pointer - * to the memory location corresponding to that location */ -static inline void* unifyfs_compute_chunk_buf(const unifyfs_filemeta_t* meta, - int cid, off_t offset) -{ - /* get pointer to chunk meta */ - const unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, cid); - - /* identify physical chunk id */ - int physical_id = chunk_meta->id; - - /* compute the start of the chunk */ - char* start = NULL; - if (physical_id < unifyfs_max_chunks) { - start = unifyfs_chunks + ((long)physical_id << unifyfs_chunk_bits); - } else { - /* chunk is in spill over */ - LOGERR("wrong chunk ID"); - return NULL; - } - - /* now add offset */ - char* buf = start + offset; - return (void*)buf; -} - -/* given a chunk id and an offset within that chunk, return the offset - * in the spillover file corresponding to that location */ -static inline off_t unifyfs_compute_spill_offset(const unifyfs_filemeta_t* meta, - int cid, off_t offset) -{ - /* get pointer to chunk meta */ - const unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, cid); - - /* identify physical chunk id */ - int physical_id = chunk_meta->id; - - /* compute start of chunk in spill over device */ - off_t start = 0; - if (physical_id < unifyfs_max_chunks) { - LOGERR("wrong spill-chunk ID"); - return -1; - } else { - /* compute buffer loc within spillover device chunk */ - /* account for the unifyfs_max_chunks added to identify location when - * grabbing this chunk */ - start = ((long)(physical_id - unifyfs_max_chunks) << unifyfs_chunk_bits); - } - - off_t buf = start + offset; - return buf; -} - -/* allocate a new chunk for the specified file and logical chunk id */ -static int unifyfs_chunk_alloc(int fid, unifyfs_filemeta_t* meta, int chunk_id) -{ - /* get pointer to chunk meta data */ - unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); - - /* allocate a chunk and record its location */ - if (unifyfs_use_memfs) { - /* allocate a new chunk from memory */ - unifyfs_stack_lock(); - int id = unifyfs_stack_pop(free_chunk_stack); - unifyfs_stack_unlock(); - - /* if we got one return, otherwise try spill over */ - if (id >= 0) { - /* got a chunk from memory */ - chunk_meta->location = CHUNK_LOCATION_MEMFS; - chunk_meta->id = id; - } else if (unifyfs_use_spillover) { - /* shm segment out of space, grab a block from spill-over device */ - LOGDBG("getting blocks from spill-over device"); - - /* TODO: missing lock calls? */ - /* add unifyfs_max_chunks to identify chunk location */ - unifyfs_stack_lock(); - id = unifyfs_stack_pop(free_spillchunk_stack) + unifyfs_max_chunks; - unifyfs_stack_unlock(); - if (id < unifyfs_max_chunks) { - LOGERR("spill-over device out of space (%d)", id); - return ENOSPC; - } - - /* got one from spill over */ - chunk_meta->location = CHUNK_LOCATION_SPILLOVER; - chunk_meta->id = id; - } else { - /* spill over isn't available, so we're out of space */ - LOGERR("memfs out of space (%d)", id); - return ENOSPC; - } - } else if (unifyfs_use_spillover) { - /* memory file system is not enabled, but spill over is */ - - /* shm segment out of space, grab a block from spill-over device */ - LOGDBG("getting blocks from spill-over device"); - - /* TODO: missing lock calls? */ - /* add unifyfs_max_chunks to identify chunk location */ - unifyfs_stack_lock(); - int id = unifyfs_stack_pop(free_spillchunk_stack) + unifyfs_max_chunks; - unifyfs_stack_unlock(); - if (id < unifyfs_max_chunks) { - LOGERR("spill-over device out of space (%d)", id); - return ENOSPC; - } - - /* got one from spill over */ - chunk_meta->location = CHUNK_LOCATION_SPILLOVER; - chunk_meta->id = id; - } else { - /* don't know how to allocate chunk */ - chunk_meta->location = CHUNK_LOCATION_NULL; - return EIO; - } - - return UNIFYFS_SUCCESS; -} - -static int unifyfs_chunk_free(int fid, unifyfs_filemeta_t* meta, int chunk_id) -{ - /* get pointer to chunk meta data */ - unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); - - /* get physical id of chunk */ - int id = chunk_meta->id; - LOGDBG("free chunk %d from location %d", id, chunk_meta->location); - - /* determine location of chunk */ - if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { - unifyfs_stack_lock(); - unifyfs_stack_push(free_chunk_stack, id); - unifyfs_stack_unlock(); - } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { - /* TODO: free spill over chunk */ - } else { - /* unkwown chunk location */ - LOGERR("unknown chunk location %d", chunk_meta->location); - return EIO; - } - - /* update location of chunk */ - chunk_meta->location = CHUNK_LOCATION_NULL; - - return UNIFYFS_SUCCESS; -} - -/* read data from specified chunk id, chunk offset, and count into user buffer, - * count should fit within chunk starting from specified offset */ -static int unifyfs_chunk_read( - unifyfs_filemeta_t* meta, /* pointer to file meta data */ - int chunk_id, /* logical chunk id to read data from */ - off_t chunk_offset, /* logical offset within chunk to read from */ - void* buf, /* buffer to store data to */ - size_t count) /* number of bytes to read */ -{ - /* get chunk meta data */ - unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); - - /* determine location of chunk */ - if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { - /* just need a memcpy to read data */ - void* chunk_buf = unifyfs_compute_chunk_buf( - meta, chunk_id, chunk_offset); - memcpy(buf, chunk_buf, count); - } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { - /* spill over to a file, so read from file descriptor */ - //MAP_OR_FAIL(pread); - off_t spill_offset = unifyfs_compute_spill_offset(meta, chunk_id, chunk_offset); - ssize_t rc = pread(unifyfs_spilloverblock, buf, count, spill_offset); - if (rc < 0) { - return errno; - } - } else { - /* unknown chunk type */ - LOGERR("unknown chunk type"); - return EIO; - } - - /* assume read was successful if we get to here */ - return UNIFYFS_SUCCESS; -} - -/* merges next_idx into prev_idx if possible, - * updates both prev_idx and next_idx leaving any +/* Merges write at next_idx into prev_idx if possible. + * Updates both prev_idx and next_idx leaving any * portion that could not be merged in next_idx */ static int unifyfs_coalesce_index( unifyfs_index_t* prev_idx, /* existing index entry to coalesce into */ @@ -341,60 +124,13 @@ static int unifyfs_coalesce_index( * Clear all entries in the log index. This only clears the metadata, * not the data itself. */ -static void unifyfs_clear_index(void) +static void clear_index(void) { *unifyfs_indices.ptr_num_entries = 0; } -/* - * Sync all the extents to the server. Clears the metadata index afterwards. - * - * Returns 0 on success, nonzero otherwise. - */ -int unifyfs_sync(int gfid) -{ - /* write contents from segment tree to index buffer - * if we're using that optimization */ - if (unifyfs_flatten_writes) { - unifyfs_rewrite_index_from_seg_tree(); - } - - /* if there are no index entries, we've got nothing to sync */ - if (*unifyfs_indices.ptr_num_entries == 0) { - return UNIFYFS_SUCCESS; - } - - /* ensure any data written to the spill over file is flushed */ - /* if using spill over, fsync spillover data to disk */ - if (unifyfs_use_spillover) { - int ret = __real_fsync(unifyfs_spilloverblock); - if (ret != 0) { - /* error, need to set errno appropriately, - * we called the real fsync which should - * have already set errno to something reasonable */ - LOGERR("failed to flush data to spill over file"); - return EIO; - } - } - - /* tell the server to grab our new extents */ - int ret = invoke_client_fsync_rpc(gfid); - if (ret != UNIFYFS_SUCCESS) { - /* something went wrong when trying to flush key/values */ - LOGERR("failed to flush key/value index to server"); - return EIO; - } - - /* flushed, clear buffer and refresh number of entries - * and number remaining */ - unifyfs_clear_index(); - - return UNIFYFS_SUCCESS; -} - -void unifyfs_add_index_entry_to_seg_tree( - unifyfs_filemeta_t* meta, - unifyfs_index_t* index) +static void add_index_entry_to_seg_tree(unifyfs_filemeta_t* meta, + unifyfs_index_t* index) { /* add index to our local log */ if (unifyfs_local_extents) { @@ -440,8 +176,10 @@ void unifyfs_add_index_entry_to_seg_tree( } /* Add the metadata for a single write to the index */ -static int unifyfs_logio_add_write_meta_to_index(unifyfs_filemeta_t* meta, - off_t file_pos, off_t log_pos, size_t length) +static int add_write_meta_to_index(unifyfs_filemeta_t* meta, + off_t file_pos, + off_t log_pos, + size_t length) { /* global file id for this entry */ int gfid = meta->gfid; @@ -472,7 +210,7 @@ static int unifyfs_logio_add_write_meta_to_index(unifyfs_filemeta_t* meta, unifyfs_key_slice_range); if (cur_idx.length == 0) { /* We were able to coalesce this write into prev_idx */ - unifyfs_add_index_entry_to_seg_tree(meta, prev_idx); + add_index_entry_to_seg_tree(meta, prev_idx); } } @@ -496,7 +234,7 @@ static int unifyfs_logio_add_write_meta_to_index(unifyfs_filemeta_t* meta, idxs[num_entries] = cur_idx; /* Add index entry to our seg_tree */ - unifyfs_add_index_entry_to_seg_tree(meta, &idxs[num_entries]); + add_index_entry_to_seg_tree(meta, &idxs[num_entries]); /* account for entries we just added */ num_entries += 1; @@ -508,84 +246,6 @@ static int unifyfs_logio_add_write_meta_to_index(unifyfs_filemeta_t* meta, return UNIFYFS_SUCCESS; } -/* write count bytes from user buffer into specified chunk id at chunk offset, - * count should fit within chunk starting from specified offset */ -static int unifyfs_logio_chunk_write( - int fid, /* local file id */ - long pos, /* write offset inside the file */ - unifyfs_filemeta_t* meta, /* pointer to file meta data */ - int chunk_id, /* logical chunk id to write to */ - off_t chunk_offset, /* logical offset within chunk to write to */ - const void* buf, /* buffer holding data to be written */ - size_t count) /* number of bytes to write */ -{ - /* get chunk meta data */ - unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); - - /* sanity check on the chunk type */ - if (chunk_meta->location != CHUNK_LOCATION_MEMFS && - chunk_meta->location != CHUNK_LOCATION_SPILLOVER) { - /* unknown chunk type */ - LOGERR("unknown chunk type"); - return EIO; - } - - /* copy data into chunk and record its starting offset within the log */ - off_t log_offset = 0; - if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { - /* store data in shared memory chunk, - * compute address to copy data to */ - char* chunk_buf = unifyfs_compute_chunk_buf( - meta, chunk_id, chunk_offset); - - /* just need a memcpy to record data */ - memcpy(chunk_buf, buf, count); - - /* record byte offset position within log, which is the number - * of bytes between the memory location we write to and the - * starting memory location of the shared memory data chunk region */ - log_offset = chunk_buf - unifyfs_chunks; - } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { - /* spill over to a file, so write to file descriptor, - * compute offset within spill over file */ - off_t spill_offset = unifyfs_compute_spill_offset(meta, chunk_id, chunk_offset); - - /* write data into file at appropriate offset, - * loop to keep trying short writes */ - //MAP_OR_FAIL(pwrite); - size_t nwritten = 0; - while (nwritten < count) { - /* attempt to write */ - void* bufpos = (void*)((char*)buf + nwritten); - size_t remaining = count - nwritten; - off_t filepos = spill_offset + (off_t)nwritten; - errno = 0; - ssize_t rc = __real_pwrite(unifyfs_spilloverblock, - bufpos, remaining, filepos); - if (rc < 0) { - LOGERR("pwrite failed: errno=%d (%s)", errno, strerror(errno)); - return EIO; - } - - /* wrote without an error, total up bytes written so far */ - nwritten += (size_t)rc; - } - - /* record byte offset position within log, for spill over - * locations we take the offset within the spill file - * added to the total size of all shared memory data chunks */ - log_offset = spill_offset + unifyfs_max_chunks * (1 << unifyfs_chunk_bits); - } - - /* get global file id for this file */ - int gfid = unifyfs_gfid_from_fid(fid); - - /* Update our write metadata with the new write */ - int rc = unifyfs_logio_add_write_meta_to_index(meta, pos, log_offset, - count); - return rc; -} - /* * Remove all entries in the current index and re-write it using the write * metadata stored in all the file's seg_trees. This only re-writes the @@ -605,7 +265,7 @@ void unifyfs_rewrite_index_from_seg_tree(void) unifyfs_index_t* indexes = unifyfs_indices.index_entry; /* Erase the index before we re-write it */ - unifyfs_clear_index(); + clear_index(); /* count up number of entries we wrote to buffer */ unsigned long idx = 0; @@ -646,209 +306,102 @@ void unifyfs_rewrite_index_from_seg_tree(void) *unifyfs_indices.ptr_num_entries = idx; } -/* write count bytes from user buffer into specified chunk id at chunk offset, - * count should fit within chunk starting from specified offset */ -static int unifyfs_chunk_write( - unifyfs_filemeta_t* meta, /* pointer to file meta data */ - int chunk_id, /* logical chunk id to write to */ - off_t chunk_offset, /* logical offset within chunk to write to */ - const void* buf, /* buffer holding data to be written */ - size_t count) /* number of bytes to write */ +/* + * Sync all the extents to the server. Clears the metadata index afterwards. + * + * Returns 0 on success, nonzero otherwise. + */ +int unifyfs_sync(int gfid) { - /* get chunk meta data */ - unifyfs_chunkmeta_t* chunk_meta = filemeta_get_chunkmeta(meta, chunk_id); - - /* determine location of chunk */ - if (chunk_meta->location == CHUNK_LOCATION_MEMFS) { - /* just need a memcpy to write data */ - void* chunk_buf = unifyfs_compute_chunk_buf( - meta, chunk_id, chunk_offset); - memcpy(chunk_buf, buf, count); -// _intel_fast_memcpy(chunk_buf, buf, count); -// unifyfs_memcpy(chunk_buf, buf, count); - } else if (chunk_meta->location == CHUNK_LOCATION_SPILLOVER) { - /* spill over to a file, so write to file descriptor */ - //MAP_OR_FAIL(pwrite); - off_t spill_offset = unifyfs_compute_spill_offset(meta, chunk_id, chunk_offset); - ssize_t rc = pwrite(unifyfs_spilloverblock, buf, count, spill_offset); - if (rc < 0) { - LOGERR("pwrite failed: errno=%d (%s)", errno, strerror(errno)); - } + /* NOTE: we currently ignore gfid and sync extents for all files in + * the index. If we ever switch to storing extents in a per-file index, + * we can support per-file sync. */ - /* TODO: check return code for errors */ - } else { - /* unknown chunk type */ - LOGERR("unknown chunk type"); - return EIO; + /* write contents from segment tree to index buffer + * if we're using that optimization */ + if (unifyfs_flatten_writes) { + unifyfs_rewrite_index_from_seg_tree(); } - /* assume write was successful if we get to here */ - return UNIFYFS_SUCCESS; -} - -/* --------------------------------------- - * Operations on file storage - * --------------------------------------- */ - -/* if length is greater than reserved space, reserve space up to length */ -int unifyfs_fid_store_fixed_extend(int fid, unifyfs_filemeta_t* meta, - off_t length) -{ - /* determine whether we need to allocate more chunks */ - off_t maxsize = meta->chunks << unifyfs_chunk_bits; - if (length > maxsize) { - /* compute number of additional bytes we need */ - off_t additional = length - maxsize; - while (additional > 0) { - /* allocate a new chunk */ - int rc = unifyfs_chunk_alloc(fid, meta, meta->chunks); - if (rc != UNIFYFS_SUCCESS) { - /* ran out of space to store data */ - LOGERR("failed to allocate chunk"); - return ENOSPC; - } - - /* increase chunk count and subtract bytes from the number we need */ - meta->chunks++; - additional -= unifyfs_chunk_size; - } + /* if there are no index entries, we've got nothing to sync */ + if (*unifyfs_indices.ptr_num_entries == 0) { + return UNIFYFS_SUCCESS; } - return UNIFYFS_SUCCESS; -} - -/* if length is shorter than reserved space, give back space down to length */ -int unifyfs_fid_store_fixed_shrink(int fid, unifyfs_filemeta_t* meta, - off_t length) -{ - /* determine the number of chunks to leave after truncating */ - off_t num_chunks = 0; - if (length > 0) { - num_chunks = (length >> unifyfs_chunk_bits) + 1; + /* ensure any data written to the spill over file is flushed */ + int ret = unifyfs_logio_sync(logio_ctx); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("failed to sync logio data"); + return EIO; } - /* clear off any extra chunks */ - while (meta->chunks > num_chunks) { - meta->chunks--; - unifyfs_chunk_free(fid, meta, meta->chunks); + /* tell the server to grab our new extents */ + ret = invoke_client_sync_rpc(); + if (ret != UNIFYFS_SUCCESS) { + /* something went wrong when trying to flush key/values */ + LOGERR("failed to flush key/value index to server"); + return EIO; } + /* flushed, clear buffer and refresh number of entries + * and number remaining */ + clear_index(); + return UNIFYFS_SUCCESS; } -/* read data from file stored as fixed-size chunks */ -int unifyfs_fid_store_fixed_read(int fid, unifyfs_filemeta_t* meta, off_t pos, - void* buf, size_t count) -{ - int rc; - - /* get pointer to position within first chunk */ - int chunk_id = pos >> unifyfs_chunk_bits; - off_t chunk_offset = pos & unifyfs_chunk_mask; - - /* determine how many bytes remain in the current chunk */ - size_t remaining = unifyfs_chunk_size - chunk_offset; - if (count <= remaining) { - /* all bytes for this read fit within the current chunk */ - rc = unifyfs_chunk_read(meta, chunk_id, chunk_offset, buf, count); - } else { - /* read what's left of current chunk */ - char* ptr = (char*) buf; - rc = unifyfs_chunk_read(meta, chunk_id, - chunk_offset, (void*)ptr, remaining); - ptr += remaining; - - /* read from the next chunk */ - size_t processed = remaining; - while (processed < count && rc == UNIFYFS_SUCCESS) { - /* get pointer to start of next chunk */ - chunk_id++; - - /* compute size to read from this chunk */ - size_t num = count - processed; - if (num > unifyfs_chunk_size) { - num = unifyfs_chunk_size; - } - - /* read data */ - rc = unifyfs_chunk_read(meta, chunk_id, 0, (void*)ptr, num); - ptr += num; - - /* update number of bytes written */ - processed += num; - } - } - return rc; -} +/* --------------------------------------- + * Operations on file storage + * --------------------------------------- */ -/* write data to file stored as fixed-size chunks */ -int unifyfs_fid_store_fixed_write(int fid, unifyfs_filemeta_t* meta, off_t pos, - const void* buf, size_t count) +/** + * Write data to file using log-based I/O + * + * @param fid file id to write to + * @param meta metadata for file + * @param pos file position to start writing at + * @param buf user buffer holding data + * @param count number of bytes to write + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_fid_logio_write(int fid, + unifyfs_filemeta_t* meta, + off_t pos, + const void* buf, + size_t count) { - int rc; + assert(meta != NULL); + if (meta->storage != FILE_STORAGE_LOGIO) { + LOGERR("file (fid=%d) storage mode != FILE_STORAGE_LOGIO", fid); + return EINVAL; + } - /* get pointer to position within first chunk */ - int chunk_id; - off_t chunk_offset; + /* allocate space in the log for this write */ + off_t log_off; + int rc = unifyfs_logio_alloc(logio_ctx, count, &log_off); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("logio_alloc(%zu) failed", count); + return rc; + } - if (meta->storage == FILE_STORAGE_LOGIO) { - chunk_id = meta->log_size >> unifyfs_chunk_bits; - chunk_offset = meta->log_size & unifyfs_chunk_mask; - } else { - return EIO; + /* do the write */ + size_t nwritten = 0; + rc = unifyfs_logio_write(logio_ctx, log_off, count, buf, &nwritten); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("logio_write(%zu, %zu) failed", log_off, count); + return rc; } - /* determine how many bytes remain in the current chunk */ - size_t remaining = unifyfs_chunk_size - chunk_offset; - if (count <= remaining) { - /* all bytes for this write fit within the current chunk */ - if (meta->storage == FILE_STORAGE_LOGIO) { - rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, chunk_offset, - buf, count); - } else { - return EIO; - } + if (nwritten < count) { + LOGWARN("partial logio_write() @ offset=%zu (%zu of %zu bytes)", + (size_t)log_off, nwritten, count); } else { - /* otherwise, fill up the remainder of the current chunk */ - char* ptr = (char*) buf; - if (meta->storage == FILE_STORAGE_LOGIO) { - rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, - chunk_offset, (void*)ptr, remaining); - } else { - return EIO; - } - - ptr += remaining; - pos += remaining; - - /* then write the rest of the bytes starting from beginning - * of chunks */ - size_t processed = remaining; - while (processed < count && rc == UNIFYFS_SUCCESS) { - /* get pointer to start of next chunk */ - chunk_id++; - - /* compute size to write to this chunk */ - size_t num = count - processed; - if (num > unifyfs_chunk_size) { - num = unifyfs_chunk_size; - } - - /* write data */ - if (meta->storage == FILE_STORAGE_LOGIO) { - rc = unifyfs_logio_chunk_write(fid, pos, meta, chunk_id, 0, - (void*)ptr, num); - } else { - return EIO; - } - ptr += num; - pos += num; - - /* update number of bytes processed */ - processed += num; - } + LOGDBG("successful logio_write() @ log offset=%zu (%zu bytes)", + (size_t)log_off, count); } + /* update our write metadata for this write */ + rc = add_write_meta_to_index(meta, pos, log_off, nwritten); return rc; } diff --git a/client/src/unifyfs-fixed.h b/client/src/unifyfs-fixed.h index 8e9bd6929..f61229756 100644 --- a/client/src/unifyfs-fixed.h +++ b/client/src/unifyfs-fixed.h @@ -45,42 +45,19 @@ #include "unifyfs-internal.h" -/* if length is greater than reserved space, - * reserve space up to length */ -int unifyfs_fid_store_fixed_extend( - int fid, /* file id to reserve space for */ - unifyfs_filemeta_t* meta, /* meta data for file */ - off_t length /* number of bytes to reserve for file */ -); - -/* if length is shorter than reserved space, - * give back space down to length */ -int unifyfs_fid_store_fixed_shrink( - int fid, /* file id to free space for */ - unifyfs_filemeta_t* meta, /* meta data for file */ - off_t length /* number of bytes to reserve for file */ -); +/* rewrite client's index of all file write extents */ +void unifyfs_rewrite_index_from_seg_tree(void); -/* read data from file stored as fixed-size chunks, - * returns UNIFYFS error code */ -int unifyfs_fid_store_fixed_read( - int fid, /* file id to read from */ - unifyfs_filemeta_t* meta, /* meta data for file */ - off_t pos, /* position within file to read from */ - void* buf, /* user buffer to store data in */ - size_t count /* number of bytes to read */ -); +/* sync all writes from client's index with local server */ +int unifyfs_sync(int gfid); -/* write data to file stored as fixed-size chunks, - * returns UNIFYFS error code */ -int unifyfs_fid_store_fixed_write( - int fid, /* file id to write to */ +/* write data to file using log-based I/O */ +int unifyfs_fid_logio_write( + int fid, /* file id to write to */ unifyfs_filemeta_t* meta, /* meta data for file */ - off_t pos, /* position within file to write to */ - const void* buf, /* user buffer holding data */ - size_t count /* number of bytes to write */ + off_t pos, /* file position to start writing at */ + const void* buf, /* user buffer holding data */ + size_t count /* number of bytes to write */ ); -void unifyfs_rewrite_index_from_seg_tree(void); -int unifyfs_sync(int gfid); #endif /* UNIFYFS_FIXED_H */ diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 30861856c..b31e3ed43 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -94,6 +94,7 @@ #include "unifyfs_const.h" #include "unifyfs_keyval.h" #include "unifyfs_log.h" +#include "unifyfs_logio.h" #include "unifyfs_meta.h" #include "unifyfs_shm.h" #include "seg_tree.h" @@ -327,7 +328,6 @@ typedef struct { extern unifyfs_index_buf_t unifyfs_indices; extern unsigned long unifyfs_max_index_entries; -extern long unifyfs_spillover_max_chunks; /* tracks total number of unsync'd segments for all files */ extern unsigned long unifyfs_segment_count; @@ -335,10 +335,12 @@ extern unsigned long unifyfs_segment_count; extern int local_rank_cnt; extern int local_rank_idx; extern int local_del_cnt; -extern int client_sockfd; -extern struct pollfd cmd_fd; -extern void* shm_req_buf; -extern void* shm_recv_buf; + +/* shmem context for read-request replies data region */ +extern shm_context* shm_recv_ctx; + +/* log-based I/O context */ +extern logio_context* logio_ctx; extern int app_id; extern size_t unifyfs_key_slice_range; @@ -390,26 +392,12 @@ extern void* unifyfs_stream_stack; * each is an index into unifyfs_dirstreams array */ extern void* unifyfs_dirstream_stack; -extern int unifyfs_use_memfs; -extern int unifyfs_use_spillover; +/* mutex to lock stack operations */ +extern pthread_mutex_t unifyfs_stack_mutex; extern int unifyfs_max_files; /* maximum number of files to store */ extern bool unifyfs_flatten_writes; /* enable write flattening */ extern bool unifyfs_local_extents; /* enable tracking of local extents */ -extern size_t -unifyfs_chunk_mem; /* number of bytes in memory to be used for chunk storage */ -extern int unifyfs_chunk_bits; /* we set chunk size = 2^unifyfs_chunk_bits */ -extern off_t unifyfs_chunk_size; /* chunk size in bytes */ -extern off_t -unifyfs_chunk_mask; /* mask applied to logical offset to determine physical offset within chunk */ -extern long -unifyfs_max_chunks; /* maximum number of chunks that fit in memory */ - -extern void* free_chunk_stack; -extern void* free_spillchunk_stack; -extern char* unifyfs_chunks; -extern unifyfs_chunkmeta_t* unifyfs_chunkmetas; -extern int unifyfs_spilloverblock; /* ------------------------------- * Common functions @@ -427,10 +415,6 @@ int unifyfs_would_overflow_offt(off_t a, off_t b); * added together */ int unifyfs_would_overflow_long(long a, long b); -/* given an input mode, mask it with umask and return, can specify - * an input mode==0 to specify all read/write bits */ -mode_t unifyfs_getmode(mode_t perms); - int unifyfs_stack_lock(); int unifyfs_stack_unlock(); @@ -536,25 +520,9 @@ int unifyfs_fid_create_file(const char* path); * returns the new fid, or a negative value on error */ int unifyfs_fid_create_directory(const char* path); -/* read count bytes from file starting from pos and store into buf, - * all bytes are assumed to exist, so checks on file size should be - * done before calling this routine */ -int unifyfs_fid_read(int fid, off_t pos, void* buf, size_t count); - -/* write count bytes from buf into file starting at offset pos, - * all bytes are assumed to be allocated to file, so file should - * be extended before calling this routine */ +/* write count bytes from buf into file starting at offset pos */ int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count); -/* given a file id, write zero bytes to region of specified offset - * and length, assumes space is already reserved */ -int unifyfs_fid_write_zero(int fid, off_t pos, off_t count); - -/* increase size of file if length is greater than current size, - * and allocate additional chunks as needed to reserve space for - * length bytes */ -int unifyfs_fid_extend(int fid, off_t length); - /* truncate file id to given length, frees resources if length is * less than size and allocates and zero-fills new bytes if length * is more than size */ diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 57da3ec90..7b598b082 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -46,12 +46,6 @@ #include "ucr_read_builder.h" #include "seg_tree.h" -/* ------------------- - * define external variables - * --------------------*/ - -extern int unifyfs_spilloverblock; -extern int unifyfs_use_spillover; #define MAX(a, b) (a > b ? a : b) @@ -654,12 +648,6 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) /* compute size log will be after we append data */ off_t newlogsize = logsize + count; - /* allocate storage space to hold data for this write */ - int extend_rc = unifyfs_fid_extend(fid, newlogsize); - if (extend_rc != UNIFYFS_SUCCESS) { - return extend_rc; - } - /* finally write specified data to file */ int write_rc = unifyfs_fid_write(fid, pos, buf, count); if (write_rc == 0) { @@ -1278,7 +1266,7 @@ static void delegator_signal(void) LOGDBG("receive buffer now empty"); /* set shm flag to signal delegator we're done */ - shm_header_t* hdr = (shm_header_t*)shm_recv_buf; + shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); hdr->state = SHMEM_REGION_EMPTY; /* TODO: MEM_FLUSH */ @@ -1297,7 +1285,7 @@ static int delegator_wait(void) shm_wait_tm.tv_nsec = SHM_WAIT_INTERVAL; /* get pointer to flag in shared memory */ - shm_header_t* hdr = (shm_header_t*)shm_recv_buf; + shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); /* wait for server to set flag to non-zero */ int max_sleep = 5000000; // 5s @@ -1326,8 +1314,8 @@ static int process_read_data(read_req_t* read_reqs, int count, int* done) int rc = UNIFYFS_SUCCESS; /* get pointer to start of shared memory buffer */ - shm_header_t* shm_hdr = (shm_header_t*)shm_recv_buf; - char* shmptr = ((char*)shm_hdr) + sizeof(shm_header_t); + shm_data_header* shm_hdr = (shm_data_header*)(shm_recv_ctx->addr); + char* shmptr = ((char*)shm_hdr) + sizeof(shm_data_header); /* get number of read replies in shared memory */ size_t num = shm_hdr->meta_cnt; @@ -1336,8 +1324,8 @@ static int process_read_data(read_req_t* read_reqs, int count, int* done) size_t i; for (i = 0; i < num; i++) { /* get pointer to current read reply header */ - shm_meta_t* rep = (shm_meta_t*)shmptr; - shmptr += sizeof(shm_meta_t); + shm_data_meta* rep = (shm_data_meta*)shmptr; + shmptr += sizeof(shm_data_meta); /* get pointer to data */ char* rep_buf = shmptr; @@ -1558,13 +1546,13 @@ static void service_local_reqs( * again search for a starting extent using a range * of just the very first byte that we need */ next = first; - while (next != NULL && next->start < req_end) { + while ((next != NULL) && (next->start < req_end)) { /* get start and end of this extent (reply) */ size_t rep_start = next->start; size_t rep_end = next->end + 1; /* get the offset into the log */ - size_t pos = next->ptr; + size_t rep_log_pos = next->ptr; /* start of overlapping segment is the maximum of * reply and request start offsets */ @@ -1596,71 +1584,26 @@ static void service_local_reqs( memset(req_ptr, 0, gap_length); } - /* copy data from reply buffer into request buffer */ + /* copy data from local write log into request buffer */ char* req_ptr = req->buf + req_offset; - - /* compute total size of shared memory data region */ - size_t chunk_mem_size = unifyfs_max_chunks * unifyfs_chunk_size; - - /* compute number of bytes we'll read from shared memory */ - size_t mem_length = 0; - if (pos < chunk_mem_size) { - /* we need to start from memory, assume we'll get it all */ - mem_length = length; - if (pos + length > chunk_mem_size) { - /* amount to read extends past end of memory segment, - * compute the number of bytes in memory */ - mem_length = chunk_mem_size - pos; + off_t log_offset = rep_log_pos + rep_offset; + size_t nread = 0; + int rc = unifyfs_logio_read(logio_ctx, log_offset, length, + req_ptr, &nread); + if (rc == UNIFYFS_SUCCESS) { + if (nread < length) { + /* account for short read by updating end offset */ + end -= (length - nread); } - } - - /* any remainder comes from the spill over file */ - size_t spill_length = length - mem_length; - - /* copy data from memory into request buffer */ - if (mem_length > 0) { - /* pointer to data is starting address of superblock - * data region (unify_chunks) + offset within the - * superblock to the start of the segment in - * reply (pos) + any offset from start of that segment - * until the first byte we are asking for (rep_offset) */ - char* mem_ptr = unifyfs_chunks + pos + rep_offset; - memcpy(req_ptr, mem_ptr, mem_length); - } - - /* copy data from spill over into request buffer */ - if (spill_length > 0) { - /* advance in user buffer past any bytes we copied in - * from memory */ - char* reqbuf = req_ptr + mem_length; - - /* offset to start of data in spill over file is - * position within log (pos) + offset from start of - * segment to the first byte in segment that overlaps - * with request (rep_offset) + any bytes copied from - * memory (mem_length) - the size of the data region - * in memory */ - off_t spill_offset = pos + rep_offset + mem_length - - chunk_mem_size; - - /* TODO: loop on this if we get a short read - * or EIO/EAGAIN */ - - /* read data from file into request buffer */ - ssize_t rc = pread(unifyfs_spilloverblock, - reqbuf, spill_length, spill_offset); - if (rc != length) { - /* had a problem reading, - * set the request error code */ - req->errcode = EIO; + /* update max number of bytes we have filled in the req buf */ + size_t req_nread = end - req_start; + if (req_nread > req->nread) { + req->nread = req_nread; } - } - - /* update max number of bytes we have written to in the - * request buffer */ - size_t nread = end - req_start; - if (nread > req->nread) { - req->nread = nread; + } else { + LOGERR("local log read failed for offset=%zu size=%zu", + (size_t)log_offset, length); + req->errcode = EIO; } /* get the next element in the tree */ @@ -1921,7 +1864,7 @@ int unifyfs_fd_logreadlist(read_req_t* in_reqs, int in_count) /* copy sever completed requests back into user's array */ if (count > 0) { /* skip past any items we copied in from the local requests */ - char* in_ptr = in_reqs + local_count * sizeof(read_req_t); + read_req_t* in_ptr = in_reqs + local_count; memcpy(in_ptr, read_reqs, count * sizeof(read_req_t)); } diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 687696ea9..c54ab345e 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -70,7 +70,6 @@ unifyfs_index_buf_t unifyfs_indices; static size_t unifyfs_index_buf_size; /* size of metadata log */ static size_t unifyfs_fattr_buf_size; unsigned long unifyfs_max_index_entries; /* max metadata log entries */ -int unifyfs_spillmetablock; /* tracks total number of unsync'd segments for all files */ unsigned long unifyfs_segment_count; @@ -79,34 +78,17 @@ int global_rank_cnt; /* count of world ranks */ int local_rank_cnt; int local_rank_idx; -int local_del_cnt = 1; -int client_sockfd = -1; -struct pollfd cmd_fd; - -/* shared memory buffer to transfer read requests - * from client to server */ -static char shm_req_name[GEN_STR_LEN] = {0}; -static size_t shm_req_size = UNIFYFS_SHMEM_REQ_SIZE; -void* shm_req_buf; +int local_del_cnt = 1; /* count of local servers */ /* shared memory buffer to transfer read replies * from server to client */ -static char shm_recv_name[GEN_STR_LEN] = {0}; -static size_t shm_recv_size = UNIFYFS_SHMEM_RECV_SIZE; -void* shm_recv_buf; +static size_t shm_recv_size = UNIFYFS_DATA_RECV_SIZE; +shm_context* shm_recv_ctx; // = NULL int client_rank; int app_id; size_t unifyfs_key_slice_range; -/* whether chunks should be allocated to - * store file contents in memory */ -int unifyfs_use_memfs = 1; - -/* whether chunks should be allocated to - * store file contents on spill over device */ -int unifyfs_use_spillover = 1; - static int unifyfs_use_single_shm = 0; static int unifyfs_page_size = 0; @@ -119,40 +101,23 @@ static off_t unifyfs_min_long; int unifyfs_max_files; /* maximum number of files to store */ bool unifyfs_flatten_writes; /* flatten our writes (true = enabled) */ bool unifyfs_local_extents; /* track data extents in client to read local */ -size_t unifyfs_chunk_mem; /* number of bytes in memory to be used for chunk storage */ -int unifyfs_chunk_bits; /* we set chunk size = 2^unifyfs_chunk_bits */ -off_t unifyfs_chunk_size; /* chunk size in bytes */ -off_t unifyfs_chunk_mask; /* mask applied to logical offset to determine physical offset within chunk */ -long unifyfs_max_chunks; /* maximum number of chunks that fit in memory */ -/* number of bytes in spillover to be used for chunk storage */ -static size_t unifyfs_spillover_size; - -/* maximum number of chunks that fit in spillover storage */ -long unifyfs_spillover_max_chunks; - -extern pthread_mutex_t unifyfs_stack_mutex; +/* log-based I/O context */ +logio_context* logio_ctx; /* keep track of what we've initialized */ int unifyfs_initialized = 0; -/* shared memory for superblock */ -static char shm_super_name[GEN_STR_LEN] = {0}; -static size_t shm_super_size; +/* superblock - persistent shared memory region (metadata + data) */ +static shm_context* shm_super_ctx; -/* global persistent memory block (metadata + data) */ -void* shm_super_buf; +/* per-file metadata */ static void* free_fid_stack; -void* free_chunk_stack; -void* free_spillchunk_stack; unifyfs_filename_t* unifyfs_filelist; static unifyfs_filemeta_t* unifyfs_filemetas; -unifyfs_chunkmeta_t* unifyfs_chunkmetas; - -char* unifyfs_chunks; -int unifyfs_spilloverblock = 0; -int unifyfs_spillmetablock = 0; /*used for log-structured i/o*/ +/* TODO: metadata spillover is not currently supported */ +int unifyfs_spillmetablock = -1; /* array of file descriptors */ unifyfs_fd_t unifyfs_fds[UNIFYFS_MAX_FILEDESCS]; @@ -187,16 +152,10 @@ void* unifyfs_dirstream_stack; /* mount point information */ char* unifyfs_mount_prefix; size_t unifyfs_mount_prefixlen = 0; -static key_t unifyfs_mount_shmget_key = 0; /* mutex to lock stack operations */ pthread_mutex_t unifyfs_stack_mutex = PTHREAD_MUTEX_INITIALIZER; -/* path of external storage's mount point*/ - -char external_data_dir[UNIFYFS_MAX_FILENAME] = {0}; -char external_meta_dir[UNIFYFS_MAX_FILENAME] = {0}; - /* single function to route all unsupported wrapper calls through */ int unifyfs_vunsupported( const char* fn_name, @@ -299,24 +258,6 @@ inline int unifyfs_would_overflow_long(long a, long b) return 0; } -/* given an input mode, mask it with umask and return, can specify - * an input mode==0 to specify all read/write bits */ -mode_t unifyfs_getmode(mode_t perms) -{ - /* perms == 0 is shorthand for all read and write bits */ - if (perms == 0) { - perms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; - } - - /* get current user mask */ - mode_t mask = umask(0); - umask(mask); - - /* mask off bits from desired permissions */ - mode_t ret = perms & ~mask & 0777; - return ret; -} - /* lock access to shared data structures in superblock */ inline int unifyfs_stack_lock() { @@ -997,11 +938,9 @@ int unifyfs_fid_create_directory(const char* path) return UNIFYFS_SUCCESS; } -/* write count bytes from buf into file starting at offset pos, - * all bytes are assumed to be allocated to file, so file should - * be extended before calling this routine. +/* Write count bytes from buf into file starting at offset pos. * - * Returns a UNIFYFS_ERROR on error, 0 on success. + * Returns UNIFYFS_SUCCESS, or an error code */ int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count) { @@ -1018,7 +957,7 @@ int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count) /* determine storage type to write file data */ if (meta->storage == FILE_STORAGE_LOGIO) { /* file stored in fixed-size chunks */ - rc = unifyfs_fid_store_fixed_write(fid, meta, pos, buf, count); + rc = unifyfs_fid_logio_write(fid, meta, pos, buf, count); } else { /* unknown storage type */ rc = EIO; @@ -1027,81 +966,6 @@ int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count) return rc; } -/* given a file id, write zero bytes to region of specified offset - * and length, assumes space is already reserved */ -int unifyfs_fid_write_zero(int fid, off_t pos, off_t count) -{ - int rc = UNIFYFS_SUCCESS; - - /* allocate an aligned chunk of memory */ - size_t buf_size = 1024 * 1024; - void* buf = (void*) malloc(buf_size); - if (buf == NULL) { - return EIO; - } - - /* set values in this buffer to zero */ - memset(buf, 0, buf_size); - - /* write zeros to file */ - off_t written = 0; - off_t curpos = pos; - while (written < count) { - /* compute number of bytes to write on this iteration */ - size_t num = buf_size; - off_t remaining = count - written; - if (remaining < (off_t) buf_size) { - num = (size_t) remaining; - } - - /* write data to file */ - int write_rc = unifyfs_fid_write(fid, curpos, buf, num); - if (write_rc != UNIFYFS_SUCCESS) { - rc = EIO; - break; - } - - /* update the number of bytes written */ - curpos += (off_t) num; - written += (off_t) num; - } - - /* free the buffer */ - free(buf); - - return rc; -} - -/* increase size of file if length is greater than current size, - * and allocate additional chunks as needed to reserve space for - * length bytes */ -int unifyfs_fid_extend(int fid, off_t length) -{ - int rc; - - /* get meta data for this file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - - /* determine file storage type */ - if (meta->storage == FILE_STORAGE_LOGIO) { - /* file stored in fixed-size chunks */ - rc = unifyfs_fid_store_fixed_extend(fid, meta, length); - } else { - /* unknown storage type */ - rc = EIO; - } - - return rc; -} - -/* if length is less than reserved space, give back space down to length */ -int unifyfs_fid_shrink(int fid, off_t length) -{ - /* TODO: implement this function */ - - return EIO; -} - /* truncate file id to given length, frees resources if length is * less than size and allocates and zero-fills new bytes if length * is more than size */ @@ -1263,10 +1127,8 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } LOGDBG("Creating a new entry for %s.", path); - LOGDBG("shm_super_buf = %p; free_fid_stack = %p; " - "free_chunk_stack = %p; unifyfs_filelist = %p; " - "chunks = %p", shm_super_buf, free_fid_stack, - free_chunk_stack, unifyfs_filelist, unifyfs_chunks); + LOGDBG("superblock addr = %p; free_fid_stack = %p; filelist = %p", + shm_super_ctx->addr, free_fid_stack, unifyfs_filelist); /* allocate a file id slot for this new file */ fid = unifyfs_fid_create_file(path); @@ -1413,33 +1275,6 @@ static size_t unifyfs_superblock_size(void) /* file metadata struct array */ sb_size += unifyfs_max_files * sizeof(unifyfs_filemeta_t); - if (unifyfs_use_memfs) { - /* memory chunk metadata struct array for each file, - * enables a file to use all space in memory */ - sb_size += unifyfs_max_files * unifyfs_max_chunks * - sizeof(unifyfs_chunkmeta_t); - } - if (unifyfs_use_spillover) { - /* spillover chunk metadata struct array for each file, - * enables a file to use all space in spillover file */ - sb_size += unifyfs_max_files * unifyfs_spillover_max_chunks * - sizeof(unifyfs_chunkmeta_t); - } - - /* free chunk stack */ - if (unifyfs_use_memfs) { - sb_size += unifyfs_stack_bytes(unifyfs_max_chunks); - } - if (unifyfs_use_spillover) { - sb_size += unifyfs_stack_bytes(unifyfs_spillover_max_chunks); - } - - /* space for memory chunks */ - if (unifyfs_use_memfs) { - sb_size += unifyfs_page_size; - sb_size += unifyfs_max_chunks * unifyfs_chunk_size; - } - /* index region size */ sb_size += unifyfs_page_size; sb_size += unifyfs_max_index_entries * sizeof(unifyfs_index_t); @@ -1482,37 +1317,6 @@ static void* unifyfs_init_pointers(void* superblock) unifyfs_filemetas = (unifyfs_filemeta_t*)ptr; ptr += unifyfs_max_files * sizeof(unifyfs_filemeta_t); - /* array of chunk meta data strucutres for each file */ - unifyfs_chunkmetas = (unifyfs_chunkmeta_t*)ptr; - if (unifyfs_use_memfs) { - ptr += unifyfs_max_files * unifyfs_max_chunks * - sizeof(unifyfs_chunkmeta_t); - } - if (unifyfs_use_spillover) { - ptr += unifyfs_max_files * unifyfs_spillover_max_chunks * - sizeof(unifyfs_chunkmeta_t); - } - - /* stack to manage free memory data chunks */ - if (unifyfs_use_memfs) { - free_chunk_stack = ptr; - ptr += unifyfs_stack_bytes(unifyfs_max_chunks); - } - if (unifyfs_use_spillover) { - free_spillchunk_stack = ptr; - ptr += unifyfs_stack_bytes(unifyfs_spillover_max_chunks); - } - - /* Only set this up if we're using memfs */ - if (unifyfs_use_memfs) { - /* pointer to start of memory data chunks */ - ptr = next_page_align(ptr); - unifyfs_chunks = ptr; - ptr += unifyfs_max_chunks * unifyfs_chunk_size; - } else { - unifyfs_chunks = NULL; - } - /* record pointer to number of index entries */ unifyfs_indices.ptr_num_entries = (size_t*)ptr; @@ -1524,7 +1328,7 @@ static void* unifyfs_init_pointers(void* superblock) /* compute size of memory we're using and check that * it matches what we allocated */ size_t ptr_size = (size_t)(ptr - (char*)superblock); - if (ptr_size > shm_super_size) { + if (ptr_size > shm_super_ctx->size) { LOGERR("Data structures in superblock extend beyond its size"); } @@ -1534,15 +1338,6 @@ static void* unifyfs_init_pointers(void* superblock) /* initialize data structures for first use */ static int unifyfs_init_structures() { - /* compute total number of storage chunks available */ - int numchunks = 0; - if (unifyfs_use_memfs) { - numchunks += unifyfs_max_chunks; - } - if (unifyfs_use_spillover) { - numchunks += unifyfs_spillover_max_chunks; - } - int i; for (i = 0; i < unifyfs_max_files; i++) { /* indicate that file id is not in use by setting flag to 0 */ @@ -1550,24 +1345,11 @@ static int unifyfs_init_structures() /* set pointer to array of chunkmeta data structures */ unifyfs_filemeta_t* filemeta = &unifyfs_filemetas[i]; - - /* compute offset to start of chunk meta list for this file */ - filemeta->chunkmeta_idx = numchunks * i; } /* initialize stack of free file ids */ unifyfs_stack_init(free_fid_stack, unifyfs_max_files); - /* initialize list of free memory chunks */ - if (unifyfs_use_memfs) { - unifyfs_stack_init(free_chunk_stack, unifyfs_max_chunks); - } - - /* initialize list of free spillover chunks */ - if (unifyfs_use_spillover) { - unifyfs_stack_init(free_spillchunk_stack, unifyfs_spillover_max_chunks); - } - /* initialize count of key/value entries */ *(unifyfs_indices.ptr_num_entries) = 0; @@ -1605,21 +1387,21 @@ static int unifyfs_get_spillblock(size_t size, const char* path) /* create superblock of specified size and name, or attach to existing * block if available */ -static void* unifyfs_superblock_shmget(size_t size, key_t key) +static int unifyfs_superblock_shmget(size_t super_sz) { - /* define name for superblock shared memory region */ - snprintf(shm_super_name, sizeof(shm_super_name), "%d-super-%d", - app_id, key); - LOGDBG("Key for superblock = %x", key); + char shm_name[SHMEM_NAME_LEN] = {0}; - /* open shared memory file */ - void* addr = unifyfs_shm_alloc(shm_super_name, size); - if (addr == NULL) { - LOGERR("Failed to create superblock"); - return NULL; + /* attach shmem region for client's superblock */ + sprintf(shm_name, SHMEM_SUPER_FMTSTR, app_id, local_rank_idx); + shm_context* shm_ctx = unifyfs_shm_alloc(shm_name, super_sz); + if (NULL == shm_ctx) { + LOGERR("Failed to attach to shmem superblock region %s", shm_name); + return UNIFYFS_ERROR_SHMEM; } + shm_super_ctx = shm_ctx; /* init our global variables to point to spots in superblock */ + void* addr = shm_ctx->addr; unifyfs_init_pointers(addr); /* initialize structures in superblock if it's newly allocated, @@ -1636,7 +1418,7 @@ static void* unifyfs_superblock_shmget(size_t size, key_t key) } /* return starting memory address of super block */ - return addr; + return UNIFYFS_SUCCESS; } static int unifyfs_init(int rank) @@ -1696,27 +1478,6 @@ static int unifyfs_init(int rank) unifyfs_max_long = LONG_MAX; unifyfs_min_long = LONG_MIN; - /* will we use spillover to store the files? */ - unifyfs_use_spillover = 1; - cfgval = client_cfg.spillover_enabled; - if (cfgval != NULL) { - rc = configurator_bool_val(cfgval, &b); - if ((rc == 0) && !b) { - unifyfs_use_spillover = 0; - } - } - LOGDBG("are we using spillover? %d", unifyfs_use_spillover); - - /* determine maximum number of bytes of spillover for chunk storage */ - unifyfs_spillover_size = UNIFYFS_SPILLOVER_SIZE; - cfgval = client_cfg.spillover_size; - if (cfgval != NULL) { - rc = configurator_int_val(cfgval, &l); - if (rc == 0) { - unifyfs_spillover_size = (size_t)l; - } - } - /* determine max number of files to store in file system */ unifyfs_max_files = UNIFYFS_MAX_FILES; cfgval = client_cfg.client_max_files; @@ -1748,39 +1509,10 @@ static int unifyfs_init(int rank) } } - /* determine number of bits for chunk size */ - unifyfs_chunk_bits = UNIFYFS_CHUNK_BITS; - cfgval = client_cfg.shmem_chunk_bits; - if (cfgval != NULL) { - rc = configurator_int_val(cfgval, &l); - if (rc == 0) { - unifyfs_chunk_bits = (int)l; - } - } - - /* determine maximum number of bytes of memory for chunk storage */ - unifyfs_chunk_mem = UNIFYFS_CHUNK_MEM; - cfgval = client_cfg.shmem_chunk_mem; - if (cfgval != NULL) { - rc = configurator_int_val(cfgval, &l); - if (rc == 0) { - unifyfs_chunk_mem = (size_t)l; - } - } - - /* set chunk size, set chunk offset mask, and set total number - * of chunks */ - unifyfs_chunk_size = 1 << unifyfs_chunk_bits; - unifyfs_chunk_mask = unifyfs_chunk_size - 1; - unifyfs_max_chunks = unifyfs_chunk_mem >> unifyfs_chunk_bits; - - /* set number of chunks in spillover device */ - unifyfs_spillover_max_chunks = unifyfs_spillover_size >> unifyfs_chunk_bits; - /* define size of buffer used to cache key/value pairs for * data offsets before passing them to the server */ unifyfs_index_buf_size = UNIFYFS_INDEX_BUF_SIZE; - cfgval = client_cfg.logfs_index_buf_size; + cfgval = client_cfg.client_write_index_size; if (cfgval != NULL) { rc = configurator_int_val(cfgval, &l); if (rc == 0) { @@ -1836,68 +1568,23 @@ static int unifyfs_init(int rank) unifyfs_stack_init(unifyfs_dirstream_stack, num_dirstreams); /* determine the size of the superblock */ - shm_super_size = unifyfs_superblock_size(); + size_t shm_super_size = unifyfs_superblock_size(); /* get a superblock of shared memory and initialize our * global variables for this block */ - shm_super_buf = unifyfs_superblock_shmget( - shm_super_size, unifyfs_mount_shmget_key); - if (shm_super_buf == NULL) { + rc = unifyfs_superblock_shmget(shm_super_size); + if (rc != UNIFYFS_SUCCESS) { LOGERR("unifyfs_superblock_shmget() failed"); - return UNIFYFS_FAILURE; + return rc; } - /* initialize spillover store */ - if (unifyfs_use_spillover) { - /* get directory in which to create spill over files */ - cfgval = client_cfg.spillover_data_dir; - if (cfgval != NULL) { - strncpy(external_data_dir, cfgval, sizeof(external_data_dir)); - } else { - LOGERR("UNIFYFS_SPILLOVER_DATA_DIR not set, must be an existing" - " writable path (e.g., /mnt/ssd):"); - return UNIFYFS_FAILURE; - } - - /* define path to the spill over file for data chunks */ - char spillfile_prefix[UNIFYFS_MAX_FILENAME]; - snprintf(spillfile_prefix, sizeof(spillfile_prefix), - "%s/spill_%d_%d.log", - external_data_dir, app_id, local_rank_idx); - - /* create the spill over file */ - unifyfs_spilloverblock = - unifyfs_get_spillblock(unifyfs_spillover_size, - spillfile_prefix); - if (unifyfs_spilloverblock < 0) { - LOGERR("unifyfs_get_spillblock() failed!"); - return UNIFYFS_FAILURE; - } - - /* get directory in which to create spill over files - * for key/value pairs */ - cfgval = client_cfg.spillover_meta_dir; - if (cfgval != NULL) { - strncpy(external_meta_dir, cfgval, sizeof(external_meta_dir)); - } else { - LOGERR("UNIFYFS_SPILLOVER_META_DIR not set, must be an existing" - " writable path (e.g., /mnt/ssd):"); - return UNIFYFS_FAILURE; - } - - /* define path to the spill over file for key/value pairs */ - snprintf(spillfile_prefix, sizeof(spillfile_prefix), - "%s/spill_index_%d_%d.log", - external_meta_dir, app_id, local_rank_idx); - - /* create the spill over file for key value data */ - unifyfs_spillmetablock = - unifyfs_get_spillblock(unifyfs_index_buf_size, - spillfile_prefix); - if (unifyfs_spillmetablock < 0) { - LOGERR("unifyfs_get_spillmetablock failed!"); - return UNIFYFS_FAILURE; - } + /* initialize log-based I/O context */ + rc = unifyfs_logio_init_client(app_id, local_rank_idx, &client_cfg, + &logio_ctx); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to initialize log-based I/O (rc = %s)", + unifyfs_rc_enum_str(rc)); + return rc; } /* remember that we've now initialized the library */ @@ -1915,25 +1602,21 @@ static int unifyfs_init(int rank) void fill_client_mount_info(unifyfs_mount_in_t* in) { size_t meta_offset = (char*)unifyfs_indices.ptr_num_entries - - (char*)shm_super_buf; + (char*)shm_super_ctx->addr; size_t meta_size = unifyfs_max_index_entries * sizeof(unifyfs_index_t); - size_t data_offset = (char*)unifyfs_chunks - (char*)shm_super_buf; - size_t data_size = (size_t)unifyfs_max_chunks * unifyfs_chunk_size; - in->app_id = app_id; - in->local_rank_idx = local_rank_idx; + in->client_id = local_rank_idx; in->dbg_rank = client_rank; in->num_procs_per_node = local_rank_cnt; - in->req_buf_sz = shm_req_size; in->recv_buf_sz = shm_recv_size; - in->superblock_sz = shm_super_size; + in->superblock_sz = shm_super_ctx->size; in->meta_offset = meta_offset; in->meta_size = meta_size; - in->data_offset = data_offset; - in->data_size = data_size; - in->external_spill_dir = strdup(external_data_dir); + in->logio_mem_size = logio_ctx->shmem->size; + in->logio_spill_size = logio_ctx->spill_sz; + in->external_spill_dir = strdup(client_cfg.logio_spill_dir); } /** @@ -1941,8 +1624,10 @@ void fill_client_mount_info(unifyfs_mount_in_t* in) */ static int unifyfs_init_recv_shm(int local_rank_idx, int app_id) { + char shm_recv_name[SHMEM_NAME_LEN] = {0}; + /* get size of shared memory region from configuration */ - char* cfgval = client_cfg.shmem_recv_size; + char* cfgval = client_cfg.client_recv_data_size; if (cfgval != NULL) { long l; int rc = configurator_int_val(cfgval, &l); @@ -1953,11 +1638,11 @@ static int unifyfs_init_recv_shm(int local_rank_idx, int app_id) /* define file name to shared memory file */ snprintf(shm_recv_name, sizeof(shm_recv_name), - "%d-recv-%d", app_id, local_rank_idx); + SHMEM_DATA_FMTSTR, app_id, local_rank_idx); /* allocate memory for shared memory receive buffer */ - shm_recv_buf = unifyfs_shm_alloc(shm_recv_name, shm_recv_size); - if (shm_recv_buf == NULL) { + shm_recv_ctx = unifyfs_shm_alloc(shm_recv_name, shm_recv_size); + if (NULL == shm_recv_ctx) { LOGERR("Failed to create buffer for read replies"); return UNIFYFS_FAILURE; } @@ -1965,74 +1650,6 @@ static int unifyfs_init_recv_shm(int local_rank_idx, int app_id) return UNIFYFS_SUCCESS; } -/** - * Initialize the shared request memory, which - * is used to buffer the list of read requests - * to be transferred to the delegator on the - * server side. - * @param local_rank_idx: local process id - * @param app_id: which application this - * process is from - * @return success/error code - */ -static int unifyfs_init_req_shm(int local_rank_idx, int app_id) -{ - /* get size of shared memory region from configuration */ - char* cfgval = client_cfg.shmem_req_size; - if (cfgval != NULL) { - long l; - int rc = configurator_int_val(cfgval, &l); - if (rc == 0) { - shm_req_size = l; - } - } - - /* define name of shared memory region for request buffer */ - snprintf(shm_req_name, sizeof(shm_req_name), - "%d-req-%d", app_id, local_rank_idx); - - /* allocate memory for shared memory receive buffer */ - shm_req_buf = unifyfs_shm_alloc(shm_req_name, shm_req_size); - if (shm_req_buf == NULL) { - LOGERR("Failed to create buffer for read requests"); - return UNIFYFS_FAILURE; - } - - return UNIFYFS_SUCCESS; -} - -static int compare_int(const void* a, const void* b) -{ - const int* ptr_a = a; - const int* ptr_b = b; - - if (*ptr_a - *ptr_b > 0) { - return 1; - } - - if (*ptr_a - *ptr_b < 0) { - return -1; - } - - return 0; -} - -static int compare_name_rank_pair(const void* a, const void* b) -{ - const name_rank_pair_t* pair_a = a; - const name_rank_pair_t* pair_b = b; - - if (strcmp(pair_a->hostname, pair_b->hostname) > 0) { - return 1; - } - - if (strcmp(pair_a->hostname, pair_b->hostname) < 0) { - return -1; - } - - return 0; -} - /** * calculate the number of ranks per node * @@ -2273,23 +1890,6 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, unifyfs_mount_prefix = strdup(prefix); unifyfs_mount_prefixlen = strlen(unifyfs_mount_prefix); - /* - * unifyfs_mount_shmget_key marks the start of - * the superblock shared memory of each rank - * each process has three types of shared memory: - * request memory, recv memory and superblock - * memory. We set unifyfs_mount_shmget_key in - * this way to avoid different ranks conflicting - * on the same name in shm_open. - */ - cfgval = client_cfg.shmem_single; - if (cfgval != NULL) { - rc = configurator_bool_val(cfgval, &b); - if ((rc == 0) && b) { - unifyfs_use_single_shm = 1; - } - } - /* compute our local rank on the node, * the following call initializes local_rank_{cnt,ndx} */ rc = CountTasksPerNode(rank, size); @@ -2298,10 +1898,6 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, return -1; } - /* use our local rank on the node in shared memory and file - * names to avoid conflicting with other procs on our node */ - unifyfs_mount_shmget_key = local_rank_idx; - /* initialize our library, creates superblock and spillover files */ int ret = unifyfs_init(rank); if (ret != UNIFYFS_SUCCESS) { @@ -2315,18 +1911,10 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, return ret; } - /* create shared memory region for read requests */ - rc = unifyfs_init_req_shm(local_rank_idx, app_id); - if (rc < 0) { - LOGERR("failed to init shared request memory"); - return UNIFYFS_FAILURE; - } - /* create shared memory region for holding data for read replies */ rc = unifyfs_init_recv_shm(local_rank_idx, app_id); if (rc < 0) { LOGERR("failed to init shared receive memory"); - unifyfs_shm_unlink(shm_req_name); return UNIFYFS_FAILURE; } @@ -2341,21 +1929,18 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, /* TODO: need more clean up here, but this at least deletes * some files we would otherwise leave behind */ - /* Delete file for shared memory regions for - * read requests and read replies */ - unifyfs_shm_unlink(shm_req_name); - unifyfs_shm_unlink(shm_recv_name); + /* Delete file for read reply shared memory region */ + unifyfs_shm_unlink(shm_recv_ctx); return ret; } /* Once we return from mount, we know the server has attached to our - * shared memory regions for read requests and read replies, so we - * can safely remove these files. The memory regions will stay active - * until both client and server unmap them. We keep the superblock file - * around so that a future client can reattach to it. */ - unifyfs_shm_unlink(shm_req_name); - unifyfs_shm_unlink(shm_recv_name); + * shared memory region for read replies, so we can safely remove the + * file. The memory region will stay active until both client and server + * unmap them. We keep the superblock file around so that a future client + * can reattach to it. */ + unifyfs_shm_unlink(shm_recv_ctx); /* add mount point as a new directory in the file list */ if (unifyfs_get_fid_from_path(prefix) < 0) { @@ -2388,18 +1973,14 @@ static int unifyfs_finalize(void) } /* close spillover files */ - if (unifyfs_spilloverblock != 0) { - close(unifyfs_spilloverblock); - unifyfs_spilloverblock = 0; - } - - if (unifyfs_spillmetablock != 0) { + unifyfs_logio_close(logio_ctx); + if (unifyfs_spillmetablock != -1) { close(unifyfs_spillmetablock); - unifyfs_spillmetablock = 0; + unifyfs_spillmetablock = -1; } /* detach from superblock */ - unifyfs_shm_free(shm_super_name, shm_super_size, &shm_super_buf); + unifyfs_shm_free(&shm_super_ctx); /* free directory stream stack */ if (unifyfs_dirstream_stack != NULL) { @@ -2445,18 +2026,7 @@ int unifyfs_unmount(void) ************************/ /* detach from shared memory regions */ - unifyfs_shm_free(shm_req_name, shm_req_size, &shm_req_buf); - unifyfs_shm_free(shm_recv_name, shm_recv_size, &shm_recv_buf); - - /* close socket to server */ - if (client_sockfd >= 0) { - errno = 0; - rc = close(client_sockfd); - if (rc != 0) { - LOGERR("Failed to close() socket to server errno=%d (%s)", - errno, strerror(errno)); - } - } + unifyfs_shm_free(&shm_recv_ctx); /* invoke unmount rpc to tell server we're disconnecting */ LOGDBG("calling unmount"); diff --git a/client/src/unifyfs.h b/client/src/unifyfs.h index d7413e498..d03c37b4c 100644 --- a/client/src/unifyfs.h +++ b/client/src/unifyfs.h @@ -43,32 +43,12 @@ #ifndef UNIFYFS_H #define UNIFYFS_H -#include -#include // size_t -#include // off_t - -#include "unifyfs_const.h" +#include #ifdef __cplusplus extern "C" { #endif -/* linked list of chunk information given to an external library wanting - * to RDMA out a file from UNIFYFS */ -typedef struct { - off_t chunk_id; - int location; - void* chunk_mr; - off_t spillover_offset; - struct chunk_list_t* next; -} chunk_list_t; - -/*data structures defined for unifyfs********************/ - -typedef struct { - char hostname[UNIFYFS_MAX_HOSTNAME]; - int rank; -} name_rank_pair_t; int unifyfs_mount(const char prefix[], int rank, size_t size, int l_app_id); diff --git a/common/src/Makefile.am b/common/src/Makefile.am index 4997a83d8..978f75849 100644 --- a/common/src/Makefile.am +++ b/common/src/Makefile.am @@ -13,10 +13,15 @@ BASE_SRCS = \ rm_enumerator.c \ flatbuffers_common_builder.h \ flatbuffers_common_reader.h \ - ucr_read_builder.h \ - ucr_read_reader.h \ + seg_tree.h \ + seg_tree.c \ + slotmap.h \ + slotmap.c \ tinyexpr.h \ tinyexpr.c \ + tree.h \ + ucr_read_builder.h \ + ucr_read_reader.h \ unifyfs_const.h \ unifyfs_configurator.h \ unifyfs_configurator.c \ @@ -24,6 +29,8 @@ BASE_SRCS = \ unifyfs_keyval.c \ unifyfs_log.h \ unifyfs_log.c \ + unifyfs_logio.h \ + unifyfs_logio.c \ unifyfs_meta.h \ unifyfs_meta.c \ unifyfs_rpc_util.h \ @@ -35,10 +42,7 @@ BASE_SRCS = \ unifyfs_runstate.h \ unifyfs_runstate.c \ unifyfs_shm.h \ - unifyfs_shm.c \ - tree.h \ - seg_tree.h \ - seg_tree.c + unifyfs_shm.c OPT_FLAGS = OPT_LIBS = @@ -69,6 +73,6 @@ libunifyfs_common_la_LDFLAGS = \ -version-info $(LIBUNIFYFS_LT_VERSION) libunifyfs_common_la_LIBADD = \ - $(OPT_LIBS) -lm -lrt -lcrypto + $(OPT_LIBS) -lm -lrt -lcrypto -lpthread AM_CFLAGS = -Wall -Wno-strict-aliasing diff --git a/common/src/slotmap.c b/common/src/slotmap.c new file mode 100644 index 000000000..408a36b3b --- /dev/null +++ b/common/src/slotmap.c @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "unifyfs_const.h" +#include "slotmap.h" + +#include +#include // bool +#include // uint8_t +#include +#include // NULL +#include // memset() + + +/* Bit-twiddling convenience macros */ +#define SLOT_BYTE(slot) ((slot) >> 3) +#define SLOT_BIT(slot) ((slot) & 0x7) +#define BYTE_BIT_TO_SLOT(byte, bit) (((byte) * 8) + (bit)) +#define BYTE_VAL_BIT(byte_val, bit) ((byte_val) & (uint8_t)(1 << (bit))) + +/* Set given byte-bit in use map */ +static inline +void set_usemap_byte_bit(uint8_t* usemap, size_t byte, int bit) +{ + uint8_t byte_val = usemap[byte]; + byte_val |= (uint8_t)(1 << bit); + usemap[byte] = byte_val; +} + +/* Clear given byte-bit in use map */ +static inline +void clear_usemap_byte_bit(uint8_t* usemap, size_t byte, int bit) +{ + uint8_t byte_val = usemap[byte]; + uint8_t byte_mask = (uint8_t)0xFF - (uint8_t)(1 << bit); + byte_val &= byte_mask; + usemap[byte] = byte_val; +} + +/* Check use map for slot used */ +static inline +int check_slot(uint8_t* usemap, size_t slot) +{ + size_t byte = SLOT_BYTE(slot); + int bit = SLOT_BIT(slot); + uint8_t byte_val = usemap[byte]; + if (BYTE_VAL_BIT(byte_val, bit)) { + return 1; + } + return 0; +} + +/* Update use map to use slot */ +static inline +void use_slot(uint8_t* usemap, size_t slot) +{ + size_t byte = SLOT_BYTE(slot); + int bit = SLOT_BIT(slot); + set_usemap_byte_bit(usemap, byte, bit); +} + +/* Update use map to release slot */ +static inline +void release_slot(uint8_t* usemap, size_t slot) +{ + size_t byte = SLOT_BYTE(slot); + int bit = SLOT_BIT(slot); + clear_usemap_byte_bit(usemap, byte, bit); +} + +/* Return bytes necessary to hold use map for given number of slots */ +static inline +size_t slot_map_bytes(size_t total_slots) +{ + size_t map_bytes = SLOT_BYTE(total_slots); + if (SLOT_BIT(total_slots)) { + map_bytes++; + } + return map_bytes; +} + +/* Slot usage bitmap immediately follows the structure in memory. + * The usage bitmap can be thought of as an uint8_t array, where + * each uint8_t represents 8 slots. + * uint8_t use_bitmap[total_slots/8] + */ +static inline +uint8_t* get_use_map(slot_map* smap) +{ + uint8_t* usemap = (uint8_t*)((char*)smap + sizeof(slot_map)); + return usemap; +} + +/* Return number of free slots */ +static inline +size_t get_free_slots(slot_map* smap) +{ + return smap->total_slots - smap->used_slots; +} + +/** + * Initialize a slot map within the given memory region, and return a pointer + * to the slot_map structure. Returns NULL if the provided memory region is not + * large enough to track the requested number of slots. + * + * @param num_slots number of slots to track + * @param region_addr address of the memory region + * @param region_sz size of the memory region + * + * @return valid slot_map pointer, or NULL on error + */ +slot_map* slotmap_init(size_t num_slots, + void* region_addr, + size_t region_sz) +{ + if (NULL == region_addr) { + return NULL; + } + + size_t avail_use_bytes = region_sz - sizeof(slot_map); + size_t needed_use_bytes = slot_map_bytes(num_slots); + if (needed_use_bytes > avail_use_bytes) { + /* not enough space for use map */ + return NULL; + } + + slot_map* smap = (slot_map*) region_addr; + smap->total_slots = num_slots; + slotmap_clear(smap); + + return smap; +} + +/** + * Clear the given slot_map. Marks all slots free. + * + * @param smap valid slot_map pointer + * + * @return UNIFYFS_SUCCESS, or error code + */ +int slotmap_clear(slot_map* smap) +{ + if (NULL == smap) { + return EINVAL; + } + + /* set used to zero */ + smap->used_slots = 0; + + /* zero-out use map */ + uint8_t* usemap = get_use_map(smap); + memset((void*)usemap, 0, slot_map_bytes(smap->total_slots)); + + return UNIFYFS_SUCCESS; +} + +static inline +size_t find_consecutive_zero_bits(uint8_t byte_val, + size_t num_bits, + int* start_bit) +{ + size_t max_free = 0; + int free_start = 0; + for (int bit = 0; bit < 8; bit++) { + /* starting at current bit, count consecutive zero bits */ + int run_start = bit; + size_t run_max = 0; + while (!BYTE_VAL_BIT(byte_val, bit)) { + run_max++; + bit++; + if (bit == 8) { + break; + } + } + if (run_max > max_free) { + max_free = run_max; + free_start = run_start; + } + } + if (NULL != start_bit) { + if (max_free >= num_bits) { + *start_bit = free_start; + } else { + *start_bit = -1; /* did not find enough bits */ + } + } + return max_free; +} + +/** + * Reserve consecutive slots in the slot_map. + * + * @param smap valid slot_map pointer + * @param num_slots number of slots to reserve + * + * @return starting slot index of reservation, or -1 on error + */ +ssize_t slotmap_reserve(slot_map* smap, + size_t num_slots) +{ + if ((NULL == smap) || (0 == num_slots)) { + return (ssize_t)-1; + } + + size_t free = get_free_slots(smap); + if (free < num_slots) { + /* not enough free slots available */ + return (ssize_t)-1; + } + + /* need this many usemap bytes for requested slots */ + size_t slot_bytes = slot_map_bytes(num_slots); + + /* these will be set if we find a spot for the reservation */ + size_t start_slot; + int found_start = 0; + + /* search for contiguous free slots */ + size_t search_start = 0; + if (slot_bytes > 1) { + /* skip past (likely) used slots */ + search_start = SLOT_BYTE(smap->used_slots); + } + uint8_t* usemap = get_use_map(smap); + size_t map_bytes = slot_map_bytes(smap->total_slots); + for (size_t byte_ndx = search_start; byte_ndx < map_bytes; byte_ndx++) { + uint8_t byte_val = usemap[byte_ndx]; + if (byte_val == UINT8_MAX) { + /* current byte is completely occupied */ + continue; + } else if ((slot_bytes > 1) && + ((byte_ndx + slot_bytes) <= map_bytes)) { + /* look for slot_bytes consecutive zero bytes */ + size_t run_start = byte_ndx; + size_t run_count = 0; + while (0 == usemap[byte_ndx]) { + run_count++; + byte_ndx++; + if (run_count == slot_bytes) { + /* success */ + start_slot = BYTE_BIT_TO_SLOT(run_start, 0); + found_start = 1; + break; + } + } + } else { + /* need at most 8 bits, spanning at most two bytes */ + int bit_in_byte; + size_t free_bits = find_consecutive_zero_bits(byte_val, num_slots, + &bit_in_byte); + if (free_bits >= num_slots) { + /* success, can reserve all slots in this byte of use map */ + assert(bit_in_byte != -1); + start_slot = BYTE_BIT_TO_SLOT(byte_ndx, bit_in_byte); + found_start = 1; + } else if ((byte_ndx + 1) < map_bytes) { + /* check if free bits are at the end of the byte */ + find_consecutive_zero_bits(byte_val, free_bits, &bit_in_byte); + assert(bit_in_byte != -1); + if (bit_in_byte == (8 - free_bits)) { + /* free bits are at tail end of byte, + * check next byte for remaining needed slots */ + int bit_in_byte2; + size_t have_bits = free_bits; + size_t need_bits = num_slots - have_bits; + byte_val = usemap[byte_ndx + 1]; + free_bits = find_consecutive_zero_bits(byte_val, need_bits, + &bit_in_byte2); + if ((free_bits >= need_bits) && (bit_in_byte2 == 0)) { + /* success, has enough free bits at start of byte */ + start_slot = BYTE_BIT_TO_SLOT(byte_ndx, bit_in_byte); + found_start = 1; + } + } + } + } + if (found_start) { + break; + } + } + + if (found_start) { + /* success, reserve bits in consecutive slots */ + for (size_t i = 0; i < num_slots; i++) { + use_slot(usemap, start_slot + i); + } + smap->used_slots += num_slots; + return (ssize_t)start_slot; + } + + /* did not find enough consecutive free slots */ + return (ssize_t)-1; +} + +/** + * Release consecutive slots in the slot_map. + * + * @param smap valid slot_map pointer + * @param start_index starting slot index + * @param num_slots number of slots to release + * + * @return UNIFYFS_SUCCESS, or error code + */ +int slotmap_release(slot_map* smap, + size_t start_index, + size_t num_slots) +{ + if (NULL == smap) { + return EINVAL; + } + + uint8_t* usemap = get_use_map(smap); + + /* make sure first bit at start slot index is actually set */ + if (!check_slot(usemap, start_index)) { + return EINVAL; + } + + /* release the slots */ + for (size_t i = 0; i < num_slots; i++) { + release_slot(usemap, start_index + i); + } + smap->used_slots -= num_slots; + + return UNIFYFS_SUCCESS; +} + +/** + * Print the slot_map to stderr (for debugging). + * + * @param smap valid slot_map pointer + */ +void slotmap_print(slot_map* smap) +{ + if (NULL == smap) { + return; + } + + uint8_t* usemap = get_use_map(smap); + + /* the '#' at the beginning of the lines is for compatibility with TAP */ + fprintf(stderr, "# Slot Map:\n"); + fprintf(stderr, "# total slots - %zu\n", smap->total_slots); + fprintf(stderr, "# used slots - %zu\n", smap->used_slots); + + for (size_t i = 0; i < smap->total_slots; i++) { + if (i % 64 == 0) { + fprintf(stderr, "\n# %8zu : ", i); + } else if (i % 8 == 0) { + fprintf(stderr, " "); + } + int bitval = check_slot(usemap, i); + fprintf(stderr, "%d", bitval); + } + fprintf(stderr, "\n#\n"); +} + diff --git a/common/src/slotmap.h b/common/src/slotmap.h new file mode 100644 index 000000000..ea937ff91 --- /dev/null +++ b/common/src/slotmap.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef SLOTMAP_H +#define SLOTMAP_H + +#include // size_t, ssize_t + +#ifdef __cplusplus +extern "C" { +#endif + +/* slot map, a simple structure that manages a bitmap of used/free slots */ +typedef struct slot_map { + size_t total_slots; + size_t used_slots; +} slot_map; + +/* The slot usage bitmap immediately follows the structure in memory. + * The usage bitmap can be thought of as an uint8_t array, where + * each uint8_t represents 8 slots. + * uint8_t use_bitmap[total_slots/8] + */ + +/** + * Initialize a slot map within the given memory region, and return a pointer + * to the slot_map structure. Returns NULL if the provided memory region is not + * large enough to track the requested number of slots. + * + * @param num_slots number of slots to track + * @param region_addr address of the memory region + * @param region_sz size of the memory region + * + * @return valid slot_map pointer, or NULL on error + */ +slot_map* slotmap_init(size_t num_slots, + void* region_addr, + size_t region_sz); + +/** + * Clear the given slot_map. Marks all slots free. + * + * @param smap valid slot_map pointer + * + * @return UNIFYFS_SUCCESS, or error code + */ +int slotmap_clear(slot_map* smap); + +/** + * Reserve consecutive slots in the slot_map. + * + * @param smap valid slot_map pointer + * @param num_slots number of slots to reserve + * + * @return starting slot index of reservation, or -1 on error + */ +ssize_t slotmap_reserve(slot_map* smap, + size_t num_slots); + +/** + * Release consecutive slots in the slot_map. + * + * @param smap valid slot_map pointer + * @param start_index starting slot index + * @param num_slots number of slots to release + * + * @return UNIFYFS_SUCCESS, or error code + */ +int slotmap_release(slot_map* smap, + size_t start_index, + size_t num_slots); + +/** + * Print the slot_map (for debugging). + * + * @param smap valid slot_map pointer + */ +void slotmap_print(slot_map* smap); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // UNIFYFS_BITMAP_H + diff --git a/common/src/unifyfs_client_rpcs.h b/common/src/unifyfs_client_rpcs.h index 7458da9a2..668d69e5c 100644 --- a/common/src/unifyfs_client_rpcs.h +++ b/common/src/unifyfs_client_rpcs.h @@ -21,17 +21,16 @@ extern "C" { * initialize shared memory state */ MERCURY_GEN_PROC(unifyfs_mount_in_t, ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) + ((int32_t)(client_id)) ((int32_t)(dbg_rank)) ((int32_t)(num_procs_per_node)) ((hg_const_string_t)(client_addr_str)) - ((hg_size_t)(req_buf_sz)) ((hg_size_t)(recv_buf_sz)) ((hg_size_t)(superblock_sz)) ((hg_size_t)(meta_offset)) ((hg_size_t)(meta_size)) - ((hg_size_t)(data_offset)) - ((hg_size_t)(data_size)) + ((hg_size_t)(logio_mem_size)) + ((hg_size_t)(logio_spill_size)) ((hg_const_string_t)(external_spill_dir))) MERCURY_GEN_PROC(unifyfs_mount_out_t, ((hg_size_t)(max_recs_per_slice)) @@ -43,7 +42,7 @@ DECLARE_MARGO_RPC_HANDLER(unifyfs_mount_rpc) * disconnect client from server */ MERCURY_GEN_PROC(unifyfs_unmount_in_t, ((int32_t)(app_id)) - ((int32_t)(local_rank_idx))) + ((int32_t)(client_id))) MERCURY_GEN_PROC(unifyfs_unmount_out_t, ((int32_t)(ret))) DECLARE_MARGO_RPC_HANDLER(unifyfs_unmount_rpc) @@ -92,17 +91,16 @@ MERCURY_GEN_PROC(unifyfs_metaget_out_t, ((uint32_t)(is_laminated))) DECLARE_MARGO_RPC_HANDLER(unifyfs_metaget_rpc) -/* unifyfs_fsync_rpc (client => server) +/* unifyfs_sync_rpc (client => server) * - * given app_id, client_id, and a global file id as input, - * read extent location metadata from client shared memory - * and insert corresponding key/value pairs into global index */ -MERCURY_GEN_PROC(unifyfs_fsync_in_t, + * given app_id and client_id as input, read all write extents + * from client index in shared memory and insert corresponding + * key/value pairs into our global metadata */ +MERCURY_GEN_PROC(unifyfs_sync_in_t, ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) - ((int32_t)(gfid))) -MERCURY_GEN_PROC(unifyfs_fsync_out_t, ((int32_t)(ret))) -DECLARE_MARGO_RPC_HANDLER(unifyfs_fsync_rpc) + ((int32_t)(client_id))) +MERCURY_GEN_PROC(unifyfs_sync_out_t, ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_sync_rpc) /* unifyfs_filesize_rpc (client => server) * @@ -110,7 +108,7 @@ DECLARE_MARGO_RPC_HANDLER(unifyfs_fsync_rpc) * return filesize for given file */ MERCURY_GEN_PROC(unifyfs_filesize_in_t, ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) + ((int32_t)(client_id)) ((int32_t)(gfid))) MERCURY_GEN_PROC(unifyfs_filesize_out_t, ((int32_t)(ret)) @@ -123,7 +121,7 @@ DECLARE_MARGO_RPC_HANDLER(unifyfs_filesize_rpc) * and a filesize, truncate file to that size */ MERCURY_GEN_PROC(unifyfs_truncate_in_t, ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) + ((int32_t)(client_id)) ((int32_t)(gfid)) ((hg_size_t)(filesize))) MERCURY_GEN_PROC(unifyfs_truncate_out_t, @@ -136,7 +134,7 @@ DECLARE_MARGO_RPC_HANDLER(unifyfs_truncate_rpc) * unlink the file */ MERCURY_GEN_PROC(unifyfs_unlink_in_t, ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) + ((int32_t)(client_id)) ((int32_t)(gfid))) MERCURY_GEN_PROC(unifyfs_unlink_out_t, ((int32_t)(ret))) @@ -148,7 +146,7 @@ DECLARE_MARGO_RPC_HANDLER(unifyfs_unlink_rpc) * laminate the file */ MERCURY_GEN_PROC(unifyfs_laminate_in_t, ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) + ((int32_t)(client_id)) ((int32_t)(gfid))) MERCURY_GEN_PROC(unifyfs_laminate_out_t, ((int32_t)(ret))) @@ -160,7 +158,7 @@ DECLARE_MARGO_RPC_HANDLER(unifyfs_laminate_rpc) * initiate read request for data */ MERCURY_GEN_PROC(unifyfs_read_in_t, ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) + ((int32_t)(client_id)) ((int32_t)(gfid)) ((hg_size_t)(offset)) ((hg_size_t)(length))) @@ -169,12 +167,12 @@ DECLARE_MARGO_RPC_HANDLER(unifyfs_read_rpc) /* unifyfs_mread_rpc (client => server) * - * given an app_id, client_id, global file id, and a count - * of read requests, followed by list of offset/length tuples + * given an app_id, client_id, and count of read requests, + * followed by list of (gfid, offset, length) tuples, * initiate read requests for data */ MERCURY_GEN_PROC(unifyfs_mread_in_t, ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) + ((int32_t)(client_id)) ((int32_t)(read_count)) ((hg_size_t)(bulk_size)) ((hg_bulk_t)(bulk_handle))) diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index f9fa68955..07c547ff8 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -72,10 +72,15 @@ UNIFYFS_CFG(client, max_files, INT, UNIFYFS_MAX_FILES, "client max file count", NULL) \ UNIFYFS_CFG(client, flatten_writes, BOOL, on, "flatten writes", NULL) \ UNIFYFS_CFG(client, local_extents, BOOL, off, "track extents to service reads of local data", NULL) \ + UNIFYFS_CFG(client, recv_data_size, INT, UNIFYFS_DATA_RECV_SIZE, "shared memory segment size in bytes for receiving data from server", NULL) \ + UNIFYFS_CFG(client, write_index_size, INT, UNIFYFS_INDEX_BUF_SIZE, "write metadata index buffer size", NULL) \ UNIFYFS_CFG_CLI(log, verbosity, INT, 0, "log verbosity level", NULL, 'v', "specify logging verbosity level") \ UNIFYFS_CFG_CLI(log, file, STRING, unifyfsd.log, "log file name", NULL, 'l', "specify log file name") \ UNIFYFS_CFG_CLI(log, dir, STRING, LOGDIR, "log file directory", configurator_directory_check, 'L', "specify full path to directory to contain log file") \ - UNIFYFS_CFG(logfs, index_buf_size, INT, UNIFYFS_INDEX_BUF_SIZE, "log file system index buffer size", NULL) \ + UNIFYFS_CFG(logio, chunk_size, INT, UNIFYFS_LOGIO_CHUNK_SIZE, "log-based I/O data chunk size", NULL) \ + UNIFYFS_CFG(logio, shmem_size, INT, UNIFYFS_LOGIO_SHMEM_SIZE, "log-based I/O shared memory region size", NULL) \ + UNIFYFS_CFG(logio, spill_size, INT, UNIFYFS_LOGIO_SPILL_SIZE, "log-based I/O spillover file size", NULL) \ + UNIFYFS_CFG(logio, spill_dir, STRING, NULLSTRING, "spillover directory", configurator_directory_check) \ UNIFYFS_CFG(margo, tcp, BOOL, on, "use TCP for server-server margo RPCs", NULL) \ UNIFYFS_CFG(meta, db_name, STRING, META_DEFAULT_DB_NAME, "metadata database name", NULL) \ UNIFYFS_CFG(meta, db_path, STRING, RUNDIR, "metadata database path", configurator_directory_check) \ @@ -84,15 +89,7 @@ UNIFYFS_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server runstate file") \ UNIFYFS_CFG_CLI(server, hostfile, STRING, NULLSTRING, "server hostfile name", NULL, 'H', "specify full path to server hostfile") \ UNIFYFS_CFG_CLI(sharedfs, dir, STRING, NULLSTRING, "shared file system directory", configurator_directory_check, 'S', "specify full path to directory to contain server shared files") \ - UNIFYFS_CFG(shmem, chunk_bits, INT, UNIFYFS_CHUNK_BITS, "shared memory data chunk size in bits (i.e., size=2^bits)", NULL) \ - UNIFYFS_CFG(shmem, chunk_mem, INT, UNIFYFS_CHUNK_MEM, "shared memory segment size for data chunks", NULL) \ - UNIFYFS_CFG(shmem, recv_size, INT, UNIFYFS_SHMEM_RECV_SIZE, "shared memory segment size in bytes for receiving data from delegators", NULL) \ - UNIFYFS_CFG(shmem, req_size, INT, UNIFYFS_SHMEM_REQ_SIZE, "shared memory segment size in bytes for sending requests to delegators", NULL) \ - UNIFYFS_CFG(shmem, single, BOOL, off, "use single shared memory region for all clients", NULL) \ - UNIFYFS_CFG(spillover, enabled, BOOL, on, "use local device for data chunk spillover", NULL) \ - UNIFYFS_CFG(spillover, data_dir, STRING, NULLSTRING, "spillover data directory", configurator_directory_check) \ - UNIFYFS_CFG(spillover, meta_dir, STRING, NULLSTRING, "spillover metadata directory", configurator_directory_check) \ - UNIFYFS_CFG(spillover, size, INT, UNIFYFS_SPILLOVER_SIZE, "spillover max data size in bytes", NULL) \ + #ifdef __cplusplus extern "C" { diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h index d11301045..8e9d9bd0c 100644 --- a/common/src/unifyfs_const.h +++ b/common/src/unifyfs_const.h @@ -49,12 +49,7 @@ #define UNIFYFS_MAX_FILENAME KIB #define UNIFYFS_MAX_HOSTNAME 64 -// Metadata -#define MAX_FILE_CNT_PER_NODE KIB - // Request Manager -#define RECV_BUF_CNT 4 /* number of remote read buffers */ -#define SENDRECV_BUF_LEN (8 * MIB) /* remote read buffer size */ #define MAX_META_PER_SEND (4 * KIB) /* max read request count per server */ #define REQ_BUF_LEN (MAX_META_PER_SEND * 64) /* chunk read reqs buffer size */ #define SHM_WAIT_INTERVAL 1000 /* unit: ns */ @@ -70,22 +65,19 @@ // Request and Service Managers, Command Handler #define MAX_NUM_CLIENTS 64 /* app processes per server */ -// Client and Command Handler -#define CMD_BUF_SIZE (2 * KIB) - // Client #define UNIFYFS_MAX_FILES 128 #define UNIFYFS_MAX_FILEDESCS UNIFYFS_MAX_FILES #define UNIFYFS_STREAM_BUFSIZE MIB -#define UNIFYFS_CHUNK_BITS 24 -#define UNIFYFS_CHUNK_MEM (256 * MIB) -#define UNIFYFS_SPILLOVER_SIZE (KIB * MIB) -#define UNIFYFS_SUPERBLOCK_KEY 4321 -#define UNIFYFS_SHMEM_REQ_SIZE (8 * MIB) -#define UNIFYFS_SHMEM_RECV_SIZE (32 * MIB) +#define UNIFYFS_DATA_RECV_SIZE (32 * MIB) #define UNIFYFS_INDEX_BUF_SIZE (20 * MIB) #define UNIFYFS_MAX_READ_CNT KIB +// Log-based I/O +#define UNIFYFS_LOGIO_CHUNK_SIZE (4 * MIB) +#define UNIFYFS_LOGIO_SHMEM_SIZE (256 * MIB) +#define UNIFYFS_LOGIO_SPILL_SIZE (GIB) + /* NOTE: max read size = UNIFYFS_MAX_SPLIT_CNT * META_DEFAULT_RANGE_SZ */ #define UNIFYFS_MAX_SPLIT_CNT (4 * KIB) diff --git a/common/src/unifyfs_logio.c b/common/src/unifyfs_logio.c new file mode 100644 index 000000000..4ccb950bd --- /dev/null +++ b/common/src/unifyfs_logio.c @@ -0,0 +1,805 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unifyfs_log.h" +#include "unifyfs_logio.h" +#include "unifyfs_meta.h" +#include "unifyfs_shm.h" +#include "slotmap.h" + +#define LOGIO_SHMEM_FMTSTR "logio_mem.%d.%d" +#define LOGIO_SPILL_FMTSTR "%s/logio_spill.%d.%d" + + +/* log-based I/O header - first page of shmem region or spill file */ +typedef struct log_header { + size_t data_sz; /* total data bytes in log */ + size_t reserved_sz; /* reserved data bytes */ + size_t chunk_sz; /* data chunk size */ + size_t max_reserved_slot; /* slot index for last reserved chunk */ + off_t data_offset; /* file/memory offset where data chunks start */ +} log_header; +/* chunk slot_map immediately follows header and occupies rest of the page */ +// slot_map chunk_map; /* chunk slot_map that tracks reservations */ + +static inline +slot_map* log_header_to_chunkmap(log_header* hdr) +{ + char* hdrp = (char*) hdr; + return (slot_map*)(hdrp + sizeof(log_header)); +} + +/* method to get page size once, then re-use it */ +size_t get_page_size(void) +{ + static size_t page_sz; // = 0 + if (0 == page_sz) { + page_sz = (size_t) getpagesize(); + } + return page_sz; +} + +/* calculate number of chunks needed for requested bytes */ +static inline +size_t bytes_to_chunks(size_t bytes, size_t chunk_sz) +{ + size_t n_chunks = bytes / chunk_sz; + if (bytes % chunk_sz) { + n_chunks++; + } + return n_chunks; +} + +/* determine shmem and spill chunk allocations based on log offset */ +static inline +void get_log_sizes(off_t log_offset, + size_t nbytes, + size_t shmem_data_sz, + size_t* sz_in_mem, + size_t* sz_in_spill, + off_t* spill_offset) +{ + assert((NULL != sz_in_mem) && + (NULL != sz_in_spill) && + (NULL != spill_offset)); + + *sz_in_mem = 0; + *sz_in_spill = 0; + *spill_offset = 0; + + if ((log_offset + (off_t)nbytes) <= shmem_data_sz) { + /* data fully in shared memory */ + *sz_in_mem = nbytes; + } else if (log_offset < shmem_data_sz) { + /* requested data spans shared memory and spillover file */ + *sz_in_mem = (size_t)(shmem_data_sz - log_offset); + *sz_in_spill = nbytes - *sz_in_mem; + } else { + /* requested data is totally in spillover file */ + *sz_in_spill = nbytes; + *spill_offset = log_offset - shmem_data_sz; + } +} + +/* open (or create) spill file at path and set its size */ +static int get_spillfile(const char* path, + const size_t spill_sz) +{ + /* try to create the spill file */ + mode_t perms = unifyfs_getmode(0640); + int spill_fd = open(path, O_RDWR | O_CREAT | O_EXCL, perms); + if (spill_fd < 0) { + if (errno == EEXIST) { + /* already exists - try simple open */ + spill_fd = open(path, O_RDWR); + } else { + int err = errno; + LOGERR("open(%s) failed: %s", path, strerror(err)); + } + } else { + /* new spillover block created, set its size */ + int rc = ftruncate(spill_fd, (off_t)spill_sz); + if (rc < 0) { + int err = errno; + LOGERR("ftruncate() failed: %s", strerror(err)); + } + } + return spill_fd; +} + +/* map log header (1st page) of spill file given by file descriptor */ +static void* map_spillfile(int spill_fd, int mmap_prot) +{ + size_t pgsz = get_page_size(); + void* addr = mmap(NULL, pgsz, mmap_prot, MAP_SHARED, spill_fd, 0); + if (NULL == addr) { + int err = errno; + LOGERR("mmap(fd=%d, sz=%zu, MAP_SHARED) failed - %s", + spill_fd, pgsz, strerror(err)); + } + return addr; +} + +/* Initialize logio context for server */ +int unifyfs_logio_init_server(const int app_id, + const int client_id, + const size_t mem_size, + const size_t spill_size, + const char* spill_dir, + logio_context** pctx) +{ + if (NULL == pctx) { + return EINVAL; + } + *pctx = NULL; + + shm_context* shm_ctx = NULL; + if (mem_size) { + /* attach to client shmem region */ + char shm_name[SHMEM_NAME_LEN] = {0}; + snprintf(shm_name, sizeof(shm_name), LOGIO_SHMEM_FMTSTR, + app_id, client_id); + shm_ctx = unifyfs_shm_alloc(shm_name, mem_size); + if (NULL == shm_ctx) { + LOGERR("Failed to attach logio shmem buffer!"); + return UNIFYFS_ERROR_SHMEM; + } + } + + void* spill_mapping = NULL; + int spill_fd = -1; + if (spill_size) { + if (NULL == spill_dir) { + LOGERR("Spill directory not given!"); + return EINVAL; + } + + /* open the spill over file */ + char spillfile[UNIFYFS_MAX_FILENAME]; + snprintf(spillfile, sizeof(spillfile), LOGIO_SPILL_FMTSTR, + spill_dir, app_id, client_id); + spill_fd = get_spillfile(spillfile, spill_size); + if (spill_fd < 0) { + LOGERR("Failed to open logio spill file!"); + return UNIFYFS_FAILURE; + } else { + /* map first page of the spill over file, which contains log header + * and chunk slot_map. server only needs read access. */ + spill_mapping = map_spillfile(spill_fd, PROT_READ); + if (NULL == spill_mapping) { + LOGERR("Failed to map logio spill file header!"); + return UNIFYFS_FAILURE; + } + } + } + + logio_context* ctx = (logio_context*) calloc(1, sizeof(logio_context)); + if (NULL == ctx) { + LOGERR("Failed to allocate logio context!"); + return ENOMEM; + } + ctx->shmem = shm_ctx; + ctx->spill_hdr = spill_mapping; + ctx->spill_fd = spill_fd; + ctx->spill_sz = spill_size; + *pctx = ctx; + + return UNIFYFS_SUCCESS; +} + + +/* initialize the log header page for given log region and size + * (note: intended for client use only) */ +static int init_log_header(char* log_region, + size_t region_size, + size_t chunk_size) +{ + size_t pgsz = get_page_size(); + + /* TODO: need to think about how to support client re-attach */ + + /* log header structure resides at start of log region */ + log_header* hdr = (log_header*) log_region; + + /* zero all log header fields */ + memset(log_region, 0, sizeof(log_header)); + + /* chunk data starts after header page */ + size_t data_size = region_size - pgsz; + hdr->data_sz = data_size; + hdr->chunk_sz = chunk_size; + hdr->data_offset = (off_t)pgsz; + + /* initialize chunk slot map (immediately follows header in memory) */ + char* slotmap = log_region + sizeof(log_header); + size_t slotmap_size = pgsz - sizeof(log_header); + size_t n_chunks = data_size / chunk_size; + slot_map* chunkmap = slotmap_init(n_chunks, (void*)slotmap, slotmap_size); + if (NULL == chunkmap) { + LOGERR("Failed to initialize chunk slotmap @ %p (sz=%zu, #chunks=%zu)", + slotmap, slotmap_size, n_chunks); + return UNIFYFS_FAILURE; + } + + return UNIFYFS_SUCCESS; +} + +/* Initialize logio for client */ +int unifyfs_logio_init_client(const int app_id, + const int client_id, + const unifyfs_cfg_t* client_cfg, + logio_context** pctx) +{ + char* cfgval; + int rc; + + if ((NULL == client_cfg) || (NULL == pctx)) { + return EINVAL; + } + *pctx = NULL; + + /* determine max memory bytes for chunk storage */ + size_t memlog_size = 0; + cfgval = client_cfg->logio_shmem_size; + if (cfgval != NULL) { + long l; + rc = configurator_int_val(cfgval, &l); + if (rc == 0) { + memlog_size = (size_t)l; + } + } + + /* get chunk size from config */ + size_t chunk_size = UNIFYFS_LOGIO_CHUNK_SIZE; + cfgval = client_cfg->logio_chunk_size; + if (cfgval != NULL) { + long l; + rc = configurator_int_val(cfgval, &l); + if (rc == 0) { + chunk_size = (size_t)l; + } + } + + shm_context* shm_ctx = NULL; + if (memlog_size) { + /* allocate logio shared memory buffer */ + char shm_name[SHMEM_NAME_LEN] = {0}; + snprintf(shm_name, sizeof(shm_name), LOGIO_SHMEM_FMTSTR, + app_id, client_id); + shm_ctx = unifyfs_shm_alloc(shm_name, memlog_size); + if (NULL == shm_ctx) { + LOGERR("Failed to create logio shmem buffer!"); + return UNIFYFS_ERROR_SHMEM; + } + + /* initialize shmem log header */ + char* memlog = (char*) shm_ctx->addr; + rc = init_log_header(memlog, memlog_size, chunk_size); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("Failed to initialize shmem logio header"); + return rc; + } + } + + /* will we use spillover to store the files? */ + size_t spill_size = 0; + cfgval = client_cfg->logio_spill_size; + if (cfgval != NULL) { + long l; + rc = configurator_int_val(cfgval, &l); + if (rc == 0) { + spill_size = (size_t)l; + } + } + int unifyfs_use_spillover = 0; + if (spill_size > 0) { + LOGDBG("using spillover - size = %zu B", spill_size); + unifyfs_use_spillover = 1; + } + + void* spill_mapping = NULL; + int spill_fd = -1; + if (unifyfs_use_spillover) { + /* get directory in which to create spill over files */ + cfgval = client_cfg->logio_spill_dir; + if (NULL == cfgval) { + LOGERR("UNIFYFS_LOGIO_SPILL_DIR configuration not set! " + "Set to an existing writable path (e.g., /mnt/ssd)"); + return UNIFYFS_ERROR_BADCONFIG; + } + + /* define path to the spill over file for data chunks */ + char spillfile[UNIFYFS_MAX_FILENAME]; + snprintf(spillfile, sizeof(spillfile), LOGIO_SPILL_FMTSTR, + cfgval, app_id, client_id); + + /* create the spill over file */ + spill_fd = get_spillfile(spillfile, spill_size); + if (spill_fd < 0) { + LOGERR("Failed to open logio spill file!"); + return UNIFYFS_FAILURE; + } else { + /* map first page of the spill over file, which contains log header + * and chunk slot_map. client needs read and write access. */ + spill_mapping = map_spillfile(spill_fd, PROT_READ|PROT_WRITE); + if (NULL == spill_mapping) { + LOGERR("Failed to map logio spill file header!"); + return UNIFYFS_FAILURE; + } + + /* initialize spill log header */ + char* spill = (char*) spill_mapping; + rc = init_log_header(spill, spill_size, chunk_size); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("Failed to initialize shmem logio header"); + return rc; + } + } + } + + logio_context* ctx = (logio_context*) calloc(1, sizeof(logio_context)); + if (NULL == ctx) { + LOGERR("Failed to allocate logio context!"); + return ENOMEM; + } + ctx->shmem = shm_ctx; + ctx->spill_hdr = spill_mapping; + ctx->spill_fd = spill_fd; + ctx->spill_sz = spill_size; + *pctx = ctx; + + return UNIFYFS_SUCCESS; +} + +/* Close logio context */ +int unifyfs_logio_close(logio_context* ctx) +{ + if (NULL == ctx) { + return EINVAL; + } + + int rc; + if (NULL != ctx->shmem) { + /* release shmem region */ + rc = unifyfs_shm_free(&(ctx->shmem)); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("Failed to release logio shmem region!"); + } + } + + if (ctx->spill_sz) { + if (NULL != ctx->spill_hdr) { + /* unmap log header page */ + rc = munmap(ctx->spill_hdr, get_page_size()); + if (rc != 0) { + int err = errno; + LOGERR("Failed to unmap logio spill file header (errno=%s)", + strerror(err)); + } + } + if (-1 != ctx->spill_fd) { + /* close spill file */ + rc = close(ctx->spill_fd); + if (rc != 0) { + int err = errno; + LOGERR("Failed to close logio spill file (errno=%s)", + strerror(err)); + } + } + } + + /* free the context struct */ + free(ctx); + + return UNIFYFS_SUCCESS; +} + +/* Allocate write space from logio context */ +int unifyfs_logio_alloc(logio_context* ctx, + const size_t nbytes, + off_t* log_offset) +{ + if ((NULL == ctx) || + ((nbytes > 0) && (NULL == log_offset))) { + return EINVAL; + } + + if (0 == nbytes) { + LOGWARN("zero bytes allocated from log!"); + return UNIFYFS_SUCCESS; + } + + size_t chunk_sz = 0; + size_t allocated_bytes = 0; + size_t needed_bytes = nbytes; + size_t needed_chunks; + size_t res_chunks; + ssize_t res_slot; + off_t res_off = -1; + + size_t mem_res_slot = 0; + size_t mem_res_nchk = 0; + int mem_res_at_end = 0; + size_t mem_allocation = 0; + + log_header* shmem_hdr = NULL; + log_header* spill_hdr = NULL; + slot_map* chunkmap; + + if (NULL != ctx->shmem) { + /* get shmem log header and chunk slotmap */ + shmem_hdr = (log_header*) ctx->shmem->addr; + chunkmap = log_header_to_chunkmap(shmem_hdr); + + /* calculate number of chunks needed for requested bytes */ + chunk_sz = shmem_hdr->chunk_sz; + needed_chunks = bytes_to_chunks(needed_bytes, chunk_sz); + + /* try to reserve all chunks from shmem */ + res_chunks = needed_chunks; + res_slot = slotmap_reserve(chunkmap, res_chunks); + if (-1 != res_slot) { + /* success, all needed chunks allocated in shmem */ + allocated_bytes = res_chunks * chunk_sz; + shmem_hdr->reserved_sz += allocated_bytes; + shmem_hdr->max_reserved_slot = (res_slot + res_chunks) - 1; + res_off = (off_t)(res_slot * chunk_sz); + *log_offset = res_off; + return UNIFYFS_SUCCESS; + } + + /* could not get full allocation in shmem, reserve any available + * chunks at the end of the shmem log */ + size_t log_end_chunks = chunkmap->total_slots - + (shmem_hdr->max_reserved_slot + 1); + if (log_end_chunks > 0) { + res_chunks = log_end_chunks; + res_slot = slotmap_reserve(chunkmap, res_chunks); + if (-1 != res_slot) { + /* reserved all chunks at end of shmem log */ + allocated_bytes = res_chunks * chunk_sz; + needed_bytes -= allocated_bytes; + res_off = (off_t)(res_slot * chunk_sz); + mem_allocation = allocated_bytes; + mem_res_slot = res_slot; + mem_res_nchk = res_chunks; + mem_res_at_end = 1; + } + } + } + + if (NULL != ctx->spill_hdr) { + /* get spill log header and chunk slotmap */ + spill_hdr = (log_header*) ctx->spill_hdr; + chunkmap = log_header_to_chunkmap(spill_hdr); + + /* calculate number of chunks needed for remaining bytes */ + chunk_sz = spill_hdr->chunk_sz; + needed_chunks = bytes_to_chunks(needed_bytes, chunk_sz); + + /* reserve the rest of the chunks from spill file */ + res_chunks = needed_chunks; + res_slot = slotmap_reserve(chunkmap, res_chunks); + if (-1 != res_slot) { + allocated_bytes = res_chunks * chunk_sz; + if (0 == mem_res_at_end) { + /* success, full reservation in spill */ + spill_hdr->reserved_sz += allocated_bytes; + spill_hdr->max_reserved_slot = (res_slot + res_chunks) - 1; + res_off = (off_t)(res_slot * chunk_sz); + if (NULL != shmem_hdr) { + /* update log offset to account for shmem log size */ + res_off += shmem_hdr->data_sz; + } + *log_offset = res_off; + return UNIFYFS_SUCCESS; + } else { + /* if we have an allocation from end of shmem log, make sure + * spill allocation starts at first chunk (slot=0) */ + if (res_slot != 0) { + /* incompatible shmem and spill reservations, release both + * and try to get the full allocation from spill */ + + /* release the spill chunks we just got */ + int rc = slotmap_release(chunkmap, res_slot, res_chunks); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("slotmap_release() for logio shmem failed"); + } + + /* release the shmem chunks */ + chunkmap = log_header_to_chunkmap(shmem_hdr); + rc = slotmap_release(chunkmap, mem_res_slot, mem_res_nchk); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("slotmap_release() for logio shmem failed"); + } + mem_res_slot = 0; + mem_res_nchk = 0; + mem_allocation = 0; + + /* try again with full reservation in spill */ + chunkmap = log_header_to_chunkmap(spill_hdr); + needed_chunks = bytes_to_chunks(nbytes, chunk_sz); + res_chunks = needed_chunks; + res_slot = slotmap_reserve(chunkmap, res_chunks); + if (-1 != res_slot) { + /* success, full reservation in spill */ + spill_hdr->reserved_sz += allocated_bytes; + spill_hdr->max_reserved_slot = + (res_slot + res_chunks) - 1; + res_off = (off_t)(res_slot * chunk_sz); + if (NULL != shmem_hdr) { + /* update log offset to include shmem log size */ + res_off += shmem_hdr->data_sz; + } + *log_offset = res_off; + return UNIFYFS_SUCCESS; + } + } else { + /* successful reservation spanning shmem and spill */ + shmem_hdr->reserved_sz += mem_allocation; + shmem_hdr->max_reserved_slot = + (mem_res_slot + mem_res_nchk) - 1; + spill_hdr->reserved_sz += allocated_bytes; + spill_hdr->max_reserved_slot = (res_slot + res_chunks) - 1; + *log_offset = res_off; + return UNIFYFS_SUCCESS; + } + } + } + } + + /* can't fulfill request from spill file, roll back any prior + * shmem reservation and return ENOSPC */ + if (mem_res_nchk) { + chunkmap = log_header_to_chunkmap(shmem_hdr); + int rc = slotmap_release(chunkmap, mem_res_slot, mem_res_nchk); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("slotmap_release() for logio shmem failed"); + } + } + return ENOSPC; +} + +/* Release previously allocated write space from logio context */ +int unifyfs_logio_free(logio_context* ctx, + const off_t log_offset, + const size_t nbytes) +{ + if (NULL == ctx) { + return EINVAL; + } + + if (0 == nbytes) { + LOGWARN("zero bytes freed from log!"); + return UNIFYFS_SUCCESS; + } + + log_header* shmem_hdr = NULL; + log_header* spill_hdr = NULL; + slot_map* chunkmap; + + off_t mem_size = 0; + if (NULL != ctx->shmem) { + shmem_hdr = (log_header*) ctx->shmem->addr; + mem_size = (off_t) shmem_hdr->data_sz; + } + + /* determine chunk allocations based on log offset */ + size_t sz_in_mem = 0; + size_t sz_in_spill = 0; + off_t spill_offset = 0; + get_log_sizes(log_offset, nbytes, mem_size, + &sz_in_mem, &sz_in_spill, &spill_offset); + + int rc = UNIFYFS_SUCCESS; + size_t chunk_sz, chunk_slot, num_chunks; + if (sz_in_mem > 0) { + /* release shared memory chunks */ + chunk_sz = shmem_hdr->chunk_sz; + chunk_slot = log_offset / chunk_sz; + num_chunks = bytes_to_chunks(sz_in_mem, chunk_sz); + chunkmap = log_header_to_chunkmap(shmem_hdr); + rc = slotmap_release(chunkmap, chunk_slot, num_chunks); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("slotmap_release() for logio shmem failed"); + } + } + if (sz_in_spill > 0) { + /* release spill chunks */ + spill_hdr = (log_header*) ctx->spill_hdr; + chunk_sz = spill_hdr->chunk_sz; + chunk_slot = spill_offset / chunk_sz; + num_chunks = bytes_to_chunks(sz_in_spill, chunk_sz); + chunkmap = log_header_to_chunkmap(spill_hdr); + rc = slotmap_release(chunkmap, chunk_slot, num_chunks); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("slotmap_release() for logio spill failed"); + } + } + return rc; +} + +/* Read data from logio context */ +int unifyfs_logio_read(logio_context* ctx, + const off_t log_offset, + const size_t nbytes, + char* obuf, + size_t* obytes) +{ + if ((NULL == ctx) || + ((nbytes > 0) && (NULL == obuf))) { + return EINVAL; + } + + if (NULL != obytes) { + *obytes = 0; + } + + if (0 == nbytes) { + LOGWARN("zero bytes read from log!"); + return UNIFYFS_SUCCESS; + } + + log_header* shmem_hdr = NULL; + off_t mem_size = 0; + if (NULL != ctx->shmem) { + shmem_hdr = (log_header*) ctx->shmem->addr; + mem_size = (off_t) shmem_hdr->data_sz; + } + + /* prepare read operations based on log offset */ + size_t nread = 0; + size_t sz_in_mem = 0; + size_t sz_in_spill = 0; + off_t spill_offset = 0; + get_log_sizes(log_offset, nbytes, mem_size, + &sz_in_mem, &sz_in_spill, &spill_offset); + + /* do reads */ + int err_rc; + if (sz_in_mem > 0) { + /* read data from shared memory */ + char* shmem_data = (char*)(ctx->shmem->addr) + shmem_hdr->data_offset; + char* log_ptr = shmem_data + log_offset; + memcpy(obuf, log_ptr, sz_in_mem); + nread += sz_in_mem; + } + if (sz_in_spill > 0) { + log_header* spill_hdr = (log_header*) ctx->spill_hdr; + spill_offset += spill_hdr->data_offset; + + /* read data from spillover file */ + ssize_t rc = pread(ctx->spill_fd, (obuf + sz_in_mem), + sz_in_spill, spill_offset); + if (-1 == rc) { + err_rc = errno; + LOGERR("pread(spillfile) failed: %s", strerror(err_rc)); + } else { + nread += rc; + } + } + + if (nread) { + if (nread != nbytes) { + LOGDBG("partial log read: %zu of %zu bytes", nread, nbytes); + } + if (NULL != obytes) { + *obytes = nread; + } + return UNIFYFS_SUCCESS; + } else { + return err_rc; + } +} + +/* Write data to logio context */ +int unifyfs_logio_write(logio_context* ctx, + const off_t log_offset, + const size_t nbytes, + const char* ibuf, + size_t* obytes) +{ + if ((NULL == ctx) || + ((nbytes > 0) && (NULL == ibuf))) { + return EINVAL; + } + + if (NULL != obytes) { + *obytes = 0; + } + + if (0 == nbytes) { + LOGWARN("zero bytes written to log!"); + return UNIFYFS_SUCCESS; + } + + log_header* shmem_hdr = NULL; + off_t mem_size = 0; + if (NULL != ctx->shmem) { + shmem_hdr = (log_header*) ctx->shmem->addr; + mem_size = (off_t) shmem_hdr->data_sz; + } + + /* prepare write operations based on log offset */ + size_t nwrite = 0; + size_t sz_in_mem = 0; + size_t sz_in_spill = 0; + off_t spill_offset = 0; + get_log_sizes(log_offset, nbytes, mem_size, + &sz_in_mem, &sz_in_spill, &spill_offset); + + /* do writes */ + int err_rc; + if (sz_in_mem > 0) { + /* write data to shared memory */ + char* shmem_data = (char*)(ctx->shmem->addr) + shmem_hdr->data_offset; + char* log_ptr = shmem_data + log_offset; + memcpy(log_ptr, ibuf, sz_in_mem); + nwrite += sz_in_mem; + } + if (sz_in_spill > 0) { + log_header* spill_hdr = (log_header*) ctx->spill_hdr; + spill_offset += spill_hdr->data_offset; + + /* write data to spillover file */ + ssize_t rc = pwrite(ctx->spill_fd, (ibuf + sz_in_mem), + sz_in_spill, spill_offset); + if (-1 == rc) { + err_rc = errno; + LOGERR("pwrite(spillfile) failed: %s", strerror(err_rc)); + } else { + nwrite += rc; + } + } + + /* update output parameter if we wrote anything */ + if (nwrite) { + if (nwrite != nbytes) { + LOGDBG("partial log write: %zu of %zu bytes", nwrite, nbytes); + } + + if (NULL != obytes) { + /* obytes is set to the number of bytes actually written */ + *obytes = nwrite; + } + return UNIFYFS_SUCCESS; + } else { + return err_rc; + } +} + +/* Sync any spill data to disk for given logio context */ +int unifyfs_logio_sync(logio_context* ctx) +{ + if ((ctx->spill_sz) && (-1 != ctx->spill_fd)) { + /* fsync spill file */ + int rc = fsync(ctx->spill_fd); + if (rc != 0) { + int err = errno; + LOGERR("Failed to fsync logio spill file (errno=%s)", + strerror(err)); + return err; + } + } + return UNIFYFS_SUCCESS; +} diff --git a/common/src/unifyfs_logio.h b/common/src/unifyfs_logio.h new file mode 100644 index 000000000..10a5a3135 --- /dev/null +++ b/common/src/unifyfs_logio.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2019, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#ifndef UNIFYFS_LOGIO_H +#define UNIFYFS_LOGIO_H + +#include + +#include "unifyfs_configurator.h" +#include "unifyfs_shm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* log-based I/O context structure */ +typedef struct logio_context { + shm_context* shmem; /* shmem region for memory storage */ + void* spill_hdr; /* mmap() address for spillover file log header */ + size_t spill_sz; /* size of spillover file */ + int spill_fd; /* spillover file descriptor */ +} logio_context; + +/** + * Initialize logio context for server. + * + * @param app_id application id + * @param client_id client id + * @param mem_sz shared memory region size for storing data + * @param spill_sz spillfile size for storing data + * @param spill_dir path to spillfile parent directory + * @param[out] ctx address of logio context pointer, set to new context + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_logio_init_server(const int app_id, + const int client_id, + const size_t mem_sz, + const size_t spill_sz, + const char* spill_dir, + logio_context** ctx); + +/** + * Initialize logio context for client. + * + * @param app_id application id + * @param client_id client id + * @param client_cfg pointer to client configuration + * @param[out] ctx address of logio context pointer, set to new context + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_logio_init_client(const int app_id, + const int client_id, + const unifyfs_cfg_t* client_cfg, + logio_context** ctx); + +/** + * Close logio context. + * + * @param ctx pointer to logio context + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_logio_close(logio_context* ctx); + +/** + * Allocate write space from logio context. + * + * @param ctx pointer to logio context + * @param nbytes size of allocation in bytes + * @param[out] log_offset set to log offset to use for writing + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_logio_alloc(logio_context* ctx, + const size_t nbytes, + off_t* log_offset); + +/** + * Release previously allocated write space from logio context. + * + * @param ctx pointer to logio context + * @param log_offset log offset of allocation to release + * @param nbytes size of allocation in bytes + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_logio_free(logio_context* ctx, + const off_t log_offset, + const size_t nbytes); + +/** + * Read data from logio context at given log offset. + * + * @param ctx pointer to logio context + * @param log_offset log offset to read from + * @param nbytes number of bytes to read + * @param buf destination data buffer + * @param[out] obytes set to number of bytes actually read + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_logio_read(logio_context* ctx, + const off_t log_offset, + const size_t nbytes, + char* buf, + size_t* obytes); + +/** + * Write data to logio context at given log offset. + * + * @param ctx pointer to logio context + * @param log_offset log offset to write to + * @param nbytes number of bytes to write + * @param buf source data buffer + * @param[out] obytes set to number of bytes actually written + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_logio_write(logio_context* ctx, + const off_t log_offset, + const size_t nbytes, + const char* buf, + size_t* obytes); + +/** + * Sync any spill data to disk for given logio context. + * + * @param ctx pointer to logio context + * @return UNIFYFS_SUCCESS, or error code + */ +int unifyfs_logio_sync(logio_context* ctx); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* UNIFYFS_LOGIO_H */ diff --git a/common/src/unifyfs_meta.h b/common/src/unifyfs_meta.h index ee24111e5..8c120074a 100644 --- a/common/src/unifyfs_meta.h +++ b/common/src/unifyfs_meta.h @@ -18,7 +18,6 @@ #include #include #include -#include #include #include "unifyfs_const.h" @@ -27,6 +26,21 @@ extern "C" { #endif +/* structure used to detect clients/servers colocated on a host */ +typedef struct { + char hostname[UNIFYFS_MAX_HOSTNAME]; + int rank; +} name_rank_pair_t; + +/* write-log metadata index structure */ +typedef struct { + off_t file_pos; /* start offset of data in file */ + off_t log_pos; /* start offset of data in write log */ + size_t length; /* length of data */ + int gfid; /* global file id */ +} unifyfs_index_t; + +/* UnifyFS file attributes */ typedef struct { int gfid; char filename[UNIFYFS_MAX_FILENAME]; @@ -81,12 +95,50 @@ void unifyfs_file_attr_to_stat(unifyfs_file_attr_t* fattr, struct stat* sb) } } -typedef struct { - off_t file_pos; /* starting logical offset of data in file */ - off_t log_pos; /* starting physical offset of data in log */ - size_t length; /* length of data */ - int gfid; /* global file id */ -} unifyfs_index_t; +/* given an input mode, mask it with umask and return. + * set perms=0 to request all read/write bits */ +static inline +mode_t unifyfs_getmode(mode_t perms) +{ + /* perms == 0 is shorthand for all read and write bits */ + if (perms == 0) { + perms = 0666; + } + + /* get current user mask */ + mode_t mask = umask(0); + umask(mask); + + /* mask off bits from desired permissions */ + mode_t ret = (perms & 0777) & ~mask; + return ret; +} + +/* qsort comparison function for name_rank_pair_t */ +static inline +int compare_name_rank_pair(const void* a, const void* b) +{ + const name_rank_pair_t* pair_a = (const name_rank_pair_t*) a; + const name_rank_pair_t* pair_b = (const name_rank_pair_t*) b; + + /* compare the hostnames */ + int cmp = strcmp(pair_a->hostname, pair_b->hostname); + if (0 == cmp) { + /* if hostnames are the same, compare the rank */ + cmp = pair_a->rank - pair_b->rank; + } + return cmp; +} + +/* qsort comparison function for int */ +static inline +int compare_int(const void* a, const void* b) +{ + int aval = *(const int*)a; + int bval = *(const int*)b; + return aval - bval; +} + /* * Hash a file path to a uint64_t using MD5 @@ -118,39 +170,6 @@ int unifyfs_generate_gfid(const char* path) return (int)hash32; } -/* Header for read request reply in client shared memory region. - * The associated data payload immediately follows the header in - * the shmem region. - * offset - offset within file - * length - data size - * gfid - global file id - * errcode - read error code (zero on success) */ -typedef struct { - size_t offset; - size_t length; - int gfid; - int errcode; -} shm_meta_t; - -/* State values for client shared memory region */ -typedef enum { - SHMEM_REGION_EMPTY = 0, // set by client to indicate drain complete - SHMEM_REGION_DATA_READY = 1, // set by server to initiate client drain - SHMEM_REGION_DATA_COMPLETE = 2 // set by server when done writing data -} shm_region_state_e; - -/* Header for client shared memory region. - * sync - for synchronizing updates/access by server threads - * meta_cnt - number of shm_meta_t (i.e., read replies) currently in shmem - * bytes - total bytes of shmem region in use (shm_meta_t + payloads) - * state - region state variable used for client-server coordination */ - typedef struct { - pthread_mutex_t sync; - volatile size_t meta_cnt; - volatile size_t bytes; - volatile shm_region_state_e state; -} shm_header_t; - #ifdef __cplusplus } // extern "C" #endif diff --git a/common/src/unifyfs_shm.c b/common/src/unifyfs_shm.c index b5ddceb1a..3903d95af 100644 --- a/common/src/unifyfs_shm.c +++ b/common/src/unifyfs_shm.c @@ -13,22 +13,24 @@ */ #include -#include -#include #include +#include +#include #include +#include #include #include #include -#include -#include "unifyfs_log.h" #include "unifyfs_const.h" +#include "unifyfs_log.h" +#include "unifyfs_shm.h" -/* Creates a shared memory of given size under specified name. - * Returns address of new shared memory if successful. - * Returns NULL on error. */ -void* unifyfs_shm_alloc(const char* name, size_t size) +/* Allocate a shared memory region with given name and size, + * and map it into memory. + * Returns a pointer to shm_context for region if successful, + * or NULL on error */ +shm_context* unifyfs_shm_alloc(const char* name, size_t size) { int ret; @@ -81,66 +83,84 @@ void* unifyfs_shm_alloc(const char* name, size_t size) errno = 0; ret = close(fd); if (ret == -1) { - /* failed to open shared memory */ - LOGERR("Failed to mmap shared memory %s (%s)", - name, strerror(errno)); + /* failed to close shared memory */ + LOGERR("Failed to close shared memory fd %d (%s)", + fd, strerror(errno)); /* not fatal, so keep going */ } - /* return address */ - return addr; + /* return pointer to new shm_context */ + shm_context* ctx = (shm_context*) calloc(1, sizeof(shm_context)); + if (NULL != ctx) { + snprintf(ctx->name, sizeof(ctx->name), "%s", name); + ctx->addr = addr; + ctx->size = size; + } + return ctx; } -/* Unmaps shared memory region from memory. - * Caller should provide the address of a pointer to the region - * in paddr. Sets paddr to NULL on return. - * Returns UNIFYFS_SUCCESS on success. */ -int unifyfs_shm_free(const char* name, size_t size, void** paddr) +/* Unmaps shared memory region and frees its context. + * The shm_context pointer is set to NULL on success. + * Returns UNIFYFS_SUCCESS on success, or error code */ +int unifyfs_shm_free(shm_context** pctx) { /* check that we got an address (to something) */ - if (paddr == NULL) { + if (pctx == NULL) { + return EINVAL; + } + + shm_context* ctx = *pctx; + if (ctx == NULL) { return EINVAL; } /* get address of shared memory region */ - void* addr = *paddr; + void* addr = ctx->addr; - /* if we have a pointer, munmap the region, when all procs - * have unmapped the region, the OS will free the memory */ + /* if we have a pointer, try to munmap it */ if (addr != NULL) { /* unmap shared memory from memory space */ errno = 0; - int rc = munmap(addr, size); + int rc = munmap(addr, ctx->size); if (rc == -1) { /* failed to unmap shared memory */ + int err = errno; LOGERR("Failed to unmap shared memory %s (%s)", - name, strerror(errno)); + ctx->name, strerror(err)); /* not fatal, so keep going */ } } + /* free shmem context structure */ + free(ctx); + /* set caller's pointer to NULL */ - *paddr = NULL; + *pctx = NULL; return UNIFYFS_SUCCESS; } /* Unlinks file used to attach to a shared memory region. * Once unlinked, no other processes may attach. - * Returns UNIFYFS_SUCCESS on success. */ -int unifyfs_shm_unlink(const char* name) + * Returns UNIFYFS_SUCCESS on success, or error code. */ +int unifyfs_shm_unlink(shm_context* ctx) { - /* delete associated file if told to unlink */ + /* check context pointer */ + if (ctx == NULL) { + return EINVAL; + } + + /* unlink file naming the shared memory region */ errno = 0; - int rc = shm_unlink(name); + int rc = shm_unlink(ctx->name); if (rc == -1) { int err = errno; if (ENOENT != err) { /* failed to remove shared memory */ LOGERR("Failed to unlink shared memory %s (%s)", - name, strerror(err)); + ctx->name, strerror(err)); } /* not fatal, so keep going */ } diff --git a/common/src/unifyfs_shm.h b/common/src/unifyfs_shm.h index afc2c3d56..f4f9ca530 100644 --- a/common/src/unifyfs_shm.h +++ b/common/src/unifyfs_shm.h @@ -15,24 +15,89 @@ #ifndef UNIFYFS_SHM_H #define UNIFYFS_SHM_H +#include +#include + +#include "unifyfs_rc.h" + +/* Set default region name len if not already defined */ +#ifndef SHMEM_NAME_LEN +#define SHMEM_NAME_LEN 64 +#endif + +/* printf() format strings used by both client and server to name shared + * memory regions. First %d is application id, second is client id. */ +#define SHMEM_DATA_FMTSTR "%d-data-%d" +#define SHMEM_SUPER_FMTSTR "%d-super-%d" + #ifdef __cplusplus extern "C" { #endif -/* Allocate and attach a named shared memory region of a particular size - * and mmap into our memory. Returns starting memory address on success. - * Returns NULL on failure. */ -void* unifyfs_shm_alloc(const char* name, size_t size); +/* Header for read-request reply in client shared memory region. + * The associated data payload immediately follows the header in + * the shmem region. + * offset - offset within file + * length - data size + * gfid - global file id + * errcode - read error code (zero on success) */ +typedef struct shm_data_meta { + size_t offset; + size_t length; + int gfid; + int errcode; +} shm_data_meta; + +/* State values for client shared memory region */ +typedef enum { + SHMEM_REGION_EMPTY = 0, // set by client to indicate drain complete + SHMEM_REGION_DATA_READY = 1, // set by server to initiate client drain + SHMEM_REGION_DATA_COMPLETE = 2 // set by server when done writing data +} shm_data_state_e; -/* Unmaps shared memory region from memory. - * Caller should povider the address of a pointer to the region - * in paddr. Sets paddr to NULL on return. - * Returns UNIFYFS_SUCCESS on success. */ -int unifyfs_shm_free(const char* name, size_t size, void** paddr); +/* Header for client shared memory region. + * sync - for synchronizing updates/access by server threads + * meta_cnt - number of shm_data_meta (i.e., read replies) currently in shmem + * bytes - total bytes of shmem region in use (shm_data_meta + payloads) + * state - region state variable used for client-server coordination */ + typedef struct shm_data_header { + pthread_mutex_t sync; + volatile size_t meta_cnt; + volatile size_t bytes; + volatile shm_data_state_e state; +} shm_data_header; -/* Delete file used to attach to shared memory segment. - * Returns UNIFYFS_SUCCESS on success. */ -int unifyfs_shm_unlink(const char* name); +/* Context structure for maintaining state on an active + * shared memory region */ +typedef struct shm_context { + char name[SHMEM_NAME_LEN]; + void* addr; /* base address of shmem region mapping */ + size_t size; /* size of shmem region */ +} shm_context; + +/** + * Allocate a shared memory region with given name and size, + * and map it into memory. + * @param name region name + * @param size region size in bytes + * @return shmem context pointer (NULL on failure) + */ +shm_context* unifyfs_shm_alloc(const char* name, size_t size); + +/** + * Unmaps shared memory region and frees its context. Context pointer + * is set to NULL on success. + * @param pctx address of pointer to the shmem context + * @return UNIFYFS_SUCCESS or error code + */ +int unifyfs_shm_free(shm_context** pctx); + +/** + * Unlinks the file used to attach to shared memory region. + * @param ctx shmem context pointer + * @return UNIFYFS_SUCCESS or error code + */ +int unifyfs_shm_unlink(shm_context* ctx); #ifdef __cplusplus } // extern "C" diff --git a/docs/add-rpcs.rst b/docs/add-rpcs.rst index e4691e145..017c05217 100644 --- a/docs/add-rpcs.rst +++ b/docs/add-rpcs.rst @@ -31,7 +31,7 @@ Common .. code-block:: C MERCURY_GEN_PROC(unifyfs_mount_in_t, ((int32_t)(app_id)) - ((int32_t)(local_rank_idx)) + ((int32_t)(client_id)) ((int32_t)(dbg_rank)) ((int32_t)(num_procs_per_node)) ((hg_const_string_t)(client_addr_str)) diff --git a/docs/configuration.rst b/docs/configuration.rst index e1e68b539..0f5104afc 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -50,26 +50,28 @@ a given section and key. .. table:: ``[unifyfs]`` section - main configuration settings :widths: auto - ============= ====== ===================================================== + ============= ====== =============================================== Key Type Description - ============= ====== ===================================================== + ============= ====== =============================================== cleanup BOOL cleanup storage on server exit (default: off) configfile STRING path to custom configuration file consistency STRING consistency model [ LAMINATED | POSIX | NONE ] daemonize BOOL enable server daemonization (default: off) mountpoint STRING mountpoint path prefix (default: /unifyfs) - ============= ====== ===================================================== + ============= ====== =============================================== .. table:: ``[client]`` section - client settings :widths: auto - ============== ====== ================================================================= - Key Type Description - ============== ====== ================================================================= - max_files INT maximum number of open files per client process - flatten_writes BOOL enable flattening writes (optimization for overwrite-heavy codes) - local_extents BOOL service reads from local data if possible (default: off) - ============== ====== ================================================================= + ================ ====== ================================================================= + Key Type Description + ================ ====== ================================================================= + max_files INT maximum number of open files per client process (default: 128) + flatten_writes BOOL enable flattening writes (optimization for overwrite-heavy codes) + local_extents BOOL service reads from local data if possible (default: off) + recv_data_size INT maximum size (B) of memory buffer for receiving data from server + write_index_size INT maximum size (B) of memory buffer for storing write log metadata + ================ ====== ================================================================= Enabling the ``local_extents`` optimization may significantly improve read performance. However, it should not be used by applications @@ -80,15 +82,27 @@ files. .. table:: ``[log]`` section - logging settings :widths: auto - ============= ====== ===================================================== - Key Type Description - ============= ====== ===================================================== - dir STRING path to directory to contain server log file - file STRING server log file base name (rank will be appended) - verbosity INT server logging verbosity level [0-5] (default: 0) - ============= ====== ===================================================== + ========== ====== ================================================== + Key Type Description + ========== ====== ================================================== + dir STRING path to directory to contain server log file + file STRING log file base name (rank will be appended) + verbosity INT logging verbosity level [0-5] (default: 0) + ========== ====== ================================================== + +.. table:: ``[logio]`` section - log-based write data storage settings + :widths: auto -.. table:: ``[meta]`` section - metadata settings + =========== ====== ============================================================ + Key Type Description + =========== ====== ============================================================ + chunk_size INT data chunk size (B) (default: 4 MiB) + shmem_size INT maximum size (B) of data in shared memory (default: 256 MiB) + spill_size INT maximum size (B) of data in spillover file (default: 1 GiB) + spill_dir STRING path to spillover data directory + =========== ====== ============================================================ + +.. table:: ``[meta]`` section - MDHIM metadata settings :widths: auto ============= ====== ===================================================== @@ -103,54 +117,29 @@ files. .. table:: ``[runstate]`` section - server runstate settings :widths: auto - ============= ====== ===================================================== - Key Type Description - ============= ====== ===================================================== - dir STRING path to directory to contain server runstate file - ============= ====== ===================================================== + ======== ====== ================================================== + Key Type Description + ======== ====== ================================================== + dir STRING path to directory to contain server runstate file + ======== ====== ================================================== .. table:: ``[server]`` section - server settings :widths: auto - ============= ====== ===================================================== - Key Type Description - ============= ====== ===================================================== - hostfile STRING path to server hostfile - ============= ====== ===================================================== + ========== ====== ======================== + Key Type Description + ========== ====== ======================== + hostfile STRING path to server hostfile + ========== ====== ======================== .. table:: ``[sharedfs]`` section - server shared files settings :widths: auto - ============= ====== ===================================================== - Key Type Description - ============= ====== ===================================================== - dir STRING path to directory to contain server shared files - ============= ====== ===================================================== - -.. table:: ``[shmem]`` section - shared memory segment usage settings - :widths: auto - - ============= ====== ===================================================== - Key Type Description - ============= ====== ===================================================== - chunk_bits INT data chunk size (bits), size = 2^bits (default: 24) - chunk_mem INT segment size (B) for data chunks (default: 256 MiB) - recv_size INT segment size (B) for receiving data from local server - req_size INT segment size (B) for sending requests to local server - single BOOL use one memory region for all clients (default: off) - ============= ====== ===================================================== - -.. table:: ``[spillover]`` section - local data storage spillover settings - :widths: auto - - ============= ====== ===================================================== - Key Type Description - ============= ====== ===================================================== - enabled BOOL use local storage for data spillover (default: on) - data_dir STRING path to spillover data directory - meta_dir STRING path to spillover metadata directory - size INT maximum size (B) of spillover data (default: 1 GiB) - ============= ====== ===================================================== + ======== ====== ================================================= + Key Type Description + ======== ====== ================================================= + dir STRING path to directory to contain server shared files + ======== ====== ================================================= ----------------------- diff --git a/examples/src/sysio-writeread.c b/examples/src/sysio-writeread.c index 899cbab50..1d7e815cd 100644 --- a/examples/src/sysio-writeread.c +++ b/examples/src/sysio-writeread.c @@ -151,7 +151,7 @@ int main(int argc, char* argv[]) #ifndef DISABLE_UNIFYFS if (use_unifyfs) { ret = unifyfs_mount(mntpt, rank, num_rank, 0); - if (UNIFYFS_SUCCESS != ret) { + if (0 != ret) { MPI_Abort(MPI_COMM_WORLD, ret); } MPI_Barrier(MPI_COMM_WORLD); diff --git a/server/src/margo_server.c b/server/src/margo_server.c index 812987b66..bd1b0f3a7 100644 --- a/server/src/margo_server.c +++ b/server/src/margo_server.c @@ -163,9 +163,9 @@ static void register_client_server_rpcs(margo_instance_id mid) unifyfs_metaset_in_t, unifyfs_metaset_out_t, unifyfs_metaset_rpc); - MARGO_REGISTER(mid, "unifyfs_fsync_rpc", - unifyfs_fsync_in_t, unifyfs_fsync_out_t, - unifyfs_fsync_rpc); + MARGO_REGISTER(mid, "unifyfs_sync_rpc", + unifyfs_sync_in_t, unifyfs_sync_out_t, + unifyfs_sync_rpc); MARGO_REGISTER(mid, "unifyfs_filesize_rpc", unifyfs_filesize_in_t, unifyfs_filesize_out_t, diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index daafd4cfd..47f331249 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -50,123 +50,54 @@ */ static int attach_to_shm(app_config_t* app_config, int app_id, - int client_side_id) + int client_id) { - char shm_name[GEN_STR_LEN] = {0}; + char shm_name[SHMEM_NAME_LEN] = {0}; /* attach shared superblock, a superblock is created by each * client to store the raw file data. * The overflowed data are spilled to SSD. */ /* define name of superblock region for this client */ - sprintf(shm_name, "%d-super-%d", app_id, client_side_id); + sprintf(shm_name, SHMEM_SUPER_FMTSTR, app_id, client_id); /* attach to superblock */ - void* addr = unifyfs_shm_alloc(shm_name, app_config->superblock_sz); - if (addr == NULL) { + shm_context* ctx = unifyfs_shm_alloc(shm_name, app_config->superblock_sz); + if (NULL == ctx) { LOGERR("Failed to attach to superblock %s", shm_name); return (int)UNIFYFS_ERROR_SHMEM; } - app_config->shm_superblocks[client_side_id] = addr; - - /* copy name of superblock region */ - strcpy(app_config->super_buf_name[client_side_id], shm_name); - - /* attach shared request buffer, a request buffer is created by each - * client to convey the client-side read request to the delegator */ - - /* define name of request buffer region for this client */ - sprintf(shm_name, "%d-req-%d", app_id, client_side_id); - - /* attach to request buffer region */ - addr = unifyfs_shm_alloc(shm_name, app_config->req_buf_sz); - if (addr == NULL) { - LOGERR("Failed to attach to request buffer %s", shm_name); - return (int)UNIFYFS_ERROR_SHMEM; - } - app_config->shm_req_bufs[client_side_id] = addr; - - /* copy name of request buffer region */ - strcpy(app_config->req_buf_name[client_side_id], shm_name); + app_config->shm_superblocks[client_id] = ctx; /* initialize shared receive buffer, a request buffer is created * by each client for the delegator to temporarily buffer the * received data for this client */ /* define name of receive buffer region for this client */ - sprintf(shm_name, "%d-recv-%d", app_id, client_side_id); + memset(shm_name, 0, sizeof(shm_name)); + sprintf(shm_name, SHMEM_DATA_FMTSTR, app_id, client_id); /* attach to request buffer region */ - addr = unifyfs_shm_alloc(shm_name, app_config->recv_buf_sz); - if (addr == NULL) { + ctx = unifyfs_shm_alloc(shm_name, app_config->recv_buf_sz); + if (NULL == ctx) { LOGERR("Failed to attach to receive buffer %s", shm_name); return (int)UNIFYFS_ERROR_SHMEM; } - app_config->shm_recv_bufs[client_side_id] = addr; - shm_header_t* shm_hdr = (shm_header_t*)addr; - pthread_mutex_init(&(shm_hdr->sync), NULL); + app_config->shm_recv_bufs[client_id] = ctx; + shm_data_header* shm_hdr = (shm_data_header*)(ctx->addr); + int rc = pthread_mutex_init(&(shm_hdr->sync), NULL); + if (rc) { + int err = errno; + LOGERR("shm_data_header mutex initialization failed (%s)", + strerror(err)); + } shm_hdr->meta_cnt = 0; shm_hdr->bytes = 0; shm_hdr->state = SHMEM_REGION_EMPTY; - /* copy name of request buffer region */ - strcpy(app_config->recv_buf_name[client_side_id], shm_name); - return UNIFYFS_SUCCESS; } -/** - * open spilled log file, spilled log file - * is created once the client-side shared superblock - * overflows. - * @param app_config: application information - * @param app_id: the server-side application id - * @param sock_id: position in poll_set in unifyfs_sock.h - * @return success/error code - */ -static int open_log_file(app_config_t* app_config, - int app_id, int client_side_id) -{ - /* build name to spill over log file, - * have one of these per app_id and client_id, - * client writes data to spill over file when it fills - * memory storage */ - char path[UNIFYFS_MAX_FILENAME] = {0}; - snprintf(path, sizeof(path), "%s/spill_%d_%d.log", - app_config->external_spill_dir, app_id, client_side_id); - - /* copy filename of spill over file into app_config */ - strcpy(app_config->spill_log_name[client_side_id], path); - - /* open spill over file for reading */ - app_config->spill_log_fds[client_side_id] = open(path, O_RDONLY, 0666); - if (app_config->spill_log_fds[client_side_id] < 0) { - LOGERR("failed to open spill file %s", path); - return ENOENT; - } - - /* build name of spill over index file, - * this contains index meta data for data the client wrote to the - * spill over file */ - snprintf(path, sizeof(path), "%s/spill_index_%d_%d.log", - app_config->external_spill_dir, app_id, client_side_id); - - /* copy name of spill over index metadata file to app_config */ - strcpy(app_config->spill_index_log_name[client_side_id], path); - - /* open spill over index file for reading */ - app_config->spill_index_log_fds[client_side_id] = - open(path, O_RDONLY, 0666); - if (app_config->spill_index_log_fds[client_side_id] < 0) { - LOGERR("failed to open spill index file %s", path); - return ENOENT; - } - - return UNIFYFS_SUCCESS; -} - - - /* BEGIN MARGO CLIENT-SERVER RPC HANDLER FUNCTIONS */ /* called by client to register with the server, client provides a @@ -192,54 +123,46 @@ static void unifyfs_mount_rpc(hg_handle_t handle) /* read app_id and client_id from input */ int app_id = in.app_id; - int client_id = in.local_rank_idx; + int client_id = in.client_id; /* lookup app_config for given app_id */ - app_config_t* tmp_config = + app_config_t* app_cfg = (app_config_t*) arraylist_get(app_config_list, app_id); /* fill in and insert a new entry for this app_id * if we don't already have one */ - if (tmp_config == NULL) { + if (app_cfg == NULL) { LOGDBG("creating app_config for app_id=%d", app_id); /* don't have an app_config for this app_id, * so allocate and fill one in */ - tmp_config = (app_config_t*)malloc(sizeof(app_config_t)); + app_cfg = (app_config_t*) malloc(sizeof(app_config_t)); /* record size of shared memory regions */ - tmp_config->req_buf_sz = in.req_buf_sz; - tmp_config->recv_buf_sz = in.recv_buf_sz; - tmp_config->superblock_sz = in.superblock_sz; + app_cfg->recv_buf_sz = in.recv_buf_sz; + app_cfg->superblock_sz = in.superblock_sz; /* record offset and size of index entries */ - tmp_config->meta_offset = in.meta_offset; - tmp_config->meta_size = in.meta_size; - - /* record offset and size of file data */ - tmp_config->data_offset = in.data_offset; - tmp_config->data_size = in.data_size; + app_cfg->meta_offset = in.meta_offset; + app_cfg->meta_size = in.meta_size; /* record directory holding spill over files */ - strcpy(tmp_config->external_spill_dir, in.external_spill_dir); + strcpy(app_cfg->external_spill_dir, in.external_spill_dir); /* record number of clients on this node */ - tmp_config->num_procs_per_node = in.num_procs_per_node; + app_cfg->num_procs_per_node = in.num_procs_per_node; /* initialize per-client fields */ int i; for (i = 0; i < MAX_NUM_CLIENTS; i++) { - tmp_config->client_ranks[i] = -1; - tmp_config->shm_req_bufs[i] = NULL; - tmp_config->shm_recv_bufs[i] = NULL; - tmp_config->shm_superblocks[i] = NULL; - tmp_config->spill_log_fds[i] = -1; - tmp_config->spill_index_log_fds[i] = -1; - tmp_config->client_addr[i] = HG_ADDR_NULL; + app_cfg->client_ranks[i] = -1; + app_cfg->shm_recv_bufs[i] = NULL; + app_cfg->shm_superblocks[i] = NULL; + app_cfg->client_addr[i] = HG_ADDR_NULL; } /* insert new app_config into our list, indexed by app_id */ - rc = arraylist_insert(app_config_list, app_id, tmp_config); + rc = arraylist_insert(app_config_list, app_id, app_cfg); if (rc != 0) { ret = rc; } @@ -251,26 +174,31 @@ static void unifyfs_mount_rpc(hg_handle_t handle) * which is the address type needed to call rpc functions, etc */ hret = margo_addr_lookup(unifyfsd_rpc_context->shm_mid, in.client_addr_str, - &(tmp_config->client_addr[client_id])); + &(app_cfg->client_addr[client_id])); /* record client id of process on this node */ - tmp_config->client_ranks[client_id] = client_id; + app_cfg->client_ranks[client_id] = client_id; /* record global rank of client process for debugging */ - tmp_config->dbg_ranks[client_id] = in.dbg_rank; + app_cfg->dbg_ranks[client_id] = in.dbg_rank; /* attach to shared memory regions of this client */ - rc = attach_to_shm(tmp_config, app_id, client_id); + rc = attach_to_shm(app_cfg, app_id, client_id); if (rc != UNIFYFS_SUCCESS) { - LOGERR("attach_to_shm() failed for app_id=%d client_id=%d rc=%d", + LOGERR("failed to attach shmem regions for app=%d client=%d rc=%d", app_id, client_id, rc); ret = rc; } - /* open spill over files for this client */ - rc = open_log_file(tmp_config, app_id, client_id); - if (rc < 0) { - LOGERR("open_log_file() failed for app_id=%d client_id=%d rc=%d", + /* initialize log-based I/O context for this client */ + size_t logio_shmem_sz = in.logio_mem_size; + size_t logio_spill_sz = in.logio_spill_size; + rc = unifyfs_logio_init_server(app_id, client_id, + logio_shmem_sz, logio_spill_sz, + app_cfg->external_spill_dir, + &(app_cfg->logio[client_id])); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to initialize log-based I/O for app=%d client=%d rc=%d", app_id, client_id, rc); ret = rc; } @@ -281,7 +209,7 @@ static void unifyfs_mount_rpc(hg_handle_t handle) /* TODO: seems like it would be cleaner to avoid thread_list * and instead just record address to struct */ /* remember id for thread control for this client */ - tmp_config->thrd_idxs[client_id] = rm_thrd->thrd_ndx; + app_cfg->thrd_idxs[client_id] = rm_thrd->thrd_ndx; } else { /* failed to create request manager thread */ LOGERR("unifyfs_rm_thrd_create() failed for app_id=%d client_id=%d", @@ -313,7 +241,7 @@ static void unifyfs_unmount_rpc(hg_handle_t handle) /* read app_id and client_id from input */ int app_id = in.app_id; - int client_id = in.local_rank_idx; + int client_id = in.client_id; /* build output structure to return to caller */ unifyfs_unmount_out_t out; @@ -340,18 +268,9 @@ static void unifyfs_unmount_rpc(hg_handle_t handle) /* shutdown the delegator thread */ rm_cmd_exit(thrd_ctrl); - /* detach from the request shared memory */ - if (NULL != app_config->shm_req_bufs[client_id]) { - unifyfs_shm_free(app_config->req_buf_name[client_id], - app_config->req_buf_sz, - (void**)&(app_config->shm_req_bufs[client_id])); - } - /* detach from the read shared memory buffer */ if (NULL != app_config->shm_recv_bufs[client_id]) { - unifyfs_shm_free(app_config->recv_buf_name[client_id], - app_config->recv_buf_sz, - (void**)&(app_config->shm_recv_bufs[client_id])); + unifyfs_shm_free(&(app_config->shm_recv_bufs[client_id])); } /* free margo hg_addr_t client addresses in app_config struct */ @@ -440,19 +359,19 @@ static void unifyfs_metaset_rpc(hg_handle_t handle) } DEFINE_MARGO_RPC_HANDLER(unifyfs_metaset_rpc) -/* given app_id, client_id, and a global file id as input, - * read extent location metadata from client shared memory - * and insert corresponding key/value pairs into global index */ -static void unifyfs_fsync_rpc(hg_handle_t handle) +/* given app_id and client_id as input, read all extents from client + * write index in shared memory and insert corresponding key/value pairs + * into the global metadata */ +static void unifyfs_sync_rpc(hg_handle_t handle) { /* get input params */ - unifyfs_fsync_in_t in; + unifyfs_sync_in_t in; hg_return_t hret = margo_get_input(handle, &in); assert(hret == HG_SUCCESS); /* given global file id, read index metadata from client and * insert into global index key/value store */ - int ret = rm_cmd_fsync(in.app_id, in.local_rank_idx, in.gfid); + int ret = rm_cmd_sync(in.app_id, in.client_id); /* build our output values */ unifyfs_metaset_out_t out; @@ -466,7 +385,7 @@ static void unifyfs_fsync_rpc(hg_handle_t handle) margo_free_input(handle, &in); margo_destroy(handle); } -DEFINE_MARGO_RPC_HANDLER(unifyfs_fsync_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_sync_rpc) /* given an app_id, client_id, global file id, @@ -481,7 +400,7 @@ static void unifyfs_filesize_rpc(hg_handle_t handle) /* read data for a single read request from client, * returns data to client through shared memory */ size_t filesize = 0; - int ret = rm_cmd_filesize(in.app_id, in.local_rank_idx, + int ret = rm_cmd_filesize(in.app_id, in.client_id, in.gfid, &filesize); /* build our output values */ @@ -509,7 +428,7 @@ static void unifyfs_truncate_rpc(hg_handle_t handle) assert(hret == HG_SUCCESS); /* truncate file to specified size */ - int ret = rm_cmd_truncate(in.app_id, in.local_rank_idx, + int ret = rm_cmd_truncate(in.app_id, in.client_id, in.gfid, in.filesize); /* build our output values */ @@ -536,7 +455,7 @@ static void unifyfs_unlink_rpc(hg_handle_t handle) assert(hret == HG_SUCCESS); /* truncate file to specified size */ - int ret = rm_cmd_unlink(in.app_id, in.local_rank_idx, in.gfid); + int ret = rm_cmd_unlink(in.app_id, in.client_id, in.gfid); /* build our output values */ unifyfs_truncate_out_t out; @@ -562,7 +481,7 @@ static void unifyfs_laminate_rpc(hg_handle_t handle) assert(hret == HG_SUCCESS); /* truncate file to specified size */ - int ret = rm_cmd_laminate(in.app_id, in.local_rank_idx, in.gfid); + int ret = rm_cmd_laminate(in.app_id, in.client_id, in.gfid); /* build our output values */ unifyfs_truncate_out_t out; @@ -591,7 +510,7 @@ static void unifyfs_read_rpc(hg_handle_t handle) /* read data for a single read request from client, * returns data to client through shared memory */ - int ret = rm_cmd_read(in.app_id, in.local_rank_idx, + int ret = rm_cmd_read(in.app_id, in.client_id, in.gfid, in.offset, in.length); /* build our output values */ @@ -643,7 +562,7 @@ static void unifyfs_mread_rpc(hg_handle_t handle) assert(hret == HG_SUCCESS); /* initiate read operations to fetch data for read requests */ - int ret = rm_cmd_mread(in.app_id, in.local_rank_idx, + int ret = rm_cmd_mread(in.app_id, in.client_id, in.read_count, buffer); /* build our output values */ diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index 2476b552c..e7626fa84 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -45,6 +45,7 @@ #include "arraylist.h" #include "unifyfs_const.h" #include "unifyfs_log.h" +#include "unifyfs_logio.h" #include "unifyfs_meta.h" #include "unifyfs_shm.h" @@ -131,9 +132,6 @@ typedef struct { size_t superblock_sz; /* size of memory region used to store data */ size_t meta_offset; /* superblock offset to index metadata */ size_t meta_size; /* size of index metadata region in bytes */ - size_t data_offset; /* superblock offset to data log */ - size_t data_size; /* size of data log in bytes */ - size_t req_buf_sz; /* buffer size for client to issue read requests */ size_t recv_buf_sz; /* buffer size for read replies to client */ /* number of clients on the node */ @@ -144,25 +142,16 @@ typedef struct { int thrd_idxs[MAX_NUM_CLIENTS]; /* map to thread id */ int dbg_ranks[MAX_NUM_CLIENTS]; /* map to client rank */ - /* file descriptors */ - int spill_log_fds[MAX_NUM_CLIENTS]; /* spillover data */ - int spill_index_log_fds[MAX_NUM_CLIENTS]; /* spillover index */ + /* shared memory context pointers */ + shm_context* shm_superblocks[MAX_NUM_CLIENTS]; /* superblock data */ + shm_context* shm_recv_bufs[MAX_NUM_CLIENTS]; /* read reply shm */ - /* shared memory pointers */ - char* shm_superblocks[MAX_NUM_CLIENTS]; /* superblock data */ - char* shm_req_bufs[MAX_NUM_CLIENTS]; /* read request shm */ - char* shm_recv_bufs[MAX_NUM_CLIENTS]; /* read reply shm */ + /* log-based I/O context pointers */ + logio_context* logio[MAX_NUM_CLIENTS]; /* client address for rpc invocation */ hg_addr_t client_addr[MAX_NUM_CLIENTS]; - /* file names */ - char super_buf_name[MAX_NUM_CLIENTS][UNIFYFS_MAX_FILENAME]; - char req_buf_name[MAX_NUM_CLIENTS][UNIFYFS_MAX_FILENAME]; - char recv_buf_name[MAX_NUM_CLIENTS][UNIFYFS_MAX_FILENAME]; - char spill_log_name[MAX_NUM_CLIENTS][UNIFYFS_MAX_FILENAME]; - char spill_index_log_name[MAX_NUM_CLIENTS][UNIFYFS_MAX_FILENAME]; - /* directory holding spill over files */ char external_spill_dir[UNIFYFS_MAX_FILENAME]; } app_config_t; diff --git a/server/src/unifyfs_init.c b/server/src/unifyfs_init.c index ed920b6fc..e794cf736 100644 --- a/server/src/unifyfs_init.c +++ b/server/src/unifyfs_init.c @@ -64,17 +64,6 @@ static int unifyfs_exit(void); int* local_rank_lst; int local_rank_cnt; -/* - * structure that records the information of - * each application launched by srun. - * */ -typedef struct { - char hostname[UNIFYFS_MAX_HOSTNAME]; - int rank; -} name_rank_pair_t; - -static int compare_name_rank_pair(const void* a, const void* b); -static int compare_int(const void* a, const void* b); static int CountTasksPerNode(int rank, int numTasks); static int find_rank_idx(int my_rank); #endif @@ -563,20 +552,6 @@ static int find_rank_idx(int my_rank) return -1; } -static int compare_name_rank_pair(const void* a, const void* b) -{ - const name_rank_pair_t* pair_a = a; - const name_rank_pair_t* pair_b = b; - return strcmp(pair_a->hostname, pair_b->hostname); -} - -static int compare_int(const void* a, const void* b) -{ - int aval = *(const int*)a; - int bval = *(const int*)b; - return aval - bval; -} - #endif // UNIFYFS_MULTIPLE_DELEGATORS #endif // UNIFYFSD_USE_MPI @@ -623,43 +598,20 @@ static int unifyfs_exit(void) /* free resources allocate for each client */ for (j = 0; j < MAX_NUM_CLIENTS; j++) { - /* Release request buffer shared memory region. Client - * should have deleted file already, but will not hurt - * to do this again. */ - if (app->shm_req_bufs[j] != NULL) { - unifyfs_shm_free(app->req_buf_name[j], app->req_buf_sz, - (void**)&(app->shm_req_bufs[j])); - unifyfs_shm_unlink(app->req_buf_name[j]); - } - /* Release receive buffer shared memory region. Client * should have deleted file already, but will not hurt * to do this again. */ if (app->shm_recv_bufs[j] != NULL) { - unifyfs_shm_free(app->recv_buf_name[j], app->recv_buf_sz, - (void**)&(app->shm_recv_bufs[j])); - unifyfs_shm_unlink(app->recv_buf_name[j]); + unifyfs_shm_unlink(app->shm_recv_bufs[j]); + unifyfs_shm_free(&(app->shm_recv_bufs[j])); } /* Release super block shared memory region. * Server is responsible for deleting superblock shared * memory file that was created by the client. */ if (app->shm_superblocks[j] != NULL) { - unifyfs_shm_free(app->super_buf_name[j], app->superblock_sz, - (void**)&(app->shm_superblocks[j])); - unifyfs_shm_unlink(app->super_buf_name[j]); - } - - /* close spill log file and delete it */ - if (app->spill_log_fds[j] > 0) { - close(app->spill_log_fds[j]); - unlink(app->spill_log_name[j]); - } - - /* close spill log index file and delete it */ - if (app->spill_index_log_fds[j] > 0) { - close(app->spill_index_log_fds[j]); - unlink(app->spill_index_log_name[j]); + unifyfs_shm_unlink(app->shm_superblocks[j]); + unifyfs_shm_free(&(app->shm_superblocks[j])); } } } diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 03100d023..6731b7ba3 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -479,8 +479,8 @@ int create_chunk_requests(reqmgr_thrd_t* thrd_ctrl, /* signal the client process for it to start processing read * data in shared memory */ -static int client_signal(shm_header_t* hdr, - shm_region_state_e flag) +static int client_signal(shm_data_header* hdr, + shm_data_state_e flag) { if (flag == SHMEM_REGION_DATA_READY) { LOGDBG("setting data-ready"); @@ -498,7 +498,7 @@ static int client_signal(shm_header_t* hdr, } /* wait until client has processed all read data in shared memory */ -static int client_wait(shm_header_t* hdr) +static int client_wait(shm_data_header* hdr) { int rc = (int)UNIFYFS_SUCCESS; @@ -1493,15 +1493,13 @@ int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl) } /* - * synchronize all the indices - * to the key-value store + * store all writes from app-client's index in the global metadata * * @param app_id: the application id - * @param client_side_id: client rank in app - * @param gfid: global file id + * @param client_id: client rank in app * @return success/error code */ -int rm_cmd_fsync(int app_id, int client_side_id, int gfid) +int rm_cmd_sync(int app_id, int client_id) { size_t i; @@ -1516,7 +1514,12 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) arraylist_get(app_config_list, app_id); /* get pointer to superblock for this client and app */ - char* superblk = app_config->shm_superblocks[client_side_id]; + shm_context* super_ctx = app_config->shm_superblocks[client_id]; + if (NULL == super_ctx) { + LOGERR("bad client id"); + return EINVAL; + } + char* superblk = (char*)(super_ctx->addr); /* get pointer to start of key/value region in superblock */ char* meta = superblk + app_config->meta_offset; @@ -1585,7 +1588,7 @@ int rm_cmd_fsync(int app_id, int client_side_id, int gfid) int used = split_index( &keys[count], &vals[count], &key_lens[count], &val_lens[count], gfid, offset, length, logpos, - glb_pmi_rank, app_id, client_side_id); + glb_pmi_rank, app_id, client_id); /* count up the number of keys we used for this index */ count += used; @@ -1769,11 +1772,14 @@ static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) } else if ((req->num_remote_reads == 0) && (req->status == READREQ_STARTED)) { /* look up client shared memory region */ + int app_id = req->app_id; + int client_id = req->client_id; app_config_t* app_config = - (app_config_t*) arraylist_get(app_config_list, req->app_id); + (app_config_t*) arraylist_get(app_config_list, app_id); assert(NULL != app_config); - shm_header_t* client_shm = - (shm_header_t*) app_config->shm_recv_bufs[req->client_id]; + shm_context* client_shm = app_config->shm_recv_bufs[client_id]; + assert(NULL != client_shm); + shm_data_header* shm_hdr = (shm_data_header*) client_shm->addr; RM_LOCK(thrd_ctrl); @@ -1781,10 +1787,10 @@ static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) req->status = READREQ_COMPLETE; /* signal client that we're now done writing data */ - client_signal(client_shm, SHMEM_REGION_DATA_COMPLETE); + client_signal(shm_hdr, SHMEM_REGION_DATA_COMPLETE); /* wait for client to read data */ - client_wait(client_shm); + client_wait(shm_hdr); rc = release_read_req(thrd_ctrl, req); if (rc != (int)UNIFYFS_SUCCESS) { @@ -1799,19 +1805,20 @@ static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) return ret; } -static shm_meta_t* reserve_shmem_meta(app_config_t* app_config, - shm_header_t* hdr, - size_t data_sz) +static shm_data_meta* reserve_shmem_meta(app_config_t* app_config, + shm_data_header* hdr, + size_t data_sz) { - shm_meta_t* meta = NULL; + shm_data_meta* meta = NULL; if (NULL == hdr) { LOGERR("invalid header"); } else { pthread_mutex_lock(&(hdr->sync)); - LOGDBG("shm_header(cnt=%zu, bytes=%zu)", hdr->meta_cnt, hdr->bytes); + LOGDBG("shm_data_header(cnt=%zu, bytes=%zu)", + hdr->meta_cnt, hdr->bytes); size_t remain_size = app_config->recv_buf_sz - - (sizeof(shm_header_t) + hdr->bytes); - size_t meta_size = sizeof(shm_meta_t) + data_sz; + (sizeof(shm_data_header) + hdr->bytes); + size_t meta_size = sizeof(shm_data_meta) + data_sz; if (meta_size > remain_size) { /* client-side receive buffer is full, * inform client to start reading */ @@ -1822,13 +1829,14 @@ static shm_meta_t* reserve_shmem_meta(app_config_t* app_config, int rc = client_wait(hdr); if (rc != (int)UNIFYFS_SUCCESS) { LOGERR("wait for client recv buffer space failed"); + pthread_mutex_unlock(&(hdr->sync)); return NULL; } } size_t shm_offset = hdr->bytes; - char* shm_buf = ((char*)hdr) + sizeof(shm_header_t); - meta = (shm_meta_t*)(shm_buf + shm_offset); - LOGDBG("reserved shm_meta[%zu] and %zu payload bytes", + char* shm_buf = ((char*)hdr) + sizeof(shm_data_header); + meta = (shm_data_meta*)(shm_buf + shm_offset); + LOGDBG("reserved shm_data_meta[%zu] and %zu payload bytes", hdr->meta_cnt, data_sz); hdr->meta_cnt++; hdr->bytes += meta_size; @@ -1908,8 +1916,9 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, int ret = (int)UNIFYFS_SUCCESS; app_config_t* app_config = NULL; chunk_read_resp_t* responses = NULL; - shm_header_t* client_shm = NULL; - shm_meta_t* shm_meta = NULL; + shm_context* client_shm = NULL; + shm_data_header* shm_hdr = NULL; + shm_data_meta* meta = NULL; void* shm_buf = NULL; char* data_buf = NULL; size_t data_sz, offset; @@ -1922,7 +1931,8 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, /* look up client shared memory region */ app_config = (app_config_t*) arraylist_get(app_config_list, rdreq->app_id); assert(NULL != app_config); - client_shm = (shm_header_t*) app_config->shm_recv_bufs[rdreq->client_id]; + client_shm = (shm_context*) app_config->shm_recv_bufs[rdreq->client_id]; + shm_hdr = (shm_data_header*) client_shm->addr; RM_LOCK(thrd_ctrl); @@ -1955,13 +1965,13 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, LOGDBG("chunk response for offset=%zu: sz=%zu", offset, data_sz); /* allocate and register local target buffer for bulk access */ - shm_meta = reserve_shmem_meta(app_config, client_shm, data_sz); - if (NULL != shm_meta) { - shm_meta->offset = offset; - shm_meta->length = data_sz; - shm_meta->gfid = gfid; - shm_meta->errcode = errcode; - shm_buf = (void*)((char*)shm_meta + sizeof(shm_meta_t)); + meta = reserve_shmem_meta(app_config, shm_hdr, data_sz); + if (NULL != meta) { + meta->offset = offset; + meta->length = data_sz; + meta->gfid = gfid; + meta->errcode = errcode; + shm_buf = (void*)((char*)meta + sizeof(shm_data_meta)); if (data_sz) { memcpy(shm_buf, data_buf, data_sz); } @@ -1991,10 +2001,10 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, rdreq->status = READREQ_COMPLETE; /* signal client that we're now done writing data */ - client_signal(client_shm, SHMEM_REGION_DATA_COMPLETE); + client_signal(shm_hdr, SHMEM_REGION_DATA_COMPLETE); /* wait for client to read data */ - client_wait(client_shm); + client_wait(shm_hdr); rc = release_read_req(thrd_ctrl, rdreq); if (rc != (int)UNIFYFS_SUCCESS) { diff --git a/server/src/unifyfs_request_manager.h b/server/src/unifyfs_request_manager.h index c6c4f7891..3c4f62edc 100644 --- a/server/src/unifyfs_request_manager.h +++ b/server/src/unifyfs_request_manager.h @@ -124,13 +124,9 @@ int rm_cmd_laminate(int app_id, int client_id, int gfid); * returns UNIFYFS_SUCCESS on success */ int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl); -/* - * synchronize all the indices and file attributes - * to the key-value store - * @param sock_id: the connection id in poll_set of the delegator - * @return success/error code - */ -int rm_cmd_fsync(int app_id, int client_side_id, int gfid); +/* retrieve all write index entries for app-client and + * store them in global metadata */ +int rm_cmd_sync(int app_id, int client_side_id); /* update state for remote chunk reads with received response data */ int rm_post_chunk_read_responses(int app_id, diff --git a/server/src/unifyfs_service_manager.c b/server/src/unifyfs_service_manager.c index a1eb3dff7..96b1c450a 100644 --- a/server/src/unifyfs_service_manager.c +++ b/server/src/unifyfs_service_manager.c @@ -169,15 +169,15 @@ int sm_issue_chunk_reads(int src_rank, chunk_read_resp_t* rresp = resp + i; /* get size and log offset of data we are to read */ - size_t size = rreq->nbytes; - size_t offset = rreq->log_offset; + size_t nbytes = rreq->nbytes; + size_t log_offset = rreq->log_offset; /* record request metadata in response */ rresp->read_rc = 0; - rresp->nbytes = size; + rresp->nbytes = nbytes; rresp->offset = rreq->offset; LOGDBG("reading chunk(offset=%zu, size=%zu)", - rreq->offset, size); + rreq->offset, nbytes); /* get app id and corresponding app_config struct */ int app_id = rreq->log_app_id; @@ -195,71 +195,29 @@ int sm_issue_chunk_reads(int src_rank, /* client id for this read task */ int cli_id = rreq->log_client_id; - /* get size of data region for this - * app_id and client_id */ - size_t data_size = app_config->data_size; - - /* prepare read opertions based on data location */ - size_t sz_from_mem = 0; - size_t sz_from_spill = 0; - if ((offset + size) <= data_size) { - /* requested data is totally in shared memory */ - sz_from_mem = size; - } else if (offset < data_size) { - /* part of the requested data is in shared memory */ - sz_from_mem = data_size - offset; - sz_from_spill = size - sz_from_mem; - } else { - /* all requested data is in spillover file */ - sz_from_spill = size; - } - /* get pointer to next position in buffer to store read data */ char* buf_ptr = databuf + buf_cursor; - /* read data from shared memory */ - if (sz_from_mem > 0) { - /* start of data within in superblock */ - char* super_addr = app_config->shm_superblocks[cli_id]; - char* data_addr = super_addr + app_config->data_offset; - char* data_ptr = data_addr + offset; - - /* copy data from superblock into read reply buffer */ - memcpy(buf_ptr, data_ptr, sz_from_mem); - - /* we assume memcpy copied everything */ - rresp->read_rc = sz_from_mem; - } - - /* read data from spillover file */ - if (sz_from_spill > 0) { - /* file descriptor for open spillover file for this - * app/client */ - int spill_fd = app_config->spill_log_fds[cli_id]; - - /* offset within spill over file, need to subtract off - * range of offsets that land in data region of - * superblock */ - off_t spill_offset = (off_t)(offset - data_size + - sz_from_mem); - - /* read data from the spillover file */ - ssize_t nread = pread(spill_fd, (buf_ptr + sz_from_mem), - sz_from_spill, spill_offset); - if (-1 == nread) { - /* pread hit an error, return error code */ - rresp->read_rc = (ssize_t)(-errno); + /* read data from log */ + logio_context* logio_ctx = app_config->logio[cli_id]; + if (NULL != logio_ctx) { + size_t nread = 0; + int rc = unifyfs_logio_read(logio_ctx, log_offset, nbytes, + buf_ptr, &nread); + if (UNIFYFS_SUCCESS == rc) { + rresp->read_rc = nread; } else { - /* add to byte counts we may have started from memcpy */ - rresp->read_rc += nread; + rresp->read_rc = (ssize_t)(-rc); } + } else { + rresp->read_rc = (ssize_t)(-UNIFYFS_FAILURE); } /* update to point to next slot in read reply buffer */ - buf_cursor += size; + buf_cursor += nbytes; /* update accounting for burst size */ - sm->burst_data_sz += size; + sm->burst_data_sz += nbytes; } if (src_rank != glb_pmi_rank) { diff --git a/t/9200-seg-tree-test.t b/t/9200-seg-tree-test.t new file mode 100755 index 000000000..f062b833c --- /dev/null +++ b/t/9200-seg-tree-test.t @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Source sharness environment scripts to pick up test environment +# and UnifyFS runtime settings. +# +. $(dirname $0)/sharness.d/00-test-env.sh +. $(dirname $0)/sharness.d/01-unifyfs-settings.sh +$UNIFYFS_BUILD_DIR/t/common/seg_tree_test.t diff --git a/t/9006-seg-tree-test.t b/t/9201-slotmap-test.t similarity index 82% rename from t/9006-seg-tree-test.t rename to t/9201-slotmap-test.t index a765eec35..0ff8efeec 100755 --- a/t/9006-seg-tree-test.t +++ b/t/9201-slotmap-test.t @@ -5,4 +5,4 @@ # . $(dirname $0)/sharness.d/00-test-env.sh . $(dirname $0)/sharness.d/01-unifyfs-settings.sh -$UNIFYFS_BUILD_DIR/t/seg_tree_test.t +$UNIFYFS_BUILD_DIR/t/common/slotmap_test.t diff --git a/t/Makefile.am b/t/Makefile.am index 4fd33f130..72811f613 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -10,10 +10,11 @@ TESTS = \ 0500-sysio-static.t \ 0600-stdio-static.t \ 9005-unifyfs-unmount.t \ - 9006-seg-tree-test.t \ 9010-stop-unifyfsd.t \ 9020-mountpoint-empty.t \ 9100-metadata-api.t \ + 9200-seg-tree-test.t \ + 9201-slotmap-test.t \ 9999-cleanup.t check_SCRIPTS = \ @@ -23,10 +24,11 @@ check_SCRIPTS = \ 0500-sysio-static.t \ 0600-stdio-static.t \ 9005-unifyfs-unmount.t \ - 9006-seg-tree-test.t \ 9010-stop-unifyfsd.t \ 9020-mountpoint-empty.t \ 9100-metadata-api.t \ + 9200-seg-tree-test.t \ + 9201-slotmap-test.t \ 9999-cleanup.t EXTRA_DIST = \ @@ -41,20 +43,30 @@ clean-local: rm -fr trash-directory.* test-results *.log test_run_env.sh libexec_PROGRAMS = \ - sys/sysio-gotcha.t \ + common/seg_tree_test.t \ + common/slotmap_test.t \ + server/metadata.t \ std/stdio-gotcha.t \ - sys/sysio-static.t \ std/stdio-static.t \ - server/metadata.t \ - unifyfs_unmount.t \ - seg_tree_test.t + sys/sysio-gotcha.t \ + sys/sysio-static.t \ + unifyfs_unmount.t + +test_common_ldadd = \ + $(top_builddir)/t/lib/libtap.la \ + $(top_builddir)/t/lib/libtestutil.la \ + $(top_builddir)/common/src/libunifyfs_common.la + +test_common_ldflags = \ + -static $(AM_LDFLAGS) test_ldadd = \ $(top_builddir)/t/lib/libtap.la \ $(top_builddir)/t/lib/libtestutil.la \ $(top_builddir)/client/src/libunifyfs_gotcha.la \ - $(MPI_CLDFLAGS) $(FLATCC_LDFLAGS) $(FLATCC_LIBS) + $(MPI_CLDFLAGS) \ + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) test_static_ldadd = \ $(top_builddir)/t/lib/libtap.la \ @@ -62,9 +74,10 @@ test_static_ldadd = \ $(top_builddir)/client/src/libunifyfs.la test_static_ldflags = \ - -static \ - $(CP_WRAPPERS) $(AM_LDFLAGS) $(MPI_CLDFLAGS) $(FLATCC_LDFLAGS) \ - $(FLATCC_LIBS) + -static $(AM_LDFLAGS) \ + $(CP_WRAPPERS) \ + $(MPI_CLDFLAGS) \ + $(FLATCC_LDFLAGS) $(FLATCC_LIBS) test_metadata_ldadd = \ $(top_builddir)/t/lib/libtap.la \ @@ -80,6 +93,12 @@ test_metadata_ldadd = \ $(FLATCC_LDFLAGS) $(FLATCC_LIBS) \ -lpthread -lm -lstdc++ -lrt +test_common_cppflags = \ + -I$(top_srcdir) \ + -I$(top_srcdir)/common/src \ + -D_GNU_SOURCE \ + $(AM_CPPFLAGS) + test_meta_cppflags = \ -I$(top_srcdir) \ -I$(top_srcdir)/server/src \ @@ -97,6 +116,7 @@ test_cppflags = \ $(AM_CPPFLAGS) \ $(MPI_CFLAGS) + sys_sysio_gotcha_t_SOURCES = sys/sysio_suite.h \ sys/sysio_suite.c \ sys/creat-close.c \ @@ -165,7 +185,12 @@ unifyfs_unmount_t_CPPFLAGS = $(test_cppflags) unifyfs_unmount_t_LDADD = $(test_static_ldadd) unifyfs_unmount_t_LDFLAGS = $(test_static_ldflags) -seg_tree_test_t_SOURCES = seg_tree_test.c -seg_tree_test_t_CPPFLAGS = $(test_cppflags) -seg_tree_test_t_LDADD = $(test_static_ldadd) -seg_tree_test_t_LDFLAGS = $(test_static_ldflags) +common_seg_tree_test_t_SOURCES = common/seg_tree_test.c +common_seg_tree_test_t_CPPFLAGS = $(test_common_cppflags) +common_seg_tree_test_t_LDADD = $(test_common_ldadd) +common_seg_tree_test_t_LDFLAGS = $(test_common_ldflags) + +common_slotmap_test_t_SOURCES = common/slotmap_test.c +common_slotmap_test_t_CPPFLAGS = $(test_common_cppflags) +common_slotmap_test_t_LDADD = $(test_common_ldadd) +common_slotmap_test_t_LDFLAGS = $(test_common_ldflags) diff --git a/t/seg_tree_test.c b/t/common/seg_tree_test.c similarity index 100% rename from t/seg_tree_test.c rename to t/common/seg_tree_test.c diff --git a/t/common/slotmap_test.c b/t/common/slotmap_test.c new file mode 100644 index 000000000..fe4a26b53 --- /dev/null +++ b/t/common/slotmap_test.c @@ -0,0 +1,125 @@ +#include "slotmap.h" + +#include +#include +#include + +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +struct reservation { + size_t slot; + size_t count; +}; + +int main(int argc, char** argv) +{ + int rc; + + /* process test args */ + size_t num_slots = 4096; + if (argc > 1) { + num_slots = (size_t) atoi(argv[1]); + } + + size_t num_inserts = 100; + if (argc > 2) { + num_inserts = (size_t) atoi(argv[2]); + } + + int page_multiple = 1; + if (argc > 3) { + page_multiple = atoi(argv[3]); + } + + unsigned int rand_seed = 12345678; + if (argc > 4) { + rand_seed = (unsigned int) atoi(argv[4]); + } + srand(rand_seed); + + plan(NO_PLAN); + + /* allocate an array of reservations to remove */ + size_t num_removes = num_inserts / 2; + struct reservation* to_remove = (struct reservation*) + calloc(num_removes, sizeof(struct reservation)); + if (NULL == to_remove) { + BAIL_OUT("calloc() for reservation array failed!"); + } + + /* allocate buffer to hold a slot map */ + size_t page_sz = sysconf(_SC_PAGESIZE); + printf("# NOTE: page size is %zu bytes\n", page_sz); + size_t buf_sz = page_sz * page_multiple; + void* buf = malloc(buf_sz); + if (NULL == buf) { + BAIL_OUT("ERROR: malloc(%zu) for slot map buffer failed!\n", buf_sz); + } + + slot_map* smap = slotmap_init(num_slots, buf, buf_sz); + ok(NULL != smap, "create slot map with %zu slots", num_slots); + if (NULL == smap) { + done_testing(); // will exit program + } + + size_t remove_ndx = 0; + size_t success_count = 0; + for (size_t i = 0; i < num_inserts; i++) { + size_t cnt = (size_t)rand() % 18; + if (0 == cnt) { + cnt++; + } + + ssize_t slot = slotmap_reserve(smap, cnt); + if (-1 != slot) { + success_count++; + printf("# - reserved %2zu slots (start = %zu)\n", + cnt, (size_t)slot); + + /* pick some successful reservations to test removal */ + if ((cnt > 4) && (remove_ndx < num_removes)) { + struct reservation* rsvp = to_remove + remove_ndx; + rsvp->slot = (size_t)slot; + rsvp->count = cnt; + remove_ndx++; + } + } else { + printf("# FAILED to reserve %2zu slots\n", cnt); + } + } + ok(success_count == num_inserts, + "all slotmap_reserve() calls succeeded (%zu of %zu), %zu slots used", + success_count, num_inserts, smap->used_slots); + + slotmap_print(smap); + + success_count = 0; + size_t release_count = 0; + for (size_t i = 0; i < num_removes; i++) { + struct reservation* rsvp = to_remove + i; + if (rsvp->count > 0) { + release_count++; + rc = slotmap_release(smap, rsvp->slot, rsvp->count); + if (0 == rc) { + success_count++; + printf("# - released %2zu slots (start = %zu)\n", + rsvp->count, rsvp->slot); + } else { + printf("# FAILED to release %2zu slots (start = %zu)\n", + rsvp->count, rsvp->slot); + } + } + } + ok(success_count == release_count, + "all slotmap_release() calls succeeded (%zu of %zu)", + success_count, release_count); + + slotmap_print(smap); + + rc = slotmap_clear(smap); + ok(rc == 0, "clear the slotmap"); + + done_testing(); +} + diff --git a/t/server/unifyfs_meta_get_test.c b/t/server/unifyfs_meta_get_test.c index 7bd7e8be5..1540013db 100644 --- a/t/server/unifyfs_meta_get_test.c +++ b/t/server/unifyfs_meta_get_test.c @@ -49,8 +49,8 @@ int unifyfs_get_file_extents_test(void) unifyfs_key_t keys[16]; unifyfs_keyval_t keyval[16]; - rc = unifyfs_get_file_extents(num_keys, &keys, key_lens, - &num_values, &keyval); + rc = unifyfs_get_file_extents(num_keys, (unifyfs_key_t**)&keys, key_lens, + &num_values, (unifyfs_keyval_t**)&keyval); ok(UNIFYFS_SUCCESS == rc, "Retrieved file extents (rc = %d)", rc ); diff --git a/t/sharness.d/01-unifyfs-settings.sh b/t/sharness.d/01-unifyfs-settings.sh index 815e8f9f8..a44d3fc1d 100644 --- a/t/sharness.d/01-unifyfs-settings.sh +++ b/t/sharness.d/01-unifyfs-settings.sh @@ -19,7 +19,5 @@ export UNIFYFS_RUNSTATE_DIR=${UNIFYFS_TEST_STATE} export UNIFYFS_SHAREDFS_DIR=${UNIFYFS_TEST_SHARE} # Client settings -export UNIFYFS_SPILLOVER_ENABLED=${UNIFYFS_SPILLOVER_ENABLED:-"Y"} -export UNIFYFS_SPILLOVER_SIZE=$((5 * (2 ** 30))) -export UNIFYFS_SPILLOVER_META_DIR=${UNIFYFS_TEST_SPILL} -export UNIFYFS_SPILLOVER_DATA_DIR=${UNIFYFS_TEST_SPILL} +export UNIFYFS_LOGIO_SPILL_SIZE=$((5 * (2 ** 30))) +export UNIFYFS_LOGIO_SPILL_DIR=${UNIFYFS_TEST_SPILL} From 872cb269f23d8e6473ba43032b47e34d5259769b Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Mon, 16 Mar 2020 22:04:47 -0400 Subject: [PATCH 077/168] Adding simul POSIX test example. TEST_CHECKPATCH_ALLOW_FAILURE=yes --- examples/src/COPYING-simul | 306 +++++++++ examples/src/Makefile.am | 12 + examples/src/simul.c | 1325 ++++++++++++++++++++++++++++++++++++ 3 files changed, 1643 insertions(+) create mode 100644 examples/src/COPYING-simul create mode 100644 examples/src/simul.c diff --git a/examples/src/COPYING-simul b/examples/src/COPYING-simul new file mode 100644 index 000000000..1dd1caa1c --- /dev/null +++ b/examples/src/COPYING-simul @@ -0,0 +1,306 @@ +Preamble Notice + +A. This notice is required to be provided under our contract with the U.S. +Department of Energy (DOE). This work was produced at the University of +California, Lawrence Livermore National Laboratory under Contract +No. W-7405-ENG-48 with the DOE. + +B. Neither the United States Government nor the University of California +nor any of their employees, makes any warranty, express or implied, or +assumes any liability or responsibility for the accuracy, completeness, or +usefulness of any information, apparatus, product, or process disclosed, +or represents that its use would not infringe privately-owned rights. + +C. Also, reference herein to any specific commercial products, process, or +services by trade name, trademark, manufacturer or otherwise does not +necessarily constitute or imply its endorsement, recommendation, or +favoring by the United States Government or the University of California. +The views and opinions of authors expressed herein do not necessarily +state or reflect those of the United States Government or the University +of California, and shall not be used for advertising or product +endorsement purposes. + +The precise terms and conditions for copying, distribution and +modification follows. + +---------------------------------------------------------------------- + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index 1f7f87777..7721fc921 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -23,6 +23,7 @@ if HAVE_LD_WRAP app-tileio-static \ transfer-static \ size-static \ + simul-static \ chmod-static \ multi-write-static endif @@ -45,6 +46,7 @@ if HAVE_GOTCHA app-tileio-gotcha \ transfer-gotcha \ size-gotcha \ + simul-gotcha \ chmod-gotcha \ multi-write-gotcha endif @@ -302,6 +304,16 @@ size_static_CPPFLAGS = $(test_cppflags) size_static_LDADD = $(test_static_ldadd) size_static_LDFLAGS = $(test_static_ldflags) +simul_gotcha_SOURCES = simul.c +simul_gotcha_CPPFLAGS = $(test_cppflags) +simul_gotcha_LDADD = $(test_gotcha_ldadd) +simul_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +simul_static_SOURCES = simul.c +simul_static_CPPFLAGS = $(test_cppflags) +simul_static_LDADD = $(test_static_ldadd) +simul_static_LDFLAGS = $(test_static_ldflags) + chmod_gotcha_SOURCES = chmod.c testutil.c chmod_gotcha_CPPFLAGS = $(test_cppflags) chmod_gotcha_LDADD = $(test_gotcha_ldadd) diff --git a/examples/src/simul.c b/examples/src/simul.c new file mode 100644 index 000000000..ef11d1e1f --- /dev/null +++ b/examples/src/simul.c @@ -0,0 +1,1325 @@ +/* + * Copyright (C) 2003, The Regents of the University of California. + * Produced at the Lawrence Livermore National Laboratory. + * Written by Christopher J. Morrone + * UCRL-CODE-2003-019 + * All rights reserved. + * + * Please read the COPYING file. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License (as published by + * the Free Software Foundation) version 2, dated June 1991. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * terms and conditions of the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* + * Some modifications including style changes have been made for testing + * unifyfs. + */ + +#include "mpi.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* unifyfs test */ +#include + +#define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH +#define DIRMODE S_IRUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IXOTH +#define SHARED 1 +#define MAX_FILENAME_LEN 512 + +int rank; +int size; +char *testdir = NULL; +char hostname[1024]; +int verbose; +int throttle = 1; +struct timeval t1, t2; +static char version[] = "1.16"; + +#ifdef __GNUC__ + /* "inline" is a keyword in GNU C */ +#elif __STDC_VERSION__ >= 199901L + /* "inline" is a keyword in C99 and later versions */ +#else +# define inline /* "inline" not available */ +#endif + +#ifndef AIX +#define FAIL(msg) do { \ + fprintf(stdout, "%s: Process %d(%s): FAILED in %s, %s: %s\n",\ + timestamp(), rank, hostname, __func__, \ + msg, strerror(errno)); \ + fflush(stdout);\ + MPI_Abort(MPI_COMM_WORLD, 1); \ +} while(0) +#else +#define FAIL(msg) do { \ + fprintf(stdout, "%s: Process %d(%s): FAILED, %s: %s\n",\ + timestamp(), rank, hostname, \ + msg, strerror(errno)); \ + fflush(stdout);\ + MPI_Abort(MPI_COMM_WORLD, 1); \ +} while(0) +#endif + +char *timestamp() { + static char datestring[80]; + time_t timestamp; + + fflush(stdout); + timestamp = time(NULL); + strftime(datestring, 80, "%T", localtime(×tamp)); + + return datestring; +} + +static inline void begin(char *str) { + if (verbose > 0 && rank == 0) { + gettimeofday(&t1, NULL); + fprintf(stdout, "%s:\tBeginning %s\n", timestamp(), str); + fflush(stdout); + } +} + +static inline void end(char *str) { + double elapsed; + + MPI_Barrier(MPI_COMM_WORLD); + if (verbose > 0 && rank == 0) { + gettimeofday(&t2, NULL); + elapsed = ((((t2.tv_sec - t1.tv_sec) * 1000000L) + + t2.tv_usec) - t1.tv_usec) + / (double)1000000; + if (elapsed >= 60) { + fprintf(stdout, "%s:\tFinished %-15s(%.2f min)\n", + timestamp(), str, elapsed / 60); + } else { + fprintf(stdout, "%s:\tFinished %-15s(%.3f sec)\n", + timestamp(), str, elapsed); + + } + fflush(stdout); + } +} + +void Seq_begin(MPI_Comm comm, int numprocs) { + int size; + int rank; + int buf; + MPI_Status status; + + MPI_Comm_size(comm, &size); + MPI_Comm_rank(comm, &rank); + + if (rank >= numprocs) { + MPI_Recv(&buf, 1, MPI_INT, rank-numprocs, 1333, comm, &status); + } +} + +void Seq_end(MPI_Comm comm, int numprocs) { + int size; + int rank; + int buf; + + MPI_Comm_size(comm, &size); + MPI_Comm_rank(comm, &rank); + + if ((rank + numprocs) < size) { + MPI_Send(&buf, 1, MPI_INT, rank+numprocs, 1333, comm); + } +} + +/* This function does not FAIL if the requested "name" does not exist. This + is just to clean up any files or directories left over from previous runs.*/ +void remove_file_or_dir(char *name) { + struct stat statbuf; + char errmsg[MAX_FILENAME_LEN+20]; + + if (stat(name, &statbuf) != -1) { + if (S_ISREG(statbuf.st_mode)) { + printf("stale file found\n"); + if (unlink(name) == -1) { + sprintf(errmsg, "unlink of %s", name); + FAIL(errmsg); + } + } + if (S_ISDIR(statbuf.st_mode)) { + printf("stale directory found\n"); + if (rmdir(name) == -1) { + sprintf(errmsg, "rmmdir of %s", name); + FAIL(errmsg); + } + } + } +} + +char *create_files(char *prefix, int filesize, int shared) { + static char filename[MAX_FILENAME_LEN]; + char errmsg[MAX_FILENAME_LEN+20]; + int fd, i; + short zero = 0; + + /* Process 0 creates the test file(s) */ + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + sprintf(filename, "%s/%s.%d", testdir, prefix, i); + remove_file_or_dir(filename); + if ((fd = creat(filename, FILEMODE)) == -1) { + sprintf(errmsg, "creat of file %s", filename); + FAIL(errmsg); + } + if (filesize > 0) { + if (lseek(fd, filesize - 1, SEEK_SET) == -1) { + sprintf(errmsg, "lseek in file %s", filename); + FAIL(errmsg); + } + if (write(fd, &zero, 1) == -1) { + sprintf(errmsg, "write in file %s", filename); + FAIL(errmsg); + } + } + if (close(fd) == -1) { + sprintf(errmsg, "close of file %s", filename); + FAIL(errmsg); + } + } + } + + if (shared) + sprintf(filename, "%s/%s.0", testdir, prefix); + else + sprintf(filename, "%s/%s.%d", testdir, prefix, rank); + + return filename; +} + +void remove_files(char *prefix, int shared) { + char filename[1024]; + int i; + + /* Process 0 removes the file(s) */ + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + sprintf(filename, "%s/%s.%d", testdir, prefix, i); + /*printf("Removing file %s\n", filename); fflush(stdout);*/ + if (unlink(filename) == -1) { + FAIL("unlink failed"); + } + } + } +} + +char *create_dirs(char *prefix, int shared) { + static char dirname[1024]; + int i; + + /* Process 0 creates the test file(s) */ + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + sprintf(dirname, "%s/%s.%d", testdir, prefix, i); + remove_file_or_dir(dirname); + if (mkdir(dirname, DIRMODE) == -1) { + FAIL("init mkdir failed"); + } + } + } + + if (shared) + sprintf(dirname, "%s/%s.0", testdir, prefix); + else + sprintf(dirname, "%s/%s.%d", testdir, prefix, rank); + + return dirname; +} + +void remove_dirs(char *prefix, int shared) { + char dirname[1024]; + int i; + + /* Process 0 removes the file(s) */ + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + sprintf(dirname, "%s/%s.%d", testdir, prefix, i); + if (rmdir(dirname) == -1) { + FAIL("rmdir failed"); + } + } + } +} + +char *create_symlinks(char *prefix, int shared) { + static char filename[1024]; + static char linkname[1024]; + int i; + + /* Process 0 creates the test file(s) */ + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + sprintf(filename, "%s/symlink_target", testdir); + sprintf(linkname, "%s/%s.%d", testdir, prefix, i); + remove_file_or_dir(linkname); + if (symlink(filename, linkname) == -1) { + FAIL("symlink failed"); + } + } + } + + if (shared) + sprintf(linkname, "%s/%s.0", testdir, prefix); + else + sprintf(linkname, "%s/%s.%d", testdir, prefix, rank); + + return linkname; +} + +void check_single_success(char *testname, int rc, int error_rc) { + int *rc_vec, i; + int fail = 0; + int pass = 0; + + if (rank == 0) { + if ((rc_vec = (int *)malloc(sizeof(int)*size)) == NULL) { + FAIL("malloc failed"); + } + } + MPI_Gather(&rc, 1, MPI_INT, rc_vec, 1, MPI_INT, 0, MPI_COMM_WORLD); + if (rank == 0) { + for (i = 0; i < size; i++) { + if (rc_vec[i] == error_rc) + fail++; + else + pass++; + } + if (!((pass == 1) && (fail == size-1))) { + fprintf(stdout, "%s: FAILED in %s: ", timestamp(), testname); + if (pass > 1) + fprintf(stdout, "too many operations succeeded (%d).\n", pass); + else + fprintf(stdout, "too many operations failed (%d).\n", fail); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, 1); + } + free(rc_vec); + } +} + +void simul_open(int shared) { + int fd; + char *filename; + + begin("setup"); + filename = create_files("simul_open", 0, shared); + end("setup"); + + /* All open the file simultaneously */ + begin("test"); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("open failed"); + } + end("test"); + + /* All close the file one at a time */ + begin("cleanup"); + Seq_begin(MPI_COMM_WORLD, throttle); + if (close(fd) == -1) { + FAIL("close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_open", shared); + end("cleanup"); +} + +void simul_close(int shared) { + int fd; + char *filename; + + begin("setup"); + filename = create_files("simul_close", 0, shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("open failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All close the file simultaneously */ + if (close(fd) == -1) { + FAIL("close failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_close", shared); + end("cleanup"); +} + +void simul_chdir(int shared) { + char cwd[1024]; + char *dirname; + + begin("setup"); + if (getcwd(cwd, 1024) == NULL) { + FAIL("init getcwd failed"); + } + dirname = create_dirs("simul_chdir", shared); + end("setup"); + + begin("test"); + /* All chdir to dirname */ + if (chdir(dirname) == -1) { + FAIL("chdir failed"); + } + end("test"); + + begin("cleanup"); + /* All chdir back to old cwd */ + if (chdir(cwd) == -1) { + FAIL("chdir back failed"); + } + MPI_Barrier(MPI_COMM_WORLD); + remove_dirs("simul_chdir", shared); + end("cleanup"); +} + +void simul_file_stat(int shared) { + char *filename; + struct stat buf; + + begin("setup"); + filename = create_files("simul_file_stat", 0, shared); + end("setup"); + + begin("test"); + /* All stat the file */ + if (stat(filename, &buf) == -1) { + FAIL("stat failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_file_stat", shared); + end("cleanup"); +} + +void simul_dir_stat(int shared) { + char *dirname; + struct stat buf; + + begin("setup"); + dirname = create_dirs("simul_dir_stat", shared); + end("setup"); + + begin("test"); + /* All stat the directory */ + if (stat(dirname, &buf) == -1) { + FAIL("stat failed"); + } + end("test"); + + begin("cleanup"); + remove_dirs("simul_dir_stat", shared); + end("cleanup"); +} + +void simul_readdir(int shared) { + DIR *dir; + char *dirname; + struct dirent *dptr; + + begin("setup"); + dirname = create_dirs("simul_readdir", shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the directory(ies) one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((dir = opendir(dirname)) == NULL) { + FAIL("init opendir failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All readdir the directory stream(s) */ + if ((dptr = readdir(dir)) == NULL) { + FAIL("readdir failed"); + } + end("test"); + + begin("cleanup"); + /* All close the directory(ies) one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (closedir(dir) == -1) { + FAIL("closedir failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_dirs("simul_readdir", shared); + end("cleanup"); +} + +void simul_statfs(int shared) { + char *filename; + struct statfs buf; + + begin("setup"); + filename = create_files("simul_statfs", 0, shared); + end("setup"); + + begin("test"); + /* All statfs the file(s) */ + if (statfs(filename, &buf) == -1) { + FAIL("statfs failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_statfs", shared); + end("cleanup"); +} + +void simul_lseek(int shared) { + int fd; + char *filename; + + begin("setup"); + filename = create_files("simul_lseek", 0, shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the file(s) one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("init open failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All lseek simultaneously */ + if (lseek(fd, 1024, SEEK_SET) == -1) { + FAIL("lseek failed"); + MPI_Abort(MPI_COMM_WORLD, 1); + } + end("test"); + + begin("cleanup"); + /* All close the file(s) one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (close(fd) == -1) { + FAIL("cleanup close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_lseek", shared); + end("cleanup"); +} + +void simul_read(int shared) { + int fd; + ssize_t fin; + char buf[1024]; + char *filename; + int i = 0; + int retry = 100; + + begin("setup"); + filename = create_files("simul_read", 1024, shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("init open failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All read simultaneously */ + for (i = 1024; (i > 0) && (retry > 0); i -= fin, retry--) { + if ((fin = read(fd, buf, (size_t)i)) == -1) { + FAIL("read failed"); + } + } + if( (retry == 0) && (i > 0) ) + FAIL("read exceeded retry count"); + end("test"); + + begin("cleanup"); + /* All close the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (close(fd) == -1) { + FAIL("cleanup close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_read", shared); + end("cleanup"); +} + +void simul_write(int shared) { + int fd; + ssize_t fin; + char *filename; + int i = 0; + int retry = 100; + + begin("setup"); + filename = create_files("simul_write", size * sizeof(int), shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the file and lseek one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("init open failed"); + } + if (lseek(fd, rank*sizeof(int), SEEK_SET) == -1) { + FAIL("init lseek failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All write simultaneously */ + for (i = sizeof(int); (i > 0) && (retry > 0); i -= fin, retry--) { + if ((fin = write(fd, &rank, (size_t)i)) == -1) { + FAIL("write failed"); + } + } + if( (retry == 0) && (i > 0) ) + FAIL("write exceeded retry count"); + end("test"); + + begin("cleanup"); + /* All close the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (close(fd) == -1) { + FAIL("cleanup close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_write", shared); + end("cleanup"); +} + +void simul_mkdir(int shared) { + int rc, i; + char dirname[MAX_FILENAME_LEN]; + + begin("setup"); + if (shared) + sprintf(dirname, "%s/simul_mkdir.0", testdir); + else + sprintf(dirname, "%s/simul_mkdir.%d", testdir, rank); + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/simul_mkdir.%d", testdir, i); + remove_file_or_dir(buf); + } + } + end("setup"); + + begin("test"); + /* All mkdir dirname */ + rc = mkdir(dirname, DIRMODE); + if (!shared) { + if (rc == -1) { + FAIL("mkdir failed"); + } + MPI_Barrier(MPI_COMM_WORLD); + } else { /* Only one should succeed */ + check_single_success("simul_mkdir", rc, -1); + } + end("test"); + + begin("cleanup"); + remove_dirs("simul_mkdir", shared); + end("cleanup"); +} + +void simul_rmdir(int shared) { + int rc; + char *dirname; + + begin("setup"); + dirname = create_dirs("simul_rmdir", shared); + MPI_Barrier(MPI_COMM_WORLD); + end("setup"); + + begin("test"); + /* All rmdir dirname */ + rc = rmdir(dirname); + if (!shared) { + if (rc == -1) { + FAIL("rmdir failed"); + } + MPI_Barrier(MPI_COMM_WORLD); + } else { /* Only one should succeed */ + check_single_success("simul_rmdir", rc, -1); + } + end("test"); + + begin("cleanup"); + end("cleanup"); +} + +void simul_creat(int shared) { + int fd, i; + char filename[1024]; + + begin("setup"); + if (shared) + sprintf(filename, "%s/simul_creat.0", testdir); + else + sprintf(filename, "%s/simul_creat.%d", testdir, rank); + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/simul_creat.%d", testdir, i); + remove_file_or_dir(buf); + } + } + end("setup"); + + begin("test"); + /* All create the files simultaneously */ + fd = creat(filename, FILEMODE); + if (fd == -1) { + FAIL("creat failed"); + } + end("test"); + + begin("cleanup"); + /* All close the files one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (close(fd) == -1) { + FAIL("close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_creat", shared); + end("cleanup"); +} + +void simul_unlink(int shared) { + int rc; + char *filename; + + begin("setup"); + filename = create_files("simul_unlink", 0, shared); + end("setup"); + + begin("test"); + /* All unlink the files simultaneously */ + rc = unlink(filename); + if (!shared) { + if (rc == -1) { + FAIL("unlink failed"); + } + } else { + check_single_success("simul_unlink", rc, -1); + } + end("test"); + + begin("cleanup"); + end("cleanup"); +} + +void simul_rename(int shared) { + int rc, i; + char *oldfilename; + char newfilename[1024]; + char *testname = "simul_rename"; + + begin("setup"); + oldfilename = create_files(testname, 0, shared); + sprintf(newfilename, "%s/%s_new.%d", testdir, testname, rank); + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/%s_new.%d", testdir, testname, i); + remove_file_or_dir(buf); + } + } + end("setup"); + + begin("test"); + /* All rename the files simultaneously */ + rc = rename(oldfilename, newfilename); + if (!shared) { + if (rc == -1) { + FAIL("stat failed"); + } + } else { + check_single_success(testname, rc, -1); + } + end("test"); + + begin("cleanup"); + if (rc == 0) { + if (unlink(newfilename) == -1) + FAIL("unlink failed"); + } + end("cleanup"); +} + +void simul_truncate(int shared) { + char *filename; + + begin("setup"); + filename = create_files("simul_truncate", 2048, shared); + end("setup"); + + begin("test"); + /* All truncate simultaneously */ + if (truncate(filename, 1024) == -1) { + FAIL("truncate failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_truncate", shared); + end("cleanup"); +} + +void simul_readlink(int shared) { + char *linkname; + char buf[1024]; + + begin("setup"); + linkname = create_symlinks("simul_readlink", shared); + end("setup"); + + begin("test"); + /* All read the symlink(s) simultaneously */ + if (readlink(linkname, buf, 1024) == -1) { + FAIL("readlink failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_readlink", shared); + end("cleanup"); +} + +void simul_symlink(int shared) { + int rc, i; + char linkname[MAX_FILENAME_LEN]; + char filename[MAX_FILENAME_LEN]; + + begin("setup"); + if (shared) + sprintf(linkname, "%s/simul_symlink.0", testdir); + else + sprintf(linkname, "%s/simul_symlink.%d", testdir, rank); + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/simul_symlink.%d", testdir, i); + remove_file_or_dir(buf); + } + } + sprintf(filename, "%s/simul_symlink_target", testdir); + end("setup"); + + begin("test"); + /* All create the symlinks simultaneously */ + rc = symlink(filename, linkname); + if (!shared) { + if (rc == -1) { + FAIL("symlink failed"); + } + } else { + check_single_success("simul_symlink", rc, -1); + } + end("test"); + + begin("cleanup"); + remove_files("simul_symlink", shared); + end("cleanup"); +} + +void simul_link_to_one(int shared) { + int rc, i; + char *filename; + char linkname[1024]; + + begin("setup"); + if (shared) + sprintf(linkname, "%s/simul_link.0", testdir); + else + sprintf(linkname, "%s/simul_link.%d", testdir, rank); + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/simul_link.%d", testdir, i); + remove_file_or_dir(buf); + } + } + filename = create_files("simul_link_target", 0, SHARED); + end("setup"); + + begin("test"); + /* All create the hard links simultaneously */ + rc = link(filename, linkname); + if (!shared) { + if (rc == -1) { + FAIL("link failed"); + } + } else { + check_single_success("simul_link_to_one", rc, -1); + } + end("test"); + + begin("cleanup"); + remove_files("simul_link_target", SHARED); + remove_files("simul_link", shared); + end("cleanup"); +} + +void simul_link_to_many(int shared) { + char *filename; + char linkname[1024]; + int i; + + if (shared) { + if (verbose > 0 && rank == 0) + printf("%s:\tThis is just a place holder; no test is run here.\n", + timestamp()); + return; + } + begin("setup"); + filename = create_files("simul_link", 0, shared); + sprintf(linkname, "%s/simul_link_target.%d", testdir, rank); + if (rank == 0) { + for (i = 0; i < size; i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/simul_link_target.%d", testdir, i); + remove_file_or_dir(buf); + } + } + end("setup"); + + begin("test"); + /* All create the hard links simultaneously */ + if (link(filename, linkname) == -1) { + FAIL("link failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_link", shared); + remove_files("simul_link_target", !SHARED); + end("cleanup"); +} + +void simul_fcntl_lock(int shared) { + int rc, fd; + char *filename; + struct flock sf_lock = { + .l_type = F_WRLCK, + .l_whence = SEEK_SET, + .l_start = 0, + .l_len = 0 + }; + struct flock sf_unlock = { + .l_type = F_UNLCK, + .l_whence = SEEK_SET, + .l_start = 0, + .l_len = 0 + }; + + begin("setup"); + filename = create_files("simul_fcntl", 0, shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("open failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All lock the file(s) simultaneously */ + rc = fcntl(fd, F_SETLK, &sf_lock); + if (!shared) { + if (rc == -1) { + if (errno == ENOSYS) { + if (rank == 0) { + fprintf(stdout, "WARNING: fcntl locking not supported.\n"); + fflush(stdout); + } + } else { + FAIL("fcntl lock failed"); + } + } + MPI_Barrier(MPI_COMM_WORLD); + } else { + int saved_errno = errno; + int *rc_vec, *er_vec, i; + int fail = 0; + int pass = 0; + int nosys = 0; + if (rank == 0) { + if ((rc_vec = (int *)malloc(sizeof(int)*size)) == NULL) + FAIL("malloc failed"); + if ((er_vec = (int *)malloc(sizeof(int)*size)) == NULL) + FAIL("malloc failed"); + } + MPI_Gather(&rc, 1, MPI_INT, rc_vec, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&saved_errno, 1, MPI_INT, er_vec, 1, MPI_INT, 0, + MPI_COMM_WORLD); + if (rank == 0) { + for (i = 0; i < size; i++) { + if (rc_vec[i] == -1) { + if (er_vec[i] == ENOSYS) { + nosys++; + } else if (er_vec[i] != EACCES && er_vec[i] != EAGAIN) { + errno = er_vec[i]; + FAIL("fcntl failed as expected, but with wrong errno"); + } + fail++; + } else { + pass++; + } + } + if (nosys == size) { + fprintf(stdout, "WARNING: fcntl locking not supported.\n"); + fflush(stdout); + } else if (!((pass == 1) && (fail == size-1))) { + fprintf(stdout, + "%s: FAILED in simul_fcntl_lock", timestamp()); + if (pass > 1) + fprintf(stdout, + "too many fcntl locks succeeded (%d).\n", pass); + else + fprintf(stdout, + "too many fcntl locks failed (%d).\n", fail); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, 1); + } + free(rc_vec); + free(er_vec); + } + } + end("test"); + + begin("cleanup"); + /* All close the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (!shared || rank == 0) { + rc = fcntl(fd, F_SETLK, &sf_unlock); + if (rc == -1 && errno != ENOSYS) + FAIL("fcntl unlock failed"); + } + if (close(fd) == -1) { + FAIL("close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_fcntl", shared); + end("cleanup"); +} + +struct test { + char *name; + void (*function) (int); + int simul; /* Flag designating support for simultaneus mode */ + int indiv; /* Flag designating support for individual mode */ +}; + +static struct test testlist[] = { + {"open", simul_open}, + {"close", simul_close}, + {"file stat", simul_file_stat}, + {"lseek", simul_lseek}, + {"read", simul_read}, + {"write", simul_write}, + {"chdir", simul_chdir}, + {"directory stat", simul_dir_stat}, + {"statfs", simul_statfs}, + {"readdir", simul_readdir}, + {"mkdir", simul_mkdir}, + {"rmdir", simul_rmdir}, + {"unlink", simul_unlink}, + {"rename", simul_rename}, + {"creat", simul_creat}, + {"truncate", simul_truncate}, + {"symlink", simul_symlink}, + {"readlink", simul_readlink}, + {"link to one file", simul_link_to_one}, + {"link to a file per process", simul_link_to_many}, + {"fcntl locking", simul_fcntl_lock}, + {0} +}; + +/* Searches an array of ints for one int. A "-1" must mark the end of the + array. */ +int int_in_list(int item, int *list) { + int *ptr; + + if (list == NULL) + return 0; + ptr = list; + while (*ptr != -1) { + if (*ptr == item) + return 1; + ptr += 1; + } + return 0; +} + +/* Breaks string of comma-sperated ints into an array of ints */ +int *string_split(char *string) { + char *ptr; + char *tmp; + int excl_cnt = 1; + int *list; + int i; + + ptr = string; + while((tmp = strchr(ptr, ','))) { + ptr = tmp + 1; + excl_cnt++; + } + + list = (int *)malloc(sizeof(int) * (excl_cnt + 1)); + if (list == NULL) FAIL("malloc failed"); + + tmp = (strtok(string, ", ")); + if (tmp == NULL) FAIL("strtok failed"); + list[0] = atoi(tmp); + for (i = 1; i < excl_cnt; i++) { + tmp = (strtok(NULL, ", ")); + if (tmp == NULL) FAIL("strtok failed"); + list[i] = atoi(tmp); + } + list[i] = -1; + + return list; +} + +void print_help(int testcnt) { + int i; + + if (rank == 0) { + printf("simul-%s\n", version); + printf("Usage: simul [-h] -d [-f firsttest] [-l lasttest]\n"); + printf(" [-n #] [-N #] [-i \"4,7,13\"] [-e \"6,22\"] [-s]\n"); + printf(" [-v] [-V #]\n"); + printf("\t-h: prints this help message\n"); + printf("\t-d: the directory in which the tests will run\n"); + printf("\t-f: the number of the first test to run (default: 0)\n"); + printf("\t-l: the number of the last test to run (default: %d)\n", + (testcnt*2)-1); + printf("\t-i: comma-sperated list of tests to include\n"); + printf("\t-e: comma-sperated list of tests to exclude\n"); + printf("\t-s: single-step through every iteration of every test\n"); + printf("\t-n: repeat each test # times (default: 1)\n"); + printf("\t-N: repeat the entire set of tests # times (default: 1)\n"); + printf("\t-v: increases the verbositly level by 1\n"); + printf("\t-u: test with unifyfs\n"); + printf("\t-V: select a specific verbosity level\n"); + printf("\nThe available tests are:\n"); + for (i = 0; i < testcnt * 2; i++) { + printf("\tTest #%d: %s, %s mode.\n", i, + testlist[i%testcnt].name, + (i < testcnt) ? "shared" : "individual"); + } + } + + MPI_Initialized(&i); + if (i) MPI_Finalize(); + exit(0); +} + +int main(int argc, char **argv) { + int testcnt; + int first; + int last; + int i, j, k, c; + int *excl_list = NULL; + int *incl_list = NULL; + int test; + int singlestep = 0; + int iterations = 1; + int set_iterations = 1; + int unifyfs = 0; + int ret = 0; + char linebuf[80]; + + /* Check for -h parameter before MPI_Init so the simul binary can be + called directly, without, for instance, mpirun. */ + for (testcnt = 1; testlist[testcnt].name != 0; testcnt++) continue; + for (i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + print_help(testcnt); + } + } + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + if (rank == 0) { + printf("Simul is running with %d process(es)\n", size); + fflush(stdout); + } + + first = 0; + last = testcnt * 2; + + /* Parse command line options */ + while (1) { + c = getopt(argc, argv, "d:e:f:hi:l:n:N:suvV:"); + if (c == -1) + break; + + switch (c) { + case 'd': + testdir = optarg; + break; + case 'e': + excl_list = string_split(optarg); + break; + case 'f': + first = atoi(optarg); + if (first >= last) { + printf("Invalid parameter, firsttest must be <= lasttest\n"); + MPI_Abort(MPI_COMM_WORLD, 2); + } + break; + case 'h': + print_help(testcnt); + break; + case 'i': + incl_list = string_split(optarg); + break; + case 'l': + last = atoi(optarg)+1; + if (last <= first) { + printf("Invalid parameter, lasttest must be >= firsttest\n"); + MPI_Abort(MPI_COMM_WORLD, 2); + } + break; + case 'n': + iterations = atoi(optarg); + break; + case 'N': + set_iterations = atoi(optarg); + break; + case 's': + singlestep = 1; + break; + case 'u': + unifyfs = 1; + break; + case 'v': + verbose += 1; + break; + case 'V': + verbose = atoi(optarg); + break; + } + } + + /* mount the unifyfs and use the testdir as the mountpoint. + * if testdir is not specified, use '/unifyfs.' */ + if (unifyfs) { + int ret = 0; + + if (!testdir) { + testdir = "/unifyfs"; + } + ret = unifyfs_mount(testdir, rank, size, 0); + if (ret && rank == 0) { + printf("unifyfs_mount failed (ret=%d)\n", ret); + MPI_Abort(MPI_COMM_WORLD, 2); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (testdir == NULL && rank == 0) { + printf("Please specify a test directory! (\"simul -h\" for help)\n"); + MPI_Abort(MPI_COMM_WORLD, 2); + } + + if (gethostname(hostname, 1024) == -1) { + perror("gethostname"); + MPI_Abort(MPI_COMM_WORLD, 2); + } + + /* If a list of tests was not specified with the -i option, then use + the first and last number to build a range of included tests. */ + if (incl_list == NULL) { + incl_list = (int *)malloc(sizeof(int) * (2+last-first)); + for (i = 0; i < last-first; i++) { + incl_list[i] = i + first; + } + incl_list[i] = -1; + } + + /* Run the tests */ + for (k = 0; k < set_iterations; k++) { + if ((rank == 0) && (set_iterations > 1)) + printf("%s: Set iteration %d\n", timestamp(), k); + for (i = 0; ; ++i) { + test = incl_list[i]; + if (test == -1) + break; + if (!int_in_list(test, excl_list)) { + for (j = 0; j < iterations; j++) { + if (singlestep) { + if (rank == 0) + printf("%s: Hit to run test #%d(iter %d): %s, %s mode.\n", + timestamp(), test, j, + testlist[test%testcnt].name, + (test < testcnt) ? "shared" : "individual"); + fgets(linebuf, 80, stdin); + } + if (rank == 0) { + printf("%s: Running test #%d(iter %d): %s, %s mode.\n", + timestamp(), test, j, testlist[test%testcnt].name, + (test < testcnt) ? "shared" : "individual"); + fflush(stdout); + } + testlist[test%testcnt].function((test < testcnt) ? SHARED : !SHARED); + MPI_Barrier(MPI_COMM_WORLD); + } + } + } + } + + if (rank == 0) printf("%s: All tests passed!\n", timestamp()); + + /* unmount unifyfs */ + if (unifyfs) { + unifyfs_unmount(); + } + + MPI_Finalize(); + exit(0); +} From a0124a18c9c3480e7a434fe8b02aafc48e318d6b Mon Sep 17 00:00:00 2001 From: CamStan Date: Mon, 23 Mar 2020 12:21:32 -0700 Subject: [PATCH 078/168] Add unit tests for lseek and fix related bugs This adds unit tests for our lseek wrapper and fixes some realated bugs. These mostly had to do with setting errno when invalid offsets are passed. Also adds tests for SEEK_DATA and SEEK_HOLE, which we currently do not support. --- client/src/unifyfs-sysio.c | 17 ++- t/Makefile.am | 2 + t/sys/lseek.c | 213 +++++++++++++++++++++++++++++++++++++ t/sys/sysio_suite.c | 2 + t/sys/sysio_suite.h | 3 + 5 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 t/sys/lseek.c diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 7b598b082..7017c48cc 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -873,7 +873,7 @@ off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); /* get current file position */ - off_t current_pos = filedesc->pos; + off_t logical_eof, current_pos = filedesc->pos; /* TODO: support SEEK_DATA and SEEK_HOLE? */ @@ -881,15 +881,28 @@ off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) switch (whence) { case SEEK_SET: /* seek to offset */ + if (offset < 0) { + errno = EINVAL; + return (off_t)(-1); + } current_pos = offset; break; case SEEK_CUR: /* seek to current position + offset */ + if (current_pos + offset < 0) { + errno = EINVAL; + return (off_t)(-1); + } current_pos += offset; break; case SEEK_END: /* seek to EOF + offset */ - current_pos = unifyfs_fid_logical_size(fid); + logical_eof = unifyfs_fid_logical_size(fid); + if (logical_eof + offset < 0) { + errno = EINVAL; + return (off_t)(-1); + } + current_pos = logical_eof + offset; break; default: errno = EINVAL; diff --git a/t/Makefile.am b/t/Makefile.am index 72811f613..aedf43a29 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -124,6 +124,7 @@ sys_sysio_gotcha_t_SOURCES = sys/sysio_suite.h \ sys/mkdir-rmdir.c \ sys/open.c \ sys/open64.c \ + sys/lseek.c \ sys/write-read.c \ sys/write-read-hole.c \ sys/truncate.c \ @@ -140,6 +141,7 @@ sys_sysio_static_t_SOURCES = sys/sysio_suite.h \ sys/mkdir-rmdir.c \ sys/open.c \ sys/open64.c \ + sys/lseek.c \ sys/write-read.c \ sys/write-read-hole.c \ sys/truncate.c \ diff --git a/t/sys/lseek.c b/t/sys/lseek.c new file mode 100644 index 000000000..7329782f8 --- /dev/null +++ b/t/sys/lseek.c @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2018, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +/* This function contains the tests for UNIFYFS_WRAP(lseek) found in + * client/src/unifyfs-sysio.c. + * + * Notice the tests are ordered in a logical testing order. Changing the order + * or adding new tests in between two others could negatively affect the + * desired results. */ +int lseek_test(char* unifyfs_root) +{ + /* Diagnostic message for reading and debugging output */ + diag("Starting UNIFYFS_WRAP(lseek) tests"); + + char path[64]; + int file_mode = 0600; + int fd, rc; + off_t ret; + + /* Create a random file and dir name at the mountpoint path to test on */ + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Open a file and write to it to test lseek() */ + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, file_mode); + rc = write(fd, "hello world", 12); + + /* lseek with invalid whence fails with errno=EINVAL. */ + errno = 0; + ret = lseek(fd, 0, -1); + ok(ret == -1 && errno == EINVAL, + "%s: lseek with invalid whence should fail (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* lseek() with SEEK_SET tests */ + /* lseek to negative offset with SEEK_SET should fail with errno=EINVAL */ + errno = 0; + ret = lseek(fd, -1, SEEK_SET); + ok(ret == -1 && errno == EINVAL, + "%s: lseek to negative offset w/ SEEK_SET fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* lseek beyond end of file with SEEK_SET succeeds */ + errno = 0; + ret = lseek(fd, 25, SEEK_SET); + ok(ret == 25, "%s: lseek beyond end of file w/ SEEK_SET (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek to beginning of file with SEEK_SET succeeds */ + errno = 0; + ret = lseek(fd, 0, SEEK_SET); + ok(ret == 0, "%s: lseek to beginning of file w/ SEEK_SET (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek() with SEEK_CUR tests */ + /* lseek to end of file with SEEK_CUR succeeds */ + errno = 0; + ret = lseek(fd, 12, SEEK_CUR); + ok(ret == 12, "%s: lseek to end of file w/ SEEK_CUR (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek to negative offset with SEEK_CUR should fail with errno=EINVAL */ + errno = 0; + ret = lseek(fd, -15, SEEK_CUR); + ok(ret == -1 && errno == EINVAL, + "%s: lseek to negative offset w/ SEEK_CUR fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* lseek to beginning of file with SEEK_CUR succeeds */ + errno = 0; + ret = lseek(fd, -12, SEEK_CUR); + ok(ret == 0, "%s: lseek to beginning of file w/ SEEK_CUR (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek beyond end of file with SEEK_CUR succeeds */ + errno = 0; + ret = lseek(fd, 25, SEEK_CUR); + ok(ret == 25, "%s: lseek beyond end of file w/ SEEK_CUR (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek() with SEEK_END tests */ + /* lseek to negative offset with SEEK_END should fail with errno=EINVAL */ + errno = 0; + ret = lseek(fd, -15, SEEK_END); + ok(ret == -1 && errno == EINVAL, + "%s: lseek to negative offset w/ SEEK_END fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* lseek back one from end of file with SEEK_END succeeds */ + errno = 0; + ret = lseek(fd, -1, SEEK_END); + ok(ret == 11, + "%s: lseek back one from end of file w/ SEEK_END (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek to beginning of file with SEEK_END succeeds */ + errno = 0; + ret = lseek(fd, -12, SEEK_END); + ok(ret == 0, "%s: lseek to beginning of file w/ SEEK_END (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek beyond end of file with SEEK_END succeeds */ + errno = 0; + ret = lseek(fd, 25, SEEK_END); + ok(ret == 37, "%s: lseek beyond end of file w/ SEEK_END (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek() with SEEK_DATA tests */ + /* Write beyond end of file to create a hole */ + rc = write(fd, "hello universe", 15); + + todo("SEEK_DATA and SEEK_HOLE not yet implemented"); + /* lseek to negative offset with SEEK_DATA should fail with errno=ENXIO */ + errno = 0; + ret = lseek(fd, -1, SEEK_DATA); + ok(ret == -1 && errno == ENXIO, + "%s: lseek to negative offset w/ SEEK_DATA fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* lseek to beginning of file with SEEK_DATA succeeds */ + errno = 0; + ret = lseek(fd, 0, SEEK_DATA); + ok(ret == 0, "%s: lseek to beginning of file w/ SEEK_DATA (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek to data after hole with SEEK_DATA returs current offset or offset + * of first set of data */ + errno = 0; + ret = lseek(fd, 15, SEEK_DATA); + ok(ret == 15 || ret == 37, + "%s: lseek to data after hole w/ SEEK_DATA (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek beyond end of file with SEEK_DATA should fail with errno=ENXIO */ + errno = 0; + ret = lseek(fd, -1, SEEK_DATA); + ok(ret == -1 && errno == ENXIO, + "%s: lseek beyond end of file w/ SEEK_DATA fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* lseek() with SEEK_HOLE tests */ + + /* lseek to negative offset with SEEK_HOLE should fail with errno=ENXIO */ + errno = 0; + ret = lseek(fd, -1, SEEK_HOLE); + ok(ret == -1 && errno == ENXIO, + "%s: lseek to negative offset w/ SEEK_HOLE fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* lseek to first hole of file with SEEK_HOLE succeeds returns start of hole + * or EOF */ + errno = 0; + ret = lseek(fd, 0, SEEK_HOLE); + ok(ret == 12 || ret == 52, + "%s: lseek to first hole in file w/ SEEK_HOLE (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek to middle of hole with SEEK_HOLE returns position in hole or EOF */ + errno = 0; + ret = lseek(fd, 18, SEEK_HOLE); + ok(ret == 18 || ret == 52, + "%s: lseek to middle of hole w/ SEEK_HOLE (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek to end of file with SEEK_HOLE succeeds */ + errno = 0; + ret = lseek(fd, 42, SEEK_HOLE); + ok(ret == 52, "%s: lseek to end of file w/ SEEK_HOLE (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* lseek beyond end of file with SEEK_HOLE should fail with errno= ENXIO */ + errno = 0; + ret = lseek(fd, 75, SEEK_HOLE); + ok(ret == -1 && errno == ENXIO, + "%s: lseek beyond end of file w/ SEEK_HOLE fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + end_todo; + + rc = close(fd); + + /* lseek in non-open file descriptor should fail with errno=EBADF */ + errno = 0; + ret = lseek(fd, 0, SEEK_SET); + ok(ret == -1 && errno == EBADF, + "%s: lseek in non-open file descriptor fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + diag("Finished UNIFYFS_WRAP(lseek) tests"); + + return 0; +} diff --git a/t/sys/sysio_suite.c b/t/sys/sysio_suite.c index 6e735c508..235b9ccbb 100644 --- a/t/sys/sysio_suite.c +++ b/t/sys/sysio_suite.c @@ -81,6 +81,8 @@ int main(int argc, char* argv[]) open64_test(unifyfs_root); + lseek_test(unifyfs_root); + write_read_test(unifyfs_root); write_read_hole_test(unifyfs_root); diff --git a/t/sys/sysio_suite.h b/t/sys/sysio_suite.h index 9fb32d1c8..b69b82212 100644 --- a/t/sys/sysio_suite.h +++ b/t/sys/sysio_suite.h @@ -45,6 +45,9 @@ int open_test(char* unifyfs_root); /* Tests for UNIFYFS_WRAP(open64) */ int open64_test(char* unifyfs_root); +/* Tests for UNIFYFS_WRAP(lseek) */ +int lseek_test(char* unifyfs_root); + int write_read_test(char* unifyfs_root); /* test reading from file with holes */ From 5ea739194caccb2dace3a9bd7785e07ec83616be Mon Sep 17 00:00:00 2001 From: CamStan Date: Mon, 23 Mar 2020 17:48:16 -0700 Subject: [PATCH 079/168] Add lseek support for SEEK_DATA and SEEK_HOLE This adds lseek support for SEEK_DATA and SEEK_HOLE using the fallback approaches. That is, SEEK_DATA always seeks to the offset and SEEK_HOLE always seeks to EOF. The -D_GNU_SOURCE is needed in Makefile.am in order to use SEEK_DATA and SEEK_HOLE. Removes todo() from related unit tests. --- client/src/Makefile.am | 3 ++- client/src/unifyfs-sysio.c | 30 ++++++++++++++++++++++++++---- t/sys/lseek.c | 23 +++++++++++++---------- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/client/src/Makefile.am b/client/src/Makefile.am index 6186d1b77..ec5b9b9a5 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -20,7 +20,8 @@ endif CLIENT_COMMON_CPPFLAGS = \ -I$(top_builddir)/client \ - -I$(top_srcdir)/common/src + -I$(top_srcdir)/common/src \ + -D_GNU_SOURCE CLIENT_COMMON_CFLAGS = \ $(MPI_CFLAGS) \ diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 7017c48cc..d25bcb326 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -873,15 +873,15 @@ off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); /* get current file position */ - off_t logical_eof, current_pos = filedesc->pos; - - /* TODO: support SEEK_DATA and SEEK_HOLE? */ + off_t current_pos = filedesc->pos; + off_t logical_eof; /* compute final file position */ switch (whence) { case SEEK_SET: /* seek to offset */ if (offset < 0) { + /* negative offset is invalid */ errno = EINVAL; return (off_t)(-1); } @@ -890,6 +890,7 @@ off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) case SEEK_CUR: /* seek to current position + offset */ if (current_pos + offset < 0) { + /* offset is negative and will result in negative position */ errno = EINVAL; return (off_t)(-1); } @@ -899,11 +900,32 @@ off_t UNIFYFS_WRAP(lseek)(int fd, off_t offset, int whence) /* seek to EOF + offset */ logical_eof = unifyfs_fid_logical_size(fid); if (logical_eof + offset < 0) { + /* offset is negative and will result in negative position */ errno = EINVAL; return (off_t)(-1); } current_pos = logical_eof + offset; break; + case SEEK_DATA: + /* Using fallback approach: always return offset */ + logical_eof = unifyfs_fid_logical_size(fid); + if (offset < 0 || offset > logical_eof) { + /* negative offset and offset beyond EOF are invalid */ + errno = ENXIO; + return (off_t)(-1); + } + current_pos = offset; + break; + case SEEK_HOLE: + /* Using fallback approach: always return offset for EOF */ + logical_eof = unifyfs_fid_logical_size(fid); + if (offset < 0 || offset > logical_eof) { + /* negative offset and offset beyond EOF are invalid */ + errno = ENXIO; + return (off_t)(-1); + } + current_pos = logical_eof; + break; default: errno = EINVAL; return (off_t)(-1); @@ -926,7 +948,7 @@ off64_t UNIFYFS_WRAP(lseek64)(int fd, off64_t offset, int whence) if (unifyfs_intercept_fd(&fd)) { if (sizeof(off_t) == sizeof(off64_t)) { /* off_t and off64_t are the same size, - * delegate to lseek warpper */ + * delegate to lseek wrapper */ off64_t ret = (off64_t)UNIFYFS_WRAP(lseek)( origfd, (off_t) offset, whence); return ret; diff --git a/t/sys/lseek.c b/t/sys/lseek.c index 7329782f8..d88666123 100644 --- a/t/sys/lseek.c +++ b/t/sys/lseek.c @@ -36,15 +36,15 @@ int lseek_test(char* unifyfs_root) char path[64]; int file_mode = 0600; - int fd, rc; + int fd; off_t ret; - /* Create a random file and dir name at the mountpoint path to test on */ + /* Create a random file at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); /* Open a file and write to it to test lseek() */ fd = open(path, O_RDWR | O_CREAT | O_TRUNC, file_mode); - rc = write(fd, "hello world", 12); + write(fd, "hello world", 12); /* lseek with invalid whence fails with errno=EINVAL. */ errno = 0; @@ -61,6 +61,12 @@ int lseek_test(char* unifyfs_root) "%s: lseek to negative offset w/ SEEK_SET fails (ret=%d, errno=%d): %s", __FILE__, ret, errno, strerror(errno)); + /* lseek to valid offset with SEEK_SET succeeds */ + errno = 0; + ret = lseek(fd, 7, SEEK_SET); + ok(ret == 7, "%s: lseek to valid offset w/ SEEK_SET (ret=%d): %s", + __FILE__, ret, strerror(errno)); + /* lseek beyond end of file with SEEK_SET succeeds */ errno = 0; ret = lseek(fd, 25, SEEK_SET); @@ -128,9 +134,8 @@ int lseek_test(char* unifyfs_root) /* lseek() with SEEK_DATA tests */ /* Write beyond end of file to create a hole */ - rc = write(fd, "hello universe", 15); + write(fd, "hello universe", 15); - todo("SEEK_DATA and SEEK_HOLE not yet implemented"); /* lseek to negative offset with SEEK_DATA should fail with errno=ENXIO */ errno = 0; ret = lseek(fd, -1, SEEK_DATA); @@ -144,7 +149,7 @@ int lseek_test(char* unifyfs_root) ok(ret == 0, "%s: lseek to beginning of file w/ SEEK_DATA (ret=%d): %s", __FILE__, ret, strerror(errno)); - /* lseek to data after hole with SEEK_DATA returs current offset or offset + /* lseek to data after hole with SEEK_DATA returns current offset or offset * of first set of data */ errno = 0; ret = lseek(fd, 15, SEEK_DATA); @@ -154,7 +159,7 @@ int lseek_test(char* unifyfs_root) /* lseek beyond end of file with SEEK_DATA should fail with errno=ENXIO */ errno = 0; - ret = lseek(fd, -1, SEEK_DATA); + ret = lseek(fd, 75, SEEK_DATA); ok(ret == -1 && errno == ENXIO, "%s: lseek beyond end of file w/ SEEK_DATA fails (ret=%d, errno=%d): %s", __FILE__, ret, errno, strerror(errno)); @@ -196,9 +201,7 @@ int lseek_test(char* unifyfs_root) "%s: lseek beyond end of file w/ SEEK_HOLE fails (ret=%d, errno=%d): %s", __FILE__, ret, errno, strerror(errno)); - end_todo; - - rc = close(fd); + close(fd); /* lseek in non-open file descriptor should fail with errno=EBADF */ errno = 0; From c1b0c41c9a7436bf929b0e55aaaeb3c94fa01670 Mon Sep 17 00:00:00 2001 From: CamStan Date: Tue, 24 Mar 2020 20:17:32 -0700 Subject: [PATCH 080/168] Add fseek/ftell/rewind tests and fix related bugs This adds unit tests for our fseek, ftell, and rewind wrappers and fixes some related bugs. These mostly have to do with setting errno when invalid offsets are passed. --- client/src/unifyfs-stdio.c | 22 +++- t/Makefile.am | 2 + t/std/fseek-ftell.c | 251 +++++++++++++++++++++++++++++++++++++ t/std/stdio_suite.c | 5 + t/std/stdio_suite.h | 9 ++ 5 files changed, 287 insertions(+), 2 deletions(-) create mode 100644 t/std/fseek-ftell.c diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index bd4e49940..d04ca1206 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -892,6 +892,11 @@ static int unifyfs_fseek(FILE* stream, off_t offset, int whence) switch (whence) { case SEEK_SET: /* seek to offset */ + if (offset < 0) { + /* negative offset is invalid */ + errno = EINVAL; + return -1; + } current_pos = offset; break; case SEEK_CUR: @@ -901,8 +906,12 @@ static int unifyfs_fseek(FILE* stream, off_t offset, int whence) errno = EOVERFLOW; return -1; } + if (current_pos + offset < 0) { + /* offset is negative and will result in a negative position */ + errno = EINVAL; + return -1; + } current_pos += offset; - break; case SEEK_END: /* seek to EOF + offset */ @@ -912,6 +921,11 @@ static int unifyfs_fseek(FILE* stream, off_t offset, int whence) errno = EOVERFLOW; return -1; } + if (filesize + offset < 0) { + /* offset is negative and will result in negative position */ + errno = EINVAL; + return -1; + } current_pos = filesize + offset; break; default: @@ -1559,7 +1573,11 @@ void UNIFYFS_WRAP(rewind)(FILE* stream) /* lookup stream */ unifyfs_stream_t* s = (unifyfs_stream_t*) stream; - /* TODO: check that stream is active */ + /* check that stream is active */ + if (s->fd < 0) { + errno = EBADF; + return; + } /* seek to front of file */ int rc = unifyfs_fseek(stream, (off_t) 0L, SEEK_SET); diff --git a/t/Makefile.am b/t/Makefile.am index aedf43a29..ffefd4ba0 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -154,6 +154,7 @@ sys_sysio_static_t_LDFLAGS = $(test_static_ldflags) std_stdio_gotcha_t_SOURCES = std/stdio_suite.h \ std/stdio_suite.c \ std/fopen-fclose.c \ + std/fseek-ftell.c \ std/fwrite-fread.c \ std/fflush.c \ std/size.c @@ -165,6 +166,7 @@ std_stdio_gotcha_t_LDFLAGS = $(AM_LDFLAGS) std_stdio_static_t_SOURCES = std/stdio_suite.h \ std/stdio_suite.c \ std/fopen-fclose.c \ + std/fseek-ftell.c \ std/fwrite-fread.c \ std/fflush.c \ std/size.c diff --git a/t/std/fseek-ftell.c b/t/std/fseek-ftell.c new file mode 100644 index 000000000..bf27af5d0 --- /dev/null +++ b/t/std/fseek-ftell.c @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2018, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2018, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include +#include +#include +#include + +#include "t/lib/tap.h" +#include "t/lib/testutil.h" + +/* This function contains the tests for UNIFYFS_WRAP(fseek), + * UNIFYFS_WRAP(ftell), and UNIFYFS_WRAP(rewind) found in + * client/src/unifyfs-stdio.c. + * + * Notice the tests are ordered in a logical testing order. Changing the order + * or adding new tests in between two others could negatively affect the + * desired results. */ +int fseek_ftell_test(char* unifyfs_root) +{ + /* Diagnostic message for reading and debugging output */ + diag("Starting UNIFYFS_WRAP(fseek/ftell/rewind) tests"); + + char path[64]; + FILE* fd; + int ret; + long pos; + + /* Create a random file at the mountpoint path to test on */ + testutil_rand_path(path, sizeof(path), unifyfs_root); + + /* Open a file and write to it to test fseek() */ + fd = fopen(path, "w"); + fwrite("hello world", 12, 1, fd); + + /* fseek with invalid whence fails with errno=EINVAL. */ + errno = 0; + ret = fseek(fd, 0, -1); + ok(ret == -1 && errno == EINVAL, + "%s: fseek with invalid whence should fail (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* fseek() with SEEK_SET tests */ + /* fseek to negative offset with SEEK_SET should fail with errno=EINVAL */ + errno = 0; + ret = fseek(fd, -1, SEEK_SET); + ok(ret == -1 && errno == EINVAL, + "%s: fseek to negative offset w/ SEEK_SET fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* ftell after invalid fseek should return last offset */ + errno = 0; + pos = ftell(fd); + ok(pos == 12, + "%s: ftell after fseek to negative offset w/ SEEK_SET (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek to valid offset with SEEK_SET succeeds */ + errno = 0; + ret = fseek(fd, 7, SEEK_SET); + ok(ret == 0, "%s: fseek to valid offset w/ SEEK_SET (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* ftell after valid fseek with SEEK_SET */ + errno = 0; + pos = ftell(fd); + ok(pos == 7, "%s: ftell after valid fseek w/ SEEK_SET (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek beyond end of file with SEEK_SET succeeds */ + errno = 0; + ret = fseek(fd, 25, SEEK_SET); + ok(ret == 0, "%s: fseek beyond end of file w/ SEEK_SET (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* ftell after fseek beyond end of file with SEEK_SET */ + errno = 0; + pos = ftell(fd); + ok(pos == 25, + "%s: ftell after fseek beyond end of file w/ SEEK_SET (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek to beginning of file with SEEK_SET succeeds */ + errno = 0; + ret = fseek(fd, 0, SEEK_SET); + ok(ret == 0, "%s: fseek to beginning of file w/ SEEK_SET (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* ftell after fseek to beginning of file with SEEK_SET */ + errno = 0; + pos = ftell(fd); + ok(pos == 0, + "%s: ftell after fseek to beginning of file w/ SEEK_SET (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek() with SEEK_CUR tests */ + /* fseek to end of file with SEEK_CUR succeeds */ + errno = 0; + ret = fseek(fd, 12, SEEK_CUR); + ok(ret == 0, "%s: fseek to end of file w/ SEEK_CUR (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* ftell after fseek to end of file with SEEK_CUR */ + errno = 0; + pos = ftell(fd); + ok(pos == 12, + "%s: ftell after fseek to beginning of file w/ SEEK_CUR (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek to negative offset with SEEK_CUR should fail with errno=EINVAL */ + errno = 0; + ret = fseek(fd, -15, SEEK_CUR); + ok(ret == -1 && errno == EINVAL, + "%s: fseek to negative offset w/ SEEK_CUR fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* ftell after fseek to negative offset with SEEK_CUR */ + errno = 0; + pos = ftell(fd); + ok(pos == 12, + "%s: ftell after fseek to negative offset w/ SEEK_CUR (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek to beginning of file with SEEK_CUR succeeds */ + errno = 0; + ret = fseek(fd, -12, SEEK_CUR); + ok(ret == 0, "%s: fseek to beginning of file w/ SEEK_CUR (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* ftell after fseek to beginning of file with SEEK_CUR */ + errno = 0; + pos = ftell(fd); + ok(pos == 0, + "%s: ftell after fseek to beginnig of file w/ SEEK_CUR (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek beyond end of file with SEEK_CUR succeeds */ + errno = 0; + ret = fseek(fd, 25, SEEK_CUR); + ok(ret == 0, "%s: fseek beyond end of file w/ SEEK_CUR (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* ftell after fseek beyond end of file with SEEK_CUR */ + errno = 0; + pos = ftell(fd); + ok(pos == 25, + "%s: ftell after fseek beyond end of file w/ SEEK_CUR (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* rewind test */ + /* ftell after rewind reports beginning of file */ + rewind(fd); + errno = 0; + pos = ftell(fd); + ok(pos == 0, + "%s: ftell after rewind reports beginning of file (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek() with SEEK_END tests */ + /* fseek to negative offset with SEEK_END should fail with errno=EINVAL */ + errno = 0; + ret = fseek(fd, -15, SEEK_END); + ok(ret == -1 && errno == EINVAL, + "%s: fseek to negative offset w/ SEEK_END fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* ftell after fseek to negative offset with SEEK_END */ + errno = 0; + pos = ftell(fd); + ok(pos == 0, + "%s: ftell after fseek to negative offset w/ SEEK_END (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek back one from end of file with SEEK_END succeeds */ + errno = 0; + ret = fseek(fd, -1, SEEK_END); + ok(ret == 0, + "%s: fseek back one from end of file w/ SEEK_END (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* ftell after fseek back one from end of file with SEEK_END */ + errno = 0; + pos = ftell(fd); + ok(pos == 11, + "%s: ftell after fseek back one from end w/ SEEK_END (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek to beginning of file with SEEK_END succeeds */ + errno = 0; + ret = fseek(fd, -12, SEEK_END); + ok(ret == 0, "%s: fseek to beginning of file w/ SEEK_END (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* ftell after fseek to beginning of file with SEEK_END */ + errno = 0; + pos = ftell(fd); + ok(pos == 0, + "%s: ftell after fseek to beginning of file w/ SEEK_END (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + /* fseek beyond end of file with SEEK_END succeeds */ + errno = 0; + ret = fseek(fd, 25, SEEK_END); + ok(ret == 0, "%s: fseek beyond end of file w/ SEEK_END (ret=%d): %s", + __FILE__, ret, strerror(errno)); + + /* ftell after fseek beyond end of file with SEEK_END */ + errno = 0; + pos = ftell(fd); + ok(pos == 37, + "%s: ftell after fseek beyond end of file w/ SEEK_END (pos=%ld): %s", + __FILE__, pos, strerror(errno)); + + fclose(fd); + + /* fseek in non-open file descriptor should fail with errno=EBADF */ + errno = 0; + ret = fseek(fd, 0, SEEK_SET); + ok(ret == -1 && errno == EBADF, + "%s: fseek in non-open file descriptor fails (ret=%d, errno=%d): %s", + __FILE__, ret, errno, strerror(errno)); + + /* ftell on non-open file descriptor should fail with errno=EBADF */ + errno = 0; + pos = ftell(fd); + ok(pos == -1 && errno == EBADF, + "%s: ftell on non-open file descriptor fails (pos=%ld, errno=%d): %s", + __FILE__, pos, errno, strerror(errno)); + + /* rewind on non-open file descriptor should fail with errno=EBADF */ + errno = 0; + rewind(fd); + ok(errno == EBADF, + "%s: rewind on non-open file descriptor fails (errno=%d): %s", + __FILE__, errno, strerror(errno)); + + diag("Finished UNIFYFS_WRAP(fseek/ftell/rewind) tests"); + + return 0; +} diff --git a/t/std/stdio_suite.c b/t/std/stdio_suite.c index 6c07c6c94..df30e4c4e 100644 --- a/t/std/stdio_suite.c +++ b/t/std/stdio_suite.c @@ -70,8 +70,13 @@ int main(int argc, char* argv[]) * failures to start passing. */ fopen_fclose_test(unifyfs_root); + + fseek_ftell_test(unifyfs_root); + fwrite_fread_test(unifyfs_root); + fflush_test(unifyfs_root); + size_test(unifyfs_root); MPI_Finalize(); diff --git a/t/std/stdio_suite.h b/t/std/stdio_suite.h index eceaefd3a..9257f5f99 100644 --- a/t/std/stdio_suite.h +++ b/t/std/stdio_suite.h @@ -32,8 +32,17 @@ /* Tests for UNIFYFS_WRAP(fopen) and UNIFYFS_WRAP(fclose) */ int fopen_fclose_test(char* unifyfs_root); + +/* Tests for UNIFYFS_WRAP(fseek/ftell/rewind) */ +int fseek_ftell_test(char* unifyfs_root); + +/* Tests for UNIFYFS_WRAP(fwrite) and UNIFYFS_WRAP(fread) */ int fwrite_fread_test(char* unifyfs_root); + +/* Tests for UNIFYFS_WRAP(fflush) */ int fflush_test(char* unifyfs_root); + +/* Tests for UNIFYFS_WRAP(size) */ int size_test(char* unifyfs_root); #endif /* STDIO_SUITE_H */ From 2f70ce4131034622cfabe870aa653c3f89a0bf62 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 14 Jan 2020 12:11:31 -0800 Subject: [PATCH 081/168] client: use global filesize to determine EOF in read path Since stat() returns the global filesize before lamination by default, this patch updates the read path to also use the global filesize to determine EOF in order to be consistent. --- client/src/unifyfs-stdio.c | 23 ++++++++++++++--- client/src/unifyfs-sysio.c | 26 +++++-------------- client/src/unifyfs.c | 51 +++++++++++++++++++++++++++++++++++--- t/std/size.c | 4 +-- 4 files changed, 75 insertions(+), 29 deletions(-) diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index d04ca1206..5a94ac6bd 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -483,6 +483,24 @@ static int unifyfs_stream_flush(FILE* stream) return write_rc; } + /* lookup file id from file descriptor attached to stream */ + int fid = unifyfs_get_fid_from_fd(s->fd); + if (fid < 0) { + s->err = 1; + errno = EBADF; + return EBADF; + } + + /* invoke fsync rpc to register index metadata with server */ + int gfid = unifyfs_gfid_from_fid(fid); + int ret = unifyfs_sync(gfid); + if (ret != UNIFYFS_SUCCESS) { + /* sync failed for some reason, set errno and return error */ + s->err = 1; + errno = unifyfs_rc_errno(ret); + return ret; + } + /* indicate that buffer is now flushed */ s->bufdirty = 0; } @@ -598,7 +616,6 @@ static int unifyfs_stream_read( } /* read data from file into buffer */ - size_t bufcount; ssize_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize); if (read_rc == -1) { @@ -615,7 +632,7 @@ static int unifyfs_stream_read( s->buflen = read_rc; /* set end-of-file flag if our read was short */ - if (bufcount < s->bufsize) { + if (s->buflen < s->bufsize) { eof = 1; } } @@ -1718,7 +1735,6 @@ int UNIFYFS_WRAP(fflush)(FILE* stream) /* TODO: check that stream is active */ /* flush output on stream */ int rc = unifyfs_stream_flush(stream); - if (rc != UNIFYFS_SUCCESS) { /* ERROR: flush sets error indicator and errno */ return EOF; @@ -2287,7 +2303,6 @@ static int __srefill(unifyfs_stream_t* stream) } /* read data from file into buffer */ - size_t bufcount; ssize_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize); if (read_rc < 0) { /* ERROR: read error, set error indicator */ diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index d25bcb326..c8d57b55d 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -559,20 +559,6 @@ ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) /* TODO: check that file is open for reading */ - /* check that we don't try to read past the end of the file */ - off_t lastread = pos + (off_t) count; - off_t filesize = unifyfs_fid_logical_size(fid); - if (filesize < lastread) { - /* adjust count so we don't read past end of file */ - if (filesize > pos) { - /* read all bytes until end of file */ - count = (size_t)(filesize - pos); - } else { - /* pos is already at or past the end of the file */ - count = 0; - } - } - /* if we don't read any bytes, return success */ if (count == 0) { LOGDBG("returning EOF"); @@ -1679,8 +1665,8 @@ int unifyfs_fd_logreadlist(read_req_t* in_reqs, int in_count) int count = in_count; read_req_t* read_reqs = in_reqs; - /* TODO: if the file is laminated and we know the file size, - * we could adjust some reads to not try reading past the EOF */ + /* TODO: if the file is laminated so that we know the file size, + * we can adjust read requests to not read past the EOF */ /* if the option is enabled to service requests locally, try it, * in this case we'll allocate a large array which we split into @@ -1849,13 +1835,13 @@ int unifyfs_fd_logreadlist(read_req_t* in_reqs, int in_count) * request buffer with zeros */ /* get file size for this file */ - size_t filesize; - int ret = invoke_client_filesize_rpc(req->gfid, &filesize); - if (ret != UNIFYFS_SUCCESS) { + off_t filesize_offt = unifyfs_gfid_filesize(req->gfid); + if (filesize_offt == (off_t)-1) { /* failed to get file size */ - req->errcode = ret; + req->errcode = ENOENT; continue; } + size_t filesize = (size_t)filesize_offt; /* get offset of where hole starts */ size_t gap_start = req->offset + req->nread; diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index c54ab345e..60dbc9b70 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -639,10 +639,56 @@ off_t unifyfs_fid_logical_size(int fid) if (unifyfs_fid_is_laminated(fid)) { return unifyfs_fid_global_size(fid); } else { - return unifyfs_fid_local_size(fid); + /* TODO: move this to a runtime configurable? */ + int use_local_size = 0; + if (use_local_size) { + return unifyfs_fid_local_size(fid); + } + + /* otherwise, we invoke an rpc to ask the server + * what the file size is */ + + /* get gfid for this file */ + int gfid = unifyfs_gfid_from_fid(fid); + + /* get file size for this file */ + size_t filesize; + int ret = invoke_client_filesize_rpc(gfid, &filesize); + if (ret != UNIFYFS_SUCCESS) { + /* failed to get file size */ + return (off_t)-1; + } + return (off_t)filesize; } } +/* if we have a local fid structure corresponding to the gfid + * in question, we attempt the file lookup with the fid method + * otherwise call back to the rpc */ +off_t unifyfs_gfid_filesize(int gfid) +{ + off_t filesize = (off_t)-1; + + /* see if we have a fid for this gfid */ + int fid = unifyfs_fid_from_gfid(gfid); + if (fid >= 0) { + /* got a fid, look up file size through that + * method, since it may avoid a server rpc call */ + filesize = unifyfs_fid_logical_size(fid); + } else { + /* no fid for this gfid, + * look it up with server rpc */ + size_t size; + int ret = invoke_client_filesize_rpc(gfid, &size); + if (ret == UNIFYFS_SUCCESS) { + /* got the file size successfully */ + filesize = size; + } + } + + return filesize; +} + /* Return the local (un-laminated) size of the file */ off_t unifyfs_fid_log_size(int fid) { @@ -1113,8 +1159,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } if (flags & O_APPEND) { - /* We only support O_APPEND on non-laminated (local) files, so - * this will use local_size here. */ + /* We only support O_APPEND on non-laminated files */ pos = unifyfs_fid_logical_size(fid); } } else { diff --git a/t/std/size.c b/t/std/size.c index b37861752..077a6043a 100644 --- a/t/std/size.c +++ b/t/std/size.c @@ -78,7 +78,7 @@ int size_test(char* unifyfs_root) ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); get_size(path, &global, &local, &log); - ok(global == 0, "%s: global size is %d: %s", __FILE__, global, + ok(global == 12, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); ok(local == 12, "%s: local size is %d: %s", __FILE__, local, strerror(errno)); @@ -119,7 +119,7 @@ int size_test(char* unifyfs_root) ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); get_size(path, &global, &local, &log); - ok(global == 0, "%s: global size is %d: %s", __FILE__, global, + ok(global == 30, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); ok(local == 30, "%s: local size is %d: %s", __FILE__, local, strerror(errno)); From 9e7eeb103b87b7ad0927605eb4e9ff039ba8c4e0 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 18 Mar 2020 16:20:28 -0700 Subject: [PATCH 082/168] client: drop local size field Now that the read path is using the actual, global filesize instead of the local filesize, it was decided that the local filesize was no longer needed. This patch removes that field and the logic that tracked its value. --- client/src/unifyfs-dirops.c | 1 - client/src/unifyfs-internal.h | 4 -- client/src/unifyfs-sysio.c | 8 +-- client/src/unifyfs.c | 25 +------- t/std/size.c | 28 +++------ t/sys/truncate.c | 112 ++++++++++------------------------ t/sys/write-read-hole.c | 30 +++------ t/sys/write-read.c | 34 ++++------- 8 files changed, 68 insertions(+), 174 deletions(-) diff --git a/client/src/unifyfs-dirops.c b/client/src/unifyfs-dirops.c index 0ad0f5f68..8d450d664 100644 --- a/client/src/unifyfs-dirops.c +++ b/client/src/unifyfs-dirops.c @@ -144,7 +144,6 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) meta->global_size = sb.st_size; meta->chunks = sb.st_blocks; - meta->local_size = 0; /* no need of local storage for dir operations */ unifyfs_dirstream_t* dirp = unifyfs_dirstream_alloc(fid); diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index b31e3ed43..569d9b187 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -268,7 +268,6 @@ typedef struct { typedef struct { off_t global_size; /* Global size of the file */ - off_t local_size; /* Local size of the file */ off_t log_size; /* Log size. This is the sum of all the * write counts. */ pthread_spinlock_t fspinlock; /* file lock variable */ @@ -490,9 +489,6 @@ int unifyfs_fid_is_dir_empty(const char* path); /* Return current global size of given file id */ off_t unifyfs_fid_global_size(int fid); -/* Return current local size of given file id */ -off_t unifyfs_fid_local_size(int fid); - /* Return current local size of given file id */ off_t unifyfs_fid_log_size(int fid); diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index c8d57b55d..856ac9893 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -410,14 +410,13 @@ static int __stat(const char* path, struct stat* buf) if (fid >= 0) { /* If we have a local file */ /* * For debugging and testing purposes, we hijack st_rdev to store our - * local size and log size. We also assume the stat struct is + * log size. We also assume the stat struct is * the 64-bit variant. The values are stored as: * - * st_rdev = log_size << 32 | local_size; + * st_rdev = log_size * */ - buf->st_rdev = (unifyfs_fid_log_size(fid) << 32); - buf->st_rdev |= (unifyfs_fid_local_size(fid) & 0xFFFFFFFF); + buf->st_rdev = unifyfs_fid_log_size(fid); } return 0; @@ -639,7 +638,6 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) if (write_rc == 0) { unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); meta->needs_sync = 1; - meta->local_size = MAX(meta->local_size, pos + count); meta->log_size = newlogsize; } return write_rc; diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 60dbc9b70..9a3088408 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -617,17 +617,6 @@ off_t unifyfs_fid_global_size(int fid) return (off_t)-1; } -/* Return the log size of the file */ -off_t unifyfs_fid_local_size(int fid) -{ - /* get meta data for this file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (NULL != meta) { - return meta->local_size; - } - return (off_t)-1; -} - /* * Return the size of the file. If the file is laminated, return the * laminated size. If the file is not laminated, return the local @@ -639,14 +628,7 @@ off_t unifyfs_fid_logical_size(int fid) if (unifyfs_fid_is_laminated(fid)) { return unifyfs_fid_global_size(fid); } else { - /* TODO: move this to a runtime configurable? */ - int use_local_size = 0; - if (use_local_size) { - return unifyfs_fid_local_size(fid); - } - - /* otherwise, we invoke an rpc to ask the server - * what the file size is */ + /* invoke an rpc to ask the server what the file size is */ /* get gfid for this file */ int gfid = unifyfs_gfid_from_fid(fid); @@ -715,7 +697,6 @@ int unifyfs_fid_update_file_meta(int fid, unifyfs_file_attr_t* gfattr) if (meta->is_laminated) { /* update file size */ meta->global_size = (off_t)gfattr->size; - meta->local_size = meta->global_size; LOGDBG("laminated file size is %zu bytes", (size_t)meta->global_size); } @@ -880,7 +861,6 @@ int unifyfs_fid_create_file(const char* path) /* initialize meta data */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); meta->global_size = 0; - meta->local_size = 0; meta->log_size = 0; meta->flock_status = UNLOCKED; meta->storage = FILE_STORAGE_NULL; @@ -1033,10 +1013,9 @@ int unifyfs_fid_truncate(int fid, off_t length) return rc; } - /* truncate succeeded, update global and local size to + /* truncate succeeded, update global size to * reflect truncated size, note log size is not affected */ meta->global_size = length; - meta->local_size = length; } else { /* unknown storage type */ return EIO; diff --git a/t/std/size.c b/t/std/size.c index 077a6043a..287e87d12 100644 --- a/t/std/size.c +++ b/t/std/size.c @@ -25,13 +25,13 @@ #include "t/lib/testutil.h" /* - * Test correctness of local and global file size. Also, test opening a file + * Test correctness of global file size. Also, test opening a file * for append, and test file positioning (fseek, ftell, etc). */ -/* Get global, local, or log sizes (or all) */ +/* Get global or log sizes (or all) */ static -void get_size(char* path, size_t* global, size_t* local, size_t* log) +void get_size(char* path, size_t* global, size_t* log) { struct stat sb = {0}; int rc; @@ -45,12 +45,8 @@ void get_size(char* path, size_t* global, size_t* local, size_t* log) *global = sb.st_size; } - if (local) { - *local = sb.st_rdev & 0xFFFFFFFF; - } - if (log) { - *log = (sb.st_rdev >> 32) & 0xFFFFFFFF; + *log = sb.st_rdev; } } @@ -61,7 +57,7 @@ int size_test(char* unifyfs_root) FILE* fp = NULL; int rc; char* tmp; - size_t global, local, log; + size_t global, log; errno = 0; @@ -77,11 +73,9 @@ int size_test(char* unifyfs_root) rc = fclose(fp); ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 12, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); - ok(local == 12, "%s: local size is %d: %s", __FILE__, local, - strerror(errno)); ok(log == 12, "%s: log size is %d: %s", __FILE__, log, strerror(errno)); @@ -118,11 +112,9 @@ int size_test(char* unifyfs_root) rc = fclose(fp); ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 30, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); - ok(local == 30, "%s: local size is %d: %s", __FILE__, local, - strerror(errno)); ok(log == 30, "%s: log size is %d: %s", __FILE__, log, strerror(errno)); @@ -140,12 +132,10 @@ int size_test(char* unifyfs_root) rc = chmod(path, 0444); ok(rc == 0, "%s: chmod(0444) (rc=%d): %s", __FILE__, rc, strerror(errno)); - /* Both local and global size should be correct */ - get_size(path, &global, &local, &log); + /* Global size should be correct */ + get_size(path, &global, &log); ok(global == 30, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); - ok(local == 30, "%s: local size is %d: %s", __FILE__, local, - strerror(errno)); ok(log == 30, "%s: log size is %d: %s", __FILE__, log, strerror(errno)); diff --git a/t/sys/truncate.c b/t/sys/truncate.c index ddb580758..33995c155 100644 --- a/t/sys/truncate.c +++ b/t/sys/truncate.c @@ -24,9 +24,9 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* Get global, local, or log sizes (or all) */ +/* Get global or log sizes (or all) */ static -void get_size(char* path, size_t* global, size_t* local, size_t* log) +void get_size(char* path, size_t* global, size_t* log) { struct stat sb = {0}; int rc; @@ -40,12 +40,8 @@ void get_size(char* path, size_t* global, size_t* local, size_t* log) *global = sb.st_size; } - if (local) { - *local = sb.st_rdev & 0xFFFFFFFF; - } - if (log) { - *log = (sb.st_rdev >> 32) & 0xFFFFFFFF; + *log = sb.st_rdev; } } @@ -54,7 +50,7 @@ int truncate_test(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, local, log; + size_t global, log; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -67,11 +63,9 @@ int truncate_test(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(local == 0, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 0); ok(log == 0, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 0); @@ -84,11 +78,9 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d fsync() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 1*bufsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 1*bufsize); - ok(local == 1*bufsize, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 1*bufsize); ok(log == 1*bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 1*bufsize); @@ -105,11 +97,9 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d fsync() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 3*bufsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 3*bufsize); - ok(local == 3*bufsize, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 3*bufsize); ok(log == 2*bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 2*bufsize); @@ -118,11 +108,9 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d ftruncate(%d) (rc=%d): %s", __FILE__, __LINE__, 5*bufsize, rc, strerror(errno)); - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 5*bufsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 5*bufsize); - ok(local == 5*bufsize, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 5*bufsize); ok(log == 2*bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 2*bufsize); @@ -133,11 +121,9 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d truncate(%d) (rc=%d): %s", __FILE__, __LINE__, bufsize/2, rc, strerror(errno)); - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == bufsize/2, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, bufsize/2); - ok(local == bufsize/2, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, bufsize/2); ok(log == 2*bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 2*bufsize); @@ -146,11 +132,9 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d truncate(%d) (rc=%d): %s", __FILE__, __LINE__, 0, rc, strerror(errno)); - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(local == 0, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 0); ok(log == 2*bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 2*bufsize); @@ -164,7 +148,7 @@ int truncate_bigempty(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, local, log; + size_t global, log; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -176,11 +160,9 @@ int truncate_bigempty(char* unifyfs_root) ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", __FILE__, __LINE__, path, fd, strerror(errno)); - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(local == 0, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 0); ok(log == 0, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 0); @@ -191,7 +173,7 @@ int truncate_bigempty(char* unifyfs_root) __FILE__, __LINE__, (unsigned long long) bigempty, rc, strerror(errno)); - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == (size_t)bigempty, "%s:%d global size is %llu expected %llu", __FILE__, __LINE__, global, (unsigned long long)bigempty, strerror(errno)); @@ -208,7 +190,7 @@ int truncate_eof(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, local, log; + size_t global, log; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -221,11 +203,9 @@ int truncate_eof(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(local == 0, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 0); ok(log == 0, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 0); @@ -289,7 +269,7 @@ int truncate_truncsync(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, local, log; + size_t global, log; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -302,11 +282,9 @@ int truncate_truncsync(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(local == 0, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 0); ok(log == 0, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 0); @@ -321,11 +299,9 @@ int truncate_truncsync(char* unifyfs_root) __FILE__, __LINE__, bufsize/2, rc, strerror(errno)); /* file should be 0.5MB bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == bufsize/2, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, bufsize/2); - ok(local == bufsize/2, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, bufsize/2); ok(log == bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, bufsize); @@ -334,11 +310,9 @@ int truncate_truncsync(char* unifyfs_root) __FILE__, __LINE__, rc, strerror(errno)); /* file should still be 0.5MB bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == bufsize/2, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, bufsize/2); - ok(local == bufsize/2, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, bufsize/2); ok(log == bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, bufsize); @@ -392,7 +366,7 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) char path[64]; int rc; int fd; - size_t global, local, log; + size_t global, log; int i; size_t bufsize = 1024*1024; @@ -406,11 +380,9 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(local == 0, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 0); ok(log == 0, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 0); @@ -438,11 +410,9 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, (int)truncsize, rc, strerror(errno)); /* file should be of size 5MB + 42 at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(local == truncsize, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, (int)truncsize); ok(log == 20*bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 20*bufsize); @@ -454,11 +424,9 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, rc, strerror(errno)); /* file should still be 5MB + 42 bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(local == truncsize, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, (int)truncsize); ok(log == 20*bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 20*bufsize); @@ -540,7 +508,7 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) char path[64]; int rc; int fd; - size_t global, local, log; + size_t global, log; int i; size_t bufsize = 1024*1024; @@ -554,11 +522,9 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(local == 0, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 0); ok(log == 0, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 0); @@ -571,11 +537,9 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, (int)truncsize, rc, strerror(errno)); /* file should be of size 5MB + 42 at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(local == truncsize, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, (int)truncsize); ok(log == 0, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 0); @@ -587,11 +551,9 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, rc, strerror(errno)); /* file should still be 5MB + 42 bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(local == truncsize, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, (int)truncsize); ok(log == 0, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 0); @@ -671,7 +633,7 @@ int truncate_ftrunc_before_sync(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, local, log; + size_t global, log; size_t bufsize = 1024; char* buf = (char*) malloc(bufsize); @@ -684,11 +646,9 @@ int truncate_ftrunc_before_sync(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(local == 0, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 0); ok(log == 0, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 0); @@ -714,11 +674,9 @@ int truncate_ftrunc_before_sync(char* unifyfs_root) /* finally, check that the file is 0 bytes, * i.e., check that the writes happened before the truncate * and not at the fsync */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(local == truncsize, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, (int)truncsize); ok(log == bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, bufsize); @@ -734,7 +692,7 @@ int truncate_trunc_before_sync(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, local, log; + size_t global, log; size_t bufsize = 1024; char* buf = (char*) malloc(bufsize); @@ -747,11 +705,9 @@ int truncate_trunc_before_sync(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(local == 0, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, 0); ok(log == 0, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, 0); @@ -777,11 +733,9 @@ int truncate_trunc_before_sync(char* unifyfs_root) /* finally, check that the file is 0 bytes, * i.e., check that the writes happened before the truncate * and not at the fsync */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(local == truncsize, "%s:%d local size is %d expected %d", - __FILE__, __LINE__, local, (int)truncsize); ok(log == bufsize, "%s:%d log size is %d expected %d", __FILE__, __LINE__, log, bufsize); diff --git a/t/sys/write-read-hole.c b/t/sys/write-read-hole.c index 59680872b..1834ba9d5 100644 --- a/t/sys/write-read-hole.c +++ b/t/sys/write-read-hole.c @@ -23,9 +23,9 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* Get global, local, or log sizes (or all) */ +/* Get global or log sizes (or all) */ static -void get_size(char* path, size_t* global, size_t* local, size_t* log) +void get_size(char* path, size_t* global, size_t* log) { struct stat sb = {0}; int rc; @@ -39,12 +39,8 @@ void get_size(char* path, size_t* global, size_t* local, size_t* log) *global = sb.st_size; } - if (local) { - *local = sb.st_rdev & 0xFFFFFFFF; - } - if (log) { - *log = (sb.st_rdev >> 32) & 0xFFFFFFFF; + *log = sb.st_rdev; } } @@ -65,7 +61,7 @@ int write_read_hole_test(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, local, log; + size_t global, log; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -102,12 +98,10 @@ int write_read_hole_test(char* unifyfs_root) ok(rc == bufsize, "%s:%d write() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); - /* Check global and local size on our un-laminated file */ - get_size(path, &global, &local, &log); + /* Check global size on our un-laminated file */ + get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(local == 3*bufsize, "%s:%d local size is %d: %s", - __FILE__, __LINE__, local, strerror(errno)); ok(log == 2*bufsize, "%s:%d log size is %d: %s", __FILE__, __LINE__, log, strerror(errno)); @@ -116,12 +110,10 @@ int write_read_hole_test(char* unifyfs_root) ok(rc == 0, "%s:%d fsync() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); - /* Check global and local size on our un-laminated file */ - get_size(path, &global, &local, &log); + /* Check global size on our un-laminated file */ + get_size(path, &global, &log); ok(global == 3*bufsize, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(local == 3*bufsize, "%s:%d local size is %d: %s", - __FILE__, __LINE__, local, strerror(errno)); ok(log == 2*bufsize, "%s:%d log size is %d: %s", __FILE__, __LINE__, log, strerror(errno)); @@ -136,12 +128,10 @@ int write_read_hole_test(char* unifyfs_root) ok(rc == 0, "%s:%d chmod(0444) (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); - /* Check global and local size on our un-laminated file */ - get_size(path, &global, &local, &log); + /* Check global size on our un-laminated file */ + get_size(path, &global, &log); ok(global == 4*bufsize, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(local == 4*bufsize, "%s:%d local size is %d: %s", - __FILE__, __LINE__, local, strerror(errno)); ok(log == 2*bufsize, "%s:%d log size is %d: %s", __FILE__, __LINE__, log, strerror(errno)); diff --git a/t/sys/write-read.c b/t/sys/write-read.c index a41cd7456..f71597f9a 100644 --- a/t/sys/write-read.c +++ b/t/sys/write-read.c @@ -24,9 +24,9 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* Get global, local, or log sizes (or all) */ +/* Get global or log sizes (or all) */ static -void get_size(char* path, size_t* global, size_t* local, size_t* log) +void get_size(char* path, size_t* global, size_t* log) { struct stat sb = {0}; int rc; @@ -40,12 +40,8 @@ void get_size(char* path, size_t* global, size_t* local, size_t* log) *global = sb.st_size; } - if (local) { - *local = sb.st_rdev & 0xFFFFFFFF; - } - if (log) { - *log = (sb.st_rdev >> 32) & 0xFFFFFFFF; + *log = sb.st_rdev; } } @@ -54,7 +50,7 @@ int write_read_test(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, local, log; + size_t global, log; testutil_rand_path(path, sizeof(path), unifyfs_root); @@ -73,12 +69,10 @@ int write_read_test(char* unifyfs_root) rc = write(fd, "universe", 9); ok(rc == 9, "%s: write() (rc=%d): %s", __FILE__, rc, strerror(errno)); - /* Check global and local size on our un-laminated file */ - get_size(path, &global, &local, &log); + /* Check global size on our un-laminated file */ + get_size(path, &global, &log); ok(global == 0, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); - ok(local == 15, "%s: local size is %d: %s", __FILE__, local, - strerror(errno)); ok(log == 21, "%s: log size is %d: %s", __FILE__, log, strerror(errno)); @@ -86,12 +80,10 @@ int write_read_test(char* unifyfs_root) rc = fsync(fd); ok(rc == 0, "%s: fsync() (rc=%d): %s", __FILE__, rc, strerror(errno)); - /* Check global and local size on our un-laminated file */ - get_size(path, &global, &local, &log); + /* Check global size on our un-laminated file */ + get_size(path, &global, &log); ok(global == 15, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); - ok(local == 15, "%s: local size is %d: %s", __FILE__, local, - strerror(errno)); ok(log == 21, "%s: log size is %d: %s", __FILE__, log, strerror(errno)); @@ -114,12 +106,10 @@ int write_read_test(char* unifyfs_root) ok(rc == 6, "%s: write() (rc=%d): %s", __FILE__, rc, strerror(errno)); close(fd); - /* Check global and local size on our un-laminated file */ - get_size(path, &global, &local, &log); + /* Check global size on our un-laminated file */ + get_size(path, &global, &log); ok(global == 21, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); - ok(local == 21, "%s: local size is %d: %s", __FILE__, local, - strerror(errno)); ok(log == 27, "%s: log size is %d: %s", __FILE__, log, strerror(errno)); @@ -129,11 +119,9 @@ int write_read_test(char* unifyfs_root) ok(rc == 0, "%s: chmod(0444) (rc=%d): %s", __FILE__, rc, strerror(errno)); /* Verify we're getting the correct file size */ - get_size(path, &global, &local, &log); + get_size(path, &global, &log); ok(global == 21, "%s: global size is %d: %s", __FILE__, global, strerror(errno)); - ok(local == 21, "%s: local size is %d: %s", __FILE__, local, - strerror(errno)); ok(log == 27, "%s: log size is %d: %s", __FILE__, log, strerror(errno)); From 5b0d2cb821982ab75e0e5a7dd95a939dee1f76be Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 25 Mar 2020 15:15:21 -0700 Subject: [PATCH 083/168] client: sync file if needed before calling filesize rpc --- client/src/unifyfs.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 9a3088408..a58f8fdf3 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -633,6 +633,16 @@ off_t unifyfs_fid_logical_size(int fid) /* get gfid for this file */ int gfid = unifyfs_gfid_from_fid(fid); + /* sync any writes to disk before requesting file size */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if (meta->needs_sync) { + /* we have some changes to sync for this file */ + unifyfs_sync(gfid); + + /* just synced writes for this file */ + meta->needs_sync = 0; + } + /* get file size for this file */ size_t filesize; int ret = invoke_client_filesize_rpc(gfid, &filesize); From b0344034cdcc23cd10c36ec1e7fa1a29441aec34 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 30 Mar 2020 16:01:39 -0700 Subject: [PATCH 084/168] Revert "Adding simul POSIX test example." This reverts commit 872cb269f23d8e6473ba43032b47e34d5259769b. --- examples/src/COPYING-simul | 306 --------- examples/src/Makefile.am | 12 - examples/src/simul.c | 1325 ------------------------------------ 3 files changed, 1643 deletions(-) delete mode 100644 examples/src/COPYING-simul delete mode 100644 examples/src/simul.c diff --git a/examples/src/COPYING-simul b/examples/src/COPYING-simul deleted file mode 100644 index 1dd1caa1c..000000000 --- a/examples/src/COPYING-simul +++ /dev/null @@ -1,306 +0,0 @@ -Preamble Notice - -A. This notice is required to be provided under our contract with the U.S. -Department of Energy (DOE). This work was produced at the University of -California, Lawrence Livermore National Laboratory under Contract -No. W-7405-ENG-48 with the DOE. - -B. Neither the United States Government nor the University of California -nor any of their employees, makes any warranty, express or implied, or -assumes any liability or responsibility for the accuracy, completeness, or -usefulness of any information, apparatus, product, or process disclosed, -or represents that its use would not infringe privately-owned rights. - -C. Also, reference herein to any specific commercial products, process, or -services by trade name, trademark, manufacturer or otherwise does not -necessarily constitute or imply its endorsement, recommendation, or -favoring by the United States Government or the University of California. -The views and opinions of authors expressed herein do not necessarily -state or reflect those of the United States Government or the University -of California, and shall not be used for advertising or product -endorsement purposes. - -The precise terms and conditions for copying, distribution and -modification follows. - ----------------------------------------------------------------------- - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index 7721fc921..1f7f87777 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -23,7 +23,6 @@ if HAVE_LD_WRAP app-tileio-static \ transfer-static \ size-static \ - simul-static \ chmod-static \ multi-write-static endif @@ -46,7 +45,6 @@ if HAVE_GOTCHA app-tileio-gotcha \ transfer-gotcha \ size-gotcha \ - simul-gotcha \ chmod-gotcha \ multi-write-gotcha endif @@ -304,16 +302,6 @@ size_static_CPPFLAGS = $(test_cppflags) size_static_LDADD = $(test_static_ldadd) size_static_LDFLAGS = $(test_static_ldflags) -simul_gotcha_SOURCES = simul.c -simul_gotcha_CPPFLAGS = $(test_cppflags) -simul_gotcha_LDADD = $(test_gotcha_ldadd) -simul_gotcha_LDFLAGS = $(test_gotcha_ldflags) - -simul_static_SOURCES = simul.c -simul_static_CPPFLAGS = $(test_cppflags) -simul_static_LDADD = $(test_static_ldadd) -simul_static_LDFLAGS = $(test_static_ldflags) - chmod_gotcha_SOURCES = chmod.c testutil.c chmod_gotcha_CPPFLAGS = $(test_cppflags) chmod_gotcha_LDADD = $(test_gotcha_ldadd) diff --git a/examples/src/simul.c b/examples/src/simul.c deleted file mode 100644 index ef11d1e1f..000000000 --- a/examples/src/simul.c +++ /dev/null @@ -1,1325 +0,0 @@ -/* - * Copyright (C) 2003, The Regents of the University of California. - * Produced at the Lawrence Livermore National Laboratory. - * Written by Christopher J. Morrone - * UCRL-CODE-2003-019 - * All rights reserved. - * - * Please read the COPYING file. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License (as published by - * the Free Software Foundation) version 2, dated June 1991. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * terms and conditions of the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/* - * Some modifications including style changes have been made for testing - * unifyfs. - */ - -#include "mpi.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* unifyfs test */ -#include - -#define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH -#define DIRMODE S_IRUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IXOTH -#define SHARED 1 -#define MAX_FILENAME_LEN 512 - -int rank; -int size; -char *testdir = NULL; -char hostname[1024]; -int verbose; -int throttle = 1; -struct timeval t1, t2; -static char version[] = "1.16"; - -#ifdef __GNUC__ - /* "inline" is a keyword in GNU C */ -#elif __STDC_VERSION__ >= 199901L - /* "inline" is a keyword in C99 and later versions */ -#else -# define inline /* "inline" not available */ -#endif - -#ifndef AIX -#define FAIL(msg) do { \ - fprintf(stdout, "%s: Process %d(%s): FAILED in %s, %s: %s\n",\ - timestamp(), rank, hostname, __func__, \ - msg, strerror(errno)); \ - fflush(stdout);\ - MPI_Abort(MPI_COMM_WORLD, 1); \ -} while(0) -#else -#define FAIL(msg) do { \ - fprintf(stdout, "%s: Process %d(%s): FAILED, %s: %s\n",\ - timestamp(), rank, hostname, \ - msg, strerror(errno)); \ - fflush(stdout);\ - MPI_Abort(MPI_COMM_WORLD, 1); \ -} while(0) -#endif - -char *timestamp() { - static char datestring[80]; - time_t timestamp; - - fflush(stdout); - timestamp = time(NULL); - strftime(datestring, 80, "%T", localtime(×tamp)); - - return datestring; -} - -static inline void begin(char *str) { - if (verbose > 0 && rank == 0) { - gettimeofday(&t1, NULL); - fprintf(stdout, "%s:\tBeginning %s\n", timestamp(), str); - fflush(stdout); - } -} - -static inline void end(char *str) { - double elapsed; - - MPI_Barrier(MPI_COMM_WORLD); - if (verbose > 0 && rank == 0) { - gettimeofday(&t2, NULL); - elapsed = ((((t2.tv_sec - t1.tv_sec) * 1000000L) - + t2.tv_usec) - t1.tv_usec) - / (double)1000000; - if (elapsed >= 60) { - fprintf(stdout, "%s:\tFinished %-15s(%.2f min)\n", - timestamp(), str, elapsed / 60); - } else { - fprintf(stdout, "%s:\tFinished %-15s(%.3f sec)\n", - timestamp(), str, elapsed); - - } - fflush(stdout); - } -} - -void Seq_begin(MPI_Comm comm, int numprocs) { - int size; - int rank; - int buf; - MPI_Status status; - - MPI_Comm_size(comm, &size); - MPI_Comm_rank(comm, &rank); - - if (rank >= numprocs) { - MPI_Recv(&buf, 1, MPI_INT, rank-numprocs, 1333, comm, &status); - } -} - -void Seq_end(MPI_Comm comm, int numprocs) { - int size; - int rank; - int buf; - - MPI_Comm_size(comm, &size); - MPI_Comm_rank(comm, &rank); - - if ((rank + numprocs) < size) { - MPI_Send(&buf, 1, MPI_INT, rank+numprocs, 1333, comm); - } -} - -/* This function does not FAIL if the requested "name" does not exist. This - is just to clean up any files or directories left over from previous runs.*/ -void remove_file_or_dir(char *name) { - struct stat statbuf; - char errmsg[MAX_FILENAME_LEN+20]; - - if (stat(name, &statbuf) != -1) { - if (S_ISREG(statbuf.st_mode)) { - printf("stale file found\n"); - if (unlink(name) == -1) { - sprintf(errmsg, "unlink of %s", name); - FAIL(errmsg); - } - } - if (S_ISDIR(statbuf.st_mode)) { - printf("stale directory found\n"); - if (rmdir(name) == -1) { - sprintf(errmsg, "rmmdir of %s", name); - FAIL(errmsg); - } - } - } -} - -char *create_files(char *prefix, int filesize, int shared) { - static char filename[MAX_FILENAME_LEN]; - char errmsg[MAX_FILENAME_LEN+20]; - int fd, i; - short zero = 0; - - /* Process 0 creates the test file(s) */ - if (rank == 0) { - for (i = 0; i < (shared ? 1 : size); i++) { - sprintf(filename, "%s/%s.%d", testdir, prefix, i); - remove_file_or_dir(filename); - if ((fd = creat(filename, FILEMODE)) == -1) { - sprintf(errmsg, "creat of file %s", filename); - FAIL(errmsg); - } - if (filesize > 0) { - if (lseek(fd, filesize - 1, SEEK_SET) == -1) { - sprintf(errmsg, "lseek in file %s", filename); - FAIL(errmsg); - } - if (write(fd, &zero, 1) == -1) { - sprintf(errmsg, "write in file %s", filename); - FAIL(errmsg); - } - } - if (close(fd) == -1) { - sprintf(errmsg, "close of file %s", filename); - FAIL(errmsg); - } - } - } - - if (shared) - sprintf(filename, "%s/%s.0", testdir, prefix); - else - sprintf(filename, "%s/%s.%d", testdir, prefix, rank); - - return filename; -} - -void remove_files(char *prefix, int shared) { - char filename[1024]; - int i; - - /* Process 0 removes the file(s) */ - if (rank == 0) { - for (i = 0; i < (shared ? 1 : size); i++) { - sprintf(filename, "%s/%s.%d", testdir, prefix, i); - /*printf("Removing file %s\n", filename); fflush(stdout);*/ - if (unlink(filename) == -1) { - FAIL("unlink failed"); - } - } - } -} - -char *create_dirs(char *prefix, int shared) { - static char dirname[1024]; - int i; - - /* Process 0 creates the test file(s) */ - if (rank == 0) { - for (i = 0; i < (shared ? 1 : size); i++) { - sprintf(dirname, "%s/%s.%d", testdir, prefix, i); - remove_file_or_dir(dirname); - if (mkdir(dirname, DIRMODE) == -1) { - FAIL("init mkdir failed"); - } - } - } - - if (shared) - sprintf(dirname, "%s/%s.0", testdir, prefix); - else - sprintf(dirname, "%s/%s.%d", testdir, prefix, rank); - - return dirname; -} - -void remove_dirs(char *prefix, int shared) { - char dirname[1024]; - int i; - - /* Process 0 removes the file(s) */ - if (rank == 0) { - for (i = 0; i < (shared ? 1 : size); i++) { - sprintf(dirname, "%s/%s.%d", testdir, prefix, i); - if (rmdir(dirname) == -1) { - FAIL("rmdir failed"); - } - } - } -} - -char *create_symlinks(char *prefix, int shared) { - static char filename[1024]; - static char linkname[1024]; - int i; - - /* Process 0 creates the test file(s) */ - if (rank == 0) { - for (i = 0; i < (shared ? 1 : size); i++) { - sprintf(filename, "%s/symlink_target", testdir); - sprintf(linkname, "%s/%s.%d", testdir, prefix, i); - remove_file_or_dir(linkname); - if (symlink(filename, linkname) == -1) { - FAIL("symlink failed"); - } - } - } - - if (shared) - sprintf(linkname, "%s/%s.0", testdir, prefix); - else - sprintf(linkname, "%s/%s.%d", testdir, prefix, rank); - - return linkname; -} - -void check_single_success(char *testname, int rc, int error_rc) { - int *rc_vec, i; - int fail = 0; - int pass = 0; - - if (rank == 0) { - if ((rc_vec = (int *)malloc(sizeof(int)*size)) == NULL) { - FAIL("malloc failed"); - } - } - MPI_Gather(&rc, 1, MPI_INT, rc_vec, 1, MPI_INT, 0, MPI_COMM_WORLD); - if (rank == 0) { - for (i = 0; i < size; i++) { - if (rc_vec[i] == error_rc) - fail++; - else - pass++; - } - if (!((pass == 1) && (fail == size-1))) { - fprintf(stdout, "%s: FAILED in %s: ", timestamp(), testname); - if (pass > 1) - fprintf(stdout, "too many operations succeeded (%d).\n", pass); - else - fprintf(stdout, "too many operations failed (%d).\n", fail); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, 1); - } - free(rc_vec); - } -} - -void simul_open(int shared) { - int fd; - char *filename; - - begin("setup"); - filename = create_files("simul_open", 0, shared); - end("setup"); - - /* All open the file simultaneously */ - begin("test"); - if ((fd = open(filename, O_RDWR)) == -1) { - FAIL("open failed"); - } - end("test"); - - /* All close the file one at a time */ - begin("cleanup"); - Seq_begin(MPI_COMM_WORLD, throttle); - if (close(fd) == -1) { - FAIL("close failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - MPI_Barrier(MPI_COMM_WORLD); - remove_files("simul_open", shared); - end("cleanup"); -} - -void simul_close(int shared) { - int fd; - char *filename; - - begin("setup"); - filename = create_files("simul_close", 0, shared); - MPI_Barrier(MPI_COMM_WORLD); - /* All open the file one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if ((fd = open(filename, O_RDWR)) == -1) { - FAIL("open failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - end("setup"); - - begin("test"); - /* All close the file simultaneously */ - if (close(fd) == -1) { - FAIL("close failed"); - } - end("test"); - - begin("cleanup"); - remove_files("simul_close", shared); - end("cleanup"); -} - -void simul_chdir(int shared) { - char cwd[1024]; - char *dirname; - - begin("setup"); - if (getcwd(cwd, 1024) == NULL) { - FAIL("init getcwd failed"); - } - dirname = create_dirs("simul_chdir", shared); - end("setup"); - - begin("test"); - /* All chdir to dirname */ - if (chdir(dirname) == -1) { - FAIL("chdir failed"); - } - end("test"); - - begin("cleanup"); - /* All chdir back to old cwd */ - if (chdir(cwd) == -1) { - FAIL("chdir back failed"); - } - MPI_Barrier(MPI_COMM_WORLD); - remove_dirs("simul_chdir", shared); - end("cleanup"); -} - -void simul_file_stat(int shared) { - char *filename; - struct stat buf; - - begin("setup"); - filename = create_files("simul_file_stat", 0, shared); - end("setup"); - - begin("test"); - /* All stat the file */ - if (stat(filename, &buf) == -1) { - FAIL("stat failed"); - } - end("test"); - - begin("cleanup"); - remove_files("simul_file_stat", shared); - end("cleanup"); -} - -void simul_dir_stat(int shared) { - char *dirname; - struct stat buf; - - begin("setup"); - dirname = create_dirs("simul_dir_stat", shared); - end("setup"); - - begin("test"); - /* All stat the directory */ - if (stat(dirname, &buf) == -1) { - FAIL("stat failed"); - } - end("test"); - - begin("cleanup"); - remove_dirs("simul_dir_stat", shared); - end("cleanup"); -} - -void simul_readdir(int shared) { - DIR *dir; - char *dirname; - struct dirent *dptr; - - begin("setup"); - dirname = create_dirs("simul_readdir", shared); - MPI_Barrier(MPI_COMM_WORLD); - /* All open the directory(ies) one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if ((dir = opendir(dirname)) == NULL) { - FAIL("init opendir failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - end("setup"); - - begin("test"); - /* All readdir the directory stream(s) */ - if ((dptr = readdir(dir)) == NULL) { - FAIL("readdir failed"); - } - end("test"); - - begin("cleanup"); - /* All close the directory(ies) one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if (closedir(dir) == -1) { - FAIL("closedir failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - MPI_Barrier(MPI_COMM_WORLD); - remove_dirs("simul_readdir", shared); - end("cleanup"); -} - -void simul_statfs(int shared) { - char *filename; - struct statfs buf; - - begin("setup"); - filename = create_files("simul_statfs", 0, shared); - end("setup"); - - begin("test"); - /* All statfs the file(s) */ - if (statfs(filename, &buf) == -1) { - FAIL("statfs failed"); - } - end("test"); - - begin("cleanup"); - remove_files("simul_statfs", shared); - end("cleanup"); -} - -void simul_lseek(int shared) { - int fd; - char *filename; - - begin("setup"); - filename = create_files("simul_lseek", 0, shared); - MPI_Barrier(MPI_COMM_WORLD); - /* All open the file(s) one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if ((fd = open(filename, O_RDWR)) == -1) { - FAIL("init open failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - end("setup"); - - begin("test"); - /* All lseek simultaneously */ - if (lseek(fd, 1024, SEEK_SET) == -1) { - FAIL("lseek failed"); - MPI_Abort(MPI_COMM_WORLD, 1); - } - end("test"); - - begin("cleanup"); - /* All close the file(s) one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if (close(fd) == -1) { - FAIL("cleanup close failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - MPI_Barrier(MPI_COMM_WORLD); - remove_files("simul_lseek", shared); - end("cleanup"); -} - -void simul_read(int shared) { - int fd; - ssize_t fin; - char buf[1024]; - char *filename; - int i = 0; - int retry = 100; - - begin("setup"); - filename = create_files("simul_read", 1024, shared); - MPI_Barrier(MPI_COMM_WORLD); - /* All open the file one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if ((fd = open(filename, O_RDWR)) == -1) { - FAIL("init open failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - end("setup"); - - begin("test"); - /* All read simultaneously */ - for (i = 1024; (i > 0) && (retry > 0); i -= fin, retry--) { - if ((fin = read(fd, buf, (size_t)i)) == -1) { - FAIL("read failed"); - } - } - if( (retry == 0) && (i > 0) ) - FAIL("read exceeded retry count"); - end("test"); - - begin("cleanup"); - /* All close the file one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if (close(fd) == -1) { - FAIL("cleanup close failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - MPI_Barrier(MPI_COMM_WORLD); - remove_files("simul_read", shared); - end("cleanup"); -} - -void simul_write(int shared) { - int fd; - ssize_t fin; - char *filename; - int i = 0; - int retry = 100; - - begin("setup"); - filename = create_files("simul_write", size * sizeof(int), shared); - MPI_Barrier(MPI_COMM_WORLD); - /* All open the file and lseek one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if ((fd = open(filename, O_RDWR)) == -1) { - FAIL("init open failed"); - } - if (lseek(fd, rank*sizeof(int), SEEK_SET) == -1) { - FAIL("init lseek failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - end("setup"); - - begin("test"); - /* All write simultaneously */ - for (i = sizeof(int); (i > 0) && (retry > 0); i -= fin, retry--) { - if ((fin = write(fd, &rank, (size_t)i)) == -1) { - FAIL("write failed"); - } - } - if( (retry == 0) && (i > 0) ) - FAIL("write exceeded retry count"); - end("test"); - - begin("cleanup"); - /* All close the file one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if (close(fd) == -1) { - FAIL("cleanup close failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - MPI_Barrier(MPI_COMM_WORLD); - remove_files("simul_write", shared); - end("cleanup"); -} - -void simul_mkdir(int shared) { - int rc, i; - char dirname[MAX_FILENAME_LEN]; - - begin("setup"); - if (shared) - sprintf(dirname, "%s/simul_mkdir.0", testdir); - else - sprintf(dirname, "%s/simul_mkdir.%d", testdir, rank); - if (rank == 0) { - for (i = 0; i < (shared ? 1 : size); i++) { - char buf[MAX_FILENAME_LEN]; - sprintf(buf, "%s/simul_mkdir.%d", testdir, i); - remove_file_or_dir(buf); - } - } - end("setup"); - - begin("test"); - /* All mkdir dirname */ - rc = mkdir(dirname, DIRMODE); - if (!shared) { - if (rc == -1) { - FAIL("mkdir failed"); - } - MPI_Barrier(MPI_COMM_WORLD); - } else { /* Only one should succeed */ - check_single_success("simul_mkdir", rc, -1); - } - end("test"); - - begin("cleanup"); - remove_dirs("simul_mkdir", shared); - end("cleanup"); -} - -void simul_rmdir(int shared) { - int rc; - char *dirname; - - begin("setup"); - dirname = create_dirs("simul_rmdir", shared); - MPI_Barrier(MPI_COMM_WORLD); - end("setup"); - - begin("test"); - /* All rmdir dirname */ - rc = rmdir(dirname); - if (!shared) { - if (rc == -1) { - FAIL("rmdir failed"); - } - MPI_Barrier(MPI_COMM_WORLD); - } else { /* Only one should succeed */ - check_single_success("simul_rmdir", rc, -1); - } - end("test"); - - begin("cleanup"); - end("cleanup"); -} - -void simul_creat(int shared) { - int fd, i; - char filename[1024]; - - begin("setup"); - if (shared) - sprintf(filename, "%s/simul_creat.0", testdir); - else - sprintf(filename, "%s/simul_creat.%d", testdir, rank); - if (rank == 0) { - for (i = 0; i < (shared ? 1 : size); i++) { - char buf[MAX_FILENAME_LEN]; - sprintf(buf, "%s/simul_creat.%d", testdir, i); - remove_file_or_dir(buf); - } - } - end("setup"); - - begin("test"); - /* All create the files simultaneously */ - fd = creat(filename, FILEMODE); - if (fd == -1) { - FAIL("creat failed"); - } - end("test"); - - begin("cleanup"); - /* All close the files one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if (close(fd) == -1) { - FAIL("close failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - MPI_Barrier(MPI_COMM_WORLD); - remove_files("simul_creat", shared); - end("cleanup"); -} - -void simul_unlink(int shared) { - int rc; - char *filename; - - begin("setup"); - filename = create_files("simul_unlink", 0, shared); - end("setup"); - - begin("test"); - /* All unlink the files simultaneously */ - rc = unlink(filename); - if (!shared) { - if (rc == -1) { - FAIL("unlink failed"); - } - } else { - check_single_success("simul_unlink", rc, -1); - } - end("test"); - - begin("cleanup"); - end("cleanup"); -} - -void simul_rename(int shared) { - int rc, i; - char *oldfilename; - char newfilename[1024]; - char *testname = "simul_rename"; - - begin("setup"); - oldfilename = create_files(testname, 0, shared); - sprintf(newfilename, "%s/%s_new.%d", testdir, testname, rank); - if (rank == 0) { - for (i = 0; i < (shared ? 1 : size); i++) { - char buf[MAX_FILENAME_LEN]; - sprintf(buf, "%s/%s_new.%d", testdir, testname, i); - remove_file_or_dir(buf); - } - } - end("setup"); - - begin("test"); - /* All rename the files simultaneously */ - rc = rename(oldfilename, newfilename); - if (!shared) { - if (rc == -1) { - FAIL("stat failed"); - } - } else { - check_single_success(testname, rc, -1); - } - end("test"); - - begin("cleanup"); - if (rc == 0) { - if (unlink(newfilename) == -1) - FAIL("unlink failed"); - } - end("cleanup"); -} - -void simul_truncate(int shared) { - char *filename; - - begin("setup"); - filename = create_files("simul_truncate", 2048, shared); - end("setup"); - - begin("test"); - /* All truncate simultaneously */ - if (truncate(filename, 1024) == -1) { - FAIL("truncate failed"); - } - end("test"); - - begin("cleanup"); - remove_files("simul_truncate", shared); - end("cleanup"); -} - -void simul_readlink(int shared) { - char *linkname; - char buf[1024]; - - begin("setup"); - linkname = create_symlinks("simul_readlink", shared); - end("setup"); - - begin("test"); - /* All read the symlink(s) simultaneously */ - if (readlink(linkname, buf, 1024) == -1) { - FAIL("readlink failed"); - } - end("test"); - - begin("cleanup"); - remove_files("simul_readlink", shared); - end("cleanup"); -} - -void simul_symlink(int shared) { - int rc, i; - char linkname[MAX_FILENAME_LEN]; - char filename[MAX_FILENAME_LEN]; - - begin("setup"); - if (shared) - sprintf(linkname, "%s/simul_symlink.0", testdir); - else - sprintf(linkname, "%s/simul_symlink.%d", testdir, rank); - if (rank == 0) { - for (i = 0; i < (shared ? 1 : size); i++) { - char buf[MAX_FILENAME_LEN]; - sprintf(buf, "%s/simul_symlink.%d", testdir, i); - remove_file_or_dir(buf); - } - } - sprintf(filename, "%s/simul_symlink_target", testdir); - end("setup"); - - begin("test"); - /* All create the symlinks simultaneously */ - rc = symlink(filename, linkname); - if (!shared) { - if (rc == -1) { - FAIL("symlink failed"); - } - } else { - check_single_success("simul_symlink", rc, -1); - } - end("test"); - - begin("cleanup"); - remove_files("simul_symlink", shared); - end("cleanup"); -} - -void simul_link_to_one(int shared) { - int rc, i; - char *filename; - char linkname[1024]; - - begin("setup"); - if (shared) - sprintf(linkname, "%s/simul_link.0", testdir); - else - sprintf(linkname, "%s/simul_link.%d", testdir, rank); - if (rank == 0) { - for (i = 0; i < (shared ? 1 : size); i++) { - char buf[MAX_FILENAME_LEN]; - sprintf(buf, "%s/simul_link.%d", testdir, i); - remove_file_or_dir(buf); - } - } - filename = create_files("simul_link_target", 0, SHARED); - end("setup"); - - begin("test"); - /* All create the hard links simultaneously */ - rc = link(filename, linkname); - if (!shared) { - if (rc == -1) { - FAIL("link failed"); - } - } else { - check_single_success("simul_link_to_one", rc, -1); - } - end("test"); - - begin("cleanup"); - remove_files("simul_link_target", SHARED); - remove_files("simul_link", shared); - end("cleanup"); -} - -void simul_link_to_many(int shared) { - char *filename; - char linkname[1024]; - int i; - - if (shared) { - if (verbose > 0 && rank == 0) - printf("%s:\tThis is just a place holder; no test is run here.\n", - timestamp()); - return; - } - begin("setup"); - filename = create_files("simul_link", 0, shared); - sprintf(linkname, "%s/simul_link_target.%d", testdir, rank); - if (rank == 0) { - for (i = 0; i < size; i++) { - char buf[MAX_FILENAME_LEN]; - sprintf(buf, "%s/simul_link_target.%d", testdir, i); - remove_file_or_dir(buf); - } - } - end("setup"); - - begin("test"); - /* All create the hard links simultaneously */ - if (link(filename, linkname) == -1) { - FAIL("link failed"); - } - end("test"); - - begin("cleanup"); - remove_files("simul_link", shared); - remove_files("simul_link_target", !SHARED); - end("cleanup"); -} - -void simul_fcntl_lock(int shared) { - int rc, fd; - char *filename; - struct flock sf_lock = { - .l_type = F_WRLCK, - .l_whence = SEEK_SET, - .l_start = 0, - .l_len = 0 - }; - struct flock sf_unlock = { - .l_type = F_UNLCK, - .l_whence = SEEK_SET, - .l_start = 0, - .l_len = 0 - }; - - begin("setup"); - filename = create_files("simul_fcntl", 0, shared); - MPI_Barrier(MPI_COMM_WORLD); - /* All open the file one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if ((fd = open(filename, O_RDWR)) == -1) { - FAIL("open failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - end("setup"); - - begin("test"); - /* All lock the file(s) simultaneously */ - rc = fcntl(fd, F_SETLK, &sf_lock); - if (!shared) { - if (rc == -1) { - if (errno == ENOSYS) { - if (rank == 0) { - fprintf(stdout, "WARNING: fcntl locking not supported.\n"); - fflush(stdout); - } - } else { - FAIL("fcntl lock failed"); - } - } - MPI_Barrier(MPI_COMM_WORLD); - } else { - int saved_errno = errno; - int *rc_vec, *er_vec, i; - int fail = 0; - int pass = 0; - int nosys = 0; - if (rank == 0) { - if ((rc_vec = (int *)malloc(sizeof(int)*size)) == NULL) - FAIL("malloc failed"); - if ((er_vec = (int *)malloc(sizeof(int)*size)) == NULL) - FAIL("malloc failed"); - } - MPI_Gather(&rc, 1, MPI_INT, rc_vec, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Gather(&saved_errno, 1, MPI_INT, er_vec, 1, MPI_INT, 0, - MPI_COMM_WORLD); - if (rank == 0) { - for (i = 0; i < size; i++) { - if (rc_vec[i] == -1) { - if (er_vec[i] == ENOSYS) { - nosys++; - } else if (er_vec[i] != EACCES && er_vec[i] != EAGAIN) { - errno = er_vec[i]; - FAIL("fcntl failed as expected, but with wrong errno"); - } - fail++; - } else { - pass++; - } - } - if (nosys == size) { - fprintf(stdout, "WARNING: fcntl locking not supported.\n"); - fflush(stdout); - } else if (!((pass == 1) && (fail == size-1))) { - fprintf(stdout, - "%s: FAILED in simul_fcntl_lock", timestamp()); - if (pass > 1) - fprintf(stdout, - "too many fcntl locks succeeded (%d).\n", pass); - else - fprintf(stdout, - "too many fcntl locks failed (%d).\n", fail); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, 1); - } - free(rc_vec); - free(er_vec); - } - } - end("test"); - - begin("cleanup"); - /* All close the file one at a time */ - Seq_begin(MPI_COMM_WORLD, throttle); - if (!shared || rank == 0) { - rc = fcntl(fd, F_SETLK, &sf_unlock); - if (rc == -1 && errno != ENOSYS) - FAIL("fcntl unlock failed"); - } - if (close(fd) == -1) { - FAIL("close failed"); - } - Seq_end(MPI_COMM_WORLD, throttle); - MPI_Barrier(MPI_COMM_WORLD); - remove_files("simul_fcntl", shared); - end("cleanup"); -} - -struct test { - char *name; - void (*function) (int); - int simul; /* Flag designating support for simultaneus mode */ - int indiv; /* Flag designating support for individual mode */ -}; - -static struct test testlist[] = { - {"open", simul_open}, - {"close", simul_close}, - {"file stat", simul_file_stat}, - {"lseek", simul_lseek}, - {"read", simul_read}, - {"write", simul_write}, - {"chdir", simul_chdir}, - {"directory stat", simul_dir_stat}, - {"statfs", simul_statfs}, - {"readdir", simul_readdir}, - {"mkdir", simul_mkdir}, - {"rmdir", simul_rmdir}, - {"unlink", simul_unlink}, - {"rename", simul_rename}, - {"creat", simul_creat}, - {"truncate", simul_truncate}, - {"symlink", simul_symlink}, - {"readlink", simul_readlink}, - {"link to one file", simul_link_to_one}, - {"link to a file per process", simul_link_to_many}, - {"fcntl locking", simul_fcntl_lock}, - {0} -}; - -/* Searches an array of ints for one int. A "-1" must mark the end of the - array. */ -int int_in_list(int item, int *list) { - int *ptr; - - if (list == NULL) - return 0; - ptr = list; - while (*ptr != -1) { - if (*ptr == item) - return 1; - ptr += 1; - } - return 0; -} - -/* Breaks string of comma-sperated ints into an array of ints */ -int *string_split(char *string) { - char *ptr; - char *tmp; - int excl_cnt = 1; - int *list; - int i; - - ptr = string; - while((tmp = strchr(ptr, ','))) { - ptr = tmp + 1; - excl_cnt++; - } - - list = (int *)malloc(sizeof(int) * (excl_cnt + 1)); - if (list == NULL) FAIL("malloc failed"); - - tmp = (strtok(string, ", ")); - if (tmp == NULL) FAIL("strtok failed"); - list[0] = atoi(tmp); - for (i = 1; i < excl_cnt; i++) { - tmp = (strtok(NULL, ", ")); - if (tmp == NULL) FAIL("strtok failed"); - list[i] = atoi(tmp); - } - list[i] = -1; - - return list; -} - -void print_help(int testcnt) { - int i; - - if (rank == 0) { - printf("simul-%s\n", version); - printf("Usage: simul [-h] -d [-f firsttest] [-l lasttest]\n"); - printf(" [-n #] [-N #] [-i \"4,7,13\"] [-e \"6,22\"] [-s]\n"); - printf(" [-v] [-V #]\n"); - printf("\t-h: prints this help message\n"); - printf("\t-d: the directory in which the tests will run\n"); - printf("\t-f: the number of the first test to run (default: 0)\n"); - printf("\t-l: the number of the last test to run (default: %d)\n", - (testcnt*2)-1); - printf("\t-i: comma-sperated list of tests to include\n"); - printf("\t-e: comma-sperated list of tests to exclude\n"); - printf("\t-s: single-step through every iteration of every test\n"); - printf("\t-n: repeat each test # times (default: 1)\n"); - printf("\t-N: repeat the entire set of tests # times (default: 1)\n"); - printf("\t-v: increases the verbositly level by 1\n"); - printf("\t-u: test with unifyfs\n"); - printf("\t-V: select a specific verbosity level\n"); - printf("\nThe available tests are:\n"); - for (i = 0; i < testcnt * 2; i++) { - printf("\tTest #%d: %s, %s mode.\n", i, - testlist[i%testcnt].name, - (i < testcnt) ? "shared" : "individual"); - } - } - - MPI_Initialized(&i); - if (i) MPI_Finalize(); - exit(0); -} - -int main(int argc, char **argv) { - int testcnt; - int first; - int last; - int i, j, k, c; - int *excl_list = NULL; - int *incl_list = NULL; - int test; - int singlestep = 0; - int iterations = 1; - int set_iterations = 1; - int unifyfs = 0; - int ret = 0; - char linebuf[80]; - - /* Check for -h parameter before MPI_Init so the simul binary can be - called directly, without, for instance, mpirun. */ - for (testcnt = 1; testlist[testcnt].name != 0; testcnt++) continue; - for (i = 1; i < argc; i++) { - if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { - print_help(testcnt); - } - } - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); - - if (rank == 0) { - printf("Simul is running with %d process(es)\n", size); - fflush(stdout); - } - - first = 0; - last = testcnt * 2; - - /* Parse command line options */ - while (1) { - c = getopt(argc, argv, "d:e:f:hi:l:n:N:suvV:"); - if (c == -1) - break; - - switch (c) { - case 'd': - testdir = optarg; - break; - case 'e': - excl_list = string_split(optarg); - break; - case 'f': - first = atoi(optarg); - if (first >= last) { - printf("Invalid parameter, firsttest must be <= lasttest\n"); - MPI_Abort(MPI_COMM_WORLD, 2); - } - break; - case 'h': - print_help(testcnt); - break; - case 'i': - incl_list = string_split(optarg); - break; - case 'l': - last = atoi(optarg)+1; - if (last <= first) { - printf("Invalid parameter, lasttest must be >= firsttest\n"); - MPI_Abort(MPI_COMM_WORLD, 2); - } - break; - case 'n': - iterations = atoi(optarg); - break; - case 'N': - set_iterations = atoi(optarg); - break; - case 's': - singlestep = 1; - break; - case 'u': - unifyfs = 1; - break; - case 'v': - verbose += 1; - break; - case 'V': - verbose = atoi(optarg); - break; - } - } - - /* mount the unifyfs and use the testdir as the mountpoint. - * if testdir is not specified, use '/unifyfs.' */ - if (unifyfs) { - int ret = 0; - - if (!testdir) { - testdir = "/unifyfs"; - } - ret = unifyfs_mount(testdir, rank, size, 0); - if (ret && rank == 0) { - printf("unifyfs_mount failed (ret=%d)\n", ret); - MPI_Abort(MPI_COMM_WORLD, 2); - } - } - - MPI_Barrier(MPI_COMM_WORLD); - - if (testdir == NULL && rank == 0) { - printf("Please specify a test directory! (\"simul -h\" for help)\n"); - MPI_Abort(MPI_COMM_WORLD, 2); - } - - if (gethostname(hostname, 1024) == -1) { - perror("gethostname"); - MPI_Abort(MPI_COMM_WORLD, 2); - } - - /* If a list of tests was not specified with the -i option, then use - the first and last number to build a range of included tests. */ - if (incl_list == NULL) { - incl_list = (int *)malloc(sizeof(int) * (2+last-first)); - for (i = 0; i < last-first; i++) { - incl_list[i] = i + first; - } - incl_list[i] = -1; - } - - /* Run the tests */ - for (k = 0; k < set_iterations; k++) { - if ((rank == 0) && (set_iterations > 1)) - printf("%s: Set iteration %d\n", timestamp(), k); - for (i = 0; ; ++i) { - test = incl_list[i]; - if (test == -1) - break; - if (!int_in_list(test, excl_list)) { - for (j = 0; j < iterations; j++) { - if (singlestep) { - if (rank == 0) - printf("%s: Hit to run test #%d(iter %d): %s, %s mode.\n", - timestamp(), test, j, - testlist[test%testcnt].name, - (test < testcnt) ? "shared" : "individual"); - fgets(linebuf, 80, stdin); - } - if (rank == 0) { - printf("%s: Running test #%d(iter %d): %s, %s mode.\n", - timestamp(), test, j, testlist[test%testcnt].name, - (test < testcnt) ? "shared" : "individual"); - fflush(stdout); - } - testlist[test%testcnt].function((test < testcnt) ? SHARED : !SHARED); - MPI_Barrier(MPI_COMM_WORLD); - } - } - } - } - - if (rank == 0) printf("%s: All tests passed!\n", timestamp()); - - /* unmount unifyfs */ - if (unifyfs) { - unifyfs_unmount(); - } - - MPI_Finalize(); - exit(0); -} From 5fbb4af6af84ff3bfd58201110e6b988eb9cab09 Mon Sep 17 00:00:00 2001 From: CamStan Date: Thu, 2 Apr 2020 14:33:46 -0700 Subject: [PATCH 085/168] Move _GNU_SOURCE define to before system headers Moves the _GNU_SOURCE definition to before the system headers so it is properly defined for the lseek cases needed in #473. In turn, removes _GNU_SOURCE from the Makefile to avoid redefine warnings. Our travis.yml runs `make distcheck` which apparently ignores configure options. Need to use `DISTCHECK_CONFIGURE_FLAGS` envar to pass config options to `make distcheck`. This also updates .travis.yml to fix warnings caused by depricated `sudo` key, and missing `os` key. --- .travis.yml | 9 +++++---- client/src/Makefile.am | 3 +-- client/src/unifyfs-internal.h | 6 ++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 736e2bd0b..d6f8b60c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: c -sudo: required dist: xenial +os: linux addons: apt: @@ -74,9 +74,10 @@ before_cache: script: # Force git to update the shallow clone and include tags so git-describe works - git fetch --unshallow --tags - - sh autogen.sh - - ./configure --enable-fortran || cat config.log - - make -k && make distcheck + - export DISTCHECK_CONFIGURE_FLAGS="--enable-fortran" + - ./autogen.sh + - ./configure $DISTCHECK_CONFIGURE_FLAGS || cat config.log + - make distcheck - ./scripts/checkpatch.sh || test "$TEST_CHECKPATCH_ALLOW_FAILURE" = yes after_failure: diff --git a/client/src/Makefile.am b/client/src/Makefile.am index ec5b9b9a5..6186d1b77 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -20,8 +20,7 @@ endif CLIENT_COMMON_CPPFLAGS = \ -I$(top_builddir)/client \ - -I$(top_srcdir)/common/src \ - -D_GNU_SOURCE + -I$(top_srcdir)/common/src CLIENT_COMMON_CFLAGS = \ $(MPI_CFLAGS) \ diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 569d9b187..755487b9e 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -57,6 +57,10 @@ * ------------------------------- */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + // system headers #include #include @@ -85,7 +89,6 @@ #include #include -#define _GNU_SOURCE #include #include @@ -147,7 +150,6 @@ * dlsym */ /* we need the dlsym function */ -#define __USE_GNU #include /* define a static variable called __real_open to record address of From feced6ac684bff9944357d5d274819c7b670459f Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Fri, 3 Apr 2020 12:26:23 -0700 Subject: [PATCH 086/168] Fix compiler warnings, set -Werror by default This enables -Werror by default and fixes all compiler warnings on GCC 9.2.1. --- .travis.yml | 2 +- client/src/Makefile.am | 4 +- client/src/unifyfs-internal.h | 5 +++ client/src/unifyfs-sysio.c | 2 +- client/src/unifyfs.c | 38 +------------------ client/src/unifyfsf.c | 6 +++ common/src/Makefile.am | 4 +- common/src/ini.c | 4 +- common/src/unifyfs_keyval.c | 30 ++++++++++----- common/src/unifyfs_log.c | 16 ++++++++ common/src/unifyfs_log.h | 13 ++----- common/src/unifyfs_logio.c | 4 +- common/src/unifyfs_meta.h | 1 + common/src/unifyfs_misc.c | 61 +++++++++++++++++++++++++++++++ common/src/unifyfs_misc.h | 5 +++ examples/src/app-hdf5-create.c | 6 +-- examples/src/app-hdf5-writeread.c | 4 +- examples/src/multi-write.c | 5 ++- m4/gotcha.m4 | 2 +- meta/src/Makefile.am | 2 +- meta/src/Mlog2/mlog2.c | 2 + meta/src/ds_leveldb.c | 1 + server/src/Makefile.am | 2 +- server/src/unifyfs_cmd_handler.c | 3 +- server/src/unifyfs_metadata.c | 1 + t/Makefile.am | 2 +- t/lib/Makefile.am | 2 +- t/server/metadata_suite.c | 2 +- t/server/metadata_suite.h | 1 - t/server/unifyfs_meta_get_test.c | 16 -------- t/sys/creat64.c | 6 +-- t/sys/lseek.c | 9 ++++- t/sys/open.c | 11 +++--- t/sys/open64.c | 5 +-- t/sys/truncate.c | 1 - util/unifyfs/src/Makefile.am | 2 +- util/unifyfs/src/unifyfs-rm.c | 2 +- 37 files changed, 172 insertions(+), 110 deletions(-) create mode 100644 common/src/unifyfs_misc.c create mode 100644 common/src/unifyfs_misc.h diff --git a/.travis.yml b/.travis.yml index d6f8b60c4..271c42023 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: c -dist: xenial +dist: bionic os: linux addons: diff --git a/client/src/Makefile.am b/client/src/Makefile.am index 6186d1b77..23428522a 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -10,7 +10,7 @@ if HAVE_FORTRAN lib_LTLIBRARIES += libunifyfsf.la endif -AM_CFLAGS = -Wall -Wno-strict-aliasing +AM_CFLAGS = -Wall -Wno-strict-aliasing -Werror include_HEADERS = unifyfs.h @@ -19,10 +19,12 @@ include_HEADERS += unifyfsf.h endif CLIENT_COMMON_CPPFLAGS = \ + -Wall -Werror \ -I$(top_builddir)/client \ -I$(top_srcdir)/common/src CLIENT_COMMON_CFLAGS = \ + -Wall -Werror \ $(MPI_CFLAGS) \ $(MERCURY_CFLAGS) \ $(ARGOBOTS_CFLAGS) \ diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 755487b9e..cd1f1fd98 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -491,6 +491,11 @@ int unifyfs_fid_is_dir_empty(const char* path); /* Return current global size of given file id */ off_t unifyfs_fid_global_size(int fid); +/* if we have a local fid structure corresponding to the gfid + * in question, we attempt the file lookup with the fid method + * otherwise call back to the rpc */ +off_t unifyfs_gfid_filesize(int gfid); + /* Return current local size of given file id */ off_t unifyfs_fid_log_size(int fid); diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 856ac9893..a7bbd0341 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -1722,7 +1722,7 @@ int unifyfs_fd_logreadlist(read_req_t* in_reqs, int in_count) if (reqs != NULL) { free(reqs); } - return read_rc; + return ENOSPC; } /* order read request by increasing file id, then increasing offset */ diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index a58f8fdf3..43776bf4c 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -68,7 +68,6 @@ unifyfs_cfg_t client_cfg; unifyfs_index_buf_t unifyfs_indices; static size_t unifyfs_index_buf_size; /* size of metadata log */ -static size_t unifyfs_fattr_buf_size; unsigned long unifyfs_max_index_entries; /* max metadata log entries */ /* tracks total number of unsync'd segments for all files */ @@ -1376,9 +1375,6 @@ static int unifyfs_init_structures() for (i = 0; i < unifyfs_max_files; i++) { /* indicate that file id is not in use by setting flag to 0 */ unifyfs_filelist[i].in_use = 0; - - /* set pointer to array of chunkmeta data structures */ - unifyfs_filemeta_t* filemeta = &unifyfs_filemetas[i]; } /* initialize stack of free file ids */ @@ -1392,33 +1388,6 @@ static int unifyfs_init_structures() return UNIFYFS_SUCCESS; } -static int unifyfs_get_spillblock(size_t size, const char* path) -{ - //MAP_OR_FAIL(open); - mode_t perms = unifyfs_getmode(0); - int spillblock_fd = __real_open(path, O_RDWR | O_CREAT | O_EXCL, perms); - if (spillblock_fd < 0) { - if (errno == EEXIST) { - /* spillover block exists; attach and return */ - spillblock_fd = __real_open(path, O_RDWR); - } else { - LOGERR("open() failed: errno=%d (%s)", errno, strerror(errno)); - return -1; - } - } else { - /* new spillover block created */ - /* TODO: align to SSD block size*/ - - /*temp*/ - off_t rc = __real_lseek(spillblock_fd, size, SEEK_SET); - if (rc < 0) { - LOGERR("lseek() failed: errno=%d (%s)", errno, strerror(errno)); - } - } - - return spillblock_fd; -} - /* create superblock of specified size and name, or attach to existing * block if available */ static int unifyfs_superblock_shmget(size_t super_sz) @@ -1699,7 +1668,7 @@ static int CountTasksPerNode(int rank, int numTasks) int resultsLen = UNIFYFS_MAX_HOSTNAME; MPI_Status status; int i, j, rc; - int* local_rank_lst; + int* local_rank_lst = NULL; if (numTasks <= 0) { LOGERR("invalid number of tasks"); @@ -1776,7 +1745,6 @@ static int CountTasksPerNode(int rank, int numTasks) set_counter++; /* broadcast the rank_cnt and rank_set information to each rank */ - int root_set_no = -1; for (i = 0; i < set_counter; i++) { /* send each rank set to all of its ranks */ for (j = 0; j < rank_cnt[i]; j++) { @@ -1794,7 +1762,6 @@ static int CountTasksPerNode(int rank, int numTasks) return -1; } } else { - root_set_no = i; local_rank_cnt = rank_cnt[i]; local_rank_lst = (int*)calloc(rank_cnt[i], sizeof(int)); memcpy(local_rank_lst, rank_set[i], @@ -1861,8 +1828,6 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, { int rc; int kv_rank, kv_nranks; - bool b; - char* cfgval; if (-1 != unifyfs_mounted) { if (l_app_id != unifyfs_mounted) { @@ -2181,7 +2146,6 @@ static int do_transfer_file_serial(const char* src, const char* dst, int ret = 0; int fd_src = 0; int fd_dst = 0; - char buf[UNIFYFS_TX_BUFSIZE] = { 0, }; /* * for now, we do not use the @dir hint. diff --git a/client/src/unifyfsf.c b/client/src/unifyfsf.c index ffc39d8f5..1480b4845 100644 --- a/client/src/unifyfsf.c +++ b/client/src/unifyfsf.c @@ -94,6 +94,11 @@ static int unifyfs_fstr2cstr(const char* fstr, int flen, char* cstr, int clen) return rc; } +/* + * Marking unifyfs_cstr2fstr() with an '#if 0' block, since this function + * isn't used yet. + */ +#if 0 /* convert a C string to a Fortran string, adding trailing spaces * as necessary */ static int unifyfs_cstr2fstr(const char* cstr, char* fstr, int flen) @@ -128,6 +133,7 @@ static int unifyfs_cstr2fstr(const char* cstr, char* fstr, int flen) return rc; } +#endif /*================================================ * Mount, Unmount diff --git a/common/src/Makefile.am b/common/src/Makefile.am index 978f75849..fa0c02df9 100644 --- a/common/src/Makefile.am +++ b/common/src/Makefile.am @@ -33,6 +33,8 @@ BASE_SRCS = \ unifyfs_logio.c \ unifyfs_meta.h \ unifyfs_meta.c \ + unifyfs_misc.c \ + unifyfs_misc.h \ unifyfs_rpc_util.h \ unifyfs_rpc_util.c \ unifyfs_client_rpcs.h \ @@ -75,4 +77,4 @@ libunifyfs_common_la_LDFLAGS = \ libunifyfs_common_la_LIBADD = \ $(OPT_LIBS) -lm -lrt -lcrypto -lpthread -AM_CFLAGS = -Wall -Wno-strict-aliasing +AM_CFLAGS = -Wall -Werror -Wno-strict-aliasing diff --git a/common/src/ini.c b/common/src/ini.c index 1e64632a4..a2865923d 100644 --- a/common/src/ini.c +++ b/common/src/ini.c @@ -17,6 +17,7 @@ Go to the project home page for more info: #include #include "ini.h" +#include "unifyfs_misc.h" #if !INI_USE_STACK #include @@ -73,8 +74,7 @@ static char *find_chars_or_comment(const char *s, const char *chars) /* Version of strncpy that ensures dest (size bytes) is null-terminated. */ static char *strncpy0(char *dest, const char *src, size_t size) { - strncpy(dest, src, size); - dest[size - 1] = '\0'; + strlcpy(dest, src, size); return dest; } diff --git a/common/src/unifyfs_keyval.c b/common/src/unifyfs_keyval.c index caaeda29c..85ef01e0f 100644 --- a/common/src/unifyfs_keyval.c +++ b/common/src/unifyfs_keyval.c @@ -15,6 +15,7 @@ #include "unifyfs_const.h" #include "unifyfs_keyval.h" #include "unifyfs_log.h" +#include "unifyfs_misc.h" //#include "config.h" @@ -543,7 +544,7 @@ static int unifyfs_fskv_init(unifyfs_cfg_t* cfg) } // find or create rank-specific subdir - snprintf(sharedfs_rank_kvdir, sizeof(sharedfs_rank_kvdir), "%s/%d", + scnprintf(sharedfs_rank_kvdir, sizeof(sharedfs_rank_kvdir), "%s/%d", sharedfs_kvdir, kv_myrank); memset(&s, 0, sizeof(struct stat)); rc = stat(sharedfs_rank_kvdir, &s); @@ -591,7 +592,7 @@ static int unifyfs_fskv_fini(void) continue; } memset(kvfile, 0, sizeof(kvfile)); - snprintf(kvfile, sizeof(kvfile), "%s/%s", + scnprintf(kvfile, sizeof(kvfile), "%s/%s", localfs_kvdir, de->d_name); rc = remove(kvfile); if (rc != 0) { @@ -628,7 +629,7 @@ static int unifyfs_fskv_fini(void) continue; } memset(rank_kvfile, 0, sizeof(rank_kvfile)); - snprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%s", + scnprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%s", sharedfs_rank_kvdir, de->d_name); rc = remove(rank_kvfile); if (rc != 0) { @@ -688,8 +689,9 @@ static int unifyfs_fskv_lookup_local(const char* key, FILE* kvf; char kvfile[UNIFYFS_MAX_FILENAME]; char kvalue[kv_max_vallen]; + int rc; - snprintf(kvfile, sizeof(kvfile), "%s/%s", + scnprintf(kvfile, sizeof(kvfile), "%s/%s", localfs_kvdir, key); kvf = fopen(kvfile, "r"); if (NULL == kvf) { @@ -697,8 +699,12 @@ static int unifyfs_fskv_lookup_local(const char* key, return (int)UNIFYFS_ERROR_KEYVAL; } memset(kvalue, 0, sizeof(kvalue)); - fscanf(kvf, "%s\n", kvalue); + rc = fscanf(kvf, "%s\n", kvalue); fclose(kvf); + if (rc != 1) { + *oval = NULL; + return (int)UNIFYFS_FAILURE; + } *oval = strdup(kvalue); return (int)UNIFYFS_SUCCESS; @@ -711,12 +717,13 @@ static int unifyfs_fskv_lookup_remote(int rank, FILE* kvf; char rank_kvfile[UNIFYFS_MAX_FILENAME]; char kvalue[kv_max_vallen]; + int rc; if (!have_sharedfs_kvstore) { return (int)UNIFYFS_ERROR_KEYVAL; } - snprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%d/%s", + scnprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%d/%s", sharedfs_kvdir, rank, key); kvf = fopen(rank_kvfile, "r"); if (NULL == kvf) { @@ -724,9 +731,14 @@ static int unifyfs_fskv_lookup_remote(int rank, return (int)UNIFYFS_ERROR_KEYVAL; } memset(kvalue, 0, sizeof(kvalue)); - fscanf(kvf, "%s\n", kvalue); + rc = fscanf(kvf, "%s\n", kvalue); fclose(kvf); + if (rc != 1) { + *oval = NULL; + return (int)UNIFYFS_FAILURE; + } + *oval = strdup(kvalue); return (int)UNIFYFS_SUCCESS; } @@ -738,7 +750,7 @@ static int unifyfs_fskv_publish_local(const char* key, FILE* kvf; char kvfile[UNIFYFS_MAX_FILENAME]; - snprintf(kvfile, sizeof(kvfile), "%s/%s", + scnprintf(kvfile, sizeof(kvfile), "%s/%s", localfs_kvdir, key); kvf = fopen(kvfile, "w"); if (NULL == kvf) { @@ -761,7 +773,7 @@ static int unifyfs_fskv_publish_remote(const char* key, return (int)UNIFYFS_ERROR_KEYVAL; } - snprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%s", + scnprintf(rank_kvfile, sizeof(rank_kvfile), "%s/%s", sharedfs_rank_kvdir, key); kvf = fopen(rank_kvfile, "w"); if (NULL == kvf) { diff --git a/common/src/unifyfs_log.c b/common/src/unifyfs_log.c index 01d5ecf93..fb639fb5c 100644 --- a/common/src/unifyfs_log.c +++ b/common/src/unifyfs_log.c @@ -29,6 +29,10 @@ #include #include +#include +#include +#include + #include "unifyfs_log.h" #include "unifyfs_const.h" @@ -100,3 +104,15 @@ void unifyfs_set_log_level(unifyfs_log_level_t lvl) unifyfs_log_level = lvl; } } + +pid_t unifyfs_gettid(void) +{ +#if defined(gettid) + return gettid(); +#elif defined(SYS_gettid) + return syscall(SYS_gettid); +#else +#error no gettid() +#endif + return 0; +} diff --git a/common/src/unifyfs_log.h b/common/src/unifyfs_log.h index d83c4d5ac..44549485d 100644 --- a/common/src/unifyfs_log.h +++ b/common/src/unifyfs_log.h @@ -32,9 +32,8 @@ #include #include -#include -#include #include +#include #ifdef __cplusplus extern "C" { @@ -56,13 +55,7 @@ extern struct tm* unifyfs_log_ltime; extern char unifyfs_log_timestamp[256]; extern size_t unifyfs_log_source_base_len; -#if defined(__NR_gettid) -#define gettid() syscall(__NR_gettid) -#elif defined(SYS_gettid) -#define gettid() syscall(SYS_gettid) -#else -#error gettid syscall is not defined -#endif +pid_t unifyfs_gettid(void); #define LOG(level, ...) \ if (level <= unifyfs_log_level) { \ @@ -75,7 +68,7 @@ extern size_t unifyfs_log_source_base_len; unifyfs_log_stream = stderr; \ } \ fprintf(unifyfs_log_stream, "%s tid=%ld @ %s() [%s:%d] ", \ - unifyfs_log_timestamp, (long)gettid(), \ + unifyfs_log_timestamp, (long)unifyfs_gettid(), \ __func__, srcfile, __LINE__); \ fprintf(unifyfs_log_stream, __VA_ARGS__); \ fprintf(unifyfs_log_stream, "\n"); \ diff --git a/common/src/unifyfs_logio.c b/common/src/unifyfs_logio.c index 4ccb950bd..6ebe0af91 100644 --- a/common/src/unifyfs_logio.c +++ b/common/src/unifyfs_logio.c @@ -676,7 +676,7 @@ int unifyfs_logio_read(logio_context* ctx, &sz_in_mem, &sz_in_spill, &spill_offset); /* do reads */ - int err_rc; + int err_rc = 0; if (sz_in_mem > 0) { /* read data from shared memory */ char* shmem_data = (char*)(ctx->shmem->addr) + shmem_hdr->data_offset; @@ -749,7 +749,7 @@ int unifyfs_logio_write(logio_context* ctx, &sz_in_mem, &sz_in_spill, &spill_offset); /* do writes */ - int err_rc; + int err_rc = 0; if (sz_in_mem > 0) { /* write data to shared memory */ char* shmem_data = (char*)(ctx->shmem->addr) + shmem_hdr->data_offset; diff --git a/common/src/unifyfs_meta.h b/common/src/unifyfs_meta.h index 8c120074a..f1fa59e76 100644 --- a/common/src/unifyfs_meta.h +++ b/common/src/unifyfs_meta.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "unifyfs_const.h" diff --git a/common/src/unifyfs_misc.c b/common/src/unifyfs_misc.c new file mode 100644 index 000000000..99ddd5422 --- /dev/null +++ b/common/src/unifyfs_misc.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * This file contains miscellaneous common functions that don't fit into a + * particular common/src/ file. + */ +#include +#include +#include + +/* + * Re-implementation of BSD's strlcpy() function. + * + * This is a basically a safer version of strlncpy() since it always + * NULL-terminates the buffer. Google 'strlcpy' for full documentation. + */ +size_t strlcpy(char* dest, const char* src, size_t size) +{ + size_t src_len = strnlen(src, size); + + memcpy(dest, src, src_len); + if (src_len == size) { + dest[size - 1] = '\0'; + } + return strlen(dest); +} + +/* + * This is a re-implementation of the Linux kernel's scnprintf() function. + * + * It's snprintf() but returns the number of chars actually written into buf[] + * not including the '\0'. It also avoids the -Wformat-truncation warnings. + */ +int scnprintf(char* buf, size_t size, const char* fmt, ...) +{ + va_list args; + int rc; + + va_start(args, fmt); + rc = vsnprintf(buf, size, fmt, args); + va_end(args); + + if (rc >= size) { + /* We truncated */ + return size - 1; + } + + return rc; +} diff --git a/common/src/unifyfs_misc.h b/common/src/unifyfs_misc.h new file mode 100644 index 000000000..5e6c0cdac --- /dev/null +++ b/common/src/unifyfs_misc.h @@ -0,0 +1,5 @@ +#ifndef __UNIFYFS_MISC__ +#define __UNIFYFS_MISC__ +size_t strlcpy(char* dest, const char* src, size_t size); +int scnprintf(char* buf, size_t size, const char* fmt, ...); +#endif diff --git a/examples/src/app-hdf5-create.c b/examples/src/app-hdf5-create.c index bf6d41333..2c21aba5e 100644 --- a/examples/src/app-hdf5-create.c +++ b/examples/src/app-hdf5-create.c @@ -162,18 +162,18 @@ int main(int argc, char** argv) /* Create a new file using default properties. */ file_id = H5Fcreate(targetfile, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - printf("H5Fcreate: %d\n", file_id); + printf("H5Fcreate: %ld\n", (long) file_id); /* Create the data space for the dataset. */ dims[0] = 4; dims[1] = 6; dataspace_id = H5Screate_simple(2, dims, NULL); - printf("H5Screate_simple: %d\n", dataspace_id); + printf("H5Screate_simple: %ld\n", (long) dataspace_id); /* Create the dataset. */ dataset_id = H5Dcreate2(file_id, "/dset", H5T_STD_I32BE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - printf("H5Dcreate2: %d\n", dataset_id); + printf("H5Dcreate2: %ld\n", (long) dataset_id); if (write_data) { int i, j; diff --git a/examples/src/app-hdf5-writeread.c b/examples/src/app-hdf5-writeread.c index 405888896..952182e60 100644 --- a/examples/src/app-hdf5-writeread.c +++ b/examples/src/app-hdf5-writeread.c @@ -186,11 +186,11 @@ int main(int argc, char** argv) /* Open an existing file. */ file_id = H5Fopen(targetfile, H5F_ACC_RDWR, H5P_DEFAULT); - printf("H5Fopen: %d\n", file_id); + printf("H5Fopen: %ld\n", (long) file_id); /* Open an existing dataset. */ dataset_id = H5Dopen2(file_id, "/dset", H5P_DEFAULT); - printf("H5open2: %d\n", dataset_id); + printf("H5open2: %ld\n", (long) dataset_id); if (!readonly) { /* Write the dataset. */ diff --git a/examples/src/multi-write.c b/examples/src/multi-write.c index 3483f3cbb..8ff9402bb 100644 --- a/examples/src/multi-write.c +++ b/examples/src/multi-write.c @@ -140,7 +140,10 @@ int do_test(test_cfg* cfg) /* + 1 so we always write at least 1 byte */ count = (rand() % (MAX_WRITE-1)) + 1; lseek(fd, start, SEEK_SET); - write(fd, &bigbuf[start], count); + if (write(fd, &bigbuf[start], count) != count) { + perror("Couldn't write"); + exit(1); + } } /* Sync extents of all our files and laminate them */ diff --git a/m4/gotcha.m4 b/m4/gotcha.m4 index 5138a7247..54e09a5db 100644 --- a/m4/gotcha.m4 +++ b/m4/gotcha.m4 @@ -7,7 +7,7 @@ AC_DEFUN([UNIFYFS_AC_GOTCHA], [ AC_ARG_WITH([gotcha], [AC_HELP_STRING([--with-gotcha=PATH], [path to installed libgotcha [default=/usr/local]])], [ GOTCHA_CFLAGS="-I${withval}/include" - GOTCHA_LDFLAGS="-L${withval}/lib64" + GOTCHA_LDFLAGS="-L${withval}/lib64 -L${withval}/lib" CFLAGS="$CFLAGS ${GOTCHA_CFLAGS}" CXXFLAGS="$CXXFLAGS ${GOTCHA_CFLAGS}" LDFLAGS="$LDFLAGS ${GOTCHA_LDFLAGS}" diff --git a/meta/src/Makefile.am b/meta/src/Makefile.am index 4f4539ff1..30d4422d7 100644 --- a/meta/src/Makefile.am +++ b/meta/src/Makefile.am @@ -36,6 +36,6 @@ AM_CPPFLAGS = -I$(top_srcdir)/meta/src/Mlog2 \ -I$(top_srcdir)/server/src AM_CFLAGS = -DLEVELDB_SUPPORT $(LEVELDB_CFLAGS) $(MPI_CFLAGS) $(MARGO_CFLAGS) -AM_CFLAGS += -Wall +AM_CFLAGS += -Wall -Werror CLEANFILES = diff --git a/meta/src/Mlog2/mlog2.c b/meta/src/Mlog2/mlog2.c index 10ea11692..cd56aa9f4 100644 --- a/meta/src/Mlog2/mlog2.c +++ b/meta/src/Mlog2/mlog2.c @@ -1174,6 +1174,8 @@ void mlog_setmasks(char *mstr, int mlen0) /* process priority */ if (prilen > 5) { /* we know it can't be longer than this */ prino = -1; + } else if (prilen < 0) { /* This if() block gets rid of a */ + prino = -1; /* compiler warning. */ } else { memset(pbuf, 0, sizeof(pbuf)); strncpy(pbuf, pri, prilen); diff --git a/meta/src/ds_leveldb.c b/meta/src/ds_leveldb.c index b28453dc0..602b38ca3 100644 --- a/meta/src/ds_leveldb.c +++ b/meta/src/ds_leveldb.c @@ -397,6 +397,7 @@ int mdhim_leveldb_put(void *dbh, void *key, int key_len, void *data, int32_t dat gettimeofday(&start, NULL); options = mdhimdb->write_options; leveldb_put(mdhimdb->db, options, key, key_len, data, data_len, &err); + gettimeofday(&end, NULL); /* * temporarily mute the error message until the file metadata * operation is fully defined and implemented */ diff --git a/server/src/Makefile.am b/server/src/Makefile.am index 31c0efae6..5d0c2793e 100644 --- a/server/src/Makefile.am +++ b/server/src/Makefile.am @@ -40,7 +40,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/meta/src/uthash \ -I$(top_srcdir)/meta/src/Mlog2 -AM_CFLAGS = -Wall \ +AM_CFLAGS = -Wall -Werror \ $(LEVELDB_CFLAGS) \ $(MPI_CFLAGS) \ $(MERCURY_CFLAGS) \ diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index 47f331249..3045fd055 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -40,6 +40,7 @@ #include "margo_server.h" #include "unifyfs_client_rpcs.h" #include "unifyfs_rpc_util.h" +#include "unifyfs_misc.h" /** * attach to the client-side shared memory @@ -331,7 +332,7 @@ static void unifyfs_metaset_rpc(hg_handle_t handle) memset(&fattr, 0, sizeof(fattr)); int create = (int) in.create; fattr.gfid = in.gfid; - strncpy(fattr.filename, in.filename, sizeof(fattr.filename)); + strlcpy(fattr.filename, in.filename, sizeof(fattr.filename)); fattr.mode = in.mode; fattr.uid = in.uid; fattr.gid = in.gid; diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index 82a475e93..68fd2f939 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -43,6 +43,7 @@ #include "indexes.h" #include "mdhim.h" + struct mdhim_t* md; /* we use two MDHIM indexes: diff --git a/t/Makefile.am b/t/Makefile.am index ffefd4ba0..5f6dd772d 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -37,7 +37,7 @@ EXTRA_DIST = \ sharness.sh \ tap-driver.sh -AM_CFLAGS = -Wall +AM_CFLAGS = -Wall -Werror clean-local: rm -fr trash-directory.* test-results *.log test_run_env.sh diff --git a/t/lib/Makefile.am b/t/lib/Makefile.am index f3ddbc7fc..aab557e23 100644 --- a/t/lib/Makefile.am +++ b/t/lib/Makefile.am @@ -1,4 +1,4 @@ -AM_CFLAGS = -Wall +AM_CFLAGS = -Wall -Werror AM_CPPFLAGS = diff --git a/t/server/metadata_suite.c b/t/server/metadata_suite.c index 4ba5fc053..20ef8ff1e 100644 --- a/t/server/metadata_suite.c +++ b/t/server/metadata_suite.c @@ -8,7 +8,7 @@ int main(int argc, char* argv[]) { /* need to initialize enough of the server to use the metadata API */ unifyfs_cfg_t server_cfg; - int rc, provided, glb_rank, glb_size; + int rc; /* get the configuration */ rc = unifyfs_config_init(&server_cfg, argc, argv); diff --git a/t/server/metadata_suite.h b/t/server/metadata_suite.h index 3afcce2c2..56bbf0ffa 100644 --- a/t/server/metadata_suite.h +++ b/t/server/metadata_suite.h @@ -24,6 +24,5 @@ int unifyfs_set_file_attribute_test(void); int unifyfs_get_file_attribute_test(void); -int unifyfs_get_file_extents_test(void); #endif /* METADATA_SUITE_H */ diff --git a/t/server/unifyfs_meta_get_test.c b/t/server/unifyfs_meta_get_test.c index 1540013db..17718245e 100644 --- a/t/server/unifyfs_meta_get_test.c +++ b/t/server/unifyfs_meta_get_test.c @@ -40,19 +40,3 @@ int unifyfs_get_file_attribute_test(void) ); return 0; } - -// this test is not run right now -int unifyfs_get_file_extents_test(void) -{ - int rc, num_values, num_keys; - int key_lens[16]; - unifyfs_key_t keys[16]; - unifyfs_keyval_t keyval[16]; - - rc = unifyfs_get_file_extents(num_keys, (unifyfs_key_t**)&keys, key_lens, - &num_values, (unifyfs_keyval_t**)&keyval); - ok(UNIFYFS_SUCCESS == rc, - "Retrieved file extents (rc = %d)", rc - ); - return 0; -} diff --git a/t/sys/creat64.c b/t/sys/creat64.c index f2a6ab126..b90a7b48f 100644 --- a/t/sys/creat64.c +++ b/t/sys/creat64.c @@ -35,7 +35,6 @@ int creat64_test(char* unifyfs_root) char path[64]; int mode = 0600; int fd; - int rc; /* Create a random file name at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); @@ -47,7 +46,7 @@ int creat64_test(char* unifyfs_root) ok(fd >= 0, "creat64 non-existing file %s (fd=%d): %s", path, fd, strerror(errno)); - rc = close(fd); + ok(close(fd) != -1, "close() worked"); /* Verify creating an already created file succeeds. */ errno = 0; @@ -55,7 +54,8 @@ int creat64_test(char* unifyfs_root) ok(fd >= 0, "creat64 existing file %s (fd=%d): %s", path, fd, strerror(errno)); - rc = close(fd); + ok(close(fd) != -1, "close() worked"); + end_skip; diag("Finished UNIFYFS_WRAP(creat64) tests"); diff --git a/t/sys/lseek.c b/t/sys/lseek.c index d88666123..5258c8ff3 100644 --- a/t/sys/lseek.c +++ b/t/sys/lseek.c @@ -44,7 +44,10 @@ int lseek_test(char* unifyfs_root) /* Open a file and write to it to test lseek() */ fd = open(path, O_RDWR | O_CREAT | O_TRUNC, file_mode); - write(fd, "hello world", 12); + + ret = write(fd, "hello world", 12); + ok(ret == 12, "%s: write works (ret=%d): %s", + __FILE__, ret, strerror(errno)); /* lseek with invalid whence fails with errno=EINVAL. */ errno = 0; @@ -134,7 +137,9 @@ int lseek_test(char* unifyfs_root) /* lseek() with SEEK_DATA tests */ /* Write beyond end of file to create a hole */ - write(fd, "hello universe", 15); + ret = write(fd, "hello universe", 15); + ok(ret == 15, "%s: write works (ret=%d): %s", + __FILE__, ret, strerror(errno)); /* lseek to negative offset with SEEK_DATA should fail with errno=ENXIO */ errno = 0; diff --git a/t/sys/open.c b/t/sys/open.c index d88a5d4a1..787fe43c7 100644 --- a/t/sys/open.c +++ b/t/sys/open.c @@ -57,7 +57,7 @@ int open_test(char* unifyfs_root) ok(fd >= 0, "open non-existing file %s flags O_CREAT|O_EXCL (fd=%d): %s", path, fd, strerror(errno)); - rc = close(fd); + ok(close(fd) != -1, "close() worked"); /* Verify opening an existing file with O_CREAT|O_EXCL fails with * errno=EEXIST. */ @@ -73,12 +73,13 @@ int open_test(char* unifyfs_root) ok(fd >= 0, "open existing file %s O_RDWR (fd=%d): %s", path, fd, strerror(errno)); - rc = close(fd); + ok(close(fd) != -1, "close() worked"); + + rc = mkdir(dir_path, dir_mode); + ok(rc == 0, "mkdir(%s, %o) worked, rc = %d", dir_path, dir_mode, rc); /* todo_open_1: Remove when issue is resolved */ todo("open_1: should fail with errno=EISDIR=21"); - /* Verify opening a dir for write fails with errno=EISDIR */ - rc = mkdir(dir_path, dir_mode); errno = 0; fd = open(dir_path, O_RDWR, file_mode); @@ -92,7 +93,7 @@ int open_test(char* unifyfs_root) * Don't unlink `path` so that the final test (9020-mountpoint-empty) can * check if open left anything in the mountpoint and thus wasn't wrapped * properly. */ - rc = rmdir(dir_path); + ok(rmdir(dir_path) != -1, "rmdir() worked"); diag("Finished UNIFYFS_WRAP(open) tests"); diff --git a/t/sys/open64.c b/t/sys/open64.c index f831e72ae..ec6b0ff83 100644 --- a/t/sys/open64.c +++ b/t/sys/open64.c @@ -37,7 +37,6 @@ int open64_test(char* unifyfs_root) char path[64]; int mode = 0600; int fd; - int rc; /* Create a random file name at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); @@ -56,7 +55,7 @@ int open64_test(char* unifyfs_root) ok(fd >= 0, "open64 non-existing file %s flags O_CREAT|O_EXCL (fd=%d): %s", path, fd, strerror(errno)); - rc = close(fd); + ok(close(fd) != -1, "close() worked"); /* Verify opening an existing file with O_CREAT|O_EXCL fails with * errno=EEXIST. */ @@ -72,7 +71,7 @@ int open64_test(char* unifyfs_root) ok(fd >= 0, "open64 existing file %s O_RDWR (fd=%d): %s", path, fd, strerror(errno)); - rc = close(fd); + ok(close(fd) != -1, "close() worked"); diag("Finished UNIFYFS_WRAP(open64) tests"); diff --git a/t/sys/truncate.c b/t/sys/truncate.c index 33995c155..b541e9731 100644 --- a/t/sys/truncate.c +++ b/t/sys/truncate.c @@ -509,7 +509,6 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) int rc; int fd; size_t global, log; - int i; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); diff --git a/util/unifyfs/src/Makefile.am b/util/unifyfs/src/Makefile.am index 9ae1304f1..770e66be0 100644 --- a/util/unifyfs/src/Makefile.am +++ b/util/unifyfs/src/Makefile.am @@ -11,7 +11,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/common/src \ -DBINDIR=\"$(bindir)\" \ -DSBINDIR=\"$(sbindir)\" -AM_CFLAGS = -Wall +AM_CFLAGS = -Wall -Werror CLEANFILES = $(bin_PROGRAMS) diff --git a/util/unifyfs/src/unifyfs-rm.c b/util/unifyfs/src/unifyfs-rm.c index cd9744ccd..da804ed38 100644 --- a/util/unifyfs/src/unifyfs-rm.c +++ b/util/unifyfs/src/unifyfs-rm.c @@ -196,7 +196,7 @@ static int lsf_read_resource(unifyfs_resource_t* resource) size_t i, n_nodes; char* val; char* node; - char* last_node; + char* last_node = NULL; char* lsb_hosts; char* pos; char** nodes; From 91fc27d20ea005a95d88ee130e015edfa789bff9 Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Tue, 7 Apr 2020 12:54:57 -0400 Subject: [PATCH 087/168] simul POSIX test example with the updated/proper license. TEST_CHECKPATCH_ALLOW_FAILURE=yes --- examples/src/Makefile.am | 12 + examples/src/simul.c | 1326 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 1338 insertions(+) create mode 100644 examples/src/simul.c diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index 1f7f87777..7721fc921 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -23,6 +23,7 @@ if HAVE_LD_WRAP app-tileio-static \ transfer-static \ size-static \ + simul-static \ chmod-static \ multi-write-static endif @@ -45,6 +46,7 @@ if HAVE_GOTCHA app-tileio-gotcha \ transfer-gotcha \ size-gotcha \ + simul-gotcha \ chmod-gotcha \ multi-write-gotcha endif @@ -302,6 +304,16 @@ size_static_CPPFLAGS = $(test_cppflags) size_static_LDADD = $(test_static_ldadd) size_static_LDFLAGS = $(test_static_ldflags) +simul_gotcha_SOURCES = simul.c +simul_gotcha_CPPFLAGS = $(test_cppflags) +simul_gotcha_LDADD = $(test_gotcha_ldadd) +simul_gotcha_LDFLAGS = $(test_gotcha_ldflags) + +simul_static_SOURCES = simul.c +simul_static_CPPFLAGS = $(test_cppflags) +simul_static_LDADD = $(test_static_ldadd) +simul_static_LDFLAGS = $(test_static_ldflags) + chmod_gotcha_SOURCES = chmod.c testutil.c chmod_gotcha_CPPFLAGS = $(test_cppflags) chmod_gotcha_LDADD = $(test_gotcha_ldadd) diff --git a/examples/src/simul.c b/examples/src/simul.c new file mode 100644 index 000000000..49e0cf5a2 --- /dev/null +++ b/examples/src/simul.c @@ -0,0 +1,1326 @@ +/* + * Copyright (c) 2017, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2017, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +/* + * Copyright (C) 2003, The Regents of the University of California. + * Produced at the Lawrence Livermore National Laboratory. + * Written by Christopher J. Morrone + * UCRL-CODE-2003-019 + * All rights reserved. + */ + +/* + * Some modifications including style changes have been made for testing + * unifyfs: + * To test with UnifyFS, pass the '-u' flag. Then, simul will set the test + * directory as /unifyfs, where unifyfs will be accordingly mounted. + */ + +#include "mpi.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* unifyfs test */ +#include + +#define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH +#define DIRMODE S_IRUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IXOTH +#define SHARED 1 +#define MAX_FILENAME_LEN 512 + +int rank; +int size; +char *testdir = NULL; +char hostname[1024]; +int verbose; +int throttle = 1; +struct timeval t1, t2; +static char version[] = "1.16"; + +#ifdef __GNUC__ + /* "inline" is a keyword in GNU C */ +#elif __STDC_VERSION__ >= 199901L + /* "inline" is a keyword in C99 and later versions */ +#else +# define inline /* "inline" not available */ +#endif + +#ifndef AIX +#define FAIL(msg) do { \ + fprintf(stdout, "%s: Process %d(%s): FAILED in %s, %s: %s\n",\ + timestamp(), rank, hostname, __func__, \ + msg, strerror(errno)); \ + fflush(stdout);\ + MPI_Abort(MPI_COMM_WORLD, 1); \ +} while(0) +#else +#define FAIL(msg) do { \ + fprintf(stdout, "%s: Process %d(%s): FAILED, %s: %s\n",\ + timestamp(), rank, hostname, \ + msg, strerror(errno)); \ + fflush(stdout);\ + MPI_Abort(MPI_COMM_WORLD, 1); \ +} while(0) +#endif + +char *timestamp() { + static char datestring[80]; + time_t timestamp; + + fflush(stdout); + timestamp = time(NULL); + strftime(datestring, 80, "%T", localtime(×tamp)); + + return datestring; +} + +static inline void begin(char *str) { + if (verbose > 0 && rank == 0) { + gettimeofday(&t1, NULL); + fprintf(stdout, "%s:\tBeginning %s\n", timestamp(), str); + fflush(stdout); + } +} + +static inline void end(char *str) { + double elapsed; + + MPI_Barrier(MPI_COMM_WORLD); + if (verbose > 0 && rank == 0) { + gettimeofday(&t2, NULL); + elapsed = ((((t2.tv_sec - t1.tv_sec) * 1000000L) + + t2.tv_usec) - t1.tv_usec) + / (double)1000000; + if (elapsed >= 60) { + fprintf(stdout, "%s:\tFinished %-15s(%.2f min)\n", + timestamp(), str, elapsed / 60); + } else { + fprintf(stdout, "%s:\tFinished %-15s(%.3f sec)\n", + timestamp(), str, elapsed); + + } + fflush(stdout); + } +} + +void Seq_begin(MPI_Comm comm, int numprocs) { + int size; + int rank; + int buf; + MPI_Status status; + + MPI_Comm_size(comm, &size); + MPI_Comm_rank(comm, &rank); + + if (rank >= numprocs) { + MPI_Recv(&buf, 1, MPI_INT, rank-numprocs, 1333, comm, &status); + } +} + +void Seq_end(MPI_Comm comm, int numprocs) { + int size; + int rank; + int buf; + + MPI_Comm_size(comm, &size); + MPI_Comm_rank(comm, &rank); + + if ((rank + numprocs) < size) { + MPI_Send(&buf, 1, MPI_INT, rank+numprocs, 1333, comm); + } +} + +/* This function does not FAIL if the requested "name" does not exist. This + is just to clean up any files or directories left over from previous runs.*/ +void remove_file_or_dir(char *name) { + struct stat statbuf; + char errmsg[MAX_FILENAME_LEN+20]; + + if (stat(name, &statbuf) != -1) { + if (S_ISREG(statbuf.st_mode)) { + printf("stale file found\n"); + if (unlink(name) == -1) { + sprintf(errmsg, "unlink of %s", name); + FAIL(errmsg); + } + } + if (S_ISDIR(statbuf.st_mode)) { + printf("stale directory found\n"); + if (rmdir(name) == -1) { + sprintf(errmsg, "rmmdir of %s", name); + FAIL(errmsg); + } + } + } +} + +char *create_files(char *prefix, int filesize, int shared) { + static char filename[MAX_FILENAME_LEN]; + char errmsg[MAX_FILENAME_LEN+20]; + int fd, i; + short zero = 0; + + /* Process 0 creates the test file(s) */ + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + sprintf(filename, "%s/%s.%d", testdir, prefix, i); + remove_file_or_dir(filename); + if ((fd = creat(filename, FILEMODE)) == -1) { + sprintf(errmsg, "creat of file %s", filename); + FAIL(errmsg); + } + if (filesize > 0) { + if (lseek(fd, filesize - 1, SEEK_SET) == -1) { + sprintf(errmsg, "lseek in file %s", filename); + FAIL(errmsg); + } + if (write(fd, &zero, 1) == -1) { + sprintf(errmsg, "write in file %s", filename); + FAIL(errmsg); + } + } + if (close(fd) == -1) { + sprintf(errmsg, "close of file %s", filename); + FAIL(errmsg); + } + } + } + + if (shared) + sprintf(filename, "%s/%s.0", testdir, prefix); + else + sprintf(filename, "%s/%s.%d", testdir, prefix, rank); + + return filename; +} + +void remove_files(char *prefix, int shared) { + char filename[1024]; + int i; + + /* Process 0 removes the file(s) */ + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + sprintf(filename, "%s/%s.%d", testdir, prefix, i); + /*printf("Removing file %s\n", filename); fflush(stdout);*/ + if (unlink(filename) == -1) { + FAIL("unlink failed"); + } + } + } +} + +char *create_dirs(char *prefix, int shared) { + static char dirname[1024]; + int i; + + /* Process 0 creates the test file(s) */ + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + sprintf(dirname, "%s/%s.%d", testdir, prefix, i); + remove_file_or_dir(dirname); + if (mkdir(dirname, DIRMODE) == -1) { + FAIL("init mkdir failed"); + } + } + } + + if (shared) + sprintf(dirname, "%s/%s.0", testdir, prefix); + else + sprintf(dirname, "%s/%s.%d", testdir, prefix, rank); + + return dirname; +} + +void remove_dirs(char *prefix, int shared) { + char dirname[1024]; + int i; + + /* Process 0 removes the file(s) */ + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + sprintf(dirname, "%s/%s.%d", testdir, prefix, i); + if (rmdir(dirname) == -1) { + FAIL("rmdir failed"); + } + } + } +} + +char *create_symlinks(char *prefix, int shared) { + static char filename[1024]; + static char linkname[1024]; + int i; + + /* Process 0 creates the test file(s) */ + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + sprintf(filename, "%s/symlink_target", testdir); + sprintf(linkname, "%s/%s.%d", testdir, prefix, i); + remove_file_or_dir(linkname); + if (symlink(filename, linkname) == -1) { + FAIL("symlink failed"); + } + } + } + + if (shared) + sprintf(linkname, "%s/%s.0", testdir, prefix); + else + sprintf(linkname, "%s/%s.%d", testdir, prefix, rank); + + return linkname; +} + +void check_single_success(char *testname, int rc, int error_rc) { + int *rc_vec, i; + int fail = 0; + int pass = 0; + + if (rank == 0) { + if ((rc_vec = (int *)malloc(sizeof(int)*size)) == NULL) { + FAIL("malloc failed"); + } + } + MPI_Gather(&rc, 1, MPI_INT, rc_vec, 1, MPI_INT, 0, MPI_COMM_WORLD); + if (rank == 0) { + for (i = 0; i < size; i++) { + if (rc_vec[i] == error_rc) + fail++; + else + pass++; + } + if (!((pass == 1) && (fail == size-1))) { + fprintf(stdout, "%s: FAILED in %s: ", timestamp(), testname); + if (pass > 1) + fprintf(stdout, "too many operations succeeded (%d).\n", pass); + else + fprintf(stdout, "too many operations failed (%d).\n", fail); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, 1); + } + free(rc_vec); + } +} + +void simul_open(int shared) { + int fd; + char *filename; + + begin("setup"); + filename = create_files("simul_open", 0, shared); + end("setup"); + + /* All open the file simultaneously */ + begin("test"); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("open failed"); + } + end("test"); + + /* All close the file one at a time */ + begin("cleanup"); + Seq_begin(MPI_COMM_WORLD, throttle); + if (close(fd) == -1) { + FAIL("close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_open", shared); + end("cleanup"); +} + +void simul_close(int shared) { + int fd; + char *filename; + + begin("setup"); + filename = create_files("simul_close", 0, shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("open failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All close the file simultaneously */ + if (close(fd) == -1) { + FAIL("close failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_close", shared); + end("cleanup"); +} + +void simul_chdir(int shared) { + char cwd[1024]; + char *dirname; + + begin("setup"); + if (getcwd(cwd, 1024) == NULL) { + FAIL("init getcwd failed"); + } + dirname = create_dirs("simul_chdir", shared); + end("setup"); + + begin("test"); + /* All chdir to dirname */ + if (chdir(dirname) == -1) { + FAIL("chdir failed"); + } + end("test"); + + begin("cleanup"); + /* All chdir back to old cwd */ + if (chdir(cwd) == -1) { + FAIL("chdir back failed"); + } + MPI_Barrier(MPI_COMM_WORLD); + remove_dirs("simul_chdir", shared); + end("cleanup"); +} + +void simul_file_stat(int shared) { + char *filename; + struct stat buf; + + begin("setup"); + filename = create_files("simul_file_stat", 0, shared); + end("setup"); + + begin("test"); + /* All stat the file */ + if (stat(filename, &buf) == -1) { + FAIL("stat failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_file_stat", shared); + end("cleanup"); +} + +void simul_dir_stat(int shared) { + char *dirname; + struct stat buf; + + begin("setup"); + dirname = create_dirs("simul_dir_stat", shared); + end("setup"); + + begin("test"); + /* All stat the directory */ + if (stat(dirname, &buf) == -1) { + FAIL("stat failed"); + } + end("test"); + + begin("cleanup"); + remove_dirs("simul_dir_stat", shared); + end("cleanup"); +} + +void simul_readdir(int shared) { + DIR *dir; + char *dirname; + struct dirent *dptr; + + begin("setup"); + dirname = create_dirs("simul_readdir", shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the directory(ies) one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((dir = opendir(dirname)) == NULL) { + FAIL("init opendir failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All readdir the directory stream(s) */ + if ((dptr = readdir(dir)) == NULL) { + FAIL("readdir failed"); + } + end("test"); + + begin("cleanup"); + /* All close the directory(ies) one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (closedir(dir) == -1) { + FAIL("closedir failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_dirs("simul_readdir", shared); + end("cleanup"); +} + +void simul_statfs(int shared) { + char *filename; + struct statfs buf; + + begin("setup"); + filename = create_files("simul_statfs", 0, shared); + end("setup"); + + begin("test"); + /* All statfs the file(s) */ + if (statfs(filename, &buf) == -1) { + FAIL("statfs failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_statfs", shared); + end("cleanup"); +} + +void simul_lseek(int shared) { + int fd; + char *filename; + + begin("setup"); + filename = create_files("simul_lseek", 0, shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the file(s) one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("init open failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All lseek simultaneously */ + if (lseek(fd, 1024, SEEK_SET) == -1) { + FAIL("lseek failed"); + MPI_Abort(MPI_COMM_WORLD, 1); + } + end("test"); + + begin("cleanup"); + /* All close the file(s) one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (close(fd) == -1) { + FAIL("cleanup close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_lseek", shared); + end("cleanup"); +} + +void simul_read(int shared) { + int fd; + ssize_t fin; + char buf[1024]; + char *filename; + int i = 0; + int retry = 100; + + begin("setup"); + filename = create_files("simul_read", 1024, shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("init open failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All read simultaneously */ + for (i = 1024; (i > 0) && (retry > 0); i -= fin, retry--) { + if ((fin = read(fd, buf, (size_t)i)) == -1) { + FAIL("read failed"); + } + } + if( (retry == 0) && (i > 0) ) + FAIL("read exceeded retry count"); + end("test"); + + begin("cleanup"); + /* All close the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (close(fd) == -1) { + FAIL("cleanup close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_read", shared); + end("cleanup"); +} + +void simul_write(int shared) { + int fd; + ssize_t fin; + char *filename; + int i = 0; + int retry = 100; + + begin("setup"); + filename = create_files("simul_write", size * sizeof(int), shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the file and lseek one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("init open failed"); + } + if (lseek(fd, rank*sizeof(int), SEEK_SET) == -1) { + FAIL("init lseek failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All write simultaneously */ + for (i = sizeof(int); (i > 0) && (retry > 0); i -= fin, retry--) { + if ((fin = write(fd, &rank, (size_t)i)) == -1) { + FAIL("write failed"); + } + } + if( (retry == 0) && (i > 0) ) + FAIL("write exceeded retry count"); + end("test"); + + begin("cleanup"); + /* All close the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (close(fd) == -1) { + FAIL("cleanup close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_write", shared); + end("cleanup"); +} + +void simul_mkdir(int shared) { + int rc, i; + char dirname[MAX_FILENAME_LEN]; + + begin("setup"); + if (shared) + sprintf(dirname, "%s/simul_mkdir.0", testdir); + else + sprintf(dirname, "%s/simul_mkdir.%d", testdir, rank); + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/simul_mkdir.%d", testdir, i); + remove_file_or_dir(buf); + } + } + end("setup"); + + begin("test"); + /* All mkdir dirname */ + rc = mkdir(dirname, DIRMODE); + if (!shared) { + if (rc == -1) { + FAIL("mkdir failed"); + } + MPI_Barrier(MPI_COMM_WORLD); + } else { /* Only one should succeed */ + check_single_success("simul_mkdir", rc, -1); + } + end("test"); + + begin("cleanup"); + remove_dirs("simul_mkdir", shared); + end("cleanup"); +} + +void simul_rmdir(int shared) { + int rc; + char *dirname; + + begin("setup"); + dirname = create_dirs("simul_rmdir", shared); + MPI_Barrier(MPI_COMM_WORLD); + end("setup"); + + begin("test"); + /* All rmdir dirname */ + rc = rmdir(dirname); + if (!shared) { + if (rc == -1) { + FAIL("rmdir failed"); + } + MPI_Barrier(MPI_COMM_WORLD); + } else { /* Only one should succeed */ + check_single_success("simul_rmdir", rc, -1); + } + end("test"); + + begin("cleanup"); + end("cleanup"); +} + +void simul_creat(int shared) { + int fd, i; + char filename[1024]; + + begin("setup"); + if (shared) + sprintf(filename, "%s/simul_creat.0", testdir); + else + sprintf(filename, "%s/simul_creat.%d", testdir, rank); + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/simul_creat.%d", testdir, i); + remove_file_or_dir(buf); + } + } + end("setup"); + + begin("test"); + /* All create the files simultaneously */ + fd = creat(filename, FILEMODE); + if (fd == -1) { + FAIL("creat failed"); + } + end("test"); + + begin("cleanup"); + /* All close the files one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (close(fd) == -1) { + FAIL("close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_creat", shared); + end("cleanup"); +} + +void simul_unlink(int shared) { + int rc; + char *filename; + + begin("setup"); + filename = create_files("simul_unlink", 0, shared); + end("setup"); + + begin("test"); + /* All unlink the files simultaneously */ + rc = unlink(filename); + if (!shared) { + if (rc == -1) { + FAIL("unlink failed"); + } + } else { + check_single_success("simul_unlink", rc, -1); + } + end("test"); + + begin("cleanup"); + end("cleanup"); +} + +void simul_rename(int shared) { + int rc, i; + char *oldfilename; + char newfilename[1024]; + char *testname = "simul_rename"; + + begin("setup"); + oldfilename = create_files(testname, 0, shared); + sprintf(newfilename, "%s/%s_new.%d", testdir, testname, rank); + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/%s_new.%d", testdir, testname, i); + remove_file_or_dir(buf); + } + } + end("setup"); + + begin("test"); + /* All rename the files simultaneously */ + rc = rename(oldfilename, newfilename); + if (!shared) { + if (rc == -1) { + FAIL("stat failed"); + } + } else { + check_single_success(testname, rc, -1); + } + end("test"); + + begin("cleanup"); + if (rc == 0) { + if (unlink(newfilename) == -1) + FAIL("unlink failed"); + } + end("cleanup"); +} + +void simul_truncate(int shared) { + char *filename; + + begin("setup"); + filename = create_files("simul_truncate", 2048, shared); + end("setup"); + + begin("test"); + /* All truncate simultaneously */ + if (truncate(filename, 1024) == -1) { + FAIL("truncate failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_truncate", shared); + end("cleanup"); +} + +void simul_readlink(int shared) { + char *linkname; + char buf[1024]; + + begin("setup"); + linkname = create_symlinks("simul_readlink", shared); + end("setup"); + + begin("test"); + /* All read the symlink(s) simultaneously */ + if (readlink(linkname, buf, 1024) == -1) { + FAIL("readlink failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_readlink", shared); + end("cleanup"); +} + +void simul_symlink(int shared) { + int rc, i; + char linkname[MAX_FILENAME_LEN]; + char filename[MAX_FILENAME_LEN]; + + begin("setup"); + if (shared) + sprintf(linkname, "%s/simul_symlink.0", testdir); + else + sprintf(linkname, "%s/simul_symlink.%d", testdir, rank); + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/simul_symlink.%d", testdir, i); + remove_file_or_dir(buf); + } + } + sprintf(filename, "%s/simul_symlink_target", testdir); + end("setup"); + + begin("test"); + /* All create the symlinks simultaneously */ + rc = symlink(filename, linkname); + if (!shared) { + if (rc == -1) { + FAIL("symlink failed"); + } + } else { + check_single_success("simul_symlink", rc, -1); + } + end("test"); + + begin("cleanup"); + remove_files("simul_symlink", shared); + end("cleanup"); +} + +void simul_link_to_one(int shared) { + int rc, i; + char *filename; + char linkname[1024]; + + begin("setup"); + if (shared) + sprintf(linkname, "%s/simul_link.0", testdir); + else + sprintf(linkname, "%s/simul_link.%d", testdir, rank); + if (rank == 0) { + for (i = 0; i < (shared ? 1 : size); i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/simul_link.%d", testdir, i); + remove_file_or_dir(buf); + } + } + filename = create_files("simul_link_target", 0, SHARED); + end("setup"); + + begin("test"); + /* All create the hard links simultaneously */ + rc = link(filename, linkname); + if (!shared) { + if (rc == -1) { + FAIL("link failed"); + } + } else { + check_single_success("simul_link_to_one", rc, -1); + } + end("test"); + + begin("cleanup"); + remove_files("simul_link_target", SHARED); + remove_files("simul_link", shared); + end("cleanup"); +} + +void simul_link_to_many(int shared) { + char *filename; + char linkname[1024]; + int i; + + if (shared) { + if (verbose > 0 && rank == 0) + printf("%s:\tThis is just a place holder; no test is run here.\n", + timestamp()); + return; + } + begin("setup"); + filename = create_files("simul_link", 0, shared); + sprintf(linkname, "%s/simul_link_target.%d", testdir, rank); + if (rank == 0) { + for (i = 0; i < size; i++) { + char buf[MAX_FILENAME_LEN]; + sprintf(buf, "%s/simul_link_target.%d", testdir, i); + remove_file_or_dir(buf); + } + } + end("setup"); + + begin("test"); + /* All create the hard links simultaneously */ + if (link(filename, linkname) == -1) { + FAIL("link failed"); + } + end("test"); + + begin("cleanup"); + remove_files("simul_link", shared); + remove_files("simul_link_target", !SHARED); + end("cleanup"); +} + +void simul_fcntl_lock(int shared) { + int rc, fd; + char *filename; + struct flock sf_lock = { + .l_type = F_WRLCK, + .l_whence = SEEK_SET, + .l_start = 0, + .l_len = 0 + }; + struct flock sf_unlock = { + .l_type = F_UNLCK, + .l_whence = SEEK_SET, + .l_start = 0, + .l_len = 0 + }; + + begin("setup"); + filename = create_files("simul_fcntl", 0, shared); + MPI_Barrier(MPI_COMM_WORLD); + /* All open the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if ((fd = open(filename, O_RDWR)) == -1) { + FAIL("open failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + end("setup"); + + begin("test"); + /* All lock the file(s) simultaneously */ + rc = fcntl(fd, F_SETLK, &sf_lock); + if (!shared) { + if (rc == -1) { + if (errno == ENOSYS) { + if (rank == 0) { + fprintf(stdout, "WARNING: fcntl locking not supported.\n"); + fflush(stdout); + } + } else { + FAIL("fcntl lock failed"); + } + } + MPI_Barrier(MPI_COMM_WORLD); + } else { + int saved_errno = errno; + int *rc_vec, *er_vec, i; + int fail = 0; + int pass = 0; + int nosys = 0; + if (rank == 0) { + if ((rc_vec = (int *)malloc(sizeof(int)*size)) == NULL) + FAIL("malloc failed"); + if ((er_vec = (int *)malloc(sizeof(int)*size)) == NULL) + FAIL("malloc failed"); + } + MPI_Gather(&rc, 1, MPI_INT, rc_vec, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&saved_errno, 1, MPI_INT, er_vec, 1, MPI_INT, 0, + MPI_COMM_WORLD); + if (rank == 0) { + for (i = 0; i < size; i++) { + if (rc_vec[i] == -1) { + if (er_vec[i] == ENOSYS) { + nosys++; + } else if (er_vec[i] != EACCES && er_vec[i] != EAGAIN) { + errno = er_vec[i]; + FAIL("fcntl failed as expected, but with wrong errno"); + } + fail++; + } else { + pass++; + } + } + if (nosys == size) { + fprintf(stdout, "WARNING: fcntl locking not supported.\n"); + fflush(stdout); + } else if (!((pass == 1) && (fail == size-1))) { + fprintf(stdout, + "%s: FAILED in simul_fcntl_lock", timestamp()); + if (pass > 1) + fprintf(stdout, + "too many fcntl locks succeeded (%d).\n", pass); + else + fprintf(stdout, + "too many fcntl locks failed (%d).\n", fail); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, 1); + } + free(rc_vec); + free(er_vec); + } + } + end("test"); + + begin("cleanup"); + /* All close the file one at a time */ + Seq_begin(MPI_COMM_WORLD, throttle); + if (!shared || rank == 0) { + rc = fcntl(fd, F_SETLK, &sf_unlock); + if (rc == -1 && errno != ENOSYS) + FAIL("fcntl unlock failed"); + } + if (close(fd) == -1) { + FAIL("close failed"); + } + Seq_end(MPI_COMM_WORLD, throttle); + MPI_Barrier(MPI_COMM_WORLD); + remove_files("simul_fcntl", shared); + end("cleanup"); +} + +struct test { + char *name; + void (*function) (int); + int simul; /* Flag designating support for simultaneus mode */ + int indiv; /* Flag designating support for individual mode */ +}; + +static struct test testlist[] = { + {"open", simul_open}, + {"close", simul_close}, + {"file stat", simul_file_stat}, + {"lseek", simul_lseek}, + {"read", simul_read}, + {"write", simul_write}, + {"chdir", simul_chdir}, + {"directory stat", simul_dir_stat}, + {"statfs", simul_statfs}, + {"readdir", simul_readdir}, + {"mkdir", simul_mkdir}, + {"rmdir", simul_rmdir}, + {"unlink", simul_unlink}, + {"rename", simul_rename}, + {"creat", simul_creat}, + {"truncate", simul_truncate}, + {"symlink", simul_symlink}, + {"readlink", simul_readlink}, + {"link to one file", simul_link_to_one}, + {"link to a file per process", simul_link_to_many}, + {"fcntl locking", simul_fcntl_lock}, + {0} +}; + +/* Searches an array of ints for one int. A "-1" must mark the end of the + array. */ +int int_in_list(int item, int *list) { + int *ptr; + + if (list == NULL) + return 0; + ptr = list; + while (*ptr != -1) { + if (*ptr == item) + return 1; + ptr += 1; + } + return 0; +} + +/* Breaks string of comma-sperated ints into an array of ints */ +int *string_split(char *string) { + char *ptr; + char *tmp; + int excl_cnt = 1; + int *list; + int i; + + ptr = string; + while((tmp = strchr(ptr, ','))) { + ptr = tmp + 1; + excl_cnt++; + } + + list = (int *)malloc(sizeof(int) * (excl_cnt + 1)); + if (list == NULL) FAIL("malloc failed"); + + tmp = (strtok(string, ", ")); + if (tmp == NULL) FAIL("strtok failed"); + list[0] = atoi(tmp); + for (i = 1; i < excl_cnt; i++) { + tmp = (strtok(NULL, ", ")); + if (tmp == NULL) FAIL("strtok failed"); + list[i] = atoi(tmp); + } + list[i] = -1; + + return list; +} + +void print_help(int testcnt) { + int i; + + if (rank == 0) { + printf("simul-%s\n", version); + printf("Usage: simul [-h] -d [-f firsttest] [-l lasttest]\n"); + printf(" [-n #] [-N #] [-i \"4,7,13\"] [-e \"6,22\"] [-s]\n"); + printf(" [-v] [-V #]\n"); + printf("\t-h: prints this help message\n"); + printf("\t-d: the directory in which the tests will run\n"); + printf("\t-f: the number of the first test to run (default: 0)\n"); + printf("\t-l: the number of the last test to run (default: %d)\n", + (testcnt*2)-1); + printf("\t-i: comma-sperated list of tests to include\n"); + printf("\t-e: comma-sperated list of tests to exclude\n"); + printf("\t-s: single-step through every iteration of every test\n"); + printf("\t-n: repeat each test # times (default: 1)\n"); + printf("\t-N: repeat the entire set of tests # times (default: 1)\n"); + printf("\t-v: increases the verbositly level by 1\n"); + printf("\t-u: test with unifyfs\n"); + printf("\t-V: select a specific verbosity level\n"); + printf("\nThe available tests are:\n"); + for (i = 0; i < testcnt * 2; i++) { + printf("\tTest #%d: %s, %s mode.\n", i, + testlist[i%testcnt].name, + (i < testcnt) ? "shared" : "individual"); + } + } + + MPI_Initialized(&i); + if (i) MPI_Finalize(); + exit(0); +} + +int main(int argc, char **argv) { + int testcnt; + int first; + int last; + int i, j, k, c; + int *excl_list = NULL; + int *incl_list = NULL; + int test; + int singlestep = 0; + int iterations = 1; + int set_iterations = 1; + int unifyfs = 0; + int ret = 0; + char linebuf[80]; + + /* Check for -h parameter before MPI_Init so the simul binary can be + called directly, without, for instance, mpirun. */ + for (testcnt = 1; testlist[testcnt].name != 0; testcnt++) continue; + for (i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + print_help(testcnt); + } + } + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + if (rank == 0) { + printf("Simul is running with %d process(es)\n", size); + fflush(stdout); + } + + first = 0; + last = testcnt * 2; + + /* Parse command line options */ + while (1) { + c = getopt(argc, argv, "d:e:f:hi:l:n:N:suvV:"); + if (c == -1) + break; + + switch (c) { + case 'd': + testdir = optarg; + break; + case 'e': + excl_list = string_split(optarg); + break; + case 'f': + first = atoi(optarg); + if (first >= last) { + printf("Invalid parameter, firsttest must be <= lasttest\n"); + MPI_Abort(MPI_COMM_WORLD, 2); + } + break; + case 'h': + print_help(testcnt); + break; + case 'i': + incl_list = string_split(optarg); + break; + case 'l': + last = atoi(optarg)+1; + if (last <= first) { + printf("Invalid parameter, lasttest must be >= firsttest\n"); + MPI_Abort(MPI_COMM_WORLD, 2); + } + break; + case 'n': + iterations = atoi(optarg); + break; + case 'N': + set_iterations = atoi(optarg); + break; + case 's': + singlestep = 1; + break; + case 'u': + unifyfs = 1; + break; + case 'v': + verbose += 1; + break; + case 'V': + verbose = atoi(optarg); + break; + } + } + + /* mount the unifyfs and use the testdir as the mountpoint. + * if testdir is not specified, use '/unifyfs.' */ + if (unifyfs) { + int ret = 0; + + if (!testdir) { + testdir = "/unifyfs"; + } + ret = unifyfs_mount(testdir, rank, size, 0); + if (ret && rank == 0) { + printf("unifyfs_mount failed (ret=%d)\n", ret); + MPI_Abort(MPI_COMM_WORLD, 2); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (testdir == NULL && rank == 0) { + printf("Please specify a test directory! (\"simul -h\" for help)\n"); + MPI_Abort(MPI_COMM_WORLD, 2); + } + + if (gethostname(hostname, 1024) == -1) { + perror("gethostname"); + MPI_Abort(MPI_COMM_WORLD, 2); + } + + /* If a list of tests was not specified with the -i option, then use + the first and last number to build a range of included tests. */ + if (incl_list == NULL) { + incl_list = (int *)malloc(sizeof(int) * (2+last-first)); + for (i = 0; i < last-first; i++) { + incl_list[i] = i + first; + } + incl_list[i] = -1; + } + + /* Run the tests */ + for (k = 0; k < set_iterations; k++) { + if ((rank == 0) && (set_iterations > 1)) + printf("%s: Set iteration %d\n", timestamp(), k); + for (i = 0; ; ++i) { + test = incl_list[i]; + if (test == -1) + break; + if (!int_in_list(test, excl_list)) { + for (j = 0; j < iterations; j++) { + if (singlestep) { + if (rank == 0) + printf("%s: Hit to run test #%d(iter %d): %s, %s mode.\n", + timestamp(), test, j, + testlist[test%testcnt].name, + (test < testcnt) ? "shared" : "individual"); + fgets(linebuf, 80, stdin); + } + if (rank == 0) { + printf("%s: Running test #%d(iter %d): %s, %s mode.\n", + timestamp(), test, j, testlist[test%testcnt].name, + (test < testcnt) ? "shared" : "individual"); + fflush(stdout); + } + testlist[test%testcnt].function((test < testcnt) ? SHARED : !SHARED); + MPI_Barrier(MPI_COMM_WORLD); + } + } + } + } + + if (rank == 0) printf("%s: All tests passed!\n", timestamp()); + + /* unmount unifyfs */ + if (unifyfs) { + unifyfs_unmount(); + } + + MPI_Finalize(); + exit(0); +} From bf5a26127b377f2f29016031c66af4fa322207cf Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Wed, 1 Apr 2020 16:21:26 -0400 Subject: [PATCH 088/168] improve server management of app/client state Application management * app_id derived from mountpoint prefix (using same method used for gfid calculation) * app_config structure holds all state for a given client * app_configs[] - simple array of app_config structure pointers (replaces app_config_list arraylist) - added methods for adding/getting structure for given app_id Client management * client_id assigned by server during mount rpc - can still be passed explicitly for re-attach * app_client structure holds all state for a given client * each app_config has an array of clients (app_client*) - added methods for adding/getting/removing clients Miscellaneous code cleanup * static functions should not use "unifyfs_" prefix, only ones used by external sources * don't use local function variables that shadow global variables * renamed max_recs_per_slice to meta_slice_sz --- client/src/margo_client.c | 90 ++- client/src/margo_client.h | 5 +- client/src/unifyfs-internal.h | 13 +- client/src/unifyfs.c | 669 +++++++++--------- common/src/unifyfs_client_rpcs.h | 35 +- common/src/unifyfs_const.h | 9 +- docs/add-rpcs.rst | 29 +- server/src/Makefile.am | 2 +- server/src/margo_server.c | 16 + server/src/unifyfs_cmd_handler.c | 243 +++---- server/src/unifyfs_global.h | 125 ++-- server/src/unifyfs_metadata.c | 4 +- server/src/unifyfs_metadata.h | 7 + server/src/unifyfs_request_manager.c | 149 ++-- server/src/unifyfs_request_manager.h | 8 +- .../src/{unifyfs_init.c => unifyfs_server.c} | 380 ++++++++-- server/src/unifyfs_service_manager.c | 46 +- 17 files changed, 1046 insertions(+), 784 deletions(-) rename server/src/{unifyfs_init.c => unifyfs_server.c} (62%) diff --git a/client/src/margo_client.c b/client/src/margo_client.c index f14570abd..ec3d9f5ec 100644 --- a/client/src/margo_client.c +++ b/client/src/margo_client.c @@ -15,6 +15,11 @@ static void register_client_rpcs(client_rpc_context_t* ctx) /* shorter name for our margo instance id */ margo_instance_id mid = ctx->mid; + ctx->rpcs.attach_id = MARGO_REGISTER(mid, "unifyfs_attach_rpc", + unifyfs_attach_in_t, + unifyfs_attach_out_t, + NULL); + ctx->rpcs.mount_id = MARGO_REGISTER(mid, "unifyfs_mount_rpc", unifyfs_mount_in_t, unifyfs_mount_out_t, @@ -182,8 +187,7 @@ int unifyfs_client_rpc_finalize(void) /* shut down margo */ margo_finalize(ctx->mid); - /* free memory allocated for context structure, - * and set caller's pointer to NULL */ + /* free memory allocated for context structure */ free(ctx->client_addr_str); free(ctx); } @@ -205,7 +209,43 @@ static hg_handle_t create_handle(hg_id_t id) return handle; } -/* invokes the mount rpc function by calling unifyfs_sync_to_del */ +/* invokes the attach rpc function */ +int invoke_client_attach_rpc(void) +{ + /* check that we have initialized margo */ + if (NULL == client_rpc_context) { + return UNIFYFS_FAILURE; + } + + /* get handle to rpc function */ + hg_handle_t handle = create_handle(client_rpc_context->rpcs.attach_id); + + /* fill in input struct */ + unifyfs_attach_in_t in; + fill_client_attach_info(&in); + + /* call rpc function */ + LOGDBG("invoking the attach rpc function in client"); + hg_return_t hret = margo_forward(handle, &in); + assert(hret == HG_SUCCESS); + + /* free memory on input struct */ + free((void*)in.logio_spill_dir); + + /* decode response */ + unifyfs_attach_out_t out; + hret = margo_get_output(handle, &out); + assert(hret == HG_SUCCESS); + int32_t ret = out.ret; + LOGDBG("Got response ret=%" PRIi32, ret); + + /* free resources */ + margo_free_output(handle, &out); + margo_destroy(handle); + return (int)ret; +} + +/* invokes the mount rpc function */ int invoke_client_mount_rpc(void) { /* check that we have initialized margo */ @@ -229,7 +269,7 @@ int invoke_client_mount_rpc(void) assert(hret == HG_SUCCESS); /* free memory on input struct */ - free((void*)in.external_spill_dir); + free((void*)in.mount_prefix); free((void*)in.client_addr_str); /* decode response */ @@ -240,9 +280,17 @@ int invoke_client_mount_rpc(void) LOGDBG("Got response ret=%" PRIi32, ret); /* get slice size for write index key/value store */ - unifyfs_key_slice_range = out.max_recs_per_slice; + unifyfs_key_slice_range = out.meta_slice_sz; LOGDBG("set unifyfs_key_slice_range=%zu", unifyfs_key_slice_range); + /* get assigned client id, and verify app_id */ + unifyfs_client_id = (int) out.client_id; + int srvr_app_id = (int) out.app_id; + if (unifyfs_app_id != srvr_app_id) { + LOGWARN("mismatch on app_id - using %d, server returned %d", + unifyfs_app_id, srvr_app_id); + } + /* free resources */ margo_free_output(handle, &out); margo_destroy(handle); @@ -262,8 +310,8 @@ int invoke_client_unmount_rpc(void) /* fill in input struct */ unifyfs_unmount_in_t in; - in.app_id = (int32_t) app_id; - in.client_id = (int32_t) local_rank_idx; + in.app_id = (int32_t) unifyfs_app_id; + in.client_id = (int32_t) unifyfs_client_id; /* call rpc function */ LOGDBG("invoking the unmount rpc function in client"); @@ -397,8 +445,8 @@ int invoke_client_filesize_rpc(int gfid, size_t* outsize) /* fill in input struct */ unifyfs_filesize_in_t in; - in.app_id = (int32_t) app_id; - in.client_id = (int32_t) local_rank_idx; + in.app_id = (int32_t) unifyfs_app_id; + in.client_id = (int32_t) unifyfs_client_id; in.gfid = (int32_t) gfid; /* call rpc function */ @@ -435,8 +483,8 @@ int invoke_client_truncate_rpc(int gfid, size_t filesize) /* fill in input struct */ unifyfs_truncate_in_t in; - in.app_id = (int32_t) app_id; - in.client_id = (int32_t) local_rank_idx; + in.app_id = (int32_t) unifyfs_app_id; + in.client_id = (int32_t) unifyfs_client_id; in.gfid = (int32_t) gfid; in.filesize = (hg_size_t) filesize; @@ -471,8 +519,8 @@ int invoke_client_unlink_rpc(int gfid) /* fill in input struct */ unifyfs_unlink_in_t in; - in.app_id = (int32_t) app_id; - in.client_id = (int32_t) local_rank_idx; + in.app_id = (int32_t) unifyfs_app_id; + in.client_id = (int32_t) unifyfs_client_id; in.gfid = (int32_t) gfid; /* call rpc function */ @@ -506,8 +554,8 @@ int invoke_client_laminate_rpc(int gfid) /* fill in input struct */ unifyfs_unlink_in_t in; - in.app_id = (int32_t) app_id; - in.client_id = (int32_t) local_rank_idx; + in.app_id = (int32_t) unifyfs_app_id; + in.client_id = (int32_t) unifyfs_client_id; in.gfid = (int32_t) gfid; /* call rpc function */ @@ -541,8 +589,8 @@ int invoke_client_sync_rpc(void) /* fill in input struct */ unifyfs_sync_in_t in; - in.app_id = (int32_t) app_id; - in.client_id = (int32_t) local_rank_idx; + in.app_id = (int32_t) unifyfs_app_id; + in.client_id = (int32_t) unifyfs_client_id; /* call rpc function */ LOGDBG("invoking the sync rpc function in client"); @@ -575,8 +623,8 @@ int invoke_client_read_rpc(int gfid, size_t offset, size_t length) /* fill in input struct */ unifyfs_read_in_t in; - in.app_id = (int32_t) app_id; - in.client_id = (int32_t) local_rank_idx; + in.app_id = (int32_t) unifyfs_app_id; + in.client_id = (int32_t) unifyfs_client_id; in.gfid = (int32_t) gfid; in.offset = (hg_size_t) offset; in.length = (hg_size_t) length; @@ -617,8 +665,8 @@ int invoke_client_mread_rpc(int read_count, size_t size, void* buffer) assert(hret == HG_SUCCESS); /* fill in input struct */ - in.app_id = (int32_t) app_id; - in.client_id = (int32_t) local_rank_idx; + in.app_id = (int32_t) unifyfs_app_id; + in.client_id = (int32_t) unifyfs_client_id; in.read_count = (int32_t) read_count; in.bulk_size = (hg_size_t) size; diff --git a/client/src/margo_client.h b/client/src/margo_client.h index 3470b4c1a..0d2139603 100644 --- a/client/src/margo_client.h +++ b/client/src/margo_client.h @@ -10,6 +10,7 @@ #include "unifyfs_client_rpcs.h" typedef struct ClientRpcIds { + hg_id_t attach_id; hg_id_t mount_id; hg_id_t unmount_id; hg_id_t metaset_id; @@ -36,8 +37,10 @@ int unifyfs_client_rpc_init(void); int unifyfs_client_rpc_finalize(void); -void fill_client_mount_info(unifyfs_mount_in_t* in); +void fill_client_attach_info(unifyfs_attach_in_t* in); +int invoke_client_attach_rpc(void); +void fill_client_mount_info(unifyfs_mount_in_t* in); int invoke_client_mount_rpc(void); int invoke_client_unmount_rpc(void); diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index cd1f1fd98..6f64f6a28 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -258,15 +258,6 @@ enum flock_enum { enum {FILE_STORAGE_NULL = 0, FILE_STORAGE_LOGIO}; -/* TODO: make this an enum */ -#define CHUNK_LOCATION_NULL 0 -#define CHUNK_LOCATION_MEMFS 1 -#define CHUNK_LOCATION_SPILLOVER 2 - -typedef struct { - int location; /* CHUNK_LOCATION type */ - off_t id; /* physical id of chunk in its respective storage */ -} unifyfs_chunkmeta_t; typedef struct { off_t global_size; /* Global size of the file */ @@ -281,7 +272,6 @@ typedef struct { int needs_sync; /* have unsynced writes */ off_t chunks; /* number of chunks allocated to file */ - off_t chunkmeta_idx; /* starting index in unifyfs_chunkmeta */ int is_laminated; /* Is this file laminated */ uint32_t mode; /* st_mode bits. This has file * permission info and will tell you if this @@ -343,7 +333,8 @@ extern shm_context* shm_recv_ctx; /* log-based I/O context */ extern logio_context* logio_ctx; -extern int app_id; +extern int unifyfs_app_id; +extern int unifyfs_client_id; extern size_t unifyfs_key_slice_range; /* ------------------------------- diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 43776bf4c..7a3260c26 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -74,18 +74,18 @@ unsigned long unifyfs_max_index_entries; /* max metadata log entries */ unsigned long unifyfs_segment_count; int global_rank_cnt; /* count of world ranks */ -int local_rank_cnt; -int local_rank_idx; +int local_rank_cnt; /* count of app ranks on local host */ +int local_rank_idx; /* index within local ranks */ +int client_rank; /* client-provided rank (for debugging) */ int local_del_cnt = 1; /* count of local servers */ /* shared memory buffer to transfer read replies * from server to client */ -static size_t shm_recv_size = UNIFYFS_DATA_RECV_SIZE; shm_context* shm_recv_ctx; // = NULL -int client_rank; -int app_id; +int unifyfs_app_id; +int unifyfs_client_id; size_t unifyfs_key_slice_range; static int unifyfs_use_single_shm = 0; @@ -484,7 +484,7 @@ int unifyfs_fd_is_laminated(int fd) * --------------------------------------- */ /* allocate and initialize data management resource for file */ -static int unifyfs_fid_store_alloc(int fid) +static int fid_store_alloc(int fid) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); @@ -496,7 +496,7 @@ static int unifyfs_fid_store_alloc(int fid) } /* free data management resource for file */ -static int unifyfs_fid_store_free(int fid) +static int fid_store_free(int fid) { /* get meta data for this file */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); @@ -517,9 +517,9 @@ static int unifyfs_fid_store_free(int fid) return UNIFYFS_SUCCESS; } -/* --------------------------------------- +/* ======================================= * Operations on file ids - * --------------------------------------- */ + * ======================================= */ /* checks to see if fid is a directory * returns 1 for yes @@ -1116,7 +1116,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } /* initialize local storage for this file */ - ret = unifyfs_fid_store_alloc(fid); + ret = fid_store_alloc(fid); if (ret != UNIFYFS_SUCCESS) { LOGERR("failed to allocate storage space for file %s (fid=%d)", path, fid); @@ -1171,7 +1171,7 @@ int unifyfs_fid_open(const char* path, int flags, mode_t mode, int* outfid, } /* initialize the storage for the file */ - int store_rc = unifyfs_fid_store_alloc(fid); + int store_rc = fid_store_alloc(fid); if (store_rc != UNIFYFS_SUCCESS) { LOGERR("Failed to create storage for file %s", path); return EIO; @@ -1221,7 +1221,7 @@ int unifyfs_fid_unlink(int fid) } /* finalize the storage we're using for this file */ - rc = unifyfs_fid_store_free(fid); + rc = fid_store_free(fid); if (rc != UNIFYFS_SUCCESS) { /* released strorage for file, but failed to release * structures tracking storage, again bail out to keep @@ -1247,9 +1247,13 @@ int unifyfs_fid_unlink(int fid) return UNIFYFS_SUCCESS; } -/* --------------------------------------- - * Operations to mount file system - * --------------------------------------- */ +/* ======================================= + * Operations to mount/unmount file system + * ======================================= */ + +/* ------------- + * static APIs + * ------------- */ /* The super block is a region of shared memory that is used to * persist file system data. It contains both room for data @@ -1290,8 +1294,8 @@ int unifyfs_fid_unlink(int fid) /* compute memory size of superblock in bytes, * critical to keep this consistent with - * unifyfs_init_pointers */ -static size_t unifyfs_superblock_size(void) + * init_superblock_pointers */ +static size_t get_superblock_size(void) { size_t sb_size = 0; @@ -1330,7 +1334,7 @@ char* next_page_align(char* ptr) } /* initialize our global pointers into the given superblock */ -static void* unifyfs_init_pointers(void* superblock) +static void init_superblock_pointers(void* superblock) { char* ptr = (char*)superblock; @@ -1364,12 +1368,10 @@ static void* unifyfs_init_pointers(void* superblock) if (ptr_size > shm_super_ctx->size) { LOGERR("Data structures in superblock extend beyond its size"); } - - return ptr; } /* initialize data structures for first use */ -static int unifyfs_init_structures() +static int init_superblock_structures(void) { int i; for (i = 0; i < unifyfs_max_files; i++) { @@ -1390,12 +1392,12 @@ static int unifyfs_init_structures() /* create superblock of specified size and name, or attach to existing * block if available */ -static int unifyfs_superblock_shmget(size_t super_sz) +static int init_superblock_shm(size_t super_sz) { char shm_name[SHMEM_NAME_LEN] = {0}; /* attach shmem region for client's superblock */ - sprintf(shm_name, SHMEM_SUPER_FMTSTR, app_id, local_rank_idx); + sprintf(shm_name, SHMEM_SUPER_FMTSTR, unifyfs_app_id, unifyfs_client_id); shm_context* shm_ctx = unifyfs_shm_alloc(shm_name, super_sz); if (NULL == shm_ctx) { LOGERR("Failed to attach to shmem superblock region %s", shm_name); @@ -1405,229 +1407,32 @@ static int unifyfs_superblock_shmget(size_t super_sz) /* init our global variables to point to spots in superblock */ void* addr = shm_ctx->addr; - unifyfs_init_pointers(addr); + init_superblock_pointers(addr); /* initialize structures in superblock if it's newly allocated, * we depend on shm_open setting all bytes to 0 to know that * it is not initialized */ - int32_t initialized = *(int32_t*)addr; + uint32_t initialized = *(uint32_t*)addr; if (initialized == 0) { /* not yet initialized, so initialize values within superblock */ - unifyfs_init_structures(); + init_superblock_structures(); /* superblock structure has been initialized, * so set flag to indicate that fact */ - *(int32_t*)addr = 0xDEADBEEF; + *(uint32_t*)addr = (uint32_t)0xDEADBEEF; } /* return starting memory address of super block */ return UNIFYFS_SUCCESS; } -static int unifyfs_init(int rank) -{ - int rc; - int i; - bool b; - long l; - unsigned long long bits; - char* cfgval; - - if (!unifyfs_initialized) { - /* unifyfs default log level is LOG_ERR */ - cfgval = client_cfg.log_verbosity; - if (cfgval != NULL) { - rc = configurator_int_val(cfgval, &l); - if (rc == 0) { - unifyfs_set_log_level((unifyfs_log_level_t)l); - } - } - -#ifdef UNIFYFS_GOTCHA - /* insert our I/O wrappers using gotcha */ - enum gotcha_error_t result; - result = gotcha_wrap(wrap_unifyfs_list, GOTCHA_NFUNCS, "unifyfs"); - if (result != GOTCHA_SUCCESS) { - LOGERR("gotcha_wrap returned %d", (int) result); - } - - /* check for an errors when registering functions with gotcha */ - for (i = 0; i < GOTCHA_NFUNCS; i++) { - if (*(void**)(wrap_unifyfs_list[i].function_address_pointer) == 0) { - LOGERR("This function name failed to be wrapped: %s", - wrap_unifyfs_list[i].name); - - } - } -#endif - - /* as a hack to support fgetpos/fsetpos, we store the value of - * a void* in an fpos_t so check that there's room and at least - * print a message if this won't work */ - if (sizeof(fpos_t) < sizeof(void*)) { - LOGERR("fgetpos/fsetpos will not work correctly"); - unifyfs_fpos_enabled = 0; - } - - /* look up page size for buffer alignment */ - unifyfs_page_size = getpagesize(); - - /* compute min and max off_t values */ - bits = sizeof(off_t) * 8; - unifyfs_max_offt = (off_t)((1ULL << (bits - 1ULL)) - 1ULL); - unifyfs_min_offt = (off_t)(-(1ULL << (bits - 1ULL))); - - /* compute min and max long values */ - unifyfs_max_long = LONG_MAX; - unifyfs_min_long = LONG_MIN; - - /* determine max number of files to store in file system */ - unifyfs_max_files = UNIFYFS_MAX_FILES; - cfgval = client_cfg.client_max_files; - if (cfgval != NULL) { - rc = configurator_int_val(cfgval, &l); - if (rc == 0) { - unifyfs_max_files = (int)l; - } - } - - /* Determine if we should flatten writes or not */ - unifyfs_flatten_writes = 1; - cfgval = client_cfg.client_flatten_writes; - if (cfgval != NULL) { - rc = configurator_bool_val(cfgval, &b); - if (rc == 0) { - unifyfs_flatten_writes = (bool)b; - } - } - - /* Determine if we should track all write extents and use them - * to service read requests if all data is local */ - unifyfs_local_extents = 0; - cfgval = client_cfg.client_local_extents; - if (cfgval != NULL) { - rc = configurator_bool_val(cfgval, &b); - if (rc == 0) { - unifyfs_local_extents = (bool)b; - } - } - - /* define size of buffer used to cache key/value pairs for - * data offsets before passing them to the server */ - unifyfs_index_buf_size = UNIFYFS_INDEX_BUF_SIZE; - cfgval = client_cfg.client_write_index_size; - if (cfgval != NULL) { - rc = configurator_int_val(cfgval, &l); - if (rc == 0) { - unifyfs_index_buf_size = (size_t)l; - } - } - unifyfs_max_index_entries = - unifyfs_index_buf_size / sizeof(unifyfs_index_t); - - /* record the max fd for the system */ - /* RLIMIT_NOFILE specifies a value one greater than the maximum - * file descriptor number that can be opened by this process */ - struct rlimit r_limit; - - if (getrlimit(RLIMIT_NOFILE, &r_limit) < 0) { - LOGERR("getrlimit failed: errno=%d (%s)", errno, strerror(errno)); - return UNIFYFS_FAILURE; - } - unifyfs_fd_limit = r_limit.rlim_cur; - LOGDBG("FD limit for system = %ld", unifyfs_fd_limit); - - /* initialize file descriptor structures */ - int num_fds = UNIFYFS_MAX_FILEDESCS; - for (i = 0; i < num_fds; i++) { - unifyfs_fd_init(i); - } - - /* initialize file stream structures */ - int num_streams = UNIFYFS_MAX_FILEDESCS; - for (i = 0; i < num_streams; i++) { - unifyfs_stream_init(i); - } - - /* initialize directory stream structures */ - int num_dirstreams = UNIFYFS_MAX_FILEDESCS; - for (i = 0; i < num_dirstreams; i++) { - unifyfs_dirstream_init(i); - } - - /* initialize stack of free fd values */ - size_t free_fd_size = unifyfs_stack_bytes(num_fds); - unifyfs_fd_stack = malloc(free_fd_size); - unifyfs_stack_init(unifyfs_fd_stack, num_fds); - - /* initialize stack of free stream values */ - size_t free_stream_size = unifyfs_stack_bytes(num_streams); - unifyfs_stream_stack = malloc(free_stream_size); - unifyfs_stack_init(unifyfs_stream_stack, num_streams); - - /* initialize stack of free directory stream values */ - size_t free_dirstream_size = unifyfs_stack_bytes(num_dirstreams); - unifyfs_dirstream_stack = malloc(free_dirstream_size); - unifyfs_stack_init(unifyfs_dirstream_stack, num_dirstreams); - - /* determine the size of the superblock */ - size_t shm_super_size = unifyfs_superblock_size(); - - /* get a superblock of shared memory and initialize our - * global variables for this block */ - rc = unifyfs_superblock_shmget(shm_super_size); - if (rc != UNIFYFS_SUCCESS) { - LOGERR("unifyfs_superblock_shmget() failed"); - return rc; - } - - /* initialize log-based I/O context */ - rc = unifyfs_logio_init_client(app_id, local_rank_idx, &client_cfg, - &logio_ctx); - if (rc != UNIFYFS_SUCCESS) { - LOGERR("failed to initialize log-based I/O (rc = %s)", - unifyfs_rc_enum_str(rc)); - return rc; - } - - /* remember that we've now initialized the library */ - unifyfs_initialized = 1; - } - - return UNIFYFS_SUCCESS; -} - -/* --------------------------------------- - * APIs exposed to external libraries - * --------------------------------------- */ - -/* Fill mount rpc input struct with client-side context info */ -void fill_client_mount_info(unifyfs_mount_in_t* in) -{ - size_t meta_offset = (char*)unifyfs_indices.ptr_num_entries - - (char*)shm_super_ctx->addr; - size_t meta_size = unifyfs_max_index_entries - * sizeof(unifyfs_index_t); - - in->app_id = app_id; - in->client_id = local_rank_idx; - in->dbg_rank = client_rank; - in->num_procs_per_node = local_rank_cnt; - in->recv_buf_sz = shm_recv_size; - in->superblock_sz = shm_super_ctx->size; - in->meta_offset = meta_offset; - in->meta_size = meta_size; - in->logio_mem_size = logio_ctx->shmem->size; - in->logio_spill_size = logio_ctx->spill_sz; - in->external_spill_dir = strdup(client_cfg.logio_spill_dir); -} - /** * Initialize the shared recv memory buffer to receive data from the delegators */ -static int unifyfs_init_recv_shm(int local_rank_idx, int app_id) +static int init_recv_shm(void) { char shm_recv_name[SHMEM_NAME_LEN] = {0}; + size_t shm_recv_size = UNIFYFS_DATA_RECV_SIZE; /* get size of shared memory region from configuration */ char* cfgval = client_cfg.client_recv_data_size; @@ -1635,13 +1440,13 @@ static int unifyfs_init_recv_shm(int local_rank_idx, int app_id) long l; int rc = configurator_int_val(cfgval, &l); if (rc == 0) { - shm_recv_size = l; + shm_recv_size = (size_t) l; } } /* define file name to shared memory file */ snprintf(shm_recv_name, sizeof(shm_recv_name), - SHMEM_DATA_FMTSTR, app_id, local_rank_idx); + SHMEM_DATA_FMTSTR, unifyfs_app_id, unifyfs_client_id); /* allocate memory for shared memory receive buffer */ shm_recv_ctx = unifyfs_shm_alloc(shm_recv_name, shm_recv_size); @@ -1814,6 +1619,260 @@ static int CountTasksPerNode(int rank, int numTasks) return 0; } +static int unifyfs_init(void) +{ + int rc; + int i; + bool b; + long l; + unsigned long long bits; + char* cfgval; + + if (!unifyfs_initialized) { + +#ifdef UNIFYFS_GOTCHA + /* insert our I/O wrappers using gotcha */ + enum gotcha_error_t result; + result = gotcha_wrap(wrap_unifyfs_list, GOTCHA_NFUNCS, "unifyfs"); + if (result != GOTCHA_SUCCESS) { + LOGERR("gotcha_wrap returned %d", (int) result); + } + + /* check for an errors when registering functions with gotcha */ + for (i = 0; i < GOTCHA_NFUNCS; i++) { + if (*(void**)(wrap_unifyfs_list[i].function_address_pointer) == 0) { + LOGERR("This function name failed to be wrapped: %s", + wrap_unifyfs_list[i].name); + + } + } +#endif + + /* as a hack to support fgetpos/fsetpos, we store the value of + * a void* in an fpos_t so check that there's room and at least + * print a message if this won't work */ + if (sizeof(fpos_t) < sizeof(void*)) { + LOGERR("fgetpos/fsetpos will not work correctly"); + unifyfs_fpos_enabled = 0; + } + + /* look up page size for buffer alignment */ + unifyfs_page_size = getpagesize(); + + /* compute min and max off_t values */ + bits = sizeof(off_t) * 8; + unifyfs_max_offt = (off_t)((1ULL << (bits - 1ULL)) - 1ULL); + unifyfs_min_offt = (off_t)(-(1ULL << (bits - 1ULL))); + + /* compute min and max long values */ + unifyfs_max_long = LONG_MAX; + unifyfs_min_long = LONG_MIN; + + /* determine max number of files to store in file system */ + unifyfs_max_files = UNIFYFS_MAX_FILES; + cfgval = client_cfg.client_max_files; + if (cfgval != NULL) { + rc = configurator_int_val(cfgval, &l); + if (rc == 0) { + unifyfs_max_files = (int)l; + } + } + + /* Determine if we should flatten writes or not */ + unifyfs_flatten_writes = 1; + cfgval = client_cfg.client_flatten_writes; + if (cfgval != NULL) { + rc = configurator_bool_val(cfgval, &b); + if (rc == 0) { + unifyfs_flatten_writes = (bool)b; + } + } + + /* Determine if we should track all write extents and use them + * to service read requests if all data is local */ + unifyfs_local_extents = 0; + cfgval = client_cfg.client_local_extents; + if (cfgval != NULL) { + rc = configurator_bool_val(cfgval, &b); + if (rc == 0) { + unifyfs_local_extents = (bool)b; + } + } + + /* define size of buffer used to cache key/value pairs for + * data offsets before passing them to the server */ + unifyfs_index_buf_size = UNIFYFS_INDEX_BUF_SIZE; + cfgval = client_cfg.client_write_index_size; + if (cfgval != NULL) { + rc = configurator_int_val(cfgval, &l); + if (rc == 0) { + unifyfs_index_buf_size = (size_t)l; + } + } + unifyfs_max_index_entries = + unifyfs_index_buf_size / sizeof(unifyfs_index_t); + + /* record the max fd for the system */ + /* RLIMIT_NOFILE specifies a value one greater than the maximum + * file descriptor number that can be opened by this process */ + struct rlimit r_limit; + + if (getrlimit(RLIMIT_NOFILE, &r_limit) < 0) { + LOGERR("getrlimit failed: errno=%d (%s)", errno, strerror(errno)); + return UNIFYFS_FAILURE; + } + unifyfs_fd_limit = r_limit.rlim_cur; + LOGDBG("FD limit for system = %ld", unifyfs_fd_limit); + + /* initialize file descriptor structures */ + int num_fds = UNIFYFS_MAX_FILEDESCS; + for (i = 0; i < num_fds; i++) { + unifyfs_fd_init(i); + } + + /* initialize file stream structures */ + int num_streams = UNIFYFS_MAX_FILEDESCS; + for (i = 0; i < num_streams; i++) { + unifyfs_stream_init(i); + } + + /* initialize directory stream structures */ + int num_dirstreams = UNIFYFS_MAX_FILEDESCS; + for (i = 0; i < num_dirstreams; i++) { + unifyfs_dirstream_init(i); + } + + /* initialize stack of free fd values */ + size_t free_fd_size = unifyfs_stack_bytes(num_fds); + unifyfs_fd_stack = malloc(free_fd_size); + unifyfs_stack_init(unifyfs_fd_stack, num_fds); + + /* initialize stack of free stream values */ + size_t free_stream_size = unifyfs_stack_bytes(num_streams); + unifyfs_stream_stack = malloc(free_stream_size); + unifyfs_stack_init(unifyfs_stream_stack, num_streams); + + /* initialize stack of free directory stream values */ + size_t free_dirstream_size = unifyfs_stack_bytes(num_dirstreams); + unifyfs_dirstream_stack = malloc(free_dirstream_size); + unifyfs_stack_init(unifyfs_dirstream_stack, num_dirstreams); + + /* determine the size of the superblock */ + size_t shm_super_size = get_superblock_size(); + + /* get a superblock of shared memory and initialize our + * global variables for this block */ + rc = init_superblock_shm(shm_super_size); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to initialize superblock shmem"); + return rc; + } + + /* create shared memory region for holding data for read replies */ + rc = init_recv_shm(); + if (rc < 0) { + LOGERR("failed to initialize data recv shmem"); + return UNIFYFS_FAILURE; + } + + /* initialize log-based I/O context */ + rc = unifyfs_logio_init_client(unifyfs_app_id, unifyfs_client_id, + &client_cfg, &logio_ctx); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to initialize log-based I/O (rc = %s)", + unifyfs_rc_enum_str(rc)); + return rc; + } + + /* remember that we've now initialized the library */ + unifyfs_initialized = 1; + } + + return UNIFYFS_SUCCESS; +} + +/* free resources allocated during unifyfs_init(). + * generally, we do this in reverse order with respect to + * how things were initialized */ +static int unifyfs_finalize(void) +{ + int rc = UNIFYFS_SUCCESS; + + if (!unifyfs_initialized) { + /* not initialized yet, so we shouldn't call finalize */ + return UNIFYFS_FAILURE; + } + + /* close spillover files */ + unifyfs_logio_close(logio_ctx); + if (unifyfs_spillmetablock != -1) { + close(unifyfs_spillmetablock); + unifyfs_spillmetablock = -1; + } + + /* detach from superblock shmem, but don't unlink the file so that + * a later client can reattach. */ + unifyfs_shm_free(&shm_super_ctx); + + /* unlink and detach from data receive shmem */ + unifyfs_shm_unlink(shm_recv_ctx); + unifyfs_shm_free(&shm_recv_ctx); + + /* free directory stream stack */ + if (unifyfs_dirstream_stack != NULL) { + free(unifyfs_dirstream_stack); + unifyfs_dirstream_stack = NULL; + } + + /* free file stream stack */ + if (unifyfs_stream_stack != NULL) { + free(unifyfs_stream_stack); + unifyfs_stream_stack = NULL; + } + + /* free file descriptor stack */ + if (unifyfs_fd_stack != NULL) { + free(unifyfs_fd_stack); + unifyfs_fd_stack = NULL; + } + + /* no longer initialized, so update the flag */ + unifyfs_initialized = 0; + + return rc; +} + + +/* --------------- + * external APIs + * --------------- */ + +/* Fill mount rpc input struct with client-side context info */ +void fill_client_mount_info(unifyfs_mount_in_t* in) +{ + in->dbg_rank = client_rank; + in->mount_prefix = strdup(client_cfg.unifyfs_mountpoint); +} + +/* Fill attach rpc input struct with client-side context info */ +void fill_client_attach_info(unifyfs_attach_in_t* in) +{ + size_t meta_offset = (char*)unifyfs_indices.ptr_num_entries - + (char*)shm_super_ctx->addr; + size_t meta_size = unifyfs_max_index_entries + * sizeof(unifyfs_index_t); + + in->app_id = unifyfs_app_id; + in->client_id = unifyfs_client_id; + in->shmem_data_size = shm_recv_ctx->size; + in->shmem_super_size = shm_super_ctx->size; + in->meta_offset = meta_offset; + in->meta_size = meta_size; + in->logio_mem_size = logio_ctx->shmem->size; + in->logio_spill_size = logio_ctx->spill_sz; + in->logio_spill_dir = strdup(client_cfg.logio_spill_dir); +} + /** * mount a file system at a given prefix * subtype: 0-> log-based file system; @@ -1839,19 +1898,13 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, } } - /* record our rank for debugging messages, - * record the value we should use for an app_id */ - app_id = l_app_id; + // record our rank for debugging messages client_rank = rank; global_rank_cnt = (int)size; - /* print log messages to stderr */ + // print log messages to stderr unifyfs_log_open(NULL); - /************************ - * read configuration values - ************************/ - // initialize configuration rc = unifyfs_config_init(&client_cfg, 0, NULL); if (rc) { @@ -1860,6 +1913,28 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, } client_cfg.ptype = UNIFYFS_CLIENT; + // set log level from config + char* cfgval = client_cfg.log_verbosity; + if (cfgval != NULL) { + long l; + rc = configurator_int_val(cfgval, &l); + if (rc == 0) { + unifyfs_set_log_level((unifyfs_log_level_t)l); + } + } + + // record mountpoint prefix string + unifyfs_mount_prefix = strdup(prefix); + unifyfs_mount_prefixlen = strlen(unifyfs_mount_prefix); + client_cfg.unifyfs_mountpoint = unifyfs_mount_prefix; + + // generate app_id from mountpoint prefix + unifyfs_app_id = unifyfs_generate_gfid(unifyfs_mount_prefix); + if (l_app_id != 0) { + LOGDBG("ignoring passed app_id=%d, using mountpoint app_id=%d", + l_app_id, unifyfs_app_id); + } + // update configuration from runstate file rc = unifyfs_read_runstate(&client_cfg, NULL); if (rc) { @@ -1879,62 +1954,49 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, LOGDBG("mismatch on mount vs kvstore rank/size"); } - /************************ - * record our mount point, and initialize structures to - * store data - ************************/ - - /* record a copy of the prefix string defining the mount point - * we should intercept */ - unifyfs_mount_prefix = strdup(prefix); - unifyfs_mount_prefixlen = strlen(unifyfs_mount_prefix); - /* compute our local rank on the node, * the following call initializes local_rank_{cnt,ndx} */ - rc = CountTasksPerNode(rank, size); + rc = CountTasksPerNode(client_rank, size); if (rc < 0) { LOGERR("cannot get the local rank list."); return -1; } - /* initialize our library, creates superblock and spillover files */ - int ret = unifyfs_init(rank); - if (ret != UNIFYFS_SUCCESS) { - return ret; - } - /* open rpc connection to server */ - ret = unifyfs_client_rpc_init(); - if (ret != UNIFYFS_SUCCESS) { - LOGERR("Failed to initialize client RPC"); - return ret; - } - - /* create shared memory region for holding data for read replies */ - rc = unifyfs_init_recv_shm(local_rank_idx, app_id); - if (rc < 0) { - LOGERR("failed to init shared receive memory"); - return UNIFYFS_FAILURE; + rc = unifyfs_client_rpc_init(); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to initialize client RPC"); + return rc; } - /* Call client mount rpc function - * to register our shared memory and files with server */ - LOGDBG("calling mount"); - ret = invoke_client_mount_rpc(); - if (ret != UNIFYFS_SUCCESS) { + /* Call client mount rpc function to get client id */ + LOGDBG("calling mount rpc"); + rc = invoke_client_mount_rpc(); + if (rc != UNIFYFS_SUCCESS) { /* If we fail to connect to the server, bail with an error */ - LOGERR("Failed to mount to server"); - - /* TODO: need more clean up here, but this at least deletes - * some files we would otherwise leave behind */ + LOGERR("failed to mount to server"); + return rc; + } - /* Delete file for read reply shared memory region */ - unifyfs_shm_unlink(shm_recv_ctx); + /* initialize our library using assigned client id, creates shared memory + * regions (e.g., superblock and data recv) and inits log-based I/O */ + rc = unifyfs_init(); + if (rc != UNIFYFS_SUCCESS) { + return rc; + } - return ret; + /* Call client attach rpc function to register our newly created shared + * memory and files with server */ + LOGDBG("calling attach rpc"); + rc = invoke_client_attach_rpc(); + if (rc != UNIFYFS_SUCCESS) { + /* If we fail, bail with an error */ + LOGERR("failed to attach to server"); + unifyfs_finalize(); + return rc; } - /* Once we return from mount, we know the server has attached to our + /* Once we return from attach, we know the server has attached to our * shared memory region for read replies, so we can safely remove the * file. The memory region will stay active until both client and server * unmap them. We keep the superblock file around so that a future client @@ -1949,62 +2011,17 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, /* if there was an error, return it */ LOGERR("failed to create directory entry for mount point: `%s'", prefix); + unifyfs_finalize(); return UNIFYFS_FAILURE; } } /* record client state as mounted for specific app_id */ - unifyfs_mounted = app_id; + unifyfs_mounted = unifyfs_app_id; return UNIFYFS_SUCCESS; } -/* free resources allocated during unifyfs_init, - * generally we do this in reverse order that - * things were initailized in */ -static int unifyfs_finalize(void) -{ - int rc = UNIFYFS_SUCCESS; - - if (!unifyfs_initialized) { - /* not initialized yet, so we shouldn't call finalize */ - return UNIFYFS_FAILURE; - } - - /* close spillover files */ - unifyfs_logio_close(logio_ctx); - if (unifyfs_spillmetablock != -1) { - close(unifyfs_spillmetablock); - unifyfs_spillmetablock = -1; - } - - /* detach from superblock */ - unifyfs_shm_free(&shm_super_ctx); - - /* free directory stream stack */ - if (unifyfs_dirstream_stack != NULL) { - free(unifyfs_dirstream_stack); - unifyfs_dirstream_stack = NULL; - } - - /* free file stream stack */ - if (unifyfs_stream_stack != NULL) { - free(unifyfs_stream_stack); - unifyfs_stream_stack = NULL; - } - - /* free file descriptor stack */ - if (unifyfs_fd_stack != NULL) { - free(unifyfs_fd_stack); - unifyfs_fd_stack = NULL; - } - - /* no longer initialized, so update the flag */ - unifyfs_initialized = 0; - - return rc; -} - /** * unmount the mounted file system * TODO: Add support for unmounting more than @@ -2024,9 +2041,6 @@ int unifyfs_unmount(void) * tear down connection to server ************************/ - /* detach from shared memory regions */ - unifyfs_shm_free(&shm_recv_ctx); - /* invoke unmount rpc to tell server we're disconnecting */ LOGDBG("calling unmount"); rc = invoke_client_unmount_rpc(); @@ -2051,6 +2065,7 @@ int unifyfs_unmount(void) free(unifyfs_mount_prefix); unifyfs_mount_prefix = NULL; unifyfs_mount_prefixlen = 0; + client_cfg.unifyfs_mountpoint = NULL; } /************************ diff --git a/common/src/unifyfs_client_rpcs.h b/common/src/unifyfs_client_rpcs.h index 668d69e5c..16db3d167 100644 --- a/common/src/unifyfs_client_rpcs.h +++ b/common/src/unifyfs_client_rpcs.h @@ -15,25 +15,34 @@ extern "C" { #endif -/* unifyfs_mount_rpc (client => server) +/* unifyfs_attach_rpc (client => server) * - * connect application client to the server, and - * initialize shared memory state */ -MERCURY_GEN_PROC(unifyfs_mount_in_t, + * initialize server access to client's shared memory and file state */ +MERCURY_GEN_PROC(unifyfs_attach_in_t, ((int32_t)(app_id)) ((int32_t)(client_id)) - ((int32_t)(dbg_rank)) - ((int32_t)(num_procs_per_node)) - ((hg_const_string_t)(client_addr_str)) - ((hg_size_t)(recv_buf_sz)) - ((hg_size_t)(superblock_sz)) + ((hg_size_t)(shmem_data_size)) + ((hg_size_t)(shmem_super_size)) ((hg_size_t)(meta_offset)) ((hg_size_t)(meta_size)) ((hg_size_t)(logio_mem_size)) ((hg_size_t)(logio_spill_size)) - ((hg_const_string_t)(external_spill_dir))) + ((hg_const_string_t)(logio_spill_dir))) +MERCURY_GEN_PROC(unifyfs_attach_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(unifyfs_attach_rpc) + +/* unifyfs_mount_rpc (client => server) + * + * connect application client to the server */ +MERCURY_GEN_PROC(unifyfs_mount_in_t, + ((int32_t)(dbg_rank)) + ((hg_const_string_t)(mount_prefix)) + ((hg_const_string_t)(client_addr_str))) MERCURY_GEN_PROC(unifyfs_mount_out_t, - ((hg_size_t)(max_recs_per_slice)) + ((hg_size_t)(meta_slice_sz)) + ((int32_t)(app_id)) + ((int32_t)(client_id)) ((int32_t)(ret))) DECLARE_MARGO_RPC_HANDLER(unifyfs_mount_rpc) @@ -41,8 +50,8 @@ DECLARE_MARGO_RPC_HANDLER(unifyfs_mount_rpc) * * disconnect client from server */ MERCURY_GEN_PROC(unifyfs_unmount_in_t, - ((int32_t)(app_id)) - ((int32_t)(client_id))) + ((int32_t)(app_id)) + ((int32_t)(client_id))) MERCURY_GEN_PROC(unifyfs_unmount_out_t, ((int32_t)(ret))) DECLARE_MARGO_RPC_HANDLER(unifyfs_unmount_rpc) diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h index 8e9d9bd0c..f4ad09401 100644 --- a/common/src/unifyfs_const.h +++ b/common/src/unifyfs_const.h @@ -49,21 +49,22 @@ #define UNIFYFS_MAX_FILENAME KIB #define UNIFYFS_MAX_HOSTNAME 64 -// Request Manager +// Server - Request Manager #define MAX_META_PER_SEND (4 * KIB) /* max read request count per server */ #define REQ_BUF_LEN (MAX_META_PER_SEND * 64) /* chunk read reqs buffer size */ #define SHM_WAIT_INTERVAL 1000 /* unit: ns */ #define RM_MAX_ACTIVE_REQUESTS 64 /* number of concurrent read requests */ -// Service Manager +// Server - Service Manager #define LARGE_BURSTY_DATA (512 * MIB) #define MAX_BURSTY_INTERVAL 10000 /* unit: us */ #define MIN_SLEEP_INTERVAL 100 /* unit: us */ #define SLEEP_INTERVAL 500 /* unit: us */ #define SLEEP_SLICE_PER_UNIT 50 /* unit: us */ -// Request and Service Managers, Command Handler -#define MAX_NUM_CLIENTS 64 /* app processes per server */ +// Server - General +#define MAX_NUM_APPS 64 /* max # apps supported by a single server */ +#define MAX_APP_CLIENTS 64 /* app processes per server */ // Client #define UNIFYFS_MAX_FILES 128 diff --git a/docs/add-rpcs.rst b/docs/add-rpcs.rst index 017c05217..3fa370e70 100644 --- a/docs/add-rpcs.rst +++ b/docs/add-rpcs.rst @@ -18,9 +18,9 @@ Common The struct definition macro `MERCURY_GEN_PROC()` is used to define both input and output parameters. For client-server RPCs, the - definitions should be placed in `common/src/unifyfs_clientcalls_rpc.h`, + definitions should be placed in `common/src/unifyfs_client_rpcs.h`, while server-server RPC structs are defined in - `common/src/unifyfs_servercalls_rpc.h`. + `common/src/unifyfs_server_rpcs.h`. The input parameters struct should contain all values the client needs to pass to the server handler function. @@ -30,23 +30,21 @@ Common .. code-block:: C MERCURY_GEN_PROC(unifyfs_mount_in_t, - ((int32_t)(app_id)) ((int32_t)(client_id)) ((int32_t)(dbg_rank)) - ((int32_t)(num_procs_per_node)) - ((hg_const_string_t)(client_addr_str)) - ((hg_size_t)(req_buf_sz)) - ((hg_size_t)(recv_buf_sz)) - ((hg_size_t)(superblock_sz)) + ((hg_size_t)(shmem_data_size)) + ((hg_size_t)(shmem_super_size)) ((hg_size_t)(meta_offset)) ((hg_size_t)(meta_size)) - ((hg_size_t)(fmeta_offset)) - ((hg_size_t)(fmeta_size)) - ((hg_size_t)(data_offset)) - ((hg_size_t)(data_size)) - ((hg_const_string_t)(external_spill_dir))) + ((hg_size_t)(logio_mem_size)) + ((hg_size_t)(logio_spill_size)) + ((hg_const_string_t)(logio_spill_dir)) + ((hg_const_string_t)(mount_prefix)) + ((hg_const_string_t)(client_addr_str))) MERCURY_GEN_PROC(unifyfs_mount_out_t, - ((hg_size_t)(max_recs_per_slice)) + ((hg_size_t)(meta_slice_sz)) + ((int32_t)(app_id)) + ((int32_t)(client_id)) ((int32_t)(ret))) .. note:: @@ -66,8 +64,7 @@ Server This is the function that will be invoked on the client and executed on the server. Client-server RPC handler functions are implemented in `server/src/unifyfs_cmd_handler.c`, while server-server RPC handlers go - in `server/src/unifyfs_service_manager.c`. The RPC handler input and output - parameters structs are defined in `common/src/unifyfs_clientcalls_rpc.h`. + in `server/src/unifyfs_service_manager.c`. All the RPC handler functions follow the same protoype, which is passed a Mercury handle as the only argument. The handler function should use diff --git a/server/src/Makefile.am b/server/src/Makefile.am index 5d0c2793e..d3a113a91 100644 --- a/server/src/Makefile.am +++ b/server/src/Makefile.am @@ -16,7 +16,7 @@ libunifyfsd_a_SOURCES = \ bin_PROGRAMS = unifyfsd -unifyfsd_SOURCES = unifyfs_init.c +unifyfsd_SOURCES = unifyfs_server.c unifyfsd_LDFLAGS = -static \ $(LEVELDB_LDFLAGS) \ diff --git a/server/src/margo_server.c b/server/src/margo_server.c index bd1b0f3a7..79325b1c6 100644 --- a/server/src/margo_server.c +++ b/server/src/margo_server.c @@ -147,6 +147,10 @@ static margo_instance_id setup_local_target(void) /* register client-server RPCs */ static void register_client_server_rpcs(margo_instance_id mid) { + MARGO_REGISTER(mid, "unifyfs_attach_rpc", + unifyfs_attach_in_t, unifyfs_attach_out_t, + unifyfs_attach_rpc); + MARGO_REGISTER(mid, "unifyfs_mount_rpc", unifyfs_mount_in_t, unifyfs_mount_out_t, unifyfs_mount_rpc); @@ -245,6 +249,18 @@ int margo_server_rpc_finalize(void) rpc_clean_local_server_addr(); + /* free global server addresses */ + for (int i = 0; i < glb_num_servers; i++) { + if (glb_servers[i].margo_svr_addr != HG_ADDR_NULL) { + margo_addr_free(ctx->svr_mid, glb_servers[i].margo_svr_addr); + glb_servers[i].margo_svr_addr = HG_ADDR_NULL; + } + if (NULL != glb_servers[i].margo_svr_addr_str) { + free(glb_servers[i].margo_svr_addr_str); + glb_servers[i].margo_svr_addr_str = NULL; + } + } + /* shut down margo */ margo_finalize(ctx->svr_mid); /* NOTE: 2nd call to margo_finalize() sometimes crashes - Margo bug? */ diff --git a/server/src/unifyfs_cmd_handler.c b/server/src/unifyfs_cmd_handler.c index 3045fd055..22636d3cc 100644 --- a/server/src/unifyfs_cmd_handler.c +++ b/server/src/unifyfs_cmd_handler.c @@ -42,62 +42,6 @@ #include "unifyfs_rpc_util.h" #include "unifyfs_misc.h" -/** - * attach to the client-side shared memory - * @param app_config: application information - * @param app_id: the server-side - * @param sock_id: position in poll_set in unifyfs_sock.h - * @return success/error code - */ -static int attach_to_shm(app_config_t* app_config, - int app_id, - int client_id) -{ - char shm_name[SHMEM_NAME_LEN] = {0}; - - /* attach shared superblock, a superblock is created by each - * client to store the raw file data. - * The overflowed data are spilled to SSD. */ - - /* define name of superblock region for this client */ - sprintf(shm_name, SHMEM_SUPER_FMTSTR, app_id, client_id); - - /* attach to superblock */ - shm_context* ctx = unifyfs_shm_alloc(shm_name, app_config->superblock_sz); - if (NULL == ctx) { - LOGERR("Failed to attach to superblock %s", shm_name); - return (int)UNIFYFS_ERROR_SHMEM; - } - app_config->shm_superblocks[client_id] = ctx; - - /* initialize shared receive buffer, a request buffer is created - * by each client for the delegator to temporarily buffer the - * received data for this client */ - - /* define name of receive buffer region for this client */ - memset(shm_name, 0, sizeof(shm_name)); - sprintf(shm_name, SHMEM_DATA_FMTSTR, app_id, client_id); - - /* attach to request buffer region */ - ctx = unifyfs_shm_alloc(shm_name, app_config->recv_buf_sz); - if (NULL == ctx) { - LOGERR("Failed to attach to receive buffer %s", shm_name); - return (int)UNIFYFS_ERROR_SHMEM; - } - app_config->shm_recv_bufs[client_id] = ctx; - shm_data_header* shm_hdr = (shm_data_header*)(ctx->addr); - int rc = pthread_mutex_init(&(shm_hdr->sync), NULL); - if (rc) { - int err = errno; - LOGERR("shm_data_header mutex initialization failed (%s)", - strerror(err)); - } - shm_hdr->meta_cnt = 0; - shm_hdr->bytes = 0; - shm_hdr->state = SHMEM_REGION_EMPTY; - - return UNIFYFS_SUCCESS; -} /* BEGIN MARGO CLIENT-SERVER RPC HANDLER FUNCTIONS */ @@ -123,105 +67,100 @@ static void unifyfs_mount_rpc(hg_handle_t handle) assert(hret == HG_SUCCESS); /* read app_id and client_id from input */ - int app_id = in.app_id; - int client_id = in.client_id; + int app_id = unifyfs_generate_gfid(in.mount_prefix); + int client_id = -1; /* lookup app_config for given app_id */ - app_config_t* app_cfg = - (app_config_t*) arraylist_get(app_config_list, app_id); - - /* fill in and insert a new entry for this app_id - * if we don't already have one */ + app_config* app_cfg = get_application(app_id); if (app_cfg == NULL) { - LOGDBG("creating app_config for app_id=%d", app_id); - /* don't have an app_config for this app_id, - * so allocate and fill one in */ - app_cfg = (app_config_t*) malloc(sizeof(app_config_t)); - - /* record size of shared memory regions */ - app_cfg->recv_buf_sz = in.recv_buf_sz; - app_cfg->superblock_sz = in.superblock_sz; - - /* record offset and size of index entries */ - app_cfg->meta_offset = in.meta_offset; - app_cfg->meta_size = in.meta_size; - - /* record directory holding spill over files */ - strcpy(app_cfg->external_spill_dir, in.external_spill_dir); - - /* record number of clients on this node */ - app_cfg->num_procs_per_node = in.num_procs_per_node; - - /* initialize per-client fields */ - int i; - for (i = 0; i < MAX_NUM_CLIENTS; i++) { - app_cfg->client_ranks[i] = -1; - app_cfg->shm_recv_bufs[i] = NULL; - app_cfg->shm_superblocks[i] = NULL; - app_cfg->client_addr[i] = HG_ADDR_NULL; - } - - /* insert new app_config into our list, indexed by app_id */ - rc = arraylist_insert(app_config_list, app_id, app_cfg); - if (rc != 0) { + * so allocate and fill a new one */ + app_cfg = (app_config*) calloc(1, sizeof(app_config)); + app_cfg->app_id = app_id; + + /* insert new app_config into our app_configs array */ + LOGDBG("creating new application"); + rc = new_application(app_cfg); + if (rc != UNIFYFS_SUCCESS) { ret = rc; + free(app_cfg); + app_cfg = NULL; } } else { LOGDBG("using existing app_config for app_id=%d", app_id); } - /* convert client_addr_str sent in input struct to margo hg_addr_t, - * which is the address type needed to call rpc functions, etc */ - hret = margo_addr_lookup(unifyfsd_rpc_context->shm_mid, - in.client_addr_str, - &(app_cfg->client_addr[client_id])); + if (NULL != app_cfg) { + LOGDBG("creating new client"); + app_client* client = create_app_client(app_cfg, + in.client_addr_str, + in.dbg_rank); + if (NULL == client) { + LOGERR("create_app_client() failed for app_id=%d dbg_rank=%d", + app_id, (int)in.dbg_rank); + ret = (int)UNIFYFS_FAILURE; + } else { + client_id = client->client_id; + } + } - /* record client id of process on this node */ - app_cfg->client_ranks[client_id] = client_id; + /* build output structure to return to caller */ + unifyfs_mount_out_t out; + out.meta_slice_sz = meta_slice_sz; + out.app_id = (int32_t) app_id; + out.client_id = (int32_t) client_id; + out.ret = ret; - /* record global rank of client process for debugging */ - app_cfg->dbg_ranks[client_id] = in.dbg_rank; + /* send output back to caller */ + hret = margo_respond(handle, &out); + assert(hret == HG_SUCCESS); - /* attach to shared memory regions of this client */ - rc = attach_to_shm(app_cfg, app_id, client_id); - if (rc != UNIFYFS_SUCCESS) { - LOGERR("failed to attach shmem regions for app=%d client=%d rc=%d", - app_id, client_id, rc); - ret = rc; - } + /* free margo resources */ + margo_free_input(handle, &in); + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(unifyfs_mount_rpc) - /* initialize log-based I/O context for this client */ - size_t logio_shmem_sz = in.logio_mem_size; - size_t logio_spill_sz = in.logio_spill_size; - rc = unifyfs_logio_init_server(app_id, client_id, - logio_shmem_sz, logio_spill_sz, - app_cfg->external_spill_dir, - &(app_cfg->logio[client_id])); - if (rc != UNIFYFS_SUCCESS) { - LOGERR("failed to initialize log-based I/O for app=%d client=%d rc=%d", - app_id, client_id, rc); - ret = rc; - } +/* server attaches to client shared memory regions, opens files + * holding spillover data */ +static void unifyfs_attach_rpc(hg_handle_t handle) +{ + int ret = (int)UNIFYFS_SUCCESS; - /* create request manager thread */ - reqmgr_thrd_t* rm_thrd = unifyfs_rm_thrd_create(app_id, client_id); - if (rm_thrd != NULL) { - /* TODO: seems like it would be cleaner to avoid thread_list - * and instead just record address to struct */ - /* remember id for thread control for this client */ - app_cfg->thrd_idxs[client_id] = rm_thrd->thrd_ndx; + /* get input params */ + unifyfs_attach_in_t in; + hg_return_t hret = margo_get_input(handle, &in); + assert(hret == HG_SUCCESS); + + /* read app_id and client_id from input */ + int app_id = in.app_id; + int client_id = in.client_id; + + /* lookup client structure and attach it */ + app_client* client = get_app_client(app_id, client_id); + if (NULL != client) { + LOGDBG("attaching client (app_id=%d, client_id=%d)", + app_id, client_id); + ret = attach_app_client(client, + in.logio_spill_dir, + in.logio_spill_size, + in.logio_mem_size, + in.shmem_data_size, + in.shmem_super_size, + in.meta_offset, + in.meta_size); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("attach_app_client() failed"); + } } else { - /* failed to create request manager thread */ - LOGERR("unifyfs_rm_thrd_create() failed for app_id=%d client_id=%d", + LOGERR("client not found (app_id=%d, client_id=%d)", app_id, client_id); - ret = UNIFYFS_FAILURE; + ret = (int)UNIFYFS_FAILURE; } /* build output structure to return to caller */ - unifyfs_mount_out_t out; + unifyfs_attach_out_t out; out.ret = ret; - out.max_recs_per_slice = max_recs_per_slice; /* send output back to caller */ hret = margo_respond(handle, &out); @@ -231,7 +170,7 @@ static void unifyfs_mount_rpc(hg_handle_t handle) margo_free_input(handle, &in); margo_destroy(handle); } -DEFINE_MARGO_RPC_HANDLER(unifyfs_mount_rpc) +DEFINE_MARGO_RPC_HANDLER(unifyfs_attach_rpc) static void unifyfs_unmount_rpc(hg_handle_t handle) { @@ -244,9 +183,19 @@ static void unifyfs_unmount_rpc(hg_handle_t handle) int app_id = in.app_id; int client_id = in.client_id; + /* disconnect app client */ + int ret = UNIFYFS_SUCCESS; + app_client* clnt = get_app_client(app_id, client_id); + if (NULL != clnt) { + ret = disconnect_app_client(clnt); + } else { + LOGERR("application client not found"); + ret = EINVAL; + } + /* build output structure to return to caller */ unifyfs_unmount_out_t out; - out.ret = UNIFYFS_SUCCESS; + out.ret = ret; /* send output back to caller */ hret = margo_respond(handle, &out); @@ -255,28 +204,6 @@ static void unifyfs_unmount_rpc(hg_handle_t handle) /* free margo resources */ margo_free_input(handle, &in); margo_destroy(handle); - - /* lookup app_config for given app_id */ - app_config_t* app_config = - (app_config_t*) arraylist_get(app_config_list, app_id); - - /* get thread id for this client */ - int thrd_id = app_config->thrd_idxs[client_id]; - - /* look up thread control structure */ - reqmgr_thrd_t* thrd_ctrl = rm_get_thread(thrd_id); - - /* shutdown the delegator thread */ - rm_cmd_exit(thrd_ctrl); - - /* detach from the read shared memory buffer */ - if (NULL != app_config->shm_recv_bufs[client_id]) { - unifyfs_shm_free(&(app_config->shm_recv_bufs[client_id])); - } - - /* free margo hg_addr_t client addresses in app_config struct */ - margo_addr_free(unifyfsd_rpc_context->shm_mid, - app_config->client_addr[client_id]); } DEFINE_MARGO_RPC_HANDLER(unifyfs_unmount_rpc) diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index e7626fa84..449ef73fc 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -56,13 +56,27 @@ # include #endif -extern arraylist_t* app_config_list; -extern arraylist_t* rm_thrd_list; + +/* Some global variables/structures used throughout the server code */ + +/* PMI server rank and server count */ +extern int glb_pmi_rank; +extern int glb_pmi_size; + +/* hostname for this server */ extern char glb_host[UNIFYFS_MAX_HOSTNAME]; -extern int glb_pmi_rank, glb_pmi_size; -extern size_t max_recs_per_slice; +typedef struct { + //char* hostname; + char* margo_svr_addr_str; + hg_addr_t margo_svr_addr; + int pmi_rank; +} server_info_t; + +extern server_info_t* glb_servers; /* array of server info structs */ +extern size_t glb_num_servers; /* number of entries in glb_servers array */ + /* defines commands for messages sent to service manager threads */ typedef enum { @@ -114,60 +128,73 @@ typedef struct { int errcode; /* request completion status */ } client_read_req_t; -/* one of these structures is created for each app id, - * it contains info for each client like names, file descriptors, - * and memory locations of file data - * - * file data stored in the superblock is in memory, - * this is mapped as a shared memory region by the delegator - * process, this data can be accessed by service manager threads - * using memcpy() - * - * when the super block is full, file data is written - * to the spillover file, data here can be accessed by - * service manager threads via read() calls */ -typedef struct { - /* global values which are identical across all clients, - * for this given app id */ - size_t superblock_sz; /* size of memory region used to store data */ - size_t meta_offset; /* superblock offset to index metadata */ - size_t meta_size; /* size of index metadata region in bytes */ - size_t recv_buf_sz; /* buffer size for read replies to client */ +// forward declaration of reqmgr_thrd +struct reqmgr_thrd; - /* number of clients on the node */ - int num_procs_per_node; +/** + * Structure to maintain application client state, including + * logio and shared memory contexts, margo rpc address, etc. + */ +typedef struct app_client { + int app_id; /* index of associated app in app_configs */ + int client_id; /* this client's index in app's clients array */ + int dbg_rank; /* client debug rank - NOT CURRENTLY USED */ + int connected; /* is client currently connected? */ - /* map from socket id to other values */ - int client_ranks[MAX_NUM_CLIENTS]; /* map to client id */ - int thrd_idxs[MAX_NUM_CLIENTS]; /* map to thread id */ - int dbg_ranks[MAX_NUM_CLIENTS]; /* map to client rank */ + hg_addr_t margo_addr; /* client Margo address */ - /* shared memory context pointers */ - shm_context* shm_superblocks[MAX_NUM_CLIENTS]; /* superblock data */ - shm_context* shm_recv_bufs[MAX_NUM_CLIENTS]; /* read reply shm */ + struct reqmgr_thrd* reqmgr; /* this client's request manager thread */ - /* log-based I/O context pointers */ - logio_context* logio[MAX_NUM_CLIENTS]; + logio_context* logio; /* logio context for write data */ - /* client address for rpc invocation */ - hg_addr_t client_addr[MAX_NUM_CLIENTS]; + shm_context* shmem_data; /* shmem context for read data */ - /* directory holding spill over files */ - char external_spill_dir[UNIFYFS_MAX_FILENAME]; -} app_config_t; + shm_context* shmem_super; /* shmem context for superblock region */ + size_t super_meta_offset; /* superblock offset to index metadata */ + size_t super_meta_size; /* size of index metadata region in bytes */ +} app_client; -typedef int fattr_key_t; +/** + * Structure to maintain application configuration state + * and track connected clients. + */ +typedef struct app_config { + /* application id - MD5(mount_prefix) */ + int app_id; -typedef struct { - //char* hostname; - char* margo_svr_addr_str; - hg_addr_t margo_svr_addr; - int pmi_rank; -} server_info_t; + /* mount prefix for application's UnifyFS files */ + char mount_prefix[UNIFYFS_MAX_FILENAME]; -extern char glb_host[UNIFYFS_MAX_HOSTNAME]; -extern size_t glb_num_servers; -extern server_info_t* glb_servers; + /* array of clients associated with this app */ + size_t num_clients; + app_client* clients[MAX_APP_CLIENTS]; +} app_config; + +extern app_config* app_configs[MAX_NUM_APPS]; /* list of apps */ + +app_config* get_application(int app_id); + +unifyfs_rc new_application(app_config* new_app); + +app_client* get_app_client(int app_id, + int client_id); + +app_client* create_app_client(app_config* app, + const char* margo_addr_str, + const int dbg_rank); + +unifyfs_rc attach_app_client(app_client* client, + const char* logio_spill_dir, + const size_t logio_spill_size, + const size_t logio_shmem_size, + const size_t shmem_data_size, + const size_t shmem_super_size, + const size_t super_meta_offset, + const size_t super_meta_size); + +unifyfs_rc disconnect_app_client(app_client* clnt); + +unifyfs_rc cleanup_app_client(app_client* clnt); #endif // UNIFYFS_GLOBAL_H diff --git a/server/src/unifyfs_metadata.c b/server/src/unifyfs_metadata.c index 68fd2f939..0adf16d5d 100644 --- a/server/src/unifyfs_metadata.c +++ b/server/src/unifyfs_metadata.c @@ -53,7 +53,7 @@ struct mdhim_t* md; #define IDX_FILE_ATTR (1) struct index_t* unifyfs_indexes[2]; -size_t max_recs_per_slice; +size_t meta_slice_sz; void debug_log_key_val(const char* ctx, unifyfs_key_t* key, @@ -141,7 +141,7 @@ int meta_init_store(unifyfs_cfg_t* cfg) if (rc != 0) { return -1; } - max_recs_per_slice = (size_t) range_sz; + meta_slice_sz = (size_t) range_sz; mdhim_options_set_max_recs_per_slice(db_opts, (uint64_t)range_sz); md = mdhimInit(&comm, db_opts); diff --git a/server/src/unifyfs_metadata.h b/server/src/unifyfs_metadata.h index 188cc8ea2..0aae5a75d 100644 --- a/server/src/unifyfs_metadata.h +++ b/server/src/unifyfs_metadata.h @@ -35,6 +35,13 @@ #define MANIFEST_FILE_NAME "mdhim_manifest_" + +/* extent slice size used for metadata */ +extern size_t meta_slice_sz; + +/* Key for file attributes */ +typedef int fattr_key_t; + /** * Key for a file extent */ diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index 6731b7ba3..da8e41e8a 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -51,17 +51,16 @@ #define RM_LOCK(rm) \ do { \ - LOGDBG("locking RM[%d] state", rm->thrd_ndx); \ + LOGDBG("locking RM[%d:%d] state", rm->app_id, rm->client_id); \ pthread_mutex_lock(&(rm->thrd_lock)); \ } while (0) #define RM_UNLOCK(rm) \ do { \ - LOGDBG("unlocking RM[%d] state", rm->thrd_ndx); \ + LOGDBG("unlocking RM[%d:%d] state", rm->app_id, rm->client_id); \ pthread_mutex_unlock(&(rm->thrd_lock)); \ } while (0) -arraylist_t* rm_thrd_list; /* One request manager thread is created for each client that a * delegator serves. The main thread of the delegator assigns @@ -143,18 +142,6 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) thrd_ctrl->has_waiting_delegator = 0; thrd_ctrl->has_waiting_dispatcher = 0; - /* insert our thread control structure into our list of - * active request manager threads, important to do this before - * launching thread since it uses list to lookup its structure */ - rc = arraylist_add(rm_thrd_list, thrd_ctrl); - if (rc != 0) { - pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); - pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); - free(thrd_ctrl); - return NULL; - } - thrd_ctrl->thrd_ndx = arraylist_size(rm_thrd_list) - 1; - /* launch request manager thread */ rc = pthread_create(&(thrd_ctrl->thrd), NULL, rm_delegate_request_thread, (void*)thrd_ctrl); @@ -171,12 +158,6 @@ reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id) return thrd_ctrl; } -/* Lookup RM thread control structure */ -reqmgr_thrd_t* rm_get_thread(int thrd_id) -{ - return (reqmgr_thrd_t*) arraylist_get(rm_thrd_list, thrd_id); -} - /* order keyvals by gfid, then host delegator rank */ static int compare_kv_gfid_rank(const void* a, const void* b) { @@ -1114,8 +1095,8 @@ int create_gfid_chunk_reads(reqmgr_thrd_t* thrd_ctrl, /* return number of slice ranges needed to cover range */ static size_t num_slices(size_t offset, size_t length) { - size_t start = offset / max_recs_per_slice; - size_t end = (offset + length - 1) / max_recs_per_slice; + size_t start = offset / meta_slice_sz; + size_t end = (offset + length - 1) / meta_slice_sz; size_t count = end - start + 1; return count; } @@ -1148,8 +1129,8 @@ static int split_request( * assume that's the last byte of the same segment * containing start, unless that happens to be * beyond the last byte of the actual request */ - size_t start_slice = start / max_recs_per_slice; - size_t end = (start_slice + 1) * max_recs_per_slice - 1; + size_t start_slice = start / meta_slice_sz; + size_t end = (start_slice + 1) * meta_slice_sz - 1; if (end > last_offset) { end = last_offset; } @@ -1178,7 +1159,7 @@ static int split_request( /* given an extent corresponding to a write index, create new key/value * pairs for that extent, splitting into multiple keys at the slice - * range boundaries (max_recs_per_slice), it returns the number of + * range boundaries (meta_slice_sz), it returns the number of * newly created key/values inserted into the given key and value * arrays */ static int split_index( @@ -1217,8 +1198,8 @@ static int split_index( * assume that's the last byte of the same slice * containing start, unless that happens to be * beyond the last byte of the actual request */ - size_t start_slice = start / max_recs_per_slice; - size_t end = (start_slice + 1) * max_recs_per_slice - 1; + size_t start_slice = start / meta_slice_sz; + size_t end = (start_slice + 1) * meta_slice_sz - 1; if (end > last_offset) { end = last_offset; } @@ -1267,15 +1248,14 @@ int rm_cmd_read( size_t offset, /* logical file offset of read request */ size_t length) /* number of bytes to read */ { - /* get pointer to app structure for this app id */ - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - /* get thread id for this client */ - int thrd_id = app_config->thrd_idxs[client_id]; + /* get application client */ + app_client* client = get_app_client(app_id, client_id); + if (NULL == client) { + return (int)UNIFYFS_FAILURE; + } - /* look up thread control structure */ - reqmgr_thrd_t* thrd_ctrl = rm_get_thread(thrd_id); + /* get thread control structure */ + reqmgr_thrd_t* thrd_ctrl = client->reqmgr; /* get chunks corresponding to requested client read extent * @@ -1338,18 +1318,14 @@ int rm_cmd_mread( { int rc = UNIFYFS_SUCCESS; - /* get pointer to app structure for this app id */ - app_config_t* app_config = - (app_config_t*)arraylist_get(app_config_list, app_id); - - /* get thread id for this client */ - int thrd_id = app_config->thrd_idxs[client_id]; - - /* look up thread control structure */ - reqmgr_thrd_t* thrd_ctrl = rm_get_thread(thrd_id); + /* get application client */ + app_client* client = get_app_client(app_id, client_id); + if (NULL == client) { + return (int)UNIFYFS_FAILURE; + } - /* get debug rank for this client */ - int cli_rank = app_config->dbg_ranks[client_id]; + /* get thread control structure */ + reqmgr_thrd_t* thrd_ctrl = client->reqmgr; /* get the locations of all the read requests from the key-value store */ unifyfs_ReadRequest_table_t readRequest = @@ -1509,20 +1485,22 @@ int rm_cmd_sync(int app_id, int client_id) /* get memory page size on this machine */ int page_sz = getpagesize(); - /* get app config struct for this client */ - app_config_t* app_config = (app_config_t*) - arraylist_get(app_config_list, app_id); + /* get application client */ + app_client* client = get_app_client(app_id, client_id); + if (NULL == client) { + return EINVAL; + } /* get pointer to superblock for this client and app */ - shm_context* super_ctx = app_config->shm_superblocks[client_id]; + shm_context* super_ctx = client->shmem_super; if (NULL == super_ctx) { - LOGERR("bad client id"); - return EINVAL; + LOGERR("missing client superblock"); + return EIO; } char* superblk = (char*)(super_ctx->addr); /* get pointer to start of key/value region in superblock */ - char* meta = superblk + app_config->meta_offset; + char* meta = superblk + client->super_meta_offset; /* get number of file extent index values client has for us, * stored as a size_t value in meta region of shared memory */ @@ -1771,26 +1749,26 @@ static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) } } else if ((req->num_remote_reads == 0) && (req->status == READREQ_STARTED)) { + RM_LOCK(thrd_ctrl); + /* look up client shared memory region */ int app_id = req->app_id; int client_id = req->client_id; - app_config_t* app_config = - (app_config_t*) arraylist_get(app_config_list, app_id); - assert(NULL != app_config); - shm_context* client_shm = app_config->shm_recv_bufs[client_id]; - assert(NULL != client_shm); - shm_data_header* shm_hdr = (shm_data_header*) client_shm->addr; + app_client* client = get_app_client(app_id, client_id); + if (NULL != client) { + shm_context* client_shm = client->shmem_data; + assert(NULL != client_shm); + shm_data_header* shm_hdr = (shm_data_header*) client_shm->addr; - RM_LOCK(thrd_ctrl); + /* mark request as complete */ + req->status = READREQ_COMPLETE; - /* mark request as complete */ - req->status = READREQ_COMPLETE; + /* signal client that we're now done writing data */ + client_signal(shm_hdr, SHMEM_REGION_DATA_COMPLETE); - /* signal client that we're now done writing data */ - client_signal(shm_hdr, SHMEM_REGION_DATA_COMPLETE); - - /* wait for client to read data */ - client_wait(shm_hdr); + /* wait for client to read data */ + client_wait(shm_hdr); + } rc = release_read_req(thrd_ctrl, req); if (rc != (int)UNIFYFS_SUCCESS) { @@ -1805,7 +1783,7 @@ static int rm_process_remote_chunk_responses(reqmgr_thrd_t* thrd_ctrl) return ret; } -static shm_data_meta* reserve_shmem_meta(app_config_t* app_config, +static shm_data_meta* reserve_shmem_meta(shm_context* shmem_data, shm_data_header* hdr, size_t data_sz) { @@ -1816,7 +1794,7 @@ static shm_data_meta* reserve_shmem_meta(app_config_t* app_config, pthread_mutex_lock(&(hdr->sync)); LOGDBG("shm_data_header(cnt=%zu, bytes=%zu)", hdr->meta_cnt, hdr->bytes); - size_t remain_size = app_config->recv_buf_sz - + size_t remain_size = shmem_data->size - (sizeof(shm_data_header) + hdr->bytes); size_t meta_size = sizeof(shm_data_meta) + data_sz; if (meta_size > remain_size) { @@ -1855,14 +1833,14 @@ int rm_post_chunk_read_responses(int app_id, { int rc; - /* lookup RM thread control structure for this app id */ - app_config_t* app_config = (app_config_t*) - arraylist_get(app_config_list, app_id); - assert(NULL != app_config); - - int thrd_id = app_config->thrd_idxs[client_id]; + /* get application client */ + app_client* client = get_app_client(app_id, client_id); + if (NULL == client) { + return (int)UNIFYFS_FAILURE; + } - reqmgr_thrd_t* thrd_ctrl = rm_get_thread(thrd_id); + /* get thread control structure */ + reqmgr_thrd_t* thrd_ctrl = client->reqmgr; assert(NULL != thrd_ctrl); RM_LOCK(thrd_ctrl); @@ -1914,7 +1892,6 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, { int errcode, gfid, i, num_chks, rc, thrd_id; int ret = (int)UNIFYFS_SUCCESS; - app_config_t* app_config = NULL; chunk_read_resp_t* responses = NULL; shm_context* client_shm = NULL; shm_data_header* shm_hdr = NULL; @@ -1929,9 +1906,12 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, (NULL != del_reads->resp)); /* look up client shared memory region */ - app_config = (app_config_t*) arraylist_get(app_config_list, rdreq->app_id); - assert(NULL != app_config); - client_shm = (shm_context*) app_config->shm_recv_bufs[rdreq->client_id]; + app_config* app_cfg = get_application(rdreq->app_id); + app_client* clnt = get_app_client(rdreq->app_id, rdreq->client_id); + if ((NULL == app_cfg) || (NULL == clnt)) { + return (int)UNIFYFS_FAILURE; + } + client_shm = clnt->shmem_data; shm_hdr = (shm_data_header*) client_shm->addr; RM_LOCK(thrd_ctrl); @@ -1965,7 +1945,7 @@ int rm_handle_chunk_read_responses(reqmgr_thrd_t* thrd_ctrl, LOGDBG("chunk response for offset=%zu: sz=%zu", offset, data_sz); /* allocate and register local target buffer for bulk access */ - meta = reserve_shmem_meta(app_config, shm_hdr, data_sz); + meta = reserve_shmem_meta(client_shm, shm_hdr, data_sz); if (NULL != meta) { meta->offset = offset; meta->length = data_sz; @@ -2059,14 +2039,15 @@ void* rm_delegate_request_thread(void* arg) } /* release lock and wait to be signaled by dispatcher */ - LOGDBG("RM[%d] waiting for work", thrd_ctrl->thrd_ndx); + LOGDBG("RM[%d:%d] waiting for work", + thrd_ctrl->app_id, thrd_ctrl->client_id); pthread_cond_wait(&thrd_ctrl->thrd_cond, &thrd_ctrl->thrd_lock); /* set flag to indicate we're no longer waiting */ thrd_ctrl->has_waiting_delegator = 0; /* go do work ... */ - LOGDBG("RM[%d] got work", thrd_ctrl->thrd_ndx); + LOGDBG("RM[%d:%d] got work", thrd_ctrl->app_id, thrd_ctrl->client_id); /* release lock and bail out if we've been told to exit */ if (thrd_ctrl->exit_flag == 1) { diff --git a/server/src/unifyfs_request_manager.h b/server/src/unifyfs_request_manager.h index 3c4f62edc..2f1caead7 100644 --- a/server/src/unifyfs_request_manager.h +++ b/server/src/unifyfs_request_manager.h @@ -47,7 +47,7 @@ typedef struct { * manager thread, contains shared data structures where main thread * issues read requests and request manager processes them, contains * condition variable and lock for coordination between threads */ -typedef struct { +typedef struct reqmgr_thrd { /* request manager thread */ pthread_t thrd; @@ -84,9 +84,6 @@ typedef struct { /* client_id this thread is serving */ int client_id; - - /* index within rm_thrd_list */ - int thrd_ndx; } reqmgr_thrd_t; @@ -94,9 +91,6 @@ typedef struct { reqmgr_thrd_t* unifyfs_rm_thrd_create(int app_id, int client_id); -/* lookup Request Manager thread by index */ -reqmgr_thrd_t* rm_get_thread(int thrd_id); - /* Request Manager pthread main */ void* rm_delegate_request_thread(void* arg); diff --git a/server/src/unifyfs_init.c b/server/src/unifyfs_server.c similarity index 62% rename from server/src/unifyfs_init.c rename to server/src/unifyfs_server.c index e794cf736..bb6da8bd0 100644 --- a/server/src/unifyfs_init.c +++ b/server/src/unifyfs_server.c @@ -45,6 +45,7 @@ // margo rpcs #include "margo_server.h" +/* PMI information */ int glb_pmi_rank; /* = 0 */ int glb_pmi_size = 1; // for standalone server tests @@ -54,10 +55,10 @@ size_t glb_host_ndx; // index of localhost in glb_servers size_t glb_num_servers; // size of glb_servers array server_info_t* glb_servers; // array of server_info_t -arraylist_t* app_config_list; - unifyfs_cfg_t server_cfg; +app_config* app_configs[MAX_NUM_APPS]; /* list of apps */ + static int unifyfs_exit(void); #if defined(UNIFYFS_MULTIPLE_DELEGATORS) @@ -281,17 +282,8 @@ int main(int argc, char* argv[]) rc = sigaction(SIGQUIT, &sa, NULL); rc = sigaction(SIGTERM, &sa, NULL); - app_config_list = arraylist_create(); - if (app_config_list == NULL) { - LOGERR("%s", unifyfs_rc_enum_description(ENOMEM)); - exit(1); - } - - rm_thrd_list = arraylist_create(); - if (rm_thrd_list == NULL) { - LOGERR("%s", unifyfs_rc_enum_description(ENOMEM)); - exit(1); - } + // initialize empty app_configs[] + memset(app_configs, 0, sizeof(app_configs)); #if defined(UNIFYFSD_USE_MPI) init_MPI(); @@ -558,64 +550,41 @@ static int find_rank_idx(int my_rank) static int unifyfs_exit(void) { - int rc = UNIFYFS_SUCCESS; - - /* shutdown rpc service */ - LOGDBG("stopping rpc service"); - margo_server_rpc_finalize(); - - /* finalize kvstore service*/ - LOGDBG("finalizing kvstore service"); - unifyfs_keyval_fini(); - - /* TODO: notify the service threads to exit */ - - /* notify the request manager threads to exit*/ - LOGDBG("stopping request manager threads"); - int i, j; - for (i = 0; i < arraylist_size(rm_thrd_list); i++) { - /* request and wait for request manager thread exit */ - reqmgr_thrd_t* thrd_ctrl = - (reqmgr_thrd_t*) arraylist_get(rm_thrd_list, i); - rm_cmd_exit(thrd_ctrl); - } - arraylist_free(rm_thrd_list); - - /* sanitize the shared memory and delete the log files - * */ - int app_sz = arraylist_size(app_config_list); + int ret = UNIFYFS_SUCCESS; /* iterate over each active application and free resources */ - for (i = 0; i < app_sz; i++) { + for (int i = 0; i < MAX_NUM_APPS; i++) { /* get pointer to app config for this app_id */ - app_config_t* app = - (app_config_t*)arraylist_get(app_config_list, i); - - /* skip to next app_id if this is empty */ - if (app == NULL) { + app_config* app = app_configs[i]; + if (NULL == app) { + /* skip to next app_id if this slot is empty */ continue; } - /* free resources allocate for each client */ - for (j = 0; j < MAX_NUM_CLIENTS; j++) { - /* Release receive buffer shared memory region. Client - * should have deleted file already, but will not hurt - * to do this again. */ - if (app->shm_recv_bufs[j] != NULL) { - unifyfs_shm_unlink(app->shm_recv_bufs[j]); - unifyfs_shm_free(&(app->shm_recv_bufs[j])); - } - - /* Release super block shared memory region. - * Server is responsible for deleting superblock shared - * memory file that was created by the client. */ - if (app->shm_superblocks[j] != NULL) { - unifyfs_shm_unlink(app->shm_superblocks[j]); - unifyfs_shm_free(&(app->shm_superblocks[j])); + /* free resources allocated for each client */ + int app_id = app->app_id; + for (int j = 1; j <= MAX_APP_CLIENTS; j++) { + app_client* client = get_app_client(app_id, j); + if (NULL != client) { + int rc = cleanup_app_client(client); + if (rc != UNIFYFS_SUCCESS) { + ret = rc; + } } } } + /* TODO: notify the service threads to exit */ + + /* finalize kvstore service*/ + LOGDBG("finalizing kvstore service"); + unifyfs_keyval_fini(); + + /* shutdown rpc service + * (note: this needs to happen after app-client cleanup above) */ + LOGDBG("stopping rpc service"); + margo_server_rpc_finalize(); + /* shutdown the metadata service*/ LOGDBG("stopping metadata service"); meta_sanitize(); @@ -628,5 +597,292 @@ static int unifyfs_exit(void) LOGDBG("all done!"); unifyfs_log_close(); - return rc; + return ret; +} + +/* get pointer to app config for this app_id */ +app_config* get_application(int app_id) +{ + for (int i = 0; i < MAX_NUM_APPS; i++) { + app_config* app_cfg = app_configs[i]; + if ((NULL != app_cfg) && (app_cfg->app_id == app_id)) { + return app_cfg; + } + } + return NULL; +} + +/* insert a new app config in app_configs[] */ +unifyfs_rc new_application(app_config* new_app) +{ + if (NULL == new_app) { + LOGERR("NULL app_config pointer"); + return EINVAL; + } + + /* check for existing app structure with given app_id */ + int app_id = new_app->app_id; + app_config* existing = get_application(app_id); + if (NULL != existing) { + LOGERR("application with app_id %d already exists", app_id); + return EINVAL; + } + + /* insert the given app_config in an empty slot */ + for (int i = 0; i < MAX_NUM_APPS; i++) { + existing = app_configs[i]; + if (NULL == existing) { + app_configs[i] = new_app; + return UNIFYFS_SUCCESS; + } + } + LOGERR("insert into app_configs[] failed"); + return UNIFYFS_FAILURE; +} + +app_client* get_app_client(int app_id, + int client_id) +{ + /* get pointer to app structure for this app id */ + app_config* app_cfg = get_application(app_id); + if ((NULL == app_cfg) || + (client_id <= 0) || + (client_id > MAX_APP_CLIENTS)) { + return NULL; + } + + /* clients array index is (id - 1) */ + int client_ndx = client_id - 1; + return app_cfg->clients[client_ndx]; +} + +/** + * Attach to the server-side of client shared memory regions. + * @param client: client information + * @return success|error code + */ +static unifyfs_rc attach_to_client_shmem(app_client* client, + size_t shmem_data_sz, + size_t shmem_super_sz) +{ + shm_context* shm_ctx; + char shm_name[SHMEM_NAME_LEN] = {0}; + + if (NULL == client) { + LOGERR("NULL client"); + return EINVAL; + } + + int app_id = client->app_id; + int client_id = client->client_id; + + /* initialize shmem region for client's superblock */ + sprintf(shm_name, SHMEM_SUPER_FMTSTR, app_id, client_id); + shm_ctx = unifyfs_shm_alloc(shm_name, shmem_super_sz); + if (NULL == shm_ctx) { + LOGERR("Failed to attach to shmem superblock region %s", shm_name); + return UNIFYFS_ERROR_SHMEM; + } + client->shmem_super = shm_ctx; + + /* initialize shmem region for read data */ + sprintf(shm_name, SHMEM_DATA_FMTSTR, app_id, client_id); + shm_ctx = unifyfs_shm_alloc(shm_name, shmem_data_sz); + if (NULL == shm_ctx) { + LOGERR("Failed to attach to shmem data region %s", shm_name); + return UNIFYFS_ERROR_SHMEM; + } + client->shmem_data = shm_ctx; + + /* initialize shmem header in data region */ + shm_data_header* shm_hdr = (shm_data_header*) client->shmem_data->addr; + pthread_mutex_init(&(shm_hdr->sync), NULL); + shm_hdr->meta_cnt = 0; + shm_hdr->bytes = 0; + shm_hdr->state = SHMEM_REGION_EMPTY; + + return UNIFYFS_SUCCESS; +} + +/** + * Initialize client state using passed values. + * + * Sets up logio and shmem region contexts, request manager thread, + * margo rpc address, etc. + */ +app_client* create_app_client(app_config* app, + const char* margo_addr_str, + const int debug_rank) +{ + if ((NULL == app) || (NULL == margo_addr_str)) { + return NULL; + } + + if (app->num_clients == MAX_APP_CLIENTS) { + LOGERR("reached maximum number of application clients"); + return NULL; + } + + int app_id = app->app_id; + int client_id = app->num_clients + 1; /* next client id */ + int client_ndx = client_id - 1; /* clients array index is (id - 1) */ + + app_client* client = (app_client*) calloc(1, sizeof(app_client)); + if (NULL != client) { + int failure = 0; + client->app_id = app_id; + client->client_id = client_id; + client->dbg_rank = debug_rank; + + /* convert client_addr_str to margo hg_addr_t */ + hg_return_t hret = margo_addr_lookup(unifyfsd_rpc_context->shm_mid, + margo_addr_str, + &(client->margo_addr)); + if (hret != HG_SUCCESS) { + failure = 1; + } + + /* create a request manager thread for this client */ + client->reqmgr = unifyfs_rm_thrd_create(app_id, client_id); + if (NULL == client->reqmgr) { + failure = 1; + } + + if (failure) { + LOGERR("failed to initialize application client"); + cleanup_app_client(client); + client = NULL; + } else { + app->num_clients++; + app->clients[client_ndx] = client; + } + } + + return client; +} + +/** + * Attaches server to shared client state (e.g., logio and shmem regions) + */ +unifyfs_rc attach_app_client(app_client* client, + const char* logio_spill_dir, + const size_t logio_spill_size, + const size_t logio_shmem_size, + const size_t shmem_data_size, + const size_t shmem_super_size, + const size_t super_meta_offset, + const size_t super_meta_size) +{ + if ((NULL == client) || (NULL == logio_spill_dir)) { + return EINVAL; + } + + int app_id = client->app_id; + int client_id = client->client_id; + int failure = 0; + + /* initialize server-side logio for this client */ + int rc = unifyfs_logio_init_server(app_id, client_id, + logio_shmem_size, + logio_spill_size, + logio_spill_dir, + &(client->logio)); + if (rc != UNIFYFS_SUCCESS) { + failure = 1; + } + + /* attach server-side shmem regions for this client */ + rc = attach_to_client_shmem(client, shmem_data_size, shmem_super_size); + if (rc != UNIFYFS_SUCCESS) { + failure = 1; + } + + if (failure) { + LOGERR("failed to attach application client"); + cleanup_app_client(client); + return UNIFYFS_FAILURE; + } + + client->super_meta_offset = super_meta_offset; + client->super_meta_size = super_meta_size; + client->connected = 1; + + return UNIFYFS_SUCCESS; +} + +/** + * Disconnect ephemeral client state, while maintaining access to any data + * the client wrote. + */ +unifyfs_rc disconnect_app_client(app_client* client) +{ + if (NULL == client) { + return EINVAL; + } + + client->connected = 0; + + /* stop client request manager thread */ + if (NULL != client->reqmgr) { + rm_cmd_exit(client->reqmgr); + free(client->reqmgr); + client->reqmgr = NULL; + } + + /* free margo client address */ + margo_addr_free(unifyfsd_rpc_context->shm_mid, + client->margo_addr); + + /* release client shared memory regions */ + if (NULL != client->shmem_data) { + /* Release read buffer shared memory region. + * Client should have deleted file already, but will not hurt + * to do this again. */ + unifyfs_shm_unlink(client->shmem_data); + unifyfs_shm_free(&(client->shmem_data)); + } + if (NULL != client->shmem_super) { + /* Release superblock shared memory region. + * Server is responsible for deleting superblock shared + * memory file that was created by the client. */ + unifyfs_shm_unlink(client->shmem_super); + unifyfs_shm_free(&(client->shmem_super)); + } + + return UNIFYFS_SUCCESS; +} + +/** + * Cleanup any client state that has been setup in preparation for + * server exit. + * + * This function may be called due to a failed initialization, so we can't + * assume any particular state is valid, other than app_id and client_id. + */ +unifyfs_rc cleanup_app_client(app_client* client) +{ + if (NULL == client) { + return EINVAL; + } + + disconnect_app_client(client); + + /* close client logio context */ + if (NULL != client->logio) { + unifyfs_logio_close(client->logio); + } + + /* reset app->clients array index if set */ + app_config* app = get_application(client->app_id); + if (NULL != app) { + int client_ndx = client->client_id - 1; /* client ids start at 1 */ + if (client == app->clients[client_ndx]) { + app->clients[client_ndx] = NULL; + } + } + + /* free client structure */ + free(client); + + return UNIFYFS_SUCCESS; } diff --git a/server/src/unifyfs_service_manager.c b/server/src/unifyfs_service_manager.c index 96b1c450a..3821708e2 100644 --- a/server/src/unifyfs_service_manager.c +++ b/server/src/unifyfs_service_manager.c @@ -159,8 +159,7 @@ int sm_issue_chunk_reads(int src_rank, size_t buf_cursor = 0; int i; - int last_app = -1; - app_config_t* app_config = NULL; + app_client* app_clnt = NULL; for (i = 0; i < num_chks; i++) { /* pointer to next read request */ chunk_read_req_t* rreq = reqs + i; @@ -179,38 +178,29 @@ int sm_issue_chunk_reads(int src_rank, LOGDBG("reading chunk(offset=%zu, size=%zu)", rreq->offset, nbytes); - /* get app id and corresponding app_config struct */ - int app_id = rreq->log_app_id; - if (app_id != last_app) { - /* look up app config for given app id */ - app_config = (app_config_t*) - arraylist_get(app_config_list, app_id); - assert(app_config); - - /* remember the current app_id to skip lookup if the next - * request is for the same app_id */ - last_app = app_id; - } - - /* client id for this read task */ - int cli_id = rreq->log_client_id; - /* get pointer to next position in buffer to store read data */ char* buf_ptr = databuf + buf_cursor; - /* read data from log */ - logio_context* logio_ctx = app_config->logio[cli_id]; - if (NULL != logio_ctx) { - size_t nread = 0; - int rc = unifyfs_logio_read(logio_ctx, log_offset, nbytes, - buf_ptr, &nread); - if (UNIFYFS_SUCCESS == rc) { - rresp->read_rc = nread; + /* read data from client log */ + int app_id = rreq->log_app_id; + int cli_id = rreq->log_client_id; + app_clnt = get_app_client(app_id, cli_id); + if (NULL != app_clnt) { + logio_context* logio_ctx = app_clnt->logio; + if (NULL != logio_ctx) { + size_t nread = 0; + int rc = unifyfs_logio_read(logio_ctx, log_offset, nbytes, + buf_ptr, &nread); + if (UNIFYFS_SUCCESS == rc) { + rresp->read_rc = nread; + } else { + rresp->read_rc = (ssize_t)(-rc); + } } else { - rresp->read_rc = (ssize_t)(-rc); + rresp->read_rc = (ssize_t)(-EINVAL); } } else { - rresp->read_rc = (ssize_t)(-UNIFYFS_FAILURE); + rresp->read_rc = (ssize_t)(-EINVAL); } /* update to point to next slot in read reply buffer */ From 599a2429ef96b5da0f6b1e02543eb6adc585a232 Mon Sep 17 00:00:00 2001 From: Craig P Steffen Date: Mon, 17 Feb 2020 17:42:39 -0500 Subject: [PATCH 089/168] doc UnifyFS as SEPARATE user file system I think that the fact that UnifyFS has a completely separate name space and a file system that's unknown by the OS is something that should be spelled out more clearly. I think this knowledge is core to understanding how UnifyFS works and how people will interact with it. --- docs/overview.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/overview.rst b/docs/overview.rst index df735beb0..b98f035ff 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -11,6 +11,17 @@ easily as they do the parallel file system. This section will provide a high level design of UnifyFS. It will describe the UnifyFS library and the UnifyFS daemon. +The file system that UnifyFS instantiates only exists in user space and is +only visible to applications linked against the UnifyFS client library. Since +traditional file system tools (ls, cd, etc.) are not linked against the +UnifyFS client library they cannot see files within UnifyFS nor can they be +used to manipulate files within UnifyFS. Each UnifyFS file system lasts as +long as the server processes are running, which is typically as long as the +job they are running within. When the servers exit the file system is +deleted. It is therefore the user's responsibility to copy out files that +need to be persisted to another permanent file system. We provide an API and +a utility to make this easier. + --------------------------- High Level Design --------------------------- From ff11788326e890a84a971f0f2936b7fc9946d652 Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Wed, 15 Apr 2020 14:02:48 -0400 Subject: [PATCH 090/168] The recent addition of `-Werror` fails the build because of redefining MIN and MAX macros. This prevents the compiler warning by `undef` those macros before defining them. --- client/src/unifyfs-sysio.c | 5 +++-- common/src/seg_tree.c | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index a7bbd0341..b62efc31d 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -46,8 +46,9 @@ #include "ucr_read_builder.h" #include "seg_tree.h" - -#define MAX(a, b) (a > b ? a : b) +#ifndef MAX +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif /* --------------------------------------- * POSIX wrappers: paths diff --git a/common/src/seg_tree.c b/common/src/seg_tree.c index 8c1e7dfbf..88d885542 100644 --- a/common/src/seg_tree.c +++ b/common/src/seg_tree.c @@ -36,8 +36,13 @@ #include "seg_tree.h" #include "tree.h" -#define MIN(a, b) (a < b ? a : b) -#define MAX(a, b) (a > b ? a : b) +#ifndef MIN +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +#ifndef MAX +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif int compare_func(struct seg_tree_node* node1, struct seg_tree_node* node2) From 0df8097b0b5e402fd3904f83ae994629579d857c Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Fri, 17 Apr 2020 13:22:08 -0400 Subject: [PATCH 091/168] Enable compile time option for using MPI for unifyfsd. This also fixes an compile error when `UNIFYFSD_USE_MPI` is defined. - configure.ac: adding the configure option - server/src/unifyfs_global.h: include config.h - server/src/unifyfs_server.c: fix MPI_Init error --- configure.ac | 9 +++++++++ server/src/unifyfs_global.h | 1 + server/src/unifyfs_server.c | 6 +++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 6e81d34d5..46d1c84bb 100755 --- a/configure.ac +++ b/configure.ac @@ -132,6 +132,15 @@ AC_ARG_ENABLE(st-dev-workaround, ] ,[]) +# use mpi for bootstraping unifyfsd +AC_ARG_ENABLE(unifyfsd-mpi, + [AS_HELP_STRING([--enable-unifyfsd-mpi],[Use MPI for bootstrapping unifyfsd])],[ + AS_IF([test "x$enableval" = "xyes"],[ + AC_DEFINE(UNIFYFSD_USE_MPI, 1, Define if unifyfsd has to use mpi for bootstrapping)],[]) + ] +,[]) + + # look for MPI and set flags LX_FIND_MPI AS_IF([test "x$enable_fortran" = "xyes"],[ diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index 449ef73fc..f75df1972 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -29,6 +29,7 @@ #ifndef UNIFYFS_GLOBAL_H #define UNIFYFS_GLOBAL_H +#include // system headers #include diff --git a/server/src/unifyfs_server.c b/server/src/unifyfs_server.c index bb6da8bd0..10412867e 100644 --- a/server/src/unifyfs_server.c +++ b/server/src/unifyfs_server.c @@ -149,10 +149,10 @@ void exit_request(int sig) } #if defined(UNIFYFSD_USE_MPI) -static void init_MPI(void) +static void init_MPI(int* argc, char*** argv) { int rc, provided; - rc = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + rc = MPI_Init_thread(argc, argv, MPI_THREAD_MULTIPLE, &provided); if (rc != MPI_SUCCESS) { exit(1); } @@ -286,7 +286,7 @@ int main(int argc, char* argv[]) memset(app_configs, 0, sizeof(app_configs)); #if defined(UNIFYFSD_USE_MPI) - init_MPI(); + init_MPI(&argc, &argv); #endif // start logging From 2c43ffa468a9ddc1f82bd8341ba77722e8cbefab Mon Sep 17 00:00:00 2001 From: CamStan Date: Thu, 16 Apr 2020 15:31:37 -0700 Subject: [PATCH 092/168] Create test get_size function and additional tests This pulls the `get_size` function out of several tests, since it is defined and used multiple times, and turns it into a testutil function. Also some additional tests are added as well as some refactoring, to include: * Add additional tests for failure cases * tests with bad parameters or call on bad file descriptors * tests for bad calls that segfault on posix. We do as well due to fallthrough. This how we want them handled? * Add __FILE__/__LINE__ to files I was already touching for better future debugging * Cleanup of redundant errno setting * errno is not reset after success, so reset after test for failures to avoid subsequent errno tests to be wrong * Turn function calls used for test setup into tests for better future debugging * Found two issues not yet addressed in this PR * Test for stat after unlink was failing, as it should, but turns out it's failing with wrong errno. Call is making it farther into our wrapper(s) than it should * Test for read from a file open as write-only reads successfully, but returns 0 bytes read. Not as issue with fread. Potentially a problem in open call when flags are set. Also updated docs to show additional examples of how to write and sharness and lib/tap test. Skip checkpatch on the docs as it is complaining about a testing call that it thinks is a typo. TEST_CHECKPATCH_SKIP_FILES=docs/testing.rst --- .github/PULL_REQUEST_TEMPLATE.md | 1 + docs/testing.rst | 98 ++++++++++++- t/lib/testutil.c | 26 +++- t/lib/testutil.h | 6 + t/std/fopen-fclose.c | 47 +++--- t/std/fseek-ftell.c | 244 +++++++++++++------------------ t/std/fwrite-fread.c | 187 +++++++++++++++++------ t/std/size.c | 188 ++++++++++-------------- t/sys/creat-close.c | 39 +++-- t/sys/lseek.c | 197 +++++++++++-------------- t/sys/mkdir-rmdir.c | 119 +++++++-------- t/sys/truncate.c | 66 +++------ t/sys/unlink.c | 146 ++++++++++++------ t/sys/write-read-hole.c | 27 +--- t/sys/write-read.c | 205 ++++++++++++++++---------- 15 files changed, 887 insertions(+), 709 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a6b6861f2..be56d54a9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -20,6 +20,7 @@ - [ ] Performance enhancement (non-breaking change which improves efficiency) - [ ] Code cleanup (non-breaking change which makes code smaller or more readable) - [ ] Breaking change (fix or feature that would cause existing functionality to change) +- [ ] Testing (addition of new tests or update to current tests) - [ ] Documentation (a change to man pages or other documentation) ### Checklist: diff --git a/docs/testing.rst b/docs/testing.rst index 5622a8c1a..1816c199e 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -55,6 +55,24 @@ Here is an example of a sharness test: test "P" == "NP" ' + # Various tests available to use inside test_expect_success/failure + test_expect_success "Show various available tests" ' + test_path_is_dir /somedir + test_must_fail test_dir_is_empty /somedir + test_path_is_file /somedir/somefile + ' + + # Use test_set_prereq/test_have_prereq to conditionally skip tests + [[ -n $(which h5cc 2>/dev/null) ]] && test_set_prereq HAVE_HDF5 + if test_have_prereq HAVE_HDF5; then + # run HDF5 tests + fi + + # Can also check for prereq in individual test + test_expect_success HAVE_HDF5 "Run HDF5 test" ' + # Run HDF5 test + ' + test_done .. _C-tests-label: @@ -62,8 +80,9 @@ Here is an example of a sharness test: C Program Tests ^^^^^^^^^^^^^^^ -C programs use the `libtap library`_ to implement test cases. Convenience -functions common to test cases written in C are implemented in the library +C programs use the `libtap library`_ to implement test cases. All available +testing functions are viewable in the `libtap README`_. Convenience functions +common to test cases written in C are implemented in the library `lib/testutil.c`_. If your C program needs to use environment variables set by sharness, it can be wrapped in a shell script that first sources `sharness.d/00-test-env.sh`_ and `sharness.d/01-unifyfs-settings.sh`_. Your @@ -74,12 +93,13 @@ The most common way to implement a test with libtap is to use the ``ok()`` function. TODO test cases that demonstrate known breakage are surrounded by the libtap library calls ``todo()`` and ``end_todo()``. -Here is an example libtap test: +Here are some examples of libtap tests: .. code-block:: C :linenos: #include "t/lib/tap.h" + #include "t/lib/testutil.h" #include int main(int argc, char *argv[]) @@ -89,16 +109,83 @@ Here is an example libtap test: result = (1 == 1); ok(result, "1 equals 1: %d", result); + /* Or put a function call directly in test */ + ok(somefunc() == 42, "somefunc() returns 42"); + ok(somefunc() == -1, "somefunc() should fail"); + + /* Use pass/fail for more complex code paths */ + int x = somefunc(); + if (x > 0) { + pass("somefunc() returned a valid value"); + } else { + fail("somefunc() returned an invalid value"); + } + + /* Use is/isnt for string comparisions */ + char buf[64] = {0}; + ok(fread(buf, 12, 1, fd) == 1, "read 12 bytes into buf); + is(buf, "hello world", "buf is \"hello world\""); + + /* Use cmp_mem to test first n bytes of memory */ + char* a = "foo"; + char* b = "bar"; + cmp_mem(a, b, 3); + + /* Use like/unlike to string match to a POSIX regex */ + like("stranger", "^s.(r).*\\1$", "matches the regex"); + + /* Use dies_ok/lives_ok to test whether code causes an exit */ + dies_ok({int x = 0/0;}, "divide by zero crashes"); + + /* Use todo for failing tests to be notified when they start passing */ todo("Prove this someday"); result = strcmp("P", "NP"); ok(result == 0, "P equals NP: %d", result); end_todo; - done_testing(); + /* Use skip/end_skip when a feature isn't implemented yet, or to + conditionally skip when a resource isn't available */ + skip(TRUE, 2, "Reason for skipping tests"); + ok(1); + ok(2); + end_skip; + + #ifdef HAVE_SOME_FEATURE + ok(somefunc()); + ok(someotherfunc()); + #else + skip(TRUE, 2, "Don't have SOME_FEATURE"); + end_skip; + #endif - return 0; + done_testing(); } +.. tip:: + + Including the file and line number, as well as any useful variable values, + in each test output can be very helpful when a test fails or needs to be + debugged. + + .. code-block:: C + + ok(somefunc() == 42, "%s:%d somefunc() returns 42", __FILE__, + __LINE__); + + Also, note that ``errno`` is only set when an error occurs and is never set + back to ``0`` implicitly. + When testing for a failure and using ``errno`` as part of the test, + resetting ``errno`` after the test will prevent subsequent tests from + appearing to error. + + .. code-block:: C + :emphasize-lines: 4 + + ok(somefunc() == -1 && errno == ENOTTY, + "%s:%d somefunc() should fail (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; + ------------ Adding Tests @@ -797,6 +884,7 @@ comments in `t/ci/ci-functions.sh`_. .. _GitLab: https://about.gitlab.com .. _examples: https://github.com/LLNL/UnifyFS/tree/dev/examples/src .. _libtap library: https://github.com/zorgnax/libtap +.. _libtap README: https://github.com/zorgnax/libtap/blob/master/README.md .. _lib/testutil.c: https://github.com/LLNL/UnifyFS/blob/dev/t/lib/testutil.c .. _PDSH: https://github.com/chaos/pdsh .. _sharness: https://github.com/chriscool/sharness diff --git a/t/lib/testutil.c b/t/lib/testutil.c index 8e66d738f..24f56e560 100644 --- a/t/lib/testutil.c +++ b/t/lib/testutil.c @@ -12,10 +12,12 @@ * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ +#include #include #include +#include #include -#include "testutil.h" +#include static unsigned long seed; @@ -97,3 +99,25 @@ char* testutil_get_mount_point(void) return path; } + +/* Stat the file associated to by path and store the global and log sizes of the + * file at path in the addresses of the respective global and log pointers + * passed in. + * User can ask for one or both sizes. */ +void testutil_get_size(char* path, size_t* global, size_t* log) +{ + struct stat sb = {0}; + int rc; + + rc = stat(path, &sb); + if (rc != 0) { + printf("Error: %s\n", strerror(errno)); + exit(1); + } + if (global) { + *global = sb.st_size; + } + if (log) { + *log = sb.st_rdev; + } +} diff --git a/t/lib/testutil.h b/t/lib/testutil.h index 001885914..8384427f2 100644 --- a/t/lib/testutil.h +++ b/t/lib/testutil.h @@ -31,3 +31,9 @@ void testutil_rand_path(char* buf, size_t len, const char* pfx); * /tmp. */ char* testutil_get_mount_point(void); + +/* Stat the file associated to by path and store the global and log sizes of the + * file at path in the addresses of the respective global and log pointers + * passed in. + * User can ask for one or both sizes. */ +void testutil_get_size(char* path, size_t* global, size_t* log); diff --git a/t/std/fopen-fclose.c b/t/std/fopen-fclose.c index 3e1995714..6d3202250 100644 --- a/t/std/fopen-fclose.c +++ b/t/std/fopen-fclose.c @@ -34,51 +34,52 @@ int fopen_fclose_test(char* unifyfs_root) char path[64]; char path2[64]; FILE* fd = NULL; - int rc; + + errno = 0; /* Generate a random file name in the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); testutil_rand_path(path2, sizeof(path2), unifyfs_root); + /* Verify fopen a non-existent file as read-only fails with errno=ENOENT. */ + fd = fopen(path, "r"); + ok(fd == NULL && errno == ENOENT, + "%s:%d fopen non-existent file %s w/ mode r: %s", + __FILE__, __LINE__, path, strerror(errno)); + errno = 0; /* Reset errno after test for failure */ + /* Verify we can create a new file. */ - errno = 0; fd = fopen(path, "w"); - ok(fd != NULL, "fopen non-existing file %s mode w: %s", - path, strerror(errno)); + ok(fd != NULL, "%s:%d fopen non-existing file %s w/ mode w: %s", + __FILE__, __LINE__, path, strerror(errno)); /* Verify close succeeds. */ - errno = 0; - rc = fclose(fd); - ok(rc == 0, "fclose new file %s (rc=%d): %s", path, rc, strerror(errno)); + ok(fclose(fd) == 0, "%s:%d fclose new file: %s", + __FILE__, __LINE__, strerror(errno)); /* Verify we can create a new file with mode "a". */ - errno = 0; fd = fopen(path2, "a"); - ok(fd != NULL, "fopen non-existing file %s mode a: %s", - path2, strerror(errno)); + ok(fd != NULL, "%s:%d fopen non-existing file %s mode a: %s", + __FILE__, __LINE__, path2, strerror(errno)); /* Verify close succeeds. */ - errno = 0; - rc = fclose(fd); - ok(rc == 0, "fclose new file %s (rc=%d): %s", path, rc, strerror(errno)); + ok(fclose(fd) == 0, "%s:%d fclose new file: %s", + __FILE__, __LINE__, strerror(errno)); /* Verify opening an existing file with mode "r" succeeds. */ - errno = 0; fd = fopen(path, "r"); - ok(fd != NULL, "fopen existing file %s mode r: %s", - path, strerror(errno)); + ok(fd != NULL, "%s:%d fopen existing file %s mode r: %s", + __FILE__, __LINE__, path, strerror(errno)); /* Verify close succeeds. */ - errno = 0; - rc = fclose(fd); - ok(rc == 0, "fclose %s (rc=%d): %s", path, rc, strerror(errno)); + ok(fclose(fd) == 0, "%s:%d fclose worked: %s", + __FILE__, __LINE__, strerror(errno)); /* Verify closing already closed file fails with errno=EBADF */ + ok(fclose(fd) == -1 && errno == EBADF, + "%s:%d fclose already closed file %s should fail (errno=%d): %s", + __FILE__, __LINE__, path, errno, strerror(errno)); errno = 0; - rc = fclose(fd); - ok(rc < 0 && errno == EBADF, - "fclose already closed file %s should fail (rc=%d, errno=%d): %s", - path, rc, errno, strerror(errno)); diag("Finished UNIFYFS_WRAP(fopen/fclose) tests"); diff --git a/t/std/fseek-ftell.c b/t/std/fseek-ftell.c index bf27af5d0..15df13b4d 100644 --- a/t/std/fseek-ftell.c +++ b/t/std/fseek-ftell.c @@ -33,217 +33,181 @@ int fseek_ftell_test(char* unifyfs_root) diag("Starting UNIFYFS_WRAP(fseek/ftell/rewind) tests"); char path[64]; - FILE* fd; - int ret; - long pos; + FILE* fp = NULL; + + errno = 0; /* Create a random file at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); + /* fseek on bad file stream should fail with errno=EBADF */ + dies_ok({ fseek(fp, 0, SEEK_SET); }, + "%s:%d fseek on bad file stream segfaults: %s", + __FILE__, __LINE__, strerror(errno)); + + /* ftell on non-open file stream should fail with errno=EBADF + * variable declaration and `ok` test are to avoid a compiler warning */ + dies_ok({ int rc = ftell(fp); ok(rc > 0); }, + "%s:%d ftell on bad file stream segfaults: %s", + __FILE__, __LINE__, strerror(errno)); + + /* rewind on non-open file stream should fail with errno=EBADF */ + dies_ok({ rewind(fp); }, "%s:%d rewind on bad file stream segfaults: %s", + __FILE__, __LINE__, strerror(errno)); + /* Open a file and write to it to test fseek() */ - fd = fopen(path, "w"); - fwrite("hello world", 12, 1, fd); + fp = fopen(path, "w"); + ok(fp != NULL, "%s:%d fopen(%s): %s", + __FILE__, __LINE__, path, strerror(errno)); + ok(fwrite("hello world", 12, 1, fp) == 1, "%s:%d fwrite() to file %s: %s", + __FILE__, __LINE__, path, strerror(errno)); /* fseek with invalid whence fails with errno=EINVAL. */ - errno = 0; - ret = fseek(fd, 0, -1); - ok(ret == -1 && errno == EINVAL, - "%s: fseek with invalid whence should fail (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); + ok(fseek(fp, 0, -1) == -1 && errno == EINVAL, + "%s:%d fseek with invalid whence should fail (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; /* Reset errno after test for failure */ /* fseek() with SEEK_SET tests */ /* fseek to negative offset with SEEK_SET should fail with errno=EINVAL */ + ok(fseek(fp, -1, SEEK_SET) == -1 && errno == EINVAL, + "%s:%d fseek(-1) to invalid offset w/ SEEK_SET fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = fseek(fd, -1, SEEK_SET); - ok(ret == -1 && errno == EINVAL, - "%s: fseek to negative offset w/ SEEK_SET fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); /* ftell after invalid fseek should return last offset */ - errno = 0; - pos = ftell(fd); - ok(pos == 12, - "%s: ftell after fseek to negative offset w/ SEEK_SET (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 12, + "%s:%d ftell after fseek(-1) to invalid offset w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek to valid offset with SEEK_SET succeeds */ - errno = 0; - ret = fseek(fd, 7, SEEK_SET); - ok(ret == 0, "%s: fseek to valid offset w/ SEEK_SET (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(fseek(fp, 7, SEEK_SET) == 0, + "%s:%d fseek(7) to valid offset w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(errno)); /* ftell after valid fseek with SEEK_SET */ - errno = 0; - pos = ftell(fd); - ok(pos == 7, "%s: ftell after valid fseek w/ SEEK_SET (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 7, "%s:%d ftell after fseek(7) w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek beyond end of file with SEEK_SET succeeds */ - errno = 0; - ret = fseek(fd, 25, SEEK_SET); - ok(ret == 0, "%s: fseek beyond end of file w/ SEEK_SET (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(fseek(fp, 25, SEEK_SET) == 0, "%s:%d fseek(25) past EOF w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(errno)); /* ftell after fseek beyond end of file with SEEK_SET */ - errno = 0; - pos = ftell(fd); - ok(pos == 25, - "%s: ftell after fseek beyond end of file w/ SEEK_SET (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 25, "%s:%d ftell after fseek(25) w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek to beginning of file with SEEK_SET succeeds */ - errno = 0; - ret = fseek(fd, 0, SEEK_SET); - ok(ret == 0, "%s: fseek to beginning of file w/ SEEK_SET (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(fseek(fp, 0, SEEK_SET) == 0, "%s:%d fseek(0) w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(errno)); /* ftell after fseek to beginning of file with SEEK_SET */ - errno = 0; - pos = ftell(fd); - ok(pos == 0, - "%s: ftell after fseek to beginning of file w/ SEEK_SET (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 0, "%s:%d ftell after fseek(0) w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek() with SEEK_CUR tests */ /* fseek to end of file with SEEK_CUR succeeds */ - errno = 0; - ret = fseek(fd, 12, SEEK_CUR); - ok(ret == 0, "%s: fseek to end of file w/ SEEK_CUR (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(fseek(fp, 12, SEEK_CUR) == 0, "%s:%d fseek(12) to EOF w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(errno)); /* ftell after fseek to end of file with SEEK_CUR */ - errno = 0; - pos = ftell(fd); - ok(pos == 12, - "%s: ftell after fseek to beginning of file w/ SEEK_CUR (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 12, "%s:%d ftell after fseek(12) w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek to negative offset with SEEK_CUR should fail with errno=EINVAL */ + ok(fseek(fp, -15, SEEK_CUR) == -1 && errno == EINVAL, + "%s:%d fseek(-15) to invalid offset w/ SEEK_CUR fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = fseek(fd, -15, SEEK_CUR); - ok(ret == -1 && errno == EINVAL, - "%s: fseek to negative offset w/ SEEK_CUR fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); /* ftell after fseek to negative offset with SEEK_CUR */ - errno = 0; - pos = ftell(fd); - ok(pos == 12, - "%s: ftell after fseek to negative offset w/ SEEK_CUR (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 12, + "%s:%d ftell after fseek(-15) to invalid offset w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek to beginning of file with SEEK_CUR succeeds */ - errno = 0; - ret = fseek(fd, -12, SEEK_CUR); - ok(ret == 0, "%s: fseek to beginning of file w/ SEEK_CUR (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(fseek(fp, -12, SEEK_CUR) == 0, + "%s:%d fseek(-12) to beginning of file w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(errno)); /* ftell after fseek to beginning of file with SEEK_CUR */ - errno = 0; - pos = ftell(fd); - ok(pos == 0, - "%s: ftell after fseek to beginnig of file w/ SEEK_CUR (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 0, + "%s:%d ftell after fseek(-12) to beginning of file w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek beyond end of file with SEEK_CUR succeeds */ - errno = 0; - ret = fseek(fd, 25, SEEK_CUR); - ok(ret == 0, "%s: fseek beyond end of file w/ SEEK_CUR (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(fseek(fp, 25, SEEK_CUR) == 0, "%s:%d fseek(25) past EOF w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(errno)); /* ftell after fseek beyond end of file with SEEK_CUR */ - errno = 0; - pos = ftell(fd); - ok(pos == 25, - "%s: ftell after fseek beyond end of file w/ SEEK_CUR (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 25, "%s:%d ftell after fseek(25) past EOF w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(errno)); /* rewind test */ /* ftell after rewind reports beginning of file */ - rewind(fd); - errno = 0; - pos = ftell(fd); - ok(pos == 0, - "%s: ftell after rewind reports beginning of file (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + rewind(fp); + ok(ftell(fp) == 0, "%s:%d ftell after rewind reports beginning of file: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek() with SEEK_END tests */ /* fseek to negative offset with SEEK_END should fail with errno=EINVAL */ + ok(fseek(fp, -15, SEEK_END) == -1 && errno == EINVAL, + "%s:%d fseek(-15) to invalid offset w/ SEEK_END fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = fseek(fd, -15, SEEK_END); - ok(ret == -1 && errno == EINVAL, - "%s: fseek to negative offset w/ SEEK_END fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); /* ftell after fseek to negative offset with SEEK_END */ - errno = 0; - pos = ftell(fd); - ok(pos == 0, - "%s: ftell after fseek to negative offset w/ SEEK_END (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 0, + "%s:%d ftell after fseek(-15) to negative offset w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek back one from end of file with SEEK_END succeeds */ - errno = 0; - ret = fseek(fd, -1, SEEK_END); - ok(ret == 0, - "%s: fseek back one from end of file w/ SEEK_END (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(fseek(fp, -1, SEEK_END) == 0, "%s:%d fseek(-1) from EOF w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(errno)); /* ftell after fseek back one from end of file with SEEK_END */ - errno = 0; - pos = ftell(fd); - ok(pos == 11, - "%s: ftell after fseek back one from end w/ SEEK_END (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 11, "%s:%d ftell after fseek(-1) from end w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek to beginning of file with SEEK_END succeeds */ - errno = 0; - ret = fseek(fd, -12, SEEK_END); - ok(ret == 0, "%s: fseek to beginning of file w/ SEEK_END (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(fseek(fp, -12, SEEK_END) == 0, + "%s:%d fseek(-12) to beginning of file w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(errno)); /* ftell after fseek to beginning of file with SEEK_END */ - errno = 0; - pos = ftell(fd); - ok(pos == 0, - "%s: ftell after fseek to beginning of file w/ SEEK_END (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 0, + "%s:%d ftell after fseek(-12) to beginning of file w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(errno)); /* fseek beyond end of file with SEEK_END succeeds */ - errno = 0; - ret = fseek(fd, 25, SEEK_END); - ok(ret == 0, "%s: fseek beyond end of file w/ SEEK_END (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(fseek(fp, 25, SEEK_END) == 0, "%s:%d fseek(25) past EOF w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(errno)); /* ftell after fseek beyond end of file with SEEK_END */ - errno = 0; - pos = ftell(fd); - ok(pos == 37, - "%s: ftell after fseek beyond end of file w/ SEEK_END (pos=%ld): %s", - __FILE__, pos, strerror(errno)); + ok(ftell(fp) == 37, "%s:%d ftell after fseek(25) past EOF w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(errno)); - fclose(fd); + ok(fclose(fp) == 0, "%s:%d fclose(): %s", + __FILE__, __LINE__, strerror(errno)); - /* fseek in non-open file descriptor should fail with errno=EBADF */ + /* fseek in non-open file stream should fail with errno=EBADF */ + ok(fseek(fp, 0, SEEK_SET) == -1 && errno == EBADF, + "%s:%d fseek in non-open file stream fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = fseek(fd, 0, SEEK_SET); - ok(ret == -1 && errno == EBADF, - "%s: fseek in non-open file descriptor fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); - /* ftell on non-open file descriptor should fail with errno=EBADF */ + /* ftell on non-open file stream should fail with errno=EBADF */ + ok(ftell(fp) == -1 && errno == EBADF, + "%s:%d ftell on non-open file stream fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - pos = ftell(fd); - ok(pos == -1 && errno == EBADF, - "%s: ftell on non-open file descriptor fails (pos=%ld, errno=%d): %s", - __FILE__, pos, errno, strerror(errno)); - /* rewind on non-open file descriptor should fail with errno=EBADF */ - errno = 0; - rewind(fd); + /* rewind on non-open file stream should fail with errno=EBADF */ + rewind(fp); ok(errno == EBADF, - "%s: rewind on non-open file descriptor fails (errno=%d): %s", - __FILE__, errno, strerror(errno)); + "%s:%d rewind on non-open file stream fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; diag("Finished UNIFYFS_WRAP(fseek/ftell/rewind) tests"); diff --git a/t/std/fwrite-fread.c b/t/std/fwrite-fread.c index 218e0971a..4809634db 100644 --- a/t/std/fwrite-fread.c +++ b/t/std/fwrite-fread.c @@ -13,8 +13,9 @@ */ /* - * Test fwrite/fread/fseek/fgets/rewind/ftell/feof/chmod + * Test fwrite/fread/fgets/rewind/feof/chmod */ + #include #include #include @@ -26,86 +27,176 @@ int fwrite_fread_test(char* unifyfs_root) { + diag("Starting UNIFYFS_WRAP(fwrite/fread/fgets/feof) tests"); + char path[64]; char buf[64] = {0}; - FILE* fp = NULL; - int rc; char* tmp; + FILE* fp = NULL; + int fd = 0; errno = 0; testutil_rand_path(path, sizeof(path), unifyfs_root); - /* Write "hello world" to a file */ - fp = fopen(path, "w"); - ok(fp != NULL, "%s: fopen(%s): %s", __FILE__, path, strerror(errno)); + /* fwrite to bad FILE stream should segfault */ + dies_ok({ FILE* p = fopen("/tmp/fakefile", "r"); + fwrite("hello world", 12, 1, p); }, + "%s:%d fwrite() to bad FILE stream segfaults: %s", + __FILE__, __LINE__, strerror(errno)); - rc = fwrite("hello world", 12, 1, fp); - ok(rc == 1, "%s: fwrite(\"hello world\"): %s", __FILE__, strerror(errno)); + /* fread from bad FILE stream should segfault */ + dies_ok({ size_t rc = fread(buf, 15, 1, fp); ok(rc > 0); }, + "%s:%d fread() from bad FILE stream segfaults: %s", + __FILE__, __LINE__, strerror(errno)); - rc = fclose(fp); - ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + /* fgets from bad FILE stream segfaults */ + memset(buf, 0, sizeof(buf)); + dies_ok({ char* tmp = fgets(buf, 15, fp); ok(tmp != NULL); }, + "%s:%d fgets() from bad FILE stream segfaults: %s", + __FILE__, __LINE__, strerror(errno)); + + dies_ok({ int rc = feof(fp); ok(rc > 0); }, + "%s:%d feof() on bad FILE stream segfaults: %s", + __FILE__, __LINE__, strerror(errno)); + + /* Write "hello world" to a file */ + fp = fopen(path, "w"); + ok(fp != NULL, "%s:%d fopen(%s): %s", + __FILE__, __LINE__, path, strerror(errno)); + ok(fwrite("hello world", 12, 1, fp) == 1, + "%s:%d fwrite(\"hello world\") to file %s: %s", + __FILE__, __LINE__, path, strerror(errno)); + + /* Seek to offset and overwrite "world" with "universe" */ + ok(fseek(fp, 6, SEEK_SET) == 0, "%s:%d fseek(6) to \"world\": %s", + __FILE__, __LINE__, strerror(errno)); + ok(fwrite("universe", 9, 1, fp) == 1, + "%s:%d overwrite \"world\" at offset 6 with \"universe\" : %s", + __FILE__, __LINE__, strerror(errno)); + + /* fread from file open as write-only should fail with errno=EBADF */ + ok(fseek(fp, 0, SEEK_SET) == 0, "%s:%d fseek(0): %s", + __FILE__, __LINE__, strerror(errno)); + ok(fread(buf, 15, 1, fp) == 0 && errno == EBADF, + "%s:%d fread() from file open as write-only should fail (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; /* reset errno after test for failure */ + + ok(fclose(fp) == 0, "%s:%d fclose(): %s", + __FILE__, __LINE__, strerror(errno)); + + /* fsync() tests */ + /* fsync on bad file descriptor should fail with errno=EINVAL */ + ok(fsync(fd) == -1 && errno == EINVAL, + "%s:%d fsync() on bad file descriptor should fail (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; /* Sync extents */ - int fd; fd = open(path, O_RDWR); - rc = fsync(fd); - ok(rc == 0, "%s: fsync() (rc=%d): %s", __FILE__, rc, strerror(errno)); - close(fd); + ok(fd != -1, "%s:%d open() (fd=%d): %s", + __FILE__, __LINE__, fd, strerror(errno)); + ok(fsync(fd) == 0, "%s:%d fsync(): %s", + __FILE__, __LINE__, strerror(errno)); + ok(close(fd) == 0, "%s:%d close(): %s", + __FILE__, __LINE__, strerror(errno)); + + /* fsync on non-open file should fail with errno=EBADF */ + ok(fsync(fd) == -1 && errno == EBADF, + "%s:%d fsync() on non-open file should fail (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; /* Laminate */ - rc = chmod(path, 0444); - ok(rc == 0, "%s: chmod(0444) (rc=%d): %s", __FILE__, strerror(errno)); + ok(chmod(path, 0444) == 0, "%s:%d chmod(0444): %s", + __FILE__, __LINE__, strerror(errno)); - /* Read it back */ + /* fopen a laminated file for write should fail with errno=EROFS */ + fp = fopen(path, "w"); + ok(fp == NULL && errno == EROFS, + "%s:%d fopen laminated file for write fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; + + /* fread() tests */ fp = fopen(path, "r"); - ok(fp != NULL, "%s: fopen(%s): %s", __FILE__, path, strerror(errno)); + ok(fp != NULL, "%s:%d fopen(%s): %s", + __FILE__, __LINE__, path, strerror(errno)); - rc = fread(buf, 12, 1, fp); - ok(rc == 1, "%s: fread() buf[]=\"%s\", (rc %d): %s", __FILE__, buf, rc, - strerror(errno)); - is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + /* fwrite to file open as read-only should fail with errno=EBADF */ + ok(fwrite("hello world", 12, 1, fp) == 0 && errno == EBADF, + "%s:%d fwrite() to file open for read-only fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; - fseek(fp, 6, SEEK_SET); - rc = ftell(fp); - ok(rc == 6, "%s: fseek() (rc %d): %s", __FILE__, rc, strerror(errno)); + ok(fread(buf, 15, 1, fp) == 1, "%s:%d fread() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + is(buf, "hello universe", "%s:%d fread() saw \"hello universe\"", + __FILE__, __LINE__); - rc = fread(buf, 6, 1, fp); - ok(rc == 1, "%s: fread() at offset 6 buf[]=\"%s\", (rc %d): %s", __FILE__, - buf, rc, strerror(errno)); - is(buf, "world", "%s: saw \"world\"", __FILE__); + ok(fseek(fp, 6, SEEK_SET) == 0, "%s:%d fseek(6): %s", + __FILE__, __LINE__, strerror(errno)); + ok(fread(buf, 9, 1, fp) == 1, "%s:%d fread() at offset 6 buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + is(buf, "universe", "%s:%d fread() saw \"universe\"", __FILE__, __LINE__); rewind(fp); - rc = fread(buf, 12, 1, fp); - ok(rc == 1, "%s: fread() after rewind() buf[]=\"%s\", (rc %d): %s", - __FILE__, buf, rc, strerror(errno)); - is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + ok(fread(buf, 15, 1, fp) == 1, + "%s:%d fread() after rewind() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + is(buf, "hello universe", "%s:%d fread() saw \"hello universe\"", + __FILE__, __LINE__); + /* fgets() tests */ rewind(fp); memset(buf, 0, sizeof(buf)); - tmp = fgets(buf, 12, fp); - ok(tmp == buf, "%s: fgets() after rewind() buf[]=\"%s\": %s", __FILE__, buf, - strerror(errno)); - is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + tmp = fgets(buf, 15, fp); + ok(tmp == buf, "%s:%d fgets() after rewind() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + is(buf, "hello universe", "%s:%d fgets() saw \"hello universe\"", + __FILE__, __LINE__); rewind(fp); memset(buf, 0, sizeof(buf)); tmp = fgets(buf, 6, fp); - ok(tmp == buf, "%s: fgets() with size = 6 after rewind() buf[]=\"%s\": %s", - __FILE__, buf, strerror(errno)); - is(buf, "hello", "%s: saw \"hello\"", __FILE__); + ok(tmp == buf, "%s:%d fgets() w/ size = 6 after rewind() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + is(buf, "hello", "%s:%d fgets() saw \"hello\"", __FILE__, __LINE__); rewind(fp); - rc = fread(buf, sizeof(buf), 1, fp); - ok(rc != 1, "%s: fread() past end of file (rc %d): %s", __FILE__, rc, - strerror(errno)); + ok(fread(buf, sizeof(buf), 1, fp) != 1, "%s:%d fread() EOF: %s", + __FILE__, __LINE__, strerror(errno)); + + ok(feof(fp) != 0, "%s:%d feof() past EOF: %s", + __FILE__, __LINE__, strerror(errno)); + + ok(fclose(fp) == 0, "%s:%d fclose(): %s", + __FILE__, __LINE__, strerror(errno)); + + /* fwrite to closed stream fails with errno=EBADF */ + ok(fwrite("hello world", 12, 1, fp) == 0 && errno == EBADF, + "%s:%d fwrite() to closed stream fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; + + /* fread from closed stream fails with errno=EBADF */ + ok(fread(buf, 15, 1, fp) == 0 && errno == EBADF, + "%s:%d fread() from closed stream fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; + + /* fgets from closed stream fails with errno=EBADF */ + memset(buf, 0, sizeof(buf)); + tmp = fgets(buf, 15, fp); + ok(errno == EBADF, "%s:%d fgets() from closed stream fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; - rc = feof(fp); - ok(rc != 0, "%s: feof() past end of file (rc %d): %s", __FILE__, rc, - strerror(errno)); + ok(feof(fp) != 0, "%s:%d feof() on closed stream: %s", + __FILE__, __LINE__, strerror(errno)); - rc = fclose(fp); - ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + diag("Finished UNIFYFS_WRAP(fwrite/fread/fgets/feof) tests"); return 0; } diff --git a/t/std/size.c b/t/std/size.c index 287e87d12..1c2059b80 100644 --- a/t/std/size.c +++ b/t/std/size.c @@ -12,9 +12,6 @@ * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ - /* - * Test fwrite/fread/fseek/fgets/rewind/ftell/feof/chmod - */ #include #include #include @@ -25,39 +22,19 @@ #include "t/lib/testutil.h" /* - * Test correctness of global file size. Also, test opening a file - * for append, and test file positioning (fseek, ftell, etc). + * Test correctness of global file size. Also, test opening a file for append */ -/* Get global or log sizes (or all) */ -static -void get_size(char* path, size_t* global, size_t* log) -{ - struct stat sb = {0}; - int rc; - - rc = stat(path, &sb); - if (rc != 0) { - printf("Error: %s\n", strerror(errno)); - exit(1); /* die on failure */ - } - if (global) { - *global = sb.st_size; - } - - if (log) { - *log = sb.st_rdev; - } -} - int size_test(char* unifyfs_root) { + diag("Starting file size and fwrite/fread with append tests"); + char path[64]; char buf[64] = {0}; FILE* fp = NULL; - int rc; char* tmp; size_t global, log; + int fd; errno = 0; @@ -65,88 +42,81 @@ int size_test(char* unifyfs_root) /* Write "hello world" to a file */ fp = fopen(path, "w"); - ok(fp != NULL, "%s: fopen(%s): %s", __FILE__, path, strerror(errno)); - - rc = fwrite("hello world", 12, 1, fp); - ok(rc == 1, "%s: fwrite(\"hello world\"): %s", __FILE__, strerror(errno)); - - rc = fclose(fp); - ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); - - get_size(path, &global, &log); - ok(global == 12, "%s: global size is %d: %s", __FILE__, global, - strerror(errno)); - ok(log == 12, "%s: log size is %d: %s", __FILE__, log, - strerror(errno)); + ok(fp != NULL, "%s:%d fopen(%s): %s", + __FILE__, __LINE__, path, strerror(errno)); + ok(fwrite("hello world", 12, 1, fp) == 1, + "%s:%d fwrite(\"hello world\": %s", __FILE__, __LINE__, strerror(errno)); + ok(fclose(fp) == 0, "%s:%d fclose(): %s", + __FILE__, __LINE__, strerror(errno)); + + testutil_get_size(path, &global, &log); + ok(global == 12, "%s:%d global size after fwrite(\"hello world\") = %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + ok(log == 12, "%s:%d log size after fwrite(\"hello world\") = %d: %s", + __FILE__, __LINE__, log, strerror(errno)); /* Open the file again with append, write to it. */ fp = fopen(path, "a"); - ok(fp != NULL, "%s: fopen(%s) in append mode: %s", __FILE__, path, - strerror(errno)); + ok(fp != NULL, "%s:%d fopen(%s) in append mode: %s", + __FILE__, __LINE__, path, strerror(errno)); + ok(fwrite("HELLO WORLD", 12, 1, fp) == 1, + "%s:%d fwrite(\"HELLO WORLD\") with file %s open for append: %s", + __FILE__, __LINE__, path, strerror(errno)); - rc = fwrite("HELLO WORLD", 12, 1, fp); - ok(rc == 1, "%s: fwrite(\"HELLO WORLD\"): %s", __FILE__, strerror(errno)); - - rc = ftell(fp); - ok(rc == 24, "%s: ftell() (rc=%d) %s", __FILE__, rc, strerror(errno)); + ok(ftell(fp) == 24, "%s:%d ftell() after appending to file: %s", + __FILE__, __LINE__, strerror(errno)); /* * Set our position to somewhere in the middle of the file. Since the file * is in append mode, this new position should be ignored, and writes * should still go to the end of the file. */ - rc = fseek(fp, 11, SEEK_SET); - ok(rc == 0, "%s: fseek(11) (rc=%d) %s", __FILE__, rc, strerror(errno)); - - rc = fwrite("", 6, 1, fp); - ok(rc == 1, "%s: fwrite(\" \") (rc=%d): %s", __FILE__, rc, strerror(errno)); - - /* Test seeking to SEEK_END */ - rc = fseek(fp, 0, SEEK_END); - ok(rc == 0, "%s: fseek(SEEK_END) (rc=%d) %s", __FILE__, rc, - strerror(errno)); - - rc = ftell(fp); - ok(rc == 30, "%s: ftell() (rc=%d) %s", __FILE__, rc, strerror(errno)); + ok(fseek(fp, 11, SEEK_SET) == 0, "%s:%d fseek(11) before append: %s", + __FILE__, __LINE__, strerror(errno)); + ok(fwrite("", 6, 1, fp) == 1, + "%s:%d fwrite(\"\") to append after seek to middle of file: %s", + __FILE__, __LINE__, strerror(errno)); - rc = fclose(fp); - ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + ok(ftell(fp) == 30, "%s:%d ftell() after seek and appending to file: %s", + __FILE__, __LINE__, strerror(errno)); - get_size(path, &global, &log); - ok(global == 30, "%s: global size is %d: %s", __FILE__, global, - strerror(errno)); - ok(log == 30, "%s: log size is %d: %s", __FILE__, log, - strerror(errno)); + ok(fclose(fp) == 0, "%s:%d fclose(): %s", + __FILE__, __LINE__, strerror(errno)); + testutil_get_size(path, &global, &log); + ok(global == 30, "%s:%d global size after append is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + ok(log == 30, "%s:%d log size after append is %d: %s", + __FILE__, __LINE__, log, strerror(errno)); /* Sync extents */ - int fd; fd = open(path, O_RDWR); - ok(fd >= 0, "%s: open() (fd=%d): %s", __FILE__, fd, strerror(errno)); - - rc = fsync(fd); - ok(rc == 0, "%s: fsync() (rc=%d): %s", __FILE__, rc, strerror(errno)); - close(fd); + ok(fd >= 0, "%s:%d open file for fsync: %s", + __FILE__, __LINE__, strerror(errno)); + ok(fsync(fd) == 0, "%s:%d fsync(): %s", + __FILE__, __LINE__, strerror(errno)); + ok(close(fd) != -1, "%s:%d close after fsync: %s", + __FILE__, __LINE__, strerror(errno)); /* Laminate */ - rc = chmod(path, 0444); - ok(rc == 0, "%s: chmod(0444) (rc=%d): %s", __FILE__, rc, strerror(errno)); + ok(chmod(path, 0444) == 0, "%s:%d chmod(0444): %s", + __FILE__, __LINE__, strerror(errno)); /* Global size should be correct */ - get_size(path, &global, &log); - ok(global == 30, "%s: global size is %d: %s", __FILE__, global, - strerror(errno)); - ok(log == 30, "%s: log size is %d: %s", __FILE__, log, - strerror(errno)); + testutil_get_size(path, &global, &log); + ok(global == 30, "%s:%d global size after laminate is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + ok(log == 30, "%s:%d log size after laminate is %d: %s", + __FILE__, __LINE__, log, strerror(errno)); /* Read it back */ fp = fopen(path, "r"); - ok(fp != NULL, "%s: fopen(%s): %s", __FILE__, path, strerror(errno)); + ok(fp != NULL, "%s%d: fopen(%s): %s", + __FILE__, __LINE__, path, strerror(errno)); memset(buf, 0, sizeof(buf)); - rc = fread(buf, 30, 1, fp); - ok(rc == 1, "%s: fread() buf[]=\"%s\", (rc %d): %s", __FILE__, buf, rc, - strerror(errno)); + ok(fread(buf, 30, 1, fp) == 1, "%s:%d fread() buf[]=\"%s\", : %s", + __FILE__, __LINE__, buf, strerror(errno)); /* * We wrote three strings to the file: "hello world" "HELLO WORLD" and @@ -157,49 +127,47 @@ int size_test(char* unifyfs_root) buf[23] = ' '; /* after "HELLO WORLD" */ is(buf, "hello world HELLO WORLD ", - "%s: saw \"hello world HELLO WORLD \"", __FILE__); + "%s:%d saw \"hello world HELLO WORLD \"", __FILE__, __LINE__); /* Try seeking and reading at various positions */ - fseek(fp, 6, SEEK_SET); - rc = ftell(fp); - ok(rc == 6, "%s: fseek() (rc %d): %s", __FILE__, rc, strerror(errno)); + ok(fseek(fp, 6, SEEK_SET) == 0, "%s:%d fseek(6): %s", + __FILE__, __LINE__, strerror(errno)); - rc = fread(buf, 6, 1, fp); - ok(rc == 1, "%s: fread() at offset 6 buf[]=\"%s\", (rc %d): %s", __FILE__, - buf, rc, strerror(errno)); - is(buf, "world", "%s: saw \"world\"", __FILE__); + ok(fread(buf, 6, 1, fp) == 1, "%s:%d fread() at offset 6 buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + is(buf, "world", "%s:%d saw \"world\"", __FILE__, __LINE__); rewind(fp); - rc = fread(buf, 12, 1, fp); - ok(rc == 1, "%s: fread() after rewind() buf[]=\"%s\", (rc %d): %s", - __FILE__, buf, rc, strerror(errno)); - is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + ok(fread(buf, 12, 1, fp) == 1, + "%s:%d fread() after rewind() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + is(buf, "hello world", "%s:%d saw \"hello world\"", __FILE__, __LINE__); rewind(fp); memset(buf, 0, sizeof(buf)); tmp = fgets(buf, 12, fp); - ok(tmp == buf, "%s: fgets() after rewind() buf[]=\"%s\": %s", __FILE__, buf, - strerror(errno)); - is(buf, "hello world", "%s: saw \"hello world\"", __FILE__); + ok(tmp == buf, "%s:%d fgets() after rewind() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + is(buf, "hello world", "%s:%d saw \"hello world\"", __FILE__, __LINE__); rewind(fp); memset(buf, 0, sizeof(buf)); tmp = fgets(buf, 6, fp); - ok(tmp == buf, "%s: fgets() with size = 6 after rewind() buf[]=\"%s\": %s", - __FILE__, buf, strerror(errno)); - is(buf, "hello", "%s: saw \"hello\"", __FILE__); + ok(tmp == buf, "%s:%d fgets() w/ size = 6 after rewind() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + is(buf, "hello", "%s:%d saw \"hello\"", __FILE__, __LINE__); rewind(fp); - rc = fread(buf, sizeof(buf), 1, fp); - ok(rc != 1, "%s: fread() past end of file (rc %d): %s", __FILE__, rc, - strerror(errno)); + ok(fread(buf, sizeof(buf), 1, fp) != 1, + "%s:%d fread() past EOF: %s", __FILE__, __LINE__, strerror(errno)); + + ok(feof(fp) != 0, "%s:%d feof() past EOF: %s", + __FILE__, __LINE__, strerror(errno)); - rc = feof(fp); - ok(rc != 0, "%s: feof() past end of file (rc %d): %s", __FILE__, rc, - strerror(errno)); + ok(fclose(fp) == 0, "%s:%d fclose(): %s", + __FILE__, __LINE__, strerror(errno)); - rc = fclose(fp); - ok(rc == 0, "%s: fclose() (rc=%d): %s", __FILE__, rc, strerror(errno)); + diag("Starting file size and fwrite/fread with append tests"); return 0; } diff --git a/t/sys/creat-close.c b/t/sys/creat-close.c index 16367036b..f0194ad49 100644 --- a/t/sys/creat-close.c +++ b/t/sys/creat-close.c @@ -34,46 +34,41 @@ int creat_close_test(char* unifyfs_root) char path[64]; int mode = 0600; int fd = -1; - int rc = -1; + + errno = 0; /* Create a random file name at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); /* Verify closing a non-existent file fails with errno=EBADF */ - errno = 0; - rc = close(fd); - ok(rc < 0 && errno == EBADF, - "close non-existing file %s should fail (rc=%d, errno=%d): %s", - path, rc, errno, strerror(errno)); + ok(close(fd) == -1 && errno == EBADF, + "%s:%d close non-existing file %s should fail (errno=%d): %s", + __FILE__, __LINE__, path, errno, strerror(errno)); + errno = 0; /* Reset errno after test for failure */ /* Verify we can create a non-existent file. */ - errno = 0; fd = creat(path, mode); - ok(fd >= 0, "creat non-existing file %s (fd=%d): %s", - path, fd, strerror(errno)); + ok(fd >= 0, "%s:%d creat non-existing file %s (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); /* Verify close succeeds. */ - errno = 0; - rc = close(fd); - ok(rc == 0, "close new file %s (rc=%d): %s", path, rc, strerror(errno)); + ok(close(fd) == 0, "%s:%d close new file: %s", + __FILE__, __LINE__, strerror(errno)); /* Verify creating an already created file succeeds. */ - errno = 0; fd = creat(path, mode); - ok(fd >= 0, "creat existing file %s (fd=%d): %s", - path, fd, strerror(errno)); + ok(fd >= 0, "%s:%d creat existing file %s (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); /* Verify close succeeds. */ - errno = 0; - rc = close(fd); - ok(rc == 0, "close %s (rc=%d): %s", path, rc, strerror(errno)); + ok(close(fd) == 0, "%s:%d close %s: %s", + __FILE__, __LINE__, path, strerror(errno)); /* Verify closing already closed file fails with errno=EBADF */ + ok(close(fd) == -1 && errno == EBADF, + "%s:%d close already closed file %s should fail (errno=%d): %s", + __FILE__, __LINE__, path, errno, strerror(errno)); errno = 0; - rc = close(fd); - ok(rc < 0 && errno == EBADF, - "close already closed file %s should fail (rc=%d, errno=%d): %s", - path, rc, errno, strerror(errno)); /* CLEANUP * diff --git a/t/sys/lseek.c b/t/sys/lseek.c index 5258c8ff3..5d8639ccd 100644 --- a/t/sys/lseek.c +++ b/t/sys/lseek.c @@ -36,184 +36,159 @@ int lseek_test(char* unifyfs_root) char path[64]; int file_mode = 0600; - int fd; - off_t ret; + int fd = -1; + + errno = 0; /* Create a random file at the mountpoint path to test on */ testutil_rand_path(path, sizeof(path), unifyfs_root); + /* lseek in bad file descriptor should fail with errno=EBADF */ + ok(lseek(fd, 0, SEEK_SET) == -1 && errno == EBADF, + "%s:%d lseek in bad file descriptor fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; /* reset errno after test for failure */ + /* Open a file and write to it to test lseek() */ fd = open(path, O_RDWR | O_CREAT | O_TRUNC, file_mode); - - ret = write(fd, "hello world", 12); - ok(ret == 12, "%s: write works (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(fd >= 0, "%s:%d open worked: %s", __FILE__, __LINE__, strerror(errno)); + ok(write(fd, "hello world", 12) == 12, "%s:%d write worked: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek with invalid whence fails with errno=EINVAL. */ + ok(lseek(fd, 0, -1) == -1 && errno == EINVAL, + "%s:%d lseek with invalid whence should fail (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = lseek(fd, 0, -1); - ok(ret == -1 && errno == EINVAL, - "%s: lseek with invalid whence should fail (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); /* lseek() with SEEK_SET tests */ /* lseek to negative offset with SEEK_SET should fail with errno=EINVAL */ + ok(lseek(fd, -1, SEEK_SET) == -1 && errno == EINVAL, + "%s:%d lseek(-1) to invalid offset w/ SEEK_SET fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = lseek(fd, -1, SEEK_SET); - ok(ret == -1 && errno == EINVAL, - "%s: lseek to negative offset w/ SEEK_SET fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); /* lseek to valid offset with SEEK_SET succeeds */ - errno = 0; - ret = lseek(fd, 7, SEEK_SET); - ok(ret == 7, "%s: lseek to valid offset w/ SEEK_SET (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, 7, SEEK_SET) == 7, + "%s:%d lseek(7) to valid offset w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek beyond end of file with SEEK_SET succeeds */ - errno = 0; - ret = lseek(fd, 25, SEEK_SET); - ok(ret == 25, "%s: lseek beyond end of file w/ SEEK_SET (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, 25, SEEK_SET) == 25, + "%s:%d lseek(25) beyond EOF w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek to beginning of file with SEEK_SET succeeds */ - errno = 0; - ret = lseek(fd, 0, SEEK_SET); - ok(ret == 0, "%s: lseek to beginning of file w/ SEEK_SET (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, 0, SEEK_SET) == 0, "%s:%d lseek(0) w/ SEEK_SET: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek() with SEEK_CUR tests */ /* lseek to end of file with SEEK_CUR succeeds */ - errno = 0; - ret = lseek(fd, 12, SEEK_CUR); - ok(ret == 12, "%s: lseek to end of file w/ SEEK_CUR (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, 12, SEEK_CUR) == 12, "%s:%d lseek(12) to EOF w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek to negative offset with SEEK_CUR should fail with errno=EINVAL */ + ok(lseek(fd, -15, SEEK_CUR) == -1 && errno == EINVAL, + "%s:%d lseek(-15) to invalid offset w/ SEEK_CUR fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = lseek(fd, -15, SEEK_CUR); - ok(ret == -1 && errno == EINVAL, - "%s: lseek to negative offset w/ SEEK_CUR fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); /* lseek to beginning of file with SEEK_CUR succeeds */ - errno = 0; - ret = lseek(fd, -12, SEEK_CUR); - ok(ret == 0, "%s: lseek to beginning of file w/ SEEK_CUR (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, -12, SEEK_CUR) == 0, + "%s:%d lseek(-12) to beginning of file w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek beyond end of file with SEEK_CUR succeeds */ - errno = 0; - ret = lseek(fd, 25, SEEK_CUR); - ok(ret == 25, "%s: lseek beyond end of file w/ SEEK_CUR (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, 25, SEEK_CUR) == 25, + "%s:%d lseek(25) beyond EOF w/ SEEK_CUR: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek() with SEEK_END tests */ /* lseek to negative offset with SEEK_END should fail with errno=EINVAL */ + ok(lseek(fd, -15, SEEK_END) == -1 && errno == EINVAL, + "%s:%d lseek(-15) to invalid offset w/ SEEK_END fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = lseek(fd, -15, SEEK_END); - ok(ret == -1 && errno == EINVAL, - "%s: lseek to negative offset w/ SEEK_END fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); /* lseek back one from end of file with SEEK_END succeeds */ - errno = 0; - ret = lseek(fd, -1, SEEK_END); - ok(ret == 11, - "%s: lseek back one from end of file w/ SEEK_END (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, -1, SEEK_END) == 11, + "%s:%d lseek(-1) from EOF w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek to beginning of file with SEEK_END succeeds */ - errno = 0; - ret = lseek(fd, -12, SEEK_END); - ok(ret == 0, "%s: lseek to beginning of file w/ SEEK_END (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, -12, SEEK_END) == 0, + "%s:%d lseek(-12) to beginning of file w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek beyond end of file with SEEK_END succeeds */ - errno = 0; - ret = lseek(fd, 25, SEEK_END); - ok(ret == 37, "%s: lseek beyond end of file w/ SEEK_END (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, 25, SEEK_END) == 37, + "%s:%d lseek(25) beyond EOF w/ SEEK_END: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek() with SEEK_DATA tests */ /* Write beyond end of file to create a hole */ - ret = write(fd, "hello universe", 15); - ok(ret == 15, "%s: write works (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(write(fd, "hello universe", 15) == 15, "%s:%d write to create hole: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek to negative offset with SEEK_DATA should fail with errno=ENXIO */ + ok(lseek(fd, -1, SEEK_DATA) == -1 && errno == ENXIO, + "%s:%d lseek(-1) to invalid offset w/ SEEK_DATA fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = lseek(fd, -1, SEEK_DATA); - ok(ret == -1 && errno == ENXIO, - "%s: lseek to negative offset w/ SEEK_DATA fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); /* lseek to beginning of file with SEEK_DATA succeeds */ - errno = 0; - ret = lseek(fd, 0, SEEK_DATA); - ok(ret == 0, "%s: lseek to beginning of file w/ SEEK_DATA (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, 0, SEEK_DATA) == 0, "%s:%d lseek(0) w/ SEEK_DATA: %s", + __FILE__, __LINE__, strerror(errno)); - /* lseek to data after hole with SEEK_DATA returns current offset or offset - * of first set of data */ - errno = 0; - ret = lseek(fd, 15, SEEK_DATA); - ok(ret == 15 || ret == 37, - "%s: lseek to data after hole w/ SEEK_DATA (ret=%d): %s", - __FILE__, ret, strerror(errno)); + /* Fallback implementation: lseek to data after hole with SEEK_DATA returns + * current offset */ + ok(lseek(fd, 15, SEEK_DATA) == 15, + "%s:%d lseek(15) to data after hole w/ SEEK_DATA returns offset: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek beyond end of file with SEEK_DATA should fail with errno=ENXIO */ + ok(lseek(fd, 75, SEEK_DATA) == -1 && errno == ENXIO, + "%s:%d lseek(75) beyond EOF w/ SEEK_DATA fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = lseek(fd, 75, SEEK_DATA); - ok(ret == -1 && errno == ENXIO, - "%s: lseek beyond end of file w/ SEEK_DATA fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); /* lseek() with SEEK_HOLE tests */ - /* lseek to negative offset with SEEK_HOLE should fail with errno=ENXIO */ + ok(lseek(fd, -1, SEEK_HOLE) == -1 && errno == ENXIO, + "%s:%d lseek(-1) to invalid offset w/ SEEK_HOLE fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = lseek(fd, -1, SEEK_HOLE); - ok(ret == -1 && errno == ENXIO, - "%s: lseek to negative offset w/ SEEK_HOLE fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); - /* lseek to first hole of file with SEEK_HOLE succeeds returns start of hole - * or EOF */ - errno = 0; - ret = lseek(fd, 0, SEEK_HOLE); - ok(ret == 12 || ret == 52, - "%s: lseek to first hole in file w/ SEEK_HOLE (ret=%d): %s", - __FILE__, ret, strerror(errno)); + /* Fallback implementation: lseek to first hole of file with SEEK_HOLE + * returns or EOF */ + ok(lseek(fd, 0, SEEK_HOLE) == 52, + "%s:%d lseek(0) to first hole in file w/ SEEK_HOLE returns EOF: %s", + __FILE__, __LINE__, strerror(errno)); - /* lseek to middle of hole with SEEK_HOLE returns position in hole or EOF */ - errno = 0; - ret = lseek(fd, 18, SEEK_HOLE); - ok(ret == 18 || ret == 52, - "%s: lseek to middle of hole w/ SEEK_HOLE (ret=%d): %s", - __FILE__, ret, strerror(errno)); + /* Fallback implementation: lseek to middle of hole with SEEK_HOLE returns + * EOF */ + ok(lseek(fd, 18, SEEK_HOLE) == 52, + "%s:%d lseek(18) to middle of hole w/ SEEK_HOLE returns EOF: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek to end of file with SEEK_HOLE succeeds */ - errno = 0; - ret = lseek(fd, 42, SEEK_HOLE); - ok(ret == 52, "%s: lseek to end of file w/ SEEK_HOLE (ret=%d): %s", - __FILE__, ret, strerror(errno)); + ok(lseek(fd, 42, SEEK_HOLE) == 52, + "%s:%d lseek(42) to EOF w/ SEEK_HOLE: %s", + __FILE__, __LINE__, strerror(errno)); /* lseek beyond end of file with SEEK_HOLE should fail with errno= ENXIO */ + ok(lseek(fd, 75, SEEK_HOLE) == -1 && errno == ENXIO, + "%s:%d lseek beyond EOF w/ SEEK_HOLE fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = lseek(fd, 75, SEEK_HOLE); - ok(ret == -1 && errno == ENXIO, - "%s: lseek beyond end of file w/ SEEK_HOLE fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); close(fd); /* lseek in non-open file descriptor should fail with errno=EBADF */ + ok(lseek(fd, 0, SEEK_SET) == -1 && errno == EBADF, + "%s:%d lseek in non-open file descriptor fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); errno = 0; - ret = lseek(fd, 0, SEEK_SET); - ok(ret == -1 && errno == EBADF, - "%s: lseek in non-open file descriptor fails (ret=%d, errno=%d): %s", - __FILE__, ret, errno, strerror(errno)); diag("Finished UNIFYFS_WRAP(lseek) tests"); diff --git a/t/sys/mkdir-rmdir.c b/t/sys/mkdir-rmdir.c index e3ac88622..817b3402c 100644 --- a/t/sys/mkdir-rmdir.c +++ b/t/sys/mkdir-rmdir.c @@ -39,7 +39,8 @@ int mkdir_rmdir_test(char* unifyfs_root) int file_mode = 0600; int dir_mode = 0700; int fd; - int rc; + + errno = 0; /* Create random dir and file path names at the mountpoint to test on */ testutil_rand_path(dir_path, sizeof(dir_path), unifyfs_root); @@ -64,115 +65,103 @@ int mkdir_rmdir_test(char* unifyfs_root) /* todo_mkdir_1: Remove when issue is resolved */ todo("mkdir_1: we currently don't support directory structure"); - /* Verify we cannot create a dir whose parent dir doesn't exist with + /* Verify creating a dir under non-existent parent dir fails with * errno=ENOENT */ - errno = 0; - rc = mkdir(subdir_path, dir_mode); - ok(rc < 0 && errno == ENOENT, - "mkdir dir %s without parent dir should fail (rc=%d, errno=%d): %s", - subdir_path, rc, errno, strerror(errno)); - + ok(mkdir(subdir_path, dir_mode) == -1 && errno == ENOENT, + "%s:%d mkdir dir %s without parent dir should fail (errno=%d): %s", + __FILE__, __LINE__, subdir_path, errno, strerror(errno)); end_todo; /* end todo_mkdir_1 */ + errno = 0; /* Reset errno after test for failure */ /* Verify rmdir a non-existing directory fails with errno=ENOENT */ + ok(rmdir(dir_path) == -1 && errno == ENOENT, + "%s:%d rmdir non-existing dir %s should fail (errno=%d): %s", + __FILE__, __LINE__, dir_path, errno, strerror(errno)); errno = 0; - rc = rmdir(dir_path); - ok(rc < 0 && errno == ENOENT, - "rmdir non-existing dir %s should fail (rc=%d, errno=%d): %s", - dir_path, rc, errno, strerror(errno)); /* Verify we can create a non-existent directory. */ - errno = 0; - rc = mkdir(dir_path, dir_mode); - ok(rc == 0, "mkdir non-existing dir %s (rc=%d): %s", - dir_path, rc, strerror(errno)); + ok(mkdir(dir_path, dir_mode) == 0, "%s:%d mkdir non-existing dir %s: %s", + __FILE__, __LINE__, dir_path, strerror(errno)); - /* Verify we cannot recreate an already created directory with - * errno=EEXIST */ + /* Verify recreating an already created directory fails with errno=EEXIST */ + ok(mkdir(dir_path, dir_mode) == -1 && errno == EEXIST, + "%s:%d mkdir existing dir %s should fail (errno=%d): %s", + __FILE__, __LINE__, dir_path, errno, strerror(errno)); errno = 0; - rc = mkdir(dir_path, dir_mode); - ok(rc < 0 && errno == EEXIST, - "mkdir existing dir %s should fail (rc=%d, errno=%d): %s", - dir_path, rc, errno, strerror(errno)); /* todo_mkdir_2: Remove when issue is resolved */ todo("mkdir_2: should fail with errno=EISDIR=21"); - /* Verify we cannot create a file with same name as a directory with - * errno=EISDIR */ - errno = 0; + /* Verify creating a file with same name as a dir fails with errno=EISDIR */ fd = creat(dir_path, file_mode); - ok(fd < 0 && errno == EISDIR, - "creat file with same name as dir %s should fail (fd=%d, errno=%d): %s", - dir_path, fd, errno, strerror(errno)); + ok(fd == -1 && errno == EISDIR, + "%s:%d creat file with same name as dir %s fails (fd=%d, errno=%d): %s", + __FILE__, __LINE__, dir_path, fd, errno, strerror(errno)); end_todo; /* end todo_mkdir_2 */ + errno = 0; /* todo_mkdir_3: Remove when issue is resolved */ todo("mkdir_3: this fails because \"TODO mkdir_1\" is failing"); /* Verify we can create a subdirectory under an existing directory */ - errno = 0; - rc = mkdir(subdir_path, dir_mode); - ok(rc == 0, "mkdir subdirectory %s in existing dir (rc=%d): %s", - subdir_path, rc, strerror(errno)); + ok(mkdir(subdir_path, dir_mode) == 0, + "%s:%d mkdir subdirectory %s in existing dir: %s", + __FILE__, __LINE__, subdir_path, strerror(errno)); end_todo; /* end todo_mkdir_3 */ + errno = 0; /* can remove when test is passing */ /* Verify we can create a subfile under an existing directory */ - errno = 0; fd = creat(subfile_path, file_mode); - ok(fd > 0, "creat subfile %s in existing dir (fd=%d): %s", - subfile_path, fd, strerror(errno)); + ok(fd >= 0, "%s:%d creat subfile %s in existing dir (fd=%d): %s", + __FILE__, __LINE__, subfile_path, fd, strerror(errno)); - rc = close(fd); + ok(close(fd) == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(errno)); - /* todo_mkdir_4: Remove when issue is resolved */ - todo("mkdir_4: unifyfs currently creates all paths as separate entities"); /* Verify creating a directory whose parent is a file fails with * errno=ENOTDIR */ fd = creat(file_path, file_mode); - rc = close(fd); + ok(fd >= 0, "%s:%d creat parent file %s (fd=%d): %s", + __FILE__, __LINE__, file_path, fd, strerror(errno)); + ok(close(fd) == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(errno)); - errno = 0; - rc = mkdir(file_subdir_path, dir_mode); - ok(rc < 0 && errno == ENOTDIR, - "mkdir dir %s whose parent is a file should fail (rc=%d, errno=%d): %s", - file_subdir_path, rc, errno, strerror(errno)); + /* todo_mkdir_4: Remove when issue is resolved */ + todo("mkdir_4: unifyfs currently creates all paths as separate entities"); + ok(mkdir(file_subdir_path, dir_mode) == -1 && errno == ENOTDIR, + "%s:%d mkdir dir %s where parent is a file should fail (errno=%d): %s", + __FILE__, __LINE__, file_subdir_path, errno, strerror(errno)); end_todo; /* end todo_mkdir_4 */ + errno = 0; /* Verify rmdir a non-directory fails with errno=ENOENT */ + ok(rmdir(file_path) == -1 && errno == ENOTDIR, + "%s:%d rmdir non-directory %s should fail (errno=%d): %s", + __FILE__, __LINE__, file_path, errno, strerror(errno)); errno = 0; - rc = rmdir(file_path); - ok(rc < 0 && errno == ENOTDIR, - "rmdir non-directory %s should fail (rc=%d, errno=%d): %s", - file_path, rc, errno, strerror(errno)); /* todo_mkdir_5: Remove when issue is resolved */ todo("mkdir_5: unifyfs currently creates all paths as separate entities"); /* Verify rmdir a non-empty directory fails with errno=ENOTEMPTY */ - errno = 0; - rc = rmdir(dir_path); - ok(rc < 0 && errno == ENOTEMPTY, - "rmdir non-empty directory %s should fail (rc=%d, errno=%d): %s", - dir_path, rc, errno, strerror(errno)); + ok(rmdir(dir_path) == -1 && errno == ENOTEMPTY, + "%s:%d rmdir non-empty directory %s should fail (errno=%d): %s", + __FILE__, __LINE__, dir_path, errno, strerror(errno)); end_todo; /* end todo_mkdir_5 */ + errno = 0; /* Verify we can rmdir an empty directory */ - errno = 0; - rc = rmdir(subdir_path); - ok(rc == 0, "rmdir an empty directory %s (rc=%d): %s", - subdir_path, rc, strerror(errno)); + ok(rmdir(subdir_path) == 0, "%s:%d rmdir an empty directory %s: %s", + __FILE__, __LINE__, subdir_path, strerror(errno)); /* Verify rmdir an already removed directory fails with errno=ENOENT */ + ok(rmdir(subdir_path) == -1 && errno == ENOENT, + "%s:%d rmdir already removed dir %s should fail (errno=%d): %s", + __FILE__, __LINE__, subdir_path, errno, strerror(errno)); errno = 0; - rc = rmdir(subdir_path); - ok(rc < 0 && errno == ENOENT, - "rmdir already removed dir %s should fail (rc=%d, errno=%d): %s", - subdir_path, rc, errno, strerror(errno)); /* Verify trying to rmdir the mount point fails with errno=EBUSY */ + ok(rmdir(unifyfs_root) == -1 && errno == EBUSY, + "%s:%d rmdir mount point %s should fail (errno=%d): %s", + __FILE__, __LINE__, unifyfs_root, errno, strerror(errno)); errno = 0; - rc = rmdir(unifyfs_root); - ok(rc < 0 && errno == EBUSY, - "rmdir mount point %s should fail (rc=%d, errno=%d): %s", - unifyfs_root, rc, errno, strerror(errno)); /* CLEANUP * diff --git a/t/sys/truncate.c b/t/sys/truncate.c index b541e9731..99a372472 100644 --- a/t/sys/truncate.c +++ b/t/sys/truncate.c @@ -20,31 +20,9 @@ #include #include #include -#include #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* Get global or log sizes (or all) */ -static -void get_size(char* path, size_t* global, size_t* log) -{ - struct stat sb = {0}; - int rc; - - rc = stat(path, &sb); - if (rc != 0) { - printf("Error: %s\n", strerror(errno)); - exit(1); /* die on failure */ - } - if (global) { - *global = sb.st_size; - } - - if (log) { - *log = sb.st_rdev; - } -} - int truncate_test(char* unifyfs_root) { char path[64]; @@ -63,7 +41,7 @@ int truncate_test(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); ok(log == 0, "%s:%d log size is %d expected %d", @@ -78,7 +56,7 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d fsync() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 1*bufsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 1*bufsize); ok(log == 1*bufsize, "%s:%d log size is %d expected %d", @@ -97,7 +75,7 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d fsync() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 3*bufsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 3*bufsize); ok(log == 2*bufsize, "%s:%d log size is %d expected %d", @@ -108,7 +86,7 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d ftruncate(%d) (rc=%d): %s", __FILE__, __LINE__, 5*bufsize, rc, strerror(errno)); - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 5*bufsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 5*bufsize); ok(log == 2*bufsize, "%s:%d log size is %d expected %d", @@ -121,7 +99,7 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d truncate(%d) (rc=%d): %s", __FILE__, __LINE__, bufsize/2, rc, strerror(errno)); - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == bufsize/2, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, bufsize/2); ok(log == 2*bufsize, "%s:%d log size is %d expected %d", @@ -132,7 +110,7 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d truncate(%d) (rc=%d): %s", __FILE__, __LINE__, 0, rc, strerror(errno)); - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); ok(log == 2*bufsize, "%s:%d log size is %d expected %d", @@ -160,7 +138,7 @@ int truncate_bigempty(char* unifyfs_root) ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", __FILE__, __LINE__, path, fd, strerror(errno)); - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); ok(log == 0, "%s:%d log size is %d expected %d", @@ -173,7 +151,7 @@ int truncate_bigempty(char* unifyfs_root) __FILE__, __LINE__, (unsigned long long) bigempty, rc, strerror(errno)); - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == (size_t)bigempty, "%s:%d global size is %llu expected %llu", __FILE__, __LINE__, global, (unsigned long long)bigempty, strerror(errno)); @@ -203,7 +181,7 @@ int truncate_eof(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); ok(log == 0, "%s:%d log size is %d expected %d", @@ -282,7 +260,7 @@ int truncate_truncsync(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); ok(log == 0, "%s:%d log size is %d expected %d", @@ -299,7 +277,7 @@ int truncate_truncsync(char* unifyfs_root) __FILE__, __LINE__, bufsize/2, rc, strerror(errno)); /* file should be 0.5MB bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == bufsize/2, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, bufsize/2); ok(log == bufsize, "%s:%d log size is %d expected %d", @@ -310,7 +288,7 @@ int truncate_truncsync(char* unifyfs_root) __FILE__, __LINE__, rc, strerror(errno)); /* file should still be 0.5MB bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == bufsize/2, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, bufsize/2); ok(log == bufsize, "%s:%d log size is %d expected %d", @@ -380,7 +358,7 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); ok(log == 0, "%s:%d log size is %d expected %d", @@ -410,7 +388,7 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, (int)truncsize, rc, strerror(errno)); /* file should be of size 5MB + 42 at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); ok(log == 20*bufsize, "%s:%d log size is %d expected %d", @@ -424,7 +402,7 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, rc, strerror(errno)); /* file should still be 5MB + 42 bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); ok(log == 20*bufsize, "%s:%d log size is %d expected %d", @@ -521,7 +499,7 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); ok(log == 0, "%s:%d log size is %d expected %d", @@ -536,7 +514,7 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, (int)truncsize, rc, strerror(errno)); /* file should be of size 5MB + 42 at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); ok(log == 0, "%s:%d log size is %d expected %d", @@ -550,7 +528,7 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, rc, strerror(errno)); /* file should still be 5MB + 42 bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); ok(log == 0, "%s:%d log size is %d expected %d", @@ -645,7 +623,7 @@ int truncate_ftrunc_before_sync(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); ok(log == 0, "%s:%d log size is %d expected %d", @@ -673,7 +651,7 @@ int truncate_ftrunc_before_sync(char* unifyfs_root) /* finally, check that the file is 0 bytes, * i.e., check that the writes happened before the truncate * and not at the fsync */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); ok(log == bufsize, "%s:%d log size is %d expected %d", @@ -704,7 +682,7 @@ int truncate_trunc_before_sync(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); ok(log == 0, "%s:%d log size is %d expected %d", @@ -732,7 +710,7 @@ int truncate_trunc_before_sync(char* unifyfs_root) /* finally, check that the file is 0 bytes, * i.e., check that the writes happened before the truncate * and not at the fsync */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); ok(log == bufsize, "%s:%d log size is %d expected %d", diff --git a/t/sys/unlink.c b/t/sys/unlink.c index 47ff40325..dd41ab08b 100644 --- a/t/sys/unlink.c +++ b/t/sys/unlink.c @@ -27,8 +27,8 @@ static int unlink_after_sync_test(char* unifyfs_root) { char path[64]; - int rc; int fd; + errno = 0; testutil_rand_path(path, sizeof(path), unifyfs_root); @@ -36,30 +36,33 @@ static int unlink_after_sync_test(char* unifyfs_root) ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", __FILE__, __LINE__, path, fd, strerror(errno)); - rc = write(fd, "hello world", 12); - ok(rc == 12, "%s:%d write() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(write(fd, "hello world", 12) == 12, "%s:%d write(): %s", + __FILE__, __LINE__, strerror(errno)); - rc = fsync(fd); - ok(rc == 0, "%s:%d fsync() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(fsync(fd) == 0, "%s:%d fsync(): %s", + __FILE__, __LINE__, strerror(errno)); - rc = close(fd); - ok(rc == 0, "%s:%d close() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(close(fd) == 0, "%s:%d close(): %s", + __FILE__, __LINE__, strerror(errno)); struct stat sb = {0}; - rc = stat(path, &sb); - ok(rc == 0, "%s:%d stat() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(stat(path, &sb) == 0, "%s:%d stat(): %s", + __FILE__, __LINE__, strerror(errno)); - rc = unlink(path); - ok(rc == 0, "%s:%d unlink() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(unlink(path) == 0, "%s:%d unlink(): %s", + __FILE__, __LINE__, strerror(errno)); - rc = stat(path, &sb); - ok(rc == -1, "%s:%d stat() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + todo("Failing with wrong errno. Should fail with errno=ENOENT=2"); + ok(stat(path, &sb) == -1 && errno == ENOENT, + "%s:%d stat() after unlink fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + end_todo; + errno = 0; /* Reset errno after test for failure */ + + ok(unlink(path) == -1 && errno == ENOENT, + "%s:%d unlink() already unlinked file fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; return 0; } @@ -67,8 +70,8 @@ static int unlink_after_sync_test(char* unifyfs_root) static int unlink_after_sync_laminate_test(char* unifyfs_root) { char path[64]; - int rc; int fd; + errno = 0; testutil_rand_path(path, sizeof(path), unifyfs_root); @@ -76,43 +79,102 @@ static int unlink_after_sync_laminate_test(char* unifyfs_root) ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", __FILE__, __LINE__, path, fd, strerror(errno)); - rc = write(fd, "hello world", 12); - ok(rc == 12, "%s:%d write() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(write(fd, "hello world", 12) == 12, "%s:%d write(): %s", + __FILE__, __LINE__, strerror(errno)); - rc = fsync(fd); - ok(rc == 0, "%s:%d fsync() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(fsync(fd) == 0, "%s:%d fsync(): %s", + __FILE__, __LINE__, strerror(errno)); - rc = close(fd); - ok(rc == 0, "%s:%d close() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(close(fd) == 0, "%s:%d close(): %s", + __FILE__, __LINE__, strerror(errno)); /* Laminate */ - rc = chmod(path, 0444); - ok(rc == 0, "%s:%d chmod(0444) (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(chmod(path, 0444) == 0, "%s:%d chmod(0444): %s", + __FILE__, __LINE__, strerror(errno)); struct stat sb = {0}; - rc = stat(path, &sb); - ok(rc == 0, "%s:%d stat() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(stat(path, &sb) == 0, "%s:%d stat(): %s", + __FILE__, __LINE__, strerror(errno)); + + ok(unlink(path) == 0, "%s:%d unlink(): %s", + __FILE__, __LINE__, strerror(errno)); - rc = unlink(path); - ok(rc == 0, "%s:%d unlink() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + todo("Failing with wrong errno. Should fail with errno=ENOENT=2"); + ok(stat(path, &sb) == -1 && errno == ENOENT, + "%s:%d stat() after unlink fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + end_todo; + errno = 0; - rc = stat(path, &sb); - ok(rc == -1, "%s:%d stat() (rc=%d): %s", - __FILE__, __LINE__, rc, strerror(errno)); + ok(unlink(path) == -1 && errno == ENOENT, + "%s:%d unlink() already unlinked, laminated file fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; return 0; } int unlink_test(char* unifyfs_root) { + diag("Finished UNIFYFS_WRAP(unlink) tests"); + + char path[64]; + char dir_path[64]; + int fd; int rc = 0; + errno = 0; + + testutil_rand_path(path, sizeof(path), unifyfs_root); + testutil_rand_path(dir_path, sizeof(dir_path), unifyfs_root); + + ok(unlink(path) == -1 && errno == ENOENT, + "%s:%d unlink() non-existent file fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; /* Reset errno after test for failure */ + + fd = creat(path, 0222); + ok(fd != -1, "%s:%d creat(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + ok(fsync(fd) == 0, "%s:%d fsync(): %s", + __FILE__, __LINE__, strerror(errno)); + ok(close(fd) == 0, "%s:%d close(): %s", + __FILE__, __LINE__, strerror(errno)); + + struct stat sb = {0}; + ok(stat(path, &sb) == 0, "%s:%d stat(): %s", + __FILE__, __LINE__, strerror(errno)); + + ok(unlink(path) == 0, "%s:%d unlink() empty file: %s", + __FILE__, __LINE__, strerror(errno)); + + todo("Failing with wrong errno. Should fail with errno=ENOENT=2"); + ok(stat(path, &sb) == -1 && errno == ENOENT, + "%s:%d stat() after unlink fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + end_todo; + errno = 0; + + ok(unlink(path) == -1 && errno == ENOENT, + "%s:%d unlink() already unlinked, empty file fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; /* Reset errno after test for failure */ + + /* Calling unlink() on a directory should fail */ + ok(mkdir(dir_path, 0777) == 0, "%s:%d mkdir(%s): %s", + __FILE__, __LINE__, dir_path, strerror(errno)); + + ok(unlink(dir_path) == -1 && errno == EISDIR, + "%s:%d unlink() a directory fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; /* Reset errno after test for failure */ + + ok(rmdir(dir_path) == 0, "%s:%d rmdir(): %s", + __FILE__, __LINE__, strerror(errno)); + + + /* Tests for unlink after writing to a file */ int ret = unlink_after_sync_test(unifyfs_root); if (ret != 0) { rc = ret; @@ -123,5 +185,7 @@ int unlink_test(char* unifyfs_root) rc = ret; } + diag("Finished UNIFYFS_WRAP(unlink) tests"); + return rc; } diff --git a/t/sys/write-read-hole.c b/t/sys/write-read-hole.c index 1834ba9d5..d1b2831ed 100644 --- a/t/sys/write-read-hole.c +++ b/t/sys/write-read-hole.c @@ -23,27 +23,6 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* Get global or log sizes (or all) */ -static -void get_size(char* path, size_t* global, size_t* log) -{ - struct stat sb = {0}; - int rc; - - rc = stat(path, &sb); - if (rc != 0) { - printf("Error: %s\n", strerror(errno)); - exit(1); /* die on failure */ - } - if (global) { - *global = sb.st_size; - } - - if (log) { - *log = sb.st_rdev; - } -} - static int check_contents(char* buf, size_t len, char c) { int valid = 1; @@ -99,7 +78,7 @@ int write_read_hole_test(char* unifyfs_root) __FILE__, __LINE__, rc, strerror(errno)); /* Check global size on our un-laminated file */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 0, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); ok(log == 2*bufsize, "%s:%d log size is %d: %s", @@ -111,7 +90,7 @@ int write_read_hole_test(char* unifyfs_root) __FILE__, __LINE__, rc, strerror(errno)); /* Check global size on our un-laminated file */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 3*bufsize, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); ok(log == 2*bufsize, "%s:%d log size is %d: %s", @@ -129,7 +108,7 @@ int write_read_hole_test(char* unifyfs_root) __FILE__, __LINE__, rc, strerror(errno)); /* Check global size on our un-laminated file */ - get_size(path, &global, &log); + testutil_get_size(path, &global, &log); ok(global == 4*bufsize, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); ok(log == 2*bufsize, "%s:%d log size is %d: %s", diff --git a/t/sys/write-read.c b/t/sys/write-read.c index f71597f9a..8245a4737 100644 --- a/t/sys/write-read.c +++ b/t/sys/write-read.c @@ -24,107 +24,162 @@ #include "t/lib/tap.h" #include "t/lib/testutil.h" -/* Get global or log sizes (or all) */ -static -void get_size(char* path, size_t* global, size_t* log) -{ - struct stat sb = {0}; - int rc; - - rc = stat(path, &sb); - if (rc != 0) { - printf("Error: %s\n", strerror(errno)); - exit(1); /* die on failure */ - } - if (global) { - *global = sb.st_size; - } - - if (log) { - *log = sb.st_rdev; - } -} - int write_read_test(char* unifyfs_root) { + diag("Starting UNIFYFS_WRAP(write/read) tests"); + char path[64]; - int rc; - int fd; + char buf[64] = {0}; + int fd = -1; size_t global, log; - testutil_rand_path(path, sizeof(path), unifyfs_root); - - /* Write to the file */ - fd = open(path, O_WRONLY | O_CREAT, 0222); - ok(fd != -1, "%s: open(%s) (fd=%d): %s", __FILE__, path, fd, - strerror(errno)); - - rc = write(fd, "hello world", 12); - ok(rc == 12, "%s: write() (rc=%d): %s", __FILE__, rc, strerror(errno)); - - /* Test writing to a different offset */ - rc = lseek(fd, 6, SEEK_SET); - ok(rc == 6, "%s: lseek() (rc=%d): %s", __FILE__, rc, strerror(errno)); + errno = 0; - rc = write(fd, "universe", 9); - ok(rc == 9, "%s: write() (rc=%d): %s", __FILE__, rc, strerror(errno)); + testutil_rand_path(path, sizeof(path), unifyfs_root); - /* Check global size on our un-laminated file */ - get_size(path, &global, &log); - ok(global == 0, "%s: global size is %d: %s", __FILE__, global, - strerror(errno)); - ok(log == 21, "%s: log size is %d: %s", __FILE__, log, - strerror(errno)); + /* write to bad file descriptor should fail with errno=EBADF */ + ok(write(fd, "hello world", 12) == -1 && errno == EBADF, + "%s:%d write() to bad file descriptor fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; /* Reset after test for failure */ + /* read from bad file descriptor should fail with errno=EBADF */ + ok(read(fd, buf, 12) == -1 && errno == EBADF, + "%s:%d read() from bad file descriptor fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; - rc = fsync(fd); - ok(rc == 0, "%s: fsync() (rc=%d): %s", __FILE__, rc, strerror(errno)); + /* Write "hello world" to the file */ + fd = open(path, O_WRONLY | O_CREAT, 0222); + ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + ok(write(fd, "hello world", 12) == 12, + "%s:%d write(\"hello world\") to file: %s", + __FILE__, __LINE__, strerror(errno)); + + /* Write to a different offset by overwriting "world" with "universe" */ + ok(lseek(fd, 6, SEEK_SET) == 6, "%s:%d lseek(6) to \"world\": %s", + __FILE__, __LINE__, strerror(errno)); + ok(write(fd, "universe", 9) == 9, + "%s:%d overwrite \"world\" at offset 6 with \"universe\": %s", + __FILE__, __LINE__, strerror(errno)); + + /* Check global size on our un-laminated and un-synced file */ + testutil_get_size(path, &global, &log); + ok(global == 0, "%s:%d global size before fsync is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + ok(log == 21, "%s:%d log size before fsync is %d: %s", + __FILE__, __LINE__, log, strerror(errno)); + + ok(fsync(fd) == 0, "%s:%d fsync() worked: %s", + __FILE__, __LINE__, strerror(errno)); /* Check global size on our un-laminated file */ - get_size(path, &global, &log); - ok(global == 15, "%s: global size is %d: %s", __FILE__, global, - strerror(errno)); - ok(log == 21, "%s: log size is %d: %s", __FILE__, log, - strerror(errno)); - - - close(fd); + testutil_get_size(path, &global, &log); + ok(global == 15, "%s:%d global size after fsync is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + ok(log == 21, "%s:%d log size after fsync is %d: %s", + __FILE__, __LINE__, log, strerror(errno)); + + /* read from file open as write-only should fail with errno=EBADF */ + ok(lseek(fd, 0, SEEK_SET) == 0, "%s:%d lseek(0): %s", + __FILE__, __LINE__, strerror(errno)); + todo("Successfully reads and gets 0 bytes back"); + ok(read(fd, buf, 15) == -1 && errno == EBADF, + "%s:%d read() from file open as write-only (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + end_todo; + errno = 0; + + ok(close(fd) == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(errno)); /* Test O_APPEND */ fd = open(path, O_WRONLY | O_APPEND, 0222); - ok(fd != -1, "%s: open(%s, O_APPEND) (fd=%d): %s", __FILE__, path, fd, - strerror(errno)); + ok(fd != -1, "%s:%d open(%s, O_APPEND) (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); /* * Seek to an offset in the file and write. Since it's O_APPEND, the * offset we seeked to doesn't matter - all writes go to the end. */ - rc = lseek(fd, 3, SEEK_SET); - ok(rc == 3, "%s: lseek() (rc=%d): %s", __FILE__, rc, strerror(errno)); - - rc = write(fd, "", 6); - ok(rc == 6, "%s: write() (rc=%d): %s", __FILE__, rc, strerror(errno)); - close(fd); + ok(lseek(fd, 3, SEEK_SET) == 3, "%s:%d lseek(3) worked: %s", + __FILE__, __LINE__, strerror(errno)); + ok(write(fd, "", 6) == 6, "%s:%d append write(\"\"): %s", + __FILE__, __LINE__, strerror(errno)); + ok(close(fd) == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(errno)); /* Check global size on our un-laminated file */ - get_size(path, &global, &log); - ok(global == 21, "%s: global size is %d: %s", __FILE__, global, - strerror(errno)); - ok(log == 27, "%s: log size is %d: %s", __FILE__, log, - strerror(errno)); - + testutil_get_size(path, &global, &log); + ok(global == 21, "%s:%d global size before laminate is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + ok(log == 27, "%s:%d log size before laminate is %d: %s", + __FILE__, __LINE__, log, strerror(errno)); /* Laminate */ - rc = chmod(path, 0444); - ok(rc == 0, "%s: chmod(0444) (rc=%d): %s", __FILE__, rc, strerror(errno)); + ok(chmod(path, 0444) == 0, "%s:%d chmod(0444): %s", + __FILE__, __LINE__, strerror(errno)); /* Verify we're getting the correct file size */ - get_size(path, &global, &log); - ok(global == 21, "%s: global size is %d: %s", __FILE__, global, - strerror(errno)); - ok(log == 27, "%s: log size is %d: %s", __FILE__, log, - strerror(errno)); + testutil_get_size(path, &global, &log); + ok(global == 21, "%s:%d global size after laminate is %d: %s", + __FILE__, __LINE__, global, strerror(errno)); + ok(log == 27, "%s:%d log size after laminate is %d: %s", + __FILE__, __LINE__, log, strerror(errno)); + /* open laminated file for write should fail with errno=EROFS */ + fd = open(path, O_WRONLY | O_CREAT, 0222); + ok(fd == -1 && errno == EROFS, + "%s:%d open() laminated file for write fails (fd=%d, errno=%d): %s", + __FILE__, __LINE__, fd, errno, strerror(errno)); + errno = 0; + + /* read() tests */ + fd = open(path, O_RDONLY, 0444); + ok(fd != -1, "%s:%d open(%s, O_RDONLY) for read (fd=%d): %s", + __FILE__, __LINE__, path, fd, strerror(errno)); + + /* write to file open as read-only should fail with errno=EBADF */ + ok(write(fd, "hello world", 12) == -1 && errno == EBADF, + "%s:%d write() to file open as read-only fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; + + ok(read(fd, buf, 21) == 21, "%s:%d read() buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + buf[14] = ' '; /* replace '\0' between initial write and append */ + is(buf, "hello universe ", "%s:%d read() saw \"hello universe \"", + __FILE__, __LINE__); + + /* Seek and read at a different position */ + ok(lseek(fd, 6, SEEK_SET) == 6, "%s:%d lseek(6) worked: %s", + __FILE__, __LINE__, strerror(errno)); + ok(read(fd, buf, 9) == 9, "%s:%d read() at offset 6 buf[]=\"%s\": %s", + __FILE__, __LINE__, buf, strerror(errno)); + is(buf, "universe", "%s:%d read() saw \"universe\"", __FILE__, __LINE__); + + ok(lseek(fd, 0, SEEK_SET) == 0, "%s:%d lseek(0) worked: %s", + __FILE__, __LINE__, strerror(errno)); + ok(read(fd, buf, sizeof(buf)) == 21, "%s:%d read() past end of file: %s", + __FILE__, __LINE__, strerror(errno)); + + ok(close(fd) == 0, "%s:%d close() worked: %s", + __FILE__, __LINE__, strerror(errno)); + + /* write to closed file descriptor should fail with errno=EBADF */ + ok(write(fd, "hello world", 12) == -1 && errno == EBADF, + "%s:%d write() to bad file descriptor fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; /* Reset after test for failure */ + + /* read from closed file descriptor should fail with errno=EBADF */ + ok(read(fd, buf, 12) == -1 && errno == EBADF, + "%s:%d read() from bad file descriptor fails (errno=%d): %s", + __FILE__, __LINE__, errno, strerror(errno)); + errno = 0; + + diag("Finished UNIFYFS_WRAP(write/read) tests"); return 0; } From 69c1d44bcfb3381b0b91c596d893a5a4fc4a17fa Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 15 Apr 2020 13:06:18 -0700 Subject: [PATCH 093/168] client: drop log_size field from meta since tracking moved to logio This drops the log_size field from the unifyfs_filemeta_t struct. This field had been used to track the next available byte offset to write to in the log, but it is no longer used since that responsibility has been moved to the new logio functions. --- client/src/unifyfs-internal.h | 5 ----- client/src/unifyfs-sysio.c | 19 ------------------- client/src/unifyfs.c | 12 ------------ 3 files changed, 36 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 6f64f6a28..b7c5f4704 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -261,8 +261,6 @@ enum {FILE_STORAGE_NULL = 0, FILE_STORAGE_LOGIO}; typedef struct { off_t global_size; /* Global size of the file */ - off_t log_size; /* Log size. This is the sum of all the - * write counts. */ pthread_spinlock_t fspinlock; /* file lock variable */ enum flock_enum flock_status; /* file lock status */ @@ -487,9 +485,6 @@ off_t unifyfs_fid_global_size(int fid); * otherwise call back to the rpc */ off_t unifyfs_gfid_filesize(int gfid); -/* Return current local size of given file id */ -off_t unifyfs_fid_log_size(int fid); - /* * Return current size of given file id. If the file is laminated, return the * global size. Otherwise, return the local size. diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index b62efc31d..1c2a4e475 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -408,18 +408,6 @@ static int __stat(const char* path, struct stat* buf) /* copy attributes to stat struct */ unifyfs_file_attr_to_stat(&fattr, buf); - if (fid >= 0) { /* If we have a local file */ - /* - * For debugging and testing purposes, we hijack st_rdev to store our - * log size. We also assume the stat struct is - * the 64-bit variant. The values are stored as: - * - * st_rdev = log_size - * - */ - buf->st_rdev = unifyfs_fid_log_size(fid); - } - return 0; } @@ -628,18 +616,11 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) return EOVERFLOW; } - /* get current log size before extending the log */ - off_t logsize = unifyfs_fid_log_size(fid); - - /* compute size log will be after we append data */ - off_t newlogsize = logsize + count; - /* finally write specified data to file */ int write_rc = unifyfs_fid_write(fid, pos, buf, count); if (write_rc == 0) { unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); meta->needs_sync = 1; - meta->log_size = newlogsize; } return write_rc; } diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 7a3260c26..45037146c 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -680,17 +680,6 @@ off_t unifyfs_gfid_filesize(int gfid) return filesize; } -/* Return the local (un-laminated) size of the file */ -off_t unifyfs_fid_log_size(int fid) -{ - /* get meta data for this file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (NULL != meta) { - return meta->log_size; - } - return (off_t)-1; -} - /* Update local metadata for file from global metadata */ int unifyfs_fid_update_file_meta(int fid, unifyfs_file_attr_t* gfattr) { @@ -870,7 +859,6 @@ int unifyfs_fid_create_file(const char* path) /* initialize meta data */ unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); meta->global_size = 0; - meta->log_size = 0; meta->flock_status = UNLOCKED; meta->storage = FILE_STORAGE_NULL; meta->gfid = unifyfs_generate_gfid(path); From ecc156ad72a5de1b8fb3d78189a2658345411eea Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 15 Apr 2020 14:06:27 -0700 Subject: [PATCH 094/168] client: set sync flag in fid_write instead of fd_write Every time a client makes a change to a file that must be eventually flushed to the server, a needs_sync flag is set in the unifyfs_filemeta_t structure. Our fd_write function delegates to fid_write. This moves the the line to set the needs_sync flag from fd_write to fid_write in case one calls fid_write directly rather than calling fd_write. --- client/src/unifyfs-stdio.c | 7 +++++-- client/src/unifyfs-sysio.c | 4 ---- client/src/unifyfs.c | 5 +++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index 5a94ac6bd..41a5351a8 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -2304,8 +2304,11 @@ static int __srefill(unifyfs_stream_t* stream) /* read data from file into buffer */ ssize_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize); - if (read_rc < 0) { - /* ERROR: read error, set error indicator */ + if (read_rc == -1) { + /* + * ERROR: read error, set error indicator. errno is already set + * by unifyfs_fd_read() + */ s->err = 1; return 1; } diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 1c2a4e475..cf846c38b 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -618,10 +618,6 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) /* finally write specified data to file */ int write_rc = unifyfs_fid_write(fid, pos, buf, count); - if (write_rc == 0) { - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - meta->needs_sync = 1; - } return write_rc; } diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 45037146c..69c3e88d3 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -981,6 +981,11 @@ int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count) if (meta->storage == FILE_STORAGE_LOGIO) { /* file stored in fixed-size chunks */ rc = unifyfs_fid_logio_write(fid, meta, pos, buf, count); + if (rc == UNIFYFS_SUCCESS) { + /* write succeeded, remember that we have new data + * that needs to be synced with the server */ + meta->needs_sync = 1; + } } else { /* unknown storage type */ rc = EIO; From 36e8622e9cd074bc4b72bec8072aa159219d546e Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 15 Apr 2020 14:47:16 -0700 Subject: [PATCH 095/168] client: rename fd_logreadlist to fid_read_reqs, move to unifyfs.c Renames fd_logreadlist to fid_read_reqs since the name "fd" was a misnomer as it operates on file ids and not file descriptors. It also moves the function from unifyfs-sysio.c to unifyfs.c to go with fid_write which is effectively the counterpart write function. --- client/src/unifyfs-internal.h | 2 + client/src/unifyfs-sysio.c | 678 +--------------------------------- client/src/unifyfs-sysio.h | 1 - client/src/unifyfs.c | 643 ++++++++++++++++++++++++++++++++ 4 files changed, 648 insertions(+), 676 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index b7c5f4704..1042cdf95 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -509,6 +509,8 @@ int unifyfs_fid_create_file(const char* path); * returns the new fid, or a negative value on error */ int unifyfs_fid_create_directory(const char* path); +int unifyfs_fid_read_reqs(read_req_t* in_reqs, int in_count); + /* write count bytes from buf into file starting at offset pos */ int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count); diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index cf846c38b..70d71d86b 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -43,12 +43,6 @@ #include "unifyfs-internal.h" #include "unifyfs-sysio.h" #include "margo_client.h" -#include "ucr_read_builder.h" -#include "seg_tree.h" - -#ifndef MAX -#define MAX(a, b) ((a) > (b) ? (a) : (b)) -#endif /* --------------------------------------- * POSIX wrappers: paths @@ -564,7 +558,7 @@ ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) /* execute read operation */ ssize_t retcount; - int ret = unifyfs_fd_logreadlist(&req, 1); + int ret = unifyfs_fid_read_reqs(&req, 1); if (ret != UNIFYFS_SUCCESS) { /* failed to issue read operation */ errno = EIO; @@ -1179,7 +1173,7 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], } if (reqcnt) { - rc = unifyfs_fd_logreadlist(reqs, reqcnt); + rc = unifyfs_fid_read_reqs(reqs, reqcnt); if (rc != UNIFYFS_SUCCESS) { /* error reading data */ ret = -1; @@ -1210,671 +1204,6 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], } #endif -/* order by file id then by file position */ -static int compare_index_entry(const void* a, const void* b) -{ - const unifyfs_index_t* ptr_a = a; - const unifyfs_index_t* ptr_b = b; - - if (ptr_a->gfid != ptr_b->gfid) { - if (ptr_a->gfid < ptr_b->gfid) { - return -1; - } else { - return 1; - } - } - - if (ptr_a->file_pos == ptr_b->file_pos) { - return 0; - } else if (ptr_a->file_pos < ptr_b->file_pos) { - return -1; - } else { - return 1; - } -} - -/* order by file id then by offset */ -static int compare_read_req(const void* a, const void* b) -{ - const read_req_t* ptr_a = a; - const read_req_t* ptr_b = b; - - if (ptr_a->gfid != ptr_b->gfid) { - if (ptr_a->gfid < ptr_b->gfid) { - return -1; - } else { - return 1; - } - } - - if (ptr_a->offset == ptr_b->offset) { - return 0; - } else if (ptr_a->offset < ptr_b->offset) { - return -1; - } else { - return 1; - } -} - -/* notify our delegator that the shared memory buffer - * is now clear and ready to hold more read data */ -static void delegator_signal(void) -{ - LOGDBG("receive buffer now empty"); - - /* set shm flag to signal delegator we're done */ - shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); - hdr->state = SHMEM_REGION_EMPTY; - - /* TODO: MEM_FLUSH */ -} - -/* wait for delegator to inform us that shared memory buffer - * is filled with read data */ -static int delegator_wait(void) -{ - int rc = (int)UNIFYFS_SUCCESS; - - /* specify time to sleep between checking flag in shared - * memory indicating server has produced */ - struct timespec shm_wait_tm; - shm_wait_tm.tv_sec = 0; - shm_wait_tm.tv_nsec = SHM_WAIT_INTERVAL; - - /* get pointer to flag in shared memory */ - shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); - - /* wait for server to set flag to non-zero */ - int max_sleep = 5000000; // 5s - volatile int* vip = (volatile int*)&(hdr->state); - while (*vip == SHMEM_REGION_EMPTY) { - /* not there yet, sleep for a while */ - nanosleep(&shm_wait_tm, NULL); - /* TODO: MEM_FETCH */ - max_sleep--; - if (0 == max_sleep) { - LOGERR("timed out waiting for non-empty"); - rc = (int)UNIFYFS_ERROR_SHMEM; - break; - } - } - - return rc; -} - -/* copy read data from shared memory buffer to user buffers from read - * calls, sets done=1 on return when delegator informs us it has no - * more data */ -static int process_read_data(read_req_t* read_reqs, int count, int* done) -{ - /* assume we'll succeed */ - int rc = UNIFYFS_SUCCESS; - - /* get pointer to start of shared memory buffer */ - shm_data_header* shm_hdr = (shm_data_header*)(shm_recv_ctx->addr); - char* shmptr = ((char*)shm_hdr) + sizeof(shm_data_header); - - /* get number of read replies in shared memory */ - size_t num = shm_hdr->meta_cnt; - - /* process each of our read replies */ - size_t i; - for (i = 0; i < num; i++) { - /* get pointer to current read reply header */ - shm_data_meta* rep = (shm_data_meta*)shmptr; - shmptr += sizeof(shm_data_meta); - - /* get pointer to data */ - char* rep_buf = shmptr; - shmptr += rep->length; - - /* get start and end offset of reply */ - size_t rep_start = rep->offset; - size_t rep_end = rep->offset + rep->length; - - /* iterate over each of our read requests */ - size_t j; - for (j = 0; j < count; j++) { - /* get pointer to read request */ - read_req_t* req = &read_reqs[j]; - - /* skip if this request if not the same file */ - if (rep->gfid != req->gfid) { - /* request and reply are for different files */ - continue; - } - - /* same file, now get start and end offsets - * of this read request */ - size_t req_start = req->offset; - size_t req_end = req->offset + req->length; - - /* test whether reply overlaps with request, - * overlap if: - * start of reply comes before the end of request - * AND - * end of reply comes after the start of request */ - int overlap = (rep_start < req_end && rep_end > req_start); - if (!overlap) { - /* reply does not overlap with this request */ - continue; - } - - /* this reply overlaps with the request, check that - * we didn't get an error */ - if (rep->errcode != UNIFYFS_SUCCESS) { - /* TODO: should we look for the reply with an errcode - * with the lowest start offset? */ - - /* read reply has an error, mark the read request - * as also having an error, then quit processing */ - req->errcode = rep->errcode; - continue; - } - - /* otherwise, we have an error-free, overlapping reply - * for this request, copy data into request buffer */ - - /* start of overlapping segment is the maximum of - * reply and request start offsets */ - size_t start = rep_start; - if (req_start > start) { - start = req_start; - } - - /* end of overlapping segment is the mimimum of - * reply and request end offsets */ - size_t end = rep_end; - if (req_end < end) { - end = req_end; - } - - /* compute length of overlapping segment */ - size_t length = end - start; - - /* get number of bytes from start of reply and request - * buffers to the start of the overlap region */ - size_t rep_offset = start - rep_start; - size_t req_offset = start - req_start; - - /* if we have a gap, fill with zeros */ - size_t gap_start = req_start + req->nread; - if (start > gap_start) { - size_t gap_length = start - gap_start; - char* req_ptr = req->buf + req->nread; - memset(req_ptr, 0, gap_length); - } - - /* copy data from reply buffer into request buffer */ - char* req_ptr = req->buf + req_offset; - char* rep_ptr = rep_buf + rep_offset; - memcpy(req_ptr, rep_ptr, length); - - /* update max number of bytes we have written to in the - * request buffer */ - size_t nread = end - req_start; - if (nread > req->nread) { - req->nread = nread; - } - } - } - - /* set done flag if there is no more data */ - if (shm_hdr->state == SHMEM_REGION_DATA_COMPLETE) { - *done = 1; - } - - return rc; -} - -/* This uses information in the extent map for a file on the client to - * complete any read requests. It only complets a request if it contains - * all of the data. Otherwise the request is copied to the list of - * requests to be handled by the server. */ -static void service_local_reqs( - read_req_t* read_reqs, /* list of input read requests */ - int count, /* number of input read requests */ - read_req_t* local_reqs, /* list to copy requests completed by client */ - read_req_t* server_reqs, /* list to copy requests to be handled by server */ - int* out_count) /* number of items copied to server list */ -{ - /* this will track the total number of requests we're passing - * on to the server */ - int local_count = 0; - int server_count = 0; - - /* iterate over each input read request, satisfy it locally if we can - * otherwise copy request into output list that the server will handle - * for us */ - int i; - for (i = 0; i < count; i++) { - /* get current read request */ - read_req_t* req = &read_reqs[i]; - - /* skip any request that's already completed or errored out, - * we pass those requests on to server */ - if (req->nread >= req->length || req->errcode != UNIFYFS_SUCCESS) { - /* copy current request into list of requests - * that we'll ask server for */ - memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); - server_count++; - continue; - } - - /* get gfid, start, and length of this request */ - int gfid = req->gfid; - size_t req_start = req->offset; - size_t req_end = req->offset + req->length; - - /* lookup local extents if we have them */ - int fid = unifyfs_fid_from_gfid(gfid); - - /* move to next request if we can't find the matching fid */ - if (fid < 0) { - /* copy current request into list of requests - * that we'll ask server for */ - memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); - server_count++; - continue; - } - - /* get pointer to extents for this file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - struct seg_tree* extents = &meta->extents; - - /* lock the extent tree for reading */ - seg_tree_rdlock(extents); - - /* identify whether we can satisfy this full request - * or not, assume we can */ - int have_local = 1; - - /* this will point to the offset of the next byte we - * need to account for */ - size_t expected_start = req_start; - - /* iterate over extents we have for this file, - * and check that there are no holes in coverage, - * we search for a starting extent using a range - * of just the very first byte that we need */ - struct seg_tree_node* first; - first = seg_tree_find_nolock(extents, req_start, req_start); - struct seg_tree_node* next = first; - while (next != NULL && next->start < req_end) { - if (expected_start >= next->start) { - /* this extent has the next byte we expect, - * bump up to the first byte past the end - * of this extent */ - expected_start = next->end + 1; - } else { - /* there is a gap between extents so we're missing - * some bytes */ - have_local = 0; - break; - } - - /* get the next element in the tree */ - next = seg_tree_iter(extents, next); - } - - /* check that we account for the full request - * up until the last byte */ - if (expected_start < req_end) { - /* missing some bytes at the end of the request */ - have_local = 0; - } - - /* if we can't fully satisfy the request, copy request to - * output array, so it can be passed on to server */ - if (!have_local) { - /* copy current request into list of requests - * that we'll ask server for */ - memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); - server_count++; - - /* release lock before we go to next request */ - seg_tree_unlock(extents); - - continue; - } - - /* otherwise we can copy the data locally, iterate - * over the extents and copy data into request buffer, - * again search for a starting extent using a range - * of just the very first byte that we need */ - next = first; - while ((next != NULL) && (next->start < req_end)) { - /* get start and end of this extent (reply) */ - size_t rep_start = next->start; - size_t rep_end = next->end + 1; - - /* get the offset into the log */ - size_t rep_log_pos = next->ptr; - - /* start of overlapping segment is the maximum of - * reply and request start offsets */ - size_t start = rep_start; - if (req_start > start) { - start = req_start; - } - - /* end of overlapping segment is the mimimum of - * reply and request end offsets */ - size_t end = rep_end; - if (req_end < end) { - end = req_end; - } - - /* compute length of overlapping segment */ - size_t length = end - start; - - /* get number of bytes from start of reply and request - * buffers to the start of the overlap region */ - size_t rep_offset = start - rep_start; - size_t req_offset = start - req_start; - - /* if we have a gap, fill with zeros */ - size_t gap_start = req_start + req->nread; - if (start > gap_start) { - size_t gap_length = start - gap_start; - char* req_ptr = req->buf + req->nread; - memset(req_ptr, 0, gap_length); - } - - /* copy data from local write log into request buffer */ - char* req_ptr = req->buf + req_offset; - off_t log_offset = rep_log_pos + rep_offset; - size_t nread = 0; - int rc = unifyfs_logio_read(logio_ctx, log_offset, length, - req_ptr, &nread); - if (rc == UNIFYFS_SUCCESS) { - if (nread < length) { - /* account for short read by updating end offset */ - end -= (length - nread); - } - /* update max number of bytes we have filled in the req buf */ - size_t req_nread = end - req_start; - if (req_nread > req->nread) { - req->nread = req_nread; - } - } else { - LOGERR("local log read failed for offset=%zu size=%zu", - (size_t)log_offset, length); - req->errcode = EIO; - } - - /* get the next element in the tree */ - next = seg_tree_iter(extents, next); - } - - /* copy request data to list we completed locally */ - memcpy(&local_reqs[local_count], req, sizeof(read_req_t)); - local_count++; - - /* done reading the tree */ - seg_tree_unlock(extents); - } - - /* return to user the number of key/values we set */ - *out_count = server_count; - - return; -} - -/* - * get data for a list of read requests from the - * delegator - * - * @param read_reqs: a list of read requests - * @param count: number of read requests - * @return error code - * */ -int unifyfs_fd_logreadlist(read_req_t* in_reqs, int in_count) -{ - int i; - int read_rc; - - /* assume we'll succeed */ - int rc = UNIFYFS_SUCCESS; - - /* assume we'll service all requests from the server */ - int count = in_count; - read_req_t* read_reqs = in_reqs; - - /* TODO: if the file is laminated so that we know the file size, - * we can adjust read requests to not read past the EOF */ - - /* if the option is enabled to service requests locally, try it, - * in this case we'll allocate a large array which we split into - * two, the first half will record requests we completed locally - * and the second half will store requests to be sent to the server */ - - /* this records the pointer to the temp request array if - * we allocate one, we should free this later if not NULL */ - read_req_t* reqs = NULL; - - /* this will point to the start of the array of requests we - * complete locally */ - read_req_t* local_reqs = NULL; - - /* attempt to complete requests locally if enabled */ - if (unifyfs_local_extents) { - /* allocate space to make local and server copies of the requests, - * each list will be at most in_count long */ - size_t reqs_size = 2 * in_count * sizeof(read_req_t); - reqs = (read_req_t*) malloc(reqs_size); - if (reqs == NULL) { - return ENOMEM; - } - - /* define pointers to space where we can build our list - * of requests handled on the client and those left - * for the server */ - local_reqs = &reqs[0]; - read_reqs = &reqs[in_count]; - - /* service reads from local extent info if we can, this copies - * completed requests from in_reqs into local_reqs, and it copies - * any requests that can't be completed locally into the read_reqs - * to be processed by the server */ - service_local_reqs(in_reqs, in_count, local_reqs, read_reqs, &count); - - /* bail early if we satisfied all requests locally */ - if (count == 0) { - /* copy completed requests back into user's array */ - memcpy(in_reqs, local_reqs, in_count * sizeof(read_req_t)); - - /* free the temporary array */ - free(reqs); - return rc; - } - } - - /* TODO: When the number of read requests exceed the - * request buffer, split list io into multiple bulk - * sends and transfer in bulks */ - - /* check that we have enough slots for all read requests */ - if (count > UNIFYFS_MAX_READ_CNT) { - LOGERR("Too many requests to pass to server"); - if (reqs != NULL) { - free(reqs); - } - return ENOSPC; - } - - /* order read request by increasing file id, then increasing offset */ - qsort(read_reqs, count, sizeof(read_req_t), compare_read_req); - - /* prepare our shared memory buffer for delegator */ - delegator_signal(); - - /* we select different rpcs depending on the number of - * read requests */ - if (count > 1) { - /* got multiple read requests, - * build up a flat buffer to include them all */ - flatcc_builder_t builder; - flatcc_builder_init(&builder); - - /* create request vector */ - unifyfs_Extent_vec_start(&builder); - - /* fill in values for each request entry */ - for (i = 0; i < count; i++) { - unifyfs_Extent_vec_push_create(&builder, - read_reqs[i].gfid, read_reqs[i].offset, read_reqs[i].length); - } - - /* complete the array */ - unifyfs_Extent_vec_ref_t extents = unifyfs_Extent_vec_end(&builder); - unifyfs_ReadRequest_create_as_root(&builder, extents); - //unifyfs_ReadRequest_end_as_root(&builder); - - /* allocate our buffer to be sent */ - size_t size = 0; - void* buffer = flatcc_builder_finalize_buffer(&builder, &size); - assert(buffer); - - LOGDBG("mread: n_reqs:%d, flatcc buffer (%p) sz:%zu", - count, buffer, size); - - /* invoke multi-read rpc */ - read_rc = invoke_client_mread_rpc(count, size, buffer); - - /* free flat buffer resources */ - flatcc_builder_clear(&builder); - free(buffer); - } else { - /* got a single read request */ - int gfid = read_reqs[0].gfid; - size_t offset = read_reqs[0].offset; - size_t length = read_reqs[0].length; - - LOGDBG("read: offset:%zu, len:%zu", offset, length); - - /* invoke single read rpc */ - read_rc = invoke_client_read_rpc(gfid, offset, length); - } - - /* bail out with error if we failed to even start the read */ - if (read_rc != UNIFYFS_SUCCESS) { - LOGERR("Failed to issue read RPC to server"); - if (reqs != NULL) { - free(reqs); - } - return read_rc; - } - - /* - * ToDo: Exception handling when some of the requests - * are missed - * */ - - /* spin waiting for read data to come back from the server, - * we process it in batches as it comes in, eventually the - * server will tell us it's sent us everything it can */ - int done = 0; - while (!done) { - int tmp_rc = delegator_wait(); - if (tmp_rc != UNIFYFS_SUCCESS) { - rc = UNIFYFS_FAILURE; - done = 1; - } else { - tmp_rc = process_read_data(read_reqs, count, &done); - if (tmp_rc != UNIFYFS_SUCCESS) { - rc = UNIFYFS_FAILURE; - } - delegator_signal(); - } - } - - /* got all of the data we'll get from the server, - * check for short reads and whether those short - * reads are from errors, holes, or the end of the file */ - for (i = 0; i < count; i++) { - /* get pointer to next read request */ - read_req_t* req = &read_reqs[i]; - - /* if we hit an error on our read, nothing else to do */ - if (req->errcode != UNIFYFS_SUCCESS) { - continue; - } - - /* if we read all of the bytes, we're done */ - if (req->nread == req->length) { - continue; - } - - /* otherwise, we have a short read, check whether there - * would be a hole after us, in which case we fill the - * request buffer with zeros */ - - /* get file size for this file */ - off_t filesize_offt = unifyfs_gfid_filesize(req->gfid); - if (filesize_offt == (off_t)-1) { - /* failed to get file size */ - req->errcode = ENOENT; - continue; - } - size_t filesize = (size_t)filesize_offt; - - /* get offset of where hole starts */ - size_t gap_start = req->offset + req->nread; - - /* get last offset of the read request */ - size_t req_end = req->offset + req->length; - - /* if file size is larger than last offset we wrote to in - * read request, then there is a hole we can fill */ - if (filesize > gap_start) { - /* assume we can fill the full request with zero */ - size_t gap_length = req_end - gap_start; - if (req_end > filesize) { - /* request is trying to read past end of file, - * so only fill zeros up to end of file */ - gap_length = filesize - gap_start; - } - - /* copy zeros into request buffer */ - char* req_ptr = req->buf + req->nread; - memset(req_ptr, 0, gap_length); - - /* update number of bytes read */ - req->nread += gap_length; - } - } - - /* if we attempted to service requests from our local extent map, - * then we need to copy the resulting read requests from the local - * and server arrays back into the user's original array */ - if (unifyfs_local_extents) { - /* TODO: would be nice to copy these back into the same order - * in which we received them. */ - - /* copy locally completed requests back into user's array */ - int local_count = in_count - count; - if (local_count > 0) { - memcpy(in_reqs, local_reqs, local_count * sizeof(read_req_t)); - } - - /* copy sever completed requests back into user's array */ - if (count > 0) { - /* skip past any items we copied in from the local requests */ - read_req_t* in_ptr = in_reqs + local_count; - memcpy(in_ptr, read_reqs, count * sizeof(read_req_t)); - } - - /* free storage we used for copies of requests */ - if (reqs != NULL) { - free(reqs); - reqs = NULL; - } - } - - return rc; -} - ssize_t UNIFYFS_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) { /* equivalent to read(), except that it shall read from a given @@ -1901,7 +1230,7 @@ ssize_t UNIFYFS_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) /* execute read operation */ ssize_t retcount; - int ret = unifyfs_fd_logreadlist(&req, 1); + int ret = unifyfs_fid_read_reqs(&req, 1); if (ret != UNIFYFS_SUCCESS) { /* error reading data */ errno = EIO; @@ -1983,7 +1312,6 @@ ssize_t UNIFYFS_WRAP(pwrite64)(int fd, const void* buf, size_t count, int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) { /* check whether we should intercept this file descriptor */ - int origfd = fd; if (unifyfs_intercept_fd(&fd)) { /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); diff --git a/client/src/unifyfs-sysio.h b/client/src/unifyfs-sysio.h index fb65d7a68..6a6b40a03 100644 --- a/client/src/unifyfs-sysio.h +++ b/client/src/unifyfs-sysio.h @@ -114,7 +114,6 @@ ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count); * allocates new bytes and updates file size as necessary, * fills any gaps with zeros */ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count); -int unifyfs_fd_logreadlist(read_req_t* read_req, int count); #include "unifyfs-dirops.h" diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 69c3e88d3..4a8c6d700 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -58,6 +58,7 @@ #include "unifyfs_rpc_util.h" #include "margo_client.h" #include "seg_tree.h" +#include "ucr_read_builder.h" /* avoid duplicate mounts (for now) */ static int unifyfs_mounted = -1; @@ -961,6 +962,648 @@ int unifyfs_fid_create_directory(const char* path) return UNIFYFS_SUCCESS; } +/* order by file id then by offset */ +static int compare_read_req(const void* a, const void* b) +{ + const read_req_t* ptr_a = a; + const read_req_t* ptr_b = b; + + if (ptr_a->gfid != ptr_b->gfid) { + if (ptr_a->gfid < ptr_b->gfid) { + return -1; + } else { + return 1; + } + } + + if (ptr_a->offset == ptr_b->offset) { + return 0; + } else if (ptr_a->offset < ptr_b->offset) { + return -1; + } else { + return 1; + } +} + +/* notify our delegator that the shared memory buffer + * is now clear and ready to hold more read data */ +static void delegator_signal(void) +{ + LOGDBG("receive buffer now empty"); + + /* set shm flag to signal delegator we're done */ + shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); + hdr->state = SHMEM_REGION_EMPTY; + + /* TODO: MEM_FLUSH */ +} + +/* wait for delegator to inform us that shared memory buffer + * is filled with read data */ +static int delegator_wait(void) +{ + int rc = (int)UNIFYFS_SUCCESS; + + /* specify time to sleep between checking flag in shared + * memory indicating server has produced */ + struct timespec shm_wait_tm; + shm_wait_tm.tv_sec = 0; + shm_wait_tm.tv_nsec = SHM_WAIT_INTERVAL; + + /* get pointer to flag in shared memory */ + shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); + + /* wait for server to set flag to non-zero */ + int max_sleep = 5000000; // 5s + volatile int* vip = (volatile int*)&(hdr->state); + while (*vip == SHMEM_REGION_EMPTY) { + /* not there yet, sleep for a while */ + nanosleep(&shm_wait_tm, NULL); + /* TODO: MEM_FETCH */ + max_sleep--; + if (0 == max_sleep) { + LOGERR("timed out waiting for non-empty"); + rc = (int)UNIFYFS_ERROR_SHMEM; + break; + } + } + + return rc; +} + +/* copy read data from shared memory buffer to user buffers from read + * calls, sets done=1 on return when delegator informs us it has no + * more data */ +static int process_read_data(read_req_t* read_reqs, int count, int* done) +{ + /* assume we'll succeed */ + int rc = UNIFYFS_SUCCESS; + + /* get pointer to start of shared memory buffer */ + shm_data_header* shm_hdr = (shm_data_header*)(shm_recv_ctx->addr); + char* shmptr = ((char*)shm_hdr) + sizeof(shm_data_header); + + /* get number of read replies in shared memory */ + size_t num = shm_hdr->meta_cnt; + + /* process each of our read replies */ + size_t i; + for (i = 0; i < num; i++) { + /* get pointer to current read reply header */ + shm_data_meta* rep = (shm_data_meta*)shmptr; + shmptr += sizeof(shm_data_meta); + + /* get pointer to data */ + char* rep_buf = shmptr; + shmptr += rep->length; + + /* get start and end offset of reply */ + size_t rep_start = rep->offset; + size_t rep_end = rep->offset + rep->length; + + /* iterate over each of our read requests */ + size_t j; + for (j = 0; j < count; j++) { + /* get pointer to read request */ + read_req_t* req = &read_reqs[j]; + + /* skip if this request if not the same file */ + if (rep->gfid != req->gfid) { + /* request and reply are for different files */ + continue; + } + + /* same file, now get start and end offsets + * of this read request */ + size_t req_start = req->offset; + size_t req_end = req->offset + req->length; + + /* test whether reply overlaps with request, + * overlap if: + * start of reply comes before the end of request + * AND + * end of reply comes after the start of request */ + int overlap = (rep_start < req_end && rep_end > req_start); + if (!overlap) { + /* reply does not overlap with this request */ + continue; + } + + /* this reply overlaps with the request, check that + * we didn't get an error */ + if (rep->errcode != UNIFYFS_SUCCESS) { + /* TODO: should we look for the reply with an errcode + * with the lowest start offset? */ + + /* read reply has an error, mark the read request + * as also having an error, then quit processing */ + req->errcode = rep->errcode; + continue; + } + + /* otherwise, we have an error-free, overlapping reply + * for this request, copy data into request buffer */ + + /* start of overlapping segment is the maximum of + * reply and request start offsets */ + size_t start = rep_start; + if (req_start > start) { + start = req_start; + } + + /* end of overlapping segment is the mimimum of + * reply and request end offsets */ + size_t end = rep_end; + if (req_end < end) { + end = req_end; + } + + /* compute length of overlapping segment */ + size_t length = end - start; + + /* get number of bytes from start of reply and request + * buffers to the start of the overlap region */ + size_t rep_offset = start - rep_start; + size_t req_offset = start - req_start; + + /* if we have a gap, fill with zeros */ + size_t gap_start = req_start + req->nread; + if (start > gap_start) { + size_t gap_length = start - gap_start; + char* req_ptr = req->buf + req->nread; + memset(req_ptr, 0, gap_length); + } + + /* copy data from reply buffer into request buffer */ + char* req_ptr = req->buf + req_offset; + char* rep_ptr = rep_buf + rep_offset; + memcpy(req_ptr, rep_ptr, length); + + /* update max number of bytes we have written to in the + * request buffer */ + size_t nread = end - req_start; + if (nread > req->nread) { + req->nread = nread; + } + } + } + + /* set done flag if there is no more data */ + if (shm_hdr->state == SHMEM_REGION_DATA_COMPLETE) { + *done = 1; + } + + return rc; +} + +/* This uses information in the extent map for a file on the client to + * complete any read requests. It only complets a request if it contains + * all of the data. Otherwise the request is copied to the list of + * requests to be handled by the server. */ +static void service_local_reqs( + read_req_t* read_reqs, /* list of input read requests */ + int count, /* number of input read requests */ + read_req_t* local_reqs, /* list to copy requests completed by client */ + read_req_t* server_reqs, /* list to copy requests to be handled by server */ + int* out_count) /* number of items copied to server list */ +{ + /* this will track the total number of requests we're passing + * on to the server */ + int local_count = 0; + int server_count = 0; + + /* iterate over each input read request, satisfy it locally if we can + * otherwise copy request into output list that the server will handle + * for us */ + int i; + for (i = 0; i < count; i++) { + /* get current read request */ + read_req_t* req = &read_reqs[i]; + + /* skip any request that's already completed or errored out, + * we pass those requests on to server */ + if (req->nread >= req->length || req->errcode != UNIFYFS_SUCCESS) { + /* copy current request into list of requests + * that we'll ask server for */ + memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); + server_count++; + continue; + } + + /* get gfid, start, and length of this request */ + int gfid = req->gfid; + size_t req_start = req->offset; + size_t req_end = req->offset + req->length; + + /* lookup local extents if we have them */ + int fid = unifyfs_fid_from_gfid(gfid); + + /* move to next request if we can't find the matching fid */ + if (fid < 0) { + /* copy current request into list of requests + * that we'll ask server for */ + memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); + server_count++; + continue; + } + + /* get pointer to extents for this file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + struct seg_tree* extents = &meta->extents; + + /* lock the extent tree for reading */ + seg_tree_rdlock(extents); + + /* identify whether we can satisfy this full request + * or not, assume we can */ + int have_local = 1; + + /* this will point to the offset of the next byte we + * need to account for */ + size_t expected_start = req_start; + + /* iterate over extents we have for this file, + * and check that there are no holes in coverage, + * we search for a starting extent using a range + * of just the very first byte that we need */ + struct seg_tree_node* first; + first = seg_tree_find_nolock(extents, req_start, req_start); + struct seg_tree_node* next = first; + while (next != NULL && next->start < req_end) { + if (expected_start >= next->start) { + /* this extent has the next byte we expect, + * bump up to the first byte past the end + * of this extent */ + expected_start = next->end + 1; + } else { + /* there is a gap between extents so we're missing + * some bytes */ + have_local = 0; + break; + } + + /* get the next element in the tree */ + next = seg_tree_iter(extents, next); + } + + /* check that we account for the full request + * up until the last byte */ + if (expected_start < req_end) { + /* missing some bytes at the end of the request */ + have_local = 0; + } + + /* if we can't fully satisfy the request, copy request to + * output array, so it can be passed on to server */ + if (!have_local) { + /* copy current request into list of requests + * that we'll ask server for */ + memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); + server_count++; + + /* release lock before we go to next request */ + seg_tree_unlock(extents); + + continue; + } + + /* otherwise we can copy the data locally, iterate + * over the extents and copy data into request buffer, + * again search for a starting extent using a range + * of just the very first byte that we need */ + next = first; + while ((next != NULL) && (next->start < req_end)) { + /* get start and end of this extent (reply) */ + size_t rep_start = next->start; + size_t rep_end = next->end + 1; + + /* get the offset into the log */ + size_t rep_log_pos = next->ptr; + + /* start of overlapping segment is the maximum of + * reply and request start offsets */ + size_t start = rep_start; + if (req_start > start) { + start = req_start; + } + + /* end of overlapping segment is the mimimum of + * reply and request end offsets */ + size_t end = rep_end; + if (req_end < end) { + end = req_end; + } + + /* compute length of overlapping segment */ + size_t length = end - start; + + /* get number of bytes from start of reply and request + * buffers to the start of the overlap region */ + size_t rep_offset = start - rep_start; + size_t req_offset = start - req_start; + + /* if we have a gap, fill with zeros */ + size_t gap_start = req_start + req->nread; + if (start > gap_start) { + size_t gap_length = start - gap_start; + char* req_ptr = req->buf + req->nread; + memset(req_ptr, 0, gap_length); + } + + /* copy data from local write log into request buffer */ + char* req_ptr = req->buf + req_offset; + off_t log_offset = rep_log_pos + rep_offset; + size_t nread = 0; + int rc = unifyfs_logio_read(logio_ctx, log_offset, length, + req_ptr, &nread); + if (rc == UNIFYFS_SUCCESS) { + if (nread < length) { + /* account for short read by updating end offset */ + end -= (length - nread); + } + /* update max number of bytes we have filled in the req buf */ + size_t req_nread = end - req_start; + if (req_nread > req->nread) { + req->nread = req_nread; + } + } else { + LOGERR("local log read failed for offset=%zu size=%zu", + (size_t)log_offset, length); + req->errcode = EIO; + } + + /* get the next element in the tree */ + next = seg_tree_iter(extents, next); + } + + /* copy request data to list we completed locally */ + memcpy(&local_reqs[local_count], req, sizeof(read_req_t)); + local_count++; + + /* done reading the tree */ + seg_tree_unlock(extents); + } + + /* return to user the number of key/values we set */ + *out_count = server_count; + + return; +} + +/* + * get data for a list of read requests from the + * delegator + * + * @param read_reqs: a list of read requests + * @param count: number of read requests + * @return error code + * */ +int unifyfs_fid_read_reqs(read_req_t* in_reqs, int in_count) +{ + int i; + int read_rc; + + /* assume we'll succeed */ + int rc = UNIFYFS_SUCCESS; + + /* assume we'll service all requests from the server */ + int count = in_count; + read_req_t* read_reqs = in_reqs; + + /* TODO: if the file is laminated so that we know the file size, + * we can adjust read requests to not read past the EOF */ + + /* if the option is enabled to service requests locally, try it, + * in this case we'll allocate a large array which we split into + * two, the first half will record requests we completed locally + * and the second half will store requests to be sent to the server */ + + /* this records the pointer to the temp request array if + * we allocate one, we should free this later if not NULL */ + read_req_t* reqs = NULL; + + /* this will point to the start of the array of requests we + * complete locally */ + read_req_t* local_reqs = NULL; + + /* attempt to complete requests locally if enabled */ + if (unifyfs_local_extents) { + /* allocate space to make local and server copies of the requests, + * each list will be at most in_count long */ + size_t reqs_size = 2 * in_count * sizeof(read_req_t); + reqs = (read_req_t*) malloc(reqs_size); + if (reqs == NULL) { + return ENOMEM; + } + + /* define pointers to space where we can build our list + * of requests handled on the client and those left + * for the server */ + local_reqs = &reqs[0]; + read_reqs = &reqs[in_count]; + + /* service reads from local extent info if we can, this copies + * completed requests from in_reqs into local_reqs, and it copies + * any requests that can't be completed locally into the read_reqs + * to be processed by the server */ + service_local_reqs(in_reqs, in_count, local_reqs, read_reqs, &count); + + /* bail early if we satisfied all requests locally */ + if (count == 0) { + /* copy completed requests back into user's array */ + memcpy(in_reqs, local_reqs, in_count * sizeof(read_req_t)); + + /* free the temporary array */ + free(reqs); + return rc; + } + } + + /* TODO: When the number of read requests exceed the + * request buffer, split list io into multiple bulk + * sends and transfer in bulks */ + + /* check that we have enough slots for all read requests */ + if (count > UNIFYFS_MAX_READ_CNT) { + LOGERR("Too many requests to pass to server"); + if (reqs != NULL) { + free(reqs); + } + return ENOSPC; + } + + /* order read request by increasing file id, then increasing offset */ + qsort(read_reqs, count, sizeof(read_req_t), compare_read_req); + + /* prepare our shared memory buffer for delegator */ + delegator_signal(); + + /* we select different rpcs depending on the number of + * read requests */ + if (count > 1) { + /* got multiple read requests, + * build up a flat buffer to include them all */ + flatcc_builder_t builder; + flatcc_builder_init(&builder); + + /* create request vector */ + unifyfs_Extent_vec_start(&builder); + + /* fill in values for each request entry */ + for (i = 0; i < count; i++) { + unifyfs_Extent_vec_push_create(&builder, + read_reqs[i].gfid, read_reqs[i].offset, read_reqs[i].length); + } + + /* complete the array */ + unifyfs_Extent_vec_ref_t extents = unifyfs_Extent_vec_end(&builder); + unifyfs_ReadRequest_create_as_root(&builder, extents); + //unifyfs_ReadRequest_end_as_root(&builder); + + /* allocate our buffer to be sent */ + size_t size = 0; + void* buffer = flatcc_builder_finalize_buffer(&builder, &size); + assert(buffer); + + LOGDBG("mread: n_reqs:%d, flatcc buffer (%p) sz:%zu", + count, buffer, size); + + /* invoke multi-read rpc */ + read_rc = invoke_client_mread_rpc(count, size, buffer); + + /* free flat buffer resources */ + flatcc_builder_clear(&builder); + free(buffer); + } else { + /* got a single read request */ + int gfid = read_reqs[0].gfid; + size_t offset = read_reqs[0].offset; + size_t length = read_reqs[0].length; + + LOGDBG("read: offset:%zu, len:%zu", offset, length); + + /* invoke single read rpc */ + read_rc = invoke_client_read_rpc(gfid, offset, length); + } + + /* bail out with error if we failed to even start the read */ + if (read_rc != UNIFYFS_SUCCESS) { + LOGERR("Failed to issue read RPC to server"); + if (reqs != NULL) { + free(reqs); + } + return read_rc; + } + + /* + * ToDo: Exception handling when some of the requests + * are missed + * */ + + /* spin waiting for read data to come back from the server, + * we process it in batches as it comes in, eventually the + * server will tell us it's sent us everything it can */ + int done = 0; + while (!done) { + int tmp_rc = delegator_wait(); + if (tmp_rc != UNIFYFS_SUCCESS) { + rc = UNIFYFS_FAILURE; + done = 1; + } else { + tmp_rc = process_read_data(read_reqs, count, &done); + if (tmp_rc != UNIFYFS_SUCCESS) { + rc = UNIFYFS_FAILURE; + } + delegator_signal(); + } + } + + /* got all of the data we'll get from the server, + * check for short reads and whether those short + * reads are from errors, holes, or the end of the file */ + for (i = 0; i < count; i++) { + /* get pointer to next read request */ + read_req_t* req = &read_reqs[i]; + + /* if we hit an error on our read, nothing else to do */ + if (req->errcode != UNIFYFS_SUCCESS) { + continue; + } + + /* if we read all of the bytes, we're done */ + if (req->nread == req->length) { + continue; + } + + /* otherwise, we have a short read, check whether there + * would be a hole after us, in which case we fill the + * request buffer with zeros */ + + /* get file size for this file */ + off_t filesize_offt = unifyfs_gfid_filesize(req->gfid); + if (filesize_offt == (off_t)-1) { + /* failed to get file size */ + req->errcode = ENOENT; + continue; + } + size_t filesize = (size_t)filesize_offt; + + /* get offset of where hole starts */ + size_t gap_start = req->offset + req->nread; + + /* get last offset of the read request */ + size_t req_end = req->offset + req->length; + + /* if file size is larger than last offset we wrote to in + * read request, then there is a hole we can fill */ + if (filesize > gap_start) { + /* assume we can fill the full request with zero */ + size_t gap_length = req_end - gap_start; + if (req_end > filesize) { + /* request is trying to read past end of file, + * so only fill zeros up to end of file */ + gap_length = filesize - gap_start; + } + + /* copy zeros into request buffer */ + char* req_ptr = req->buf + req->nread; + memset(req_ptr, 0, gap_length); + + /* update number of bytes read */ + req->nread += gap_length; + } + } + + /* if we attempted to service requests from our local extent map, + * then we need to copy the resulting read requests from the local + * and server arrays back into the user's original array */ + if (unifyfs_local_extents) { + /* TODO: would be nice to copy these back into the same order + * in which we received them. */ + + /* copy locally completed requests back into user's array */ + int local_count = in_count - count; + if (local_count > 0) { + memcpy(in_reqs, local_reqs, local_count * sizeof(read_req_t)); + } + + /* copy sever completed requests back into user's array */ + if (count > 0) { + /* skip past any items we copied in from the local requests */ + read_req_t* in_ptr = in_reqs + local_count; + memcpy(in_ptr, read_reqs, count * sizeof(read_req_t)); + } + + /* free storage we used for copies of requests */ + if (reqs != NULL) { + free(reqs); + reqs = NULL; + } + } + + return rc; +} + /* Write count bytes from buf into file starting at offset pos. * * Returns UNIFYFS_SUCCESS, or an error code From c747935296261c5e76275611155880a708d4a23e Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 15 Apr 2020 17:02:39 -0700 Subject: [PATCH 096/168] client: return bytes read and written in fd_read/write Changes fd_read/write to both return either UNIFYFS_SUCCESS or an error code, also adds a new output argument to return number of bytes actually read/written which may be less than the count requested. Sets errno within wrapper functions instead of fd_read. --- client/src/unifyfs-stdio.c | 47 +++++++++++--------- client/src/unifyfs-sysio.c | 91 ++++++++++++++++++++++---------------- client/src/unifyfs-sysio.h | 28 +++++++++--- 3 files changed, 101 insertions(+), 65 deletions(-) diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index 41a5351a8..b3a25b207 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -476,13 +476,18 @@ static int unifyfs_stream_flush(FILE* stream) /* if buffer is dirty, write data to file */ if (s->buf != NULL && s->bufdirty) { - int write_rc = unifyfs_fd_write(s->fd, s->bufpos, s->buf, s->buflen); + size_t bytes; + int write_rc = unifyfs_fd_write(s->fd, s->bufpos, s->buf, s->buflen, + &bytes); if (write_rc != UNIFYFS_SUCCESS) { + /* ERROR: set stream error indicator and errno */ s->err = 1; errno = unifyfs_rc_errno(write_rc); return write_rc; } + /* TODO: handle short writes as error? */ + /* lookup file id from file descriptor attached to stream */ int fid = unifyfs_get_fid_from_fd(s->fd); if (fid < 0) { @@ -616,20 +621,19 @@ static int unifyfs_stream_read( } /* read data from file into buffer */ - ssize_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, - s->bufsize); - if (read_rc == -1) { - /* - * ERROR: read error, set error indicator. errno is already set - * by unifyfs_fd_read() - */ + size_t bytes; + int read_rc = unifyfs_fd_read(s->fd, current, s->buf, + s->bufsize, &bytes); + if (read_rc != UNIFYFS_SUCCESS) { + /* ERROR: set error indicator and errno */ s->err = 1; + errno = unifyfs_rc_errno(read_rc); return EIO; } /* record new buffer range within file */ - s->bufpos = current; - s->buflen = read_rc; + s->bufpos = current; + s->buflen = bytes; /* set end-of-file flag if our read was short */ if (s->buflen < s->bufsize) { @@ -761,16 +765,19 @@ static int unifyfs_stream_write( /* if unbuffered, write data directly to file */ if (s->buftype == _IONBF) { /* write data directly to file */ - int write_rc = unifyfs_fd_write(s->fd, current, buf, count); + size_t bytes; + int write_rc = unifyfs_fd_write(s->fd, current, buf, count, &bytes); if (write_rc != UNIFYFS_SUCCESS) { - /* ERROR: write error, set error indicator and errno */ + /* ERROR: set stream error indicator and errno */ s->err = 1; errno = unifyfs_rc_errno(write_rc); return write_rc; } + /* TODO: handle short writes as error? */ + /* update file position */ - filedesc->pos = current + (off_t) count; + filedesc->pos = current + (off_t) bytes; return UNIFYFS_SUCCESS; } @@ -2303,19 +2310,19 @@ static int __srefill(unifyfs_stream_t* stream) } /* read data from file into buffer */ - ssize_t read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize); - if (read_rc == -1) { - /* - * ERROR: read error, set error indicator. errno is already set - * by unifyfs_fd_read() - */ + size_t bytes; + int read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize, + &bytes); + if (read_rc != UNIFYFS_SUCCESS) { + /* ERROR: set error indicator and errno */ s->err = 1; + errno = unifyfs_rc_errno(read_rc); return 1; } /* record new buffer range within file */ s->bufpos = current; - s->buflen = read_rc; + s->buflen = bytes; } /* determine number of bytes to copy from stream buffer */ diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 70d71d86b..02501f49f 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -509,34 +509,33 @@ int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) * Returns number of bytes actually read, or -1 on error, in which * case errno will be set. */ -ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) +int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* bytes) { + /* assume we'll fail, set bytes to 0 as a clue */ + *bytes = 0; + /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { - errno = EBADF; - return -1; + return EBADF; } /* it's an error to read from a directory */ if (unifyfs_fid_is_dir(fid)) { /* TODO: note that read/pread can return this, but not fread */ - errno = EISDIR; - return -1; + return EISDIR; } /* check that file descriptor is open for read */ unifyfs_fd_t* filedesc = unifyfs_get_filedesc_from_fd(fd); if (!filedesc->read) { - errno = EBADF; - return -1; + return EBADF; } /* TODO: is it safe to assume that off_t is bigger than size_t? */ /* check that we don't overflow the file length */ if (unifyfs_would_overflow_offt(pos, (off_t) count)) { - errno = EOVERFLOW; - return -1; + return EOVERFLOW; } /* TODO: check that file is open for reading */ @@ -544,7 +543,7 @@ ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) /* if we don't read any bytes, return success */ if (count == 0) { LOGDBG("returning EOF"); - return 0; + return UNIFYFS_SUCCESS; } /* fill in read request */ @@ -557,25 +556,19 @@ ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) req.buf = buf; /* execute read operation */ - ssize_t retcount; int ret = unifyfs_fid_read_reqs(&req, 1); if (ret != UNIFYFS_SUCCESS) { /* failed to issue read operation */ - errno = EIO; - retcount = -1; + return EIO; } else if (req.errcode != UNIFYFS_SUCCESS) { /* read executed, but failed */ - errno = EIO; - retcount = -1; - } else { - /* success, get number of bytes read from read request field */ - retcount = (ssize_t) req.nread; - - /* update file pointer position */ - filedesc->pos += (off_t) retcount; + return EIO; } - return retcount; + /* success, get number of bytes read from read request field */ + *bytes = req.nread; + + return UNIFYFS_SUCCESS; } /* @@ -584,8 +577,12 @@ ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count) * that 'pos' is actually where you want to write, and so O_APPEND behavior * is ignored. Fills any gaps with zeros */ -int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) +int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count, + size_t* bytes) { + /* assume we'll fail, set bytes to 0 as a clue */ + *bytes = 0; + /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); if (fid < 0) { @@ -612,6 +609,10 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count) /* finally write specified data to file */ int write_rc = unifyfs_fid_write(fid, pos, buf, count); + if (write_rc == UNIFYFS_SUCCESS) { + *bytes = count; + } + return write_rc; } @@ -994,8 +995,19 @@ ssize_t UNIFYFS_WRAP(read)(int fd, void* buf, size_t count) } /* execute read */ - ssize_t ret = unifyfs_fd_read(fd, filedesc->pos, buf, count); - return ret; + size_t bytes; + int read_rc = unifyfs_fd_read(fd, filedesc->pos, buf, count, &bytes); + if (read_rc != UNIFYFS_SUCCESS) { + /* read operation failed */ + errno = unifyfs_rc_errno(read_rc); + return (ssize_t)(-1); + } + + /* success, update file pointer position */ + filedesc->pos += (off_t)bytes; + + /* return number of bytes read */ + return (ssize_t)bytes; } else { MAP_OR_FAIL(read); ssize_t ret = UNIFYFS_REAL(read)(fd, buf, count); @@ -1007,8 +1019,6 @@ ssize_t UNIFYFS_WRAP(read)(int fd, void* buf, size_t count) ssize_t UNIFYFS_WRAP(write)(int fd, const void* buf, size_t count) { LOGDBG("write was called for fd %d", fd); - size_t ret; - off_t pos; /* check whether we should intercept this file descriptor */ if (unifyfs_intercept_fd(&fd)) { @@ -1020,6 +1030,9 @@ ssize_t UNIFYFS_WRAP(write)(int fd, const void* buf, size_t count) return (ssize_t)(-1); } + /* compute starting position to write within file, + * assume at current position on file descriptor */ + off_t pos = filedesc->pos; if (filedesc->append) { /* * With O_APPEND we always write to the end, despite the current @@ -1027,26 +1040,27 @@ ssize_t UNIFYFS_WRAP(write)(int fd, const void* buf, size_t count) */ int fid = unifyfs_get_fid_from_fd(fd); pos = unifyfs_fid_logical_size(fid); - } else { - pos = filedesc->pos; } /* write data to file */ - int write_rc = unifyfs_fd_write(fd, pos, buf, count); + size_t bytes; + int write_rc = unifyfs_fd_write(fd, pos, buf, count, &bytes); if (write_rc != UNIFYFS_SUCCESS) { + /* write failed */ errno = unifyfs_rc_errno(write_rc); return (ssize_t)(-1); } - ret = count; /* update file position */ - filedesc->pos = pos + count; + filedesc->pos = pos + bytes; + + /* return number of bytes written */ + return (ssize_t)bytes; } else { MAP_OR_FAIL(write); - ret = UNIFYFS_REAL(write)(fd, buf, count); + ssize_t ret = UNIFYFS_REAL(write)(fd, buf, count); + return ret; } - - return ret; } ssize_t UNIFYFS_WRAP(readv)(int fd, const struct iovec* iov, int iovcnt) @@ -1281,14 +1295,15 @@ ssize_t UNIFYFS_WRAP(pwrite)(int fd, const void* buf, size_t count, } /* write data to file */ - int write_rc = unifyfs_fd_write(fd, offset, buf, count); + size_t bytes; + int write_rc = unifyfs_fd_write(fd, offset, buf, count, &bytes); if (write_rc != UNIFYFS_SUCCESS) { errno = unifyfs_rc_errno(write_rc); return (ssize_t)(-1); } - /* return number of bytes read */ - return (ssize_t) count; + /* return number of bytes written */ + return (ssize_t)bytes; } else { MAP_OR_FAIL(pwrite); ssize_t ret = UNIFYFS_REAL(pwrite)(fd, buf, count, offset); diff --git a/client/src/unifyfs-sysio.h b/client/src/unifyfs-sysio.h index 6a6b40a03..17879c053 100644 --- a/client/src/unifyfs-sysio.h +++ b/client/src/unifyfs-sysio.h @@ -105,15 +105,29 @@ UNIFYFS_DECL(lio_listio, int, (int mode, struct aiocb* const aiocb_list[], /* * Read 'count' bytes info 'buf' from file starting at offset 'pos'. - * Returns number of bytes actually read, or -1 on error, in which - * case errno will be set. + * Returns UNIFYFS_SUCCESS and sets number of bytes actually read in bytes + * on success. Otherwise returns error code on error. */ -ssize_t unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count); +int unifyfs_fd_read( + int fd, /* file descriptor to read from */ + off_t pos, /* offset within file to read from */ + void* buf, /* buffer to hold data */ + size_t count, /* number of bytes to read */ + size_t* bytes /* number of bytes read */ +); -/* write count bytes from buf into file starting at offset pos, - * allocates new bytes and updates file size as necessary, - * fills any gaps with zeros */ -int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count); +/* + * Write 'count' bytes from 'buf' into file starting at offset 'pos'. + * Returns UNIFYFS_SUCCESS and sets number of bytes actually written in bytes + * on success. Otherwise returns error code on error. + */ +int unifyfs_fd_write( + int fd, /* file descriptor to write to */ + off_t pos, /* offset within file to write to */ + const void* buf, /* buffer holding data to write */ + size_t count, /* number of bytes to write */ + size_t* bytes /* number of bytes written */ +); #include "unifyfs-dirops.h" From cb10976fb50d4ba10523a80b2d4691efdfcb472f Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 15 Apr 2020 17:09:36 -0700 Subject: [PATCH 097/168] client: fix to use original file descriptor when delegating to a wrapper Fix a number of wrappers to use the original fd from the caller rather than the translated fd that is returned from the intercept_fd function when one wrapper delegates calls to another wrapper. --- client/src/unifyfs-sysio.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 02501f49f..4522b9aab 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -538,8 +538,6 @@ int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* bytes) return EOVERFLOW; } - /* TODO: check that file is open for reading */ - /* if we don't read any bytes, return success */ if (count == 0) { LOGDBG("returning EOF"); @@ -907,7 +905,7 @@ off64_t UNIFYFS_WRAP(lseek64)(int fd, off64_t offset, int whence) /* off_t and off64_t are the same size, * delegate to lseek wrapper */ off64_t ret = (off64_t)UNIFYFS_WRAP(lseek)( - origfd, (off_t) offset, whence); + origfd, (off_t) offset, whence); return ret; } else { /* ERROR: fn not yet supported */ @@ -1068,13 +1066,14 @@ ssize_t UNIFYFS_WRAP(readv)(int fd, const struct iovec* iov, int iovcnt) ssize_t ret; /* check whether we should intercept this file descriptor */ + int origfd = fd; if (unifyfs_intercept_fd(&fd)) { ssize_t rret; int i; ret = 0; for (i = 0; i < iovcnt; i++) { - rret = UNIFYFS_WRAP(read)(fd, (void*)iov[i].iov_base, - iov[i].iov_len); + rret = UNIFYFS_WRAP(read)(origfd, (void*)iov[i].iov_base, + iov[i].iov_len); if (-1 == rret) { return -1; } else if (0 == rret) { @@ -1096,13 +1095,14 @@ ssize_t UNIFYFS_WRAP(writev)(int fd, const struct iovec* iov, int iovcnt) ssize_t ret; /* check whether we should intercept this file descriptor */ + int origfd = fd; if (unifyfs_intercept_fd(&fd)) { ssize_t wret; int i; ret = 0; for (i = 0; i < iovcnt; i++) { - wret = UNIFYFS_WRAP(write)(fd, (const void*)iov[i].iov_base, - iov[i].iov_len); + wret = UNIFYFS_WRAP(write)(origfd, (const void*)iov[i].iov_base, + iov[i].iov_len); if (-1 == wret) { return -1; } else { @@ -1270,8 +1270,9 @@ ssize_t UNIFYFS_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) ssize_t UNIFYFS_WRAP(pread64)(int fd, void* buf, size_t count, off64_t offset) { /* check whether we should intercept this file descriptor */ + int origfd = fd; if (unifyfs_intercept_fd(&fd)) { - return UNIFYFS_WRAP(pread)(fd, buf, count, (off_t)offset); + return UNIFYFS_WRAP(pread)(origfd, buf, count, (off_t)offset); } else { MAP_OR_FAIL(pread64); ssize_t ret = UNIFYFS_REAL(pread64)(fd, buf, count, offset); @@ -1315,8 +1316,9 @@ ssize_t UNIFYFS_WRAP(pwrite64)(int fd, const void* buf, size_t count, off64_t offset) { /* check whether we should intercept this file descriptor */ + int origfd = fd; if (unifyfs_intercept_fd(&fd)) { - return UNIFYFS_WRAP(pwrite)(fd, buf, count, (off_t)offset); + return UNIFYFS_WRAP(pwrite)(origfd, buf, count, (off_t)offset); } else { MAP_OR_FAIL(pwrite64); ssize_t ret = UNIFYFS_REAL(pwrite64)(fd, buf, count, offset); @@ -1568,8 +1570,11 @@ void* UNIFYFS_WRAP(mmap64)(void* addr, size_t length, int prot, int flags, int fd, off64_t offset) { /* check whether we should intercept this file descriptor */ + int origfd = fd; if (unifyfs_intercept_fd(&fd)) { - return UNIFYFS_WRAP(mmap)(addr, length, prot, flags, fd, (off_t)offset); + void* ret = UNIFYFS_WRAP(mmap)(addr, length, prot, flags, origfd, + (off_t)offset); + return ret; } else { MAP_OR_FAIL(mmap64); void* ret = UNIFYFS_REAL(mmap64)(addr, length, prot, flags, fd, offset); From 7754b0fe8722cb70f6cea7447c9097b2cffc4bf0 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 15 Apr 2020 17:30:59 -0700 Subject: [PATCH 098/168] client: return number of bytes written in fid_write and fid_logio_write Adds output parameter to fid_write and fid_logio_write to return number of bytes actually written which may be less than the number requested. --- client/src/unifyfs-fixed.c | 6 +++++- client/src/unifyfs-fixed.h | 3 ++- client/src/unifyfs-internal.h | 8 +++++++- client/src/unifyfs-stdio.c | 35 +++++++++++++++++++++-------------- client/src/unifyfs-sysio.c | 6 +----- client/src/unifyfs.c | 12 ++++++++++-- 6 files changed, 46 insertions(+), 24 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index 4d7141b80..d704c42ef 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -369,7 +369,8 @@ int unifyfs_fid_logio_write(int fid, unifyfs_filemeta_t* meta, off_t pos, const void* buf, - size_t count) + size_t count, + size_t* bytes) { assert(meta != NULL); if (meta->storage != FILE_STORAGE_LOGIO) { @@ -401,6 +402,9 @@ int unifyfs_fid_logio_write(int fid, (size_t)log_off, count); } + /* return number of bytes written */ + *bytes = nwritten; + /* update our write metadata for this write */ rc = add_write_meta_to_index(meta, pos, log_off, nwritten); return rc; diff --git a/client/src/unifyfs-fixed.h b/client/src/unifyfs-fixed.h index f61229756..f05abf1dc 100644 --- a/client/src/unifyfs-fixed.h +++ b/client/src/unifyfs-fixed.h @@ -57,7 +57,8 @@ int unifyfs_fid_logio_write( unifyfs_filemeta_t* meta, /* meta data for file */ off_t pos, /* file position to start writing at */ const void* buf, /* user buffer holding data */ - size_t count /* number of bytes to write */ + size_t count, /* number of bytes to write */ + size_t* bytes /* returns number of bytes written */ ); #endif /* UNIFYFS_FIXED_H */ diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 1042cdf95..df9315308 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -512,7 +512,13 @@ int unifyfs_fid_create_directory(const char* path); int unifyfs_fid_read_reqs(read_req_t* in_reqs, int in_count); /* write count bytes from buf into file starting at offset pos */ -int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count); +int unifyfs_fid_write( + int fid, /* global file id to write to */ + off_t pos, /* starting offset within file */ + const void* buf, /* buffer of data to be written */ + size_t count, /* number of bytes to write */ + size_t* bytes /* returns number of bytes written */ +); /* truncate file id to given length, frees resources if length is * less than size and allocates and zero-fills new bytes if length diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index b3a25b207..a2d99f4dd 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -476,9 +476,9 @@ static int unifyfs_stream_flush(FILE* stream) /* if buffer is dirty, write data to file */ if (s->buf != NULL && s->bufdirty) { - size_t bytes; + size_t nwritten = 0; int write_rc = unifyfs_fd_write(s->fd, s->bufpos, s->buf, s->buflen, - &bytes); + &nwritten); if (write_rc != UNIFYFS_SUCCESS) { /* ERROR: set stream error indicator and errno */ s->err = 1; @@ -486,7 +486,11 @@ static int unifyfs_stream_flush(FILE* stream) return write_rc; } - /* TODO: handle short writes as error? */ + /* TODO: treat short writes as error? */ + + /* note there is no need to update the file descriptor position here + * since we wrote to a specific offset independent of the file + * descriptor position */ /* lookup file id from file descriptor attached to stream */ int fid = unifyfs_get_fid_from_fd(s->fd); @@ -621,9 +625,9 @@ static int unifyfs_stream_read( } /* read data from file into buffer */ - size_t bytes; + size_t nread = 0; int read_rc = unifyfs_fd_read(s->fd, current, s->buf, - s->bufsize, &bytes); + s->bufsize, &nread); if (read_rc != UNIFYFS_SUCCESS) { /* ERROR: set error indicator and errno */ s->err = 1; @@ -633,7 +637,7 @@ static int unifyfs_stream_read( /* record new buffer range within file */ s->bufpos = current; - s->buflen = bytes; + s->buflen = nread; /* set end-of-file flag if our read was short */ if (s->buflen < s->bufsize) { @@ -666,7 +670,7 @@ static int unifyfs_stream_read( *retcount = (count - remaining); /* update file position */ - filedesc->pos += (off_t) * retcount; + filedesc->pos += (off_t) *retcount; /* set end of file indicator if we hit the end */ if (*retcount < count) { @@ -765,8 +769,8 @@ static int unifyfs_stream_write( /* if unbuffered, write data directly to file */ if (s->buftype == _IONBF) { /* write data directly to file */ - size_t bytes; - int write_rc = unifyfs_fd_write(s->fd, current, buf, count, &bytes); + size_t nwritten = 0; + int write_rc = unifyfs_fd_write(s->fd, current, buf, count, &nwritten); if (write_rc != UNIFYFS_SUCCESS) { /* ERROR: set stream error indicator and errno */ s->err = 1; @@ -774,10 +778,10 @@ static int unifyfs_stream_write( return write_rc; } - /* TODO: handle short writes as error? */ + /* TODO: treat short writes as error? */ /* update file position */ - filedesc->pos = current + (off_t) bytes; + filedesc->pos = current + (off_t) nwritten; return UNIFYFS_SUCCESS; } @@ -2310,9 +2314,9 @@ static int __srefill(unifyfs_stream_t* stream) } /* read data from file into buffer */ - size_t bytes; + size_t nread = 0; int read_rc = unifyfs_fd_read(s->fd, current, s->buf, s->bufsize, - &bytes); + &nread); if (read_rc != UNIFYFS_SUCCESS) { /* ERROR: set error indicator and errno */ s->err = 1; @@ -2320,9 +2324,12 @@ static int __srefill(unifyfs_stream_t* stream) return 1; } + /* update file descriptor position to account for bytes we just read */ + filedesc->pos = current + nread; + /* record new buffer range within file */ s->bufpos = current; - s->buflen = bytes; + s->buflen = nread; } /* determine number of bytes to copy from stream buffer */ diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 4522b9aab..5039d94f9 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -606,11 +606,7 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count, } /* finally write specified data to file */ - int write_rc = unifyfs_fid_write(fid, pos, buf, count); - if (write_rc == UNIFYFS_SUCCESS) { - *bytes = count; - } - + int write_rc = unifyfs_fid_write(fid, pos, buf, count, bytes); return write_rc; } diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 4a8c6d700..b8e117fd6 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -1608,10 +1608,18 @@ int unifyfs_fid_read_reqs(read_req_t* in_reqs, int in_count) * * Returns UNIFYFS_SUCCESS, or an error code */ -int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count) +int unifyfs_fid_write( + int fid, /* global file id to write to */ + off_t pos, /* starting position in file */ + const void* buf, /* buffer to be written */ + size_t count, /* number of bytes to write */ + size_t* bytes) /* returns number of bytes written */ { int rc; + /* assume we won't write anything */ + *bytes = 0; + /* short-circuit a 0-byte write */ if (count == 0) { return UNIFYFS_SUCCESS; @@ -1623,7 +1631,7 @@ int unifyfs_fid_write(int fid, off_t pos, const void* buf, size_t count) /* determine storage type to write file data */ if (meta->storage == FILE_STORAGE_LOGIO) { /* file stored in fixed-size chunks */ - rc = unifyfs_fid_logio_write(fid, meta, pos, buf, count); + rc = unifyfs_fid_logio_write(fid, meta, pos, buf, count, bytes); if (rc == UNIFYFS_SUCCESS) { /* write succeeded, remember that we have new data * that needs to be synced with the server */ From 811de4071ea46f366b42a72b749f92e4eaeb90d4 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 17 Apr 2020 12:55:28 -0700 Subject: [PATCH 099/168] tests: drop log size checks With log management having moved to logio, there is no simple method to track the expected log size from within the test cases or to query the actual current log size consumed by an individual file. This drops those checks. --- t/lib/testutil.c | 11 ++--- t/lib/testutil.h | 8 ++-- t/std/size.c | 14 ++---- t/sys/truncate.c | 102 ++++++++++++---------------------------- t/sys/write-read-hole.c | 14 ++---- t/sys/write-read.c | 18 ++----- 6 files changed, 49 insertions(+), 118 deletions(-) diff --git a/t/lib/testutil.c b/t/lib/testutil.c index 24f56e560..54af04cc1 100644 --- a/t/lib/testutil.c +++ b/t/lib/testutil.c @@ -100,11 +100,9 @@ char* testutil_get_mount_point(void) return path; } -/* Stat the file associated to by path and store the global and log sizes of the - * file at path in the addresses of the respective global and log pointers - * passed in. - * User can ask for one or both sizes. */ -void testutil_get_size(char* path, size_t* global, size_t* log) +/* Stat the file associated to by path and store the global size of the + * file at path in the address of the global pointer passed in. */ +void testutil_get_size(char* path, size_t* global) { struct stat sb = {0}; int rc; @@ -117,7 +115,4 @@ void testutil_get_size(char* path, size_t* global, size_t* log) if (global) { *global = sb.st_size; } - if (log) { - *log = sb.st_rdev; - } } diff --git a/t/lib/testutil.h b/t/lib/testutil.h index 8384427f2..58b628df6 100644 --- a/t/lib/testutil.h +++ b/t/lib/testutil.h @@ -32,8 +32,6 @@ void testutil_rand_path(char* buf, size_t len, const char* pfx); */ char* testutil_get_mount_point(void); -/* Stat the file associated to by path and store the global and log sizes of the - * file at path in the addresses of the respective global and log pointers - * passed in. - * User can ask for one or both sizes. */ -void testutil_get_size(char* path, size_t* global, size_t* log); +/* Stat the file associated to by path and store the global size of the + * file at path in the address of the global pointer passed in. */ +void testutil_get_size(char* path, size_t* global); diff --git a/t/std/size.c b/t/std/size.c index 1c2059b80..1ff91cccd 100644 --- a/t/std/size.c +++ b/t/std/size.c @@ -33,7 +33,7 @@ int size_test(char* unifyfs_root) char buf[64] = {0}; FILE* fp = NULL; char* tmp; - size_t global, log; + size_t global; int fd; errno = 0; @@ -49,11 +49,9 @@ int size_test(char* unifyfs_root) ok(fclose(fp) == 0, "%s:%d fclose(): %s", __FILE__, __LINE__, strerror(errno)); - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 12, "%s:%d global size after fwrite(\"hello world\") = %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(log == 12, "%s:%d log size after fwrite(\"hello world\") = %d: %s", - __FILE__, __LINE__, log, strerror(errno)); /* Open the file again with append, write to it. */ fp = fopen(path, "a"); @@ -83,11 +81,9 @@ int size_test(char* unifyfs_root) ok(fclose(fp) == 0, "%s:%d fclose(): %s", __FILE__, __LINE__, strerror(errno)); - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 30, "%s:%d global size after append is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(log == 30, "%s:%d log size after append is %d: %s", - __FILE__, __LINE__, log, strerror(errno)); /* Sync extents */ fd = open(path, O_RDWR); @@ -103,11 +99,9 @@ int size_test(char* unifyfs_root) __FILE__, __LINE__, strerror(errno)); /* Global size should be correct */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 30, "%s:%d global size after laminate is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(log == 30, "%s:%d log size after laminate is %d: %s", - __FILE__, __LINE__, log, strerror(errno)); /* Read it back */ fp = fopen(path, "r"); diff --git a/t/sys/truncate.c b/t/sys/truncate.c index 99a372472..0487032d8 100644 --- a/t/sys/truncate.c +++ b/t/sys/truncate.c @@ -28,7 +28,7 @@ int truncate_test(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, log; + size_t global; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -41,11 +41,9 @@ int truncate_test(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(log == 0, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 0); /* write 1MB and fsync, expect 1MB */ rc = write(fd, buf, bufsize); @@ -56,11 +54,9 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d fsync() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 1*bufsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 1*bufsize); - ok(log == 1*bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 1*bufsize); /* skip a 1MB hole, write another 1MB, and fsync expect 3MB */ rc = lseek(fd, 2*bufsize, SEEK_SET); @@ -75,22 +71,18 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d fsync() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 3*bufsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 3*bufsize); - ok(log == 2*bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 2*bufsize); /* ftruncate at 5MB, expect 5MB */ rc = ftruncate(fd, 5*bufsize); ok(rc == 0, "%s:%d ftruncate(%d) (rc=%d): %s", __FILE__, __LINE__, 5*bufsize, rc, strerror(errno)); - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 5*bufsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 5*bufsize); - ok(log == 2*bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 2*bufsize); close(fd); @@ -99,22 +91,18 @@ int truncate_test(char* unifyfs_root) ok(rc == 0, "%s:%d truncate(%d) (rc=%d): %s", __FILE__, __LINE__, bufsize/2, rc, strerror(errno)); - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == bufsize/2, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, bufsize/2); - ok(log == 2*bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 2*bufsize); /* truncate to 0, expect 0 */ rc = truncate(path, 0); ok(rc == 0, "%s:%d truncate(%d) (rc=%d): %s", __FILE__, __LINE__, 0, rc, strerror(errno)); - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(log == 2*bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 2*bufsize); free(buf); @@ -126,7 +114,7 @@ int truncate_bigempty(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, log; + size_t global; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -138,11 +126,9 @@ int truncate_bigempty(char* unifyfs_root) ok(fd != -1, "%s:%d open(%s) (fd=%d): %s", __FILE__, __LINE__, path, fd, strerror(errno)); - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(log == 0, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 0); /* ftruncate at 1TB, expect 1TB */ off_t bigempty = 1024*1024*1024*1024ULL; @@ -151,7 +137,7 @@ int truncate_bigempty(char* unifyfs_root) __FILE__, __LINE__, (unsigned long long) bigempty, rc, strerror(errno)); - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == (size_t)bigempty, "%s:%d global size is %llu expected %llu", __FILE__, __LINE__, global, (unsigned long long)bigempty, strerror(errno)); @@ -168,7 +154,7 @@ int truncate_eof(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, log; + size_t global; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -181,11 +167,9 @@ int truncate_eof(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(log == 0, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 0); /* write 1MB */ rc = write(fd, buf, bufsize); @@ -247,7 +231,7 @@ int truncate_truncsync(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, log; + size_t global; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -260,11 +244,9 @@ int truncate_truncsync(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(log == 0, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 0); /* write 1MB */ rc = write(fd, buf, bufsize); @@ -277,22 +259,18 @@ int truncate_truncsync(char* unifyfs_root) __FILE__, __LINE__, bufsize/2, rc, strerror(errno)); /* file should be 0.5MB bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == bufsize/2, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, bufsize/2); - ok(log == bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, bufsize); rc = fsync(fd); ok(rc == 0, "%s:%d fsync() (rc=%d): %s", __FILE__, __LINE__, rc, strerror(errno)); /* file should still be 0.5MB bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == bufsize/2, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, bufsize/2); - ok(log == bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, bufsize); close(fd); @@ -344,7 +322,7 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) char path[64]; int rc; int fd; - size_t global, log; + size_t global; int i; size_t bufsize = 1024*1024; @@ -358,11 +336,9 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(log == 0, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 0); /* write pattern out of 20 MB in size */ size_t nwritten = 0; @@ -388,11 +364,9 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, (int)truncsize, rc, strerror(errno)); /* file should be of size 5MB + 42 at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(log == 20*bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 20*bufsize); /* this kind of tests that the ftruncate above implied an fsync, * can't really since the writes may have gone to disk on their @@ -402,11 +376,9 @@ int truncate_pattern_size(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, rc, strerror(errno)); /* file should still be 5MB + 42 bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(log == 20*bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 20*bufsize); close(fd); @@ -486,7 +458,7 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) char path[64]; int rc; int fd; - size_t global, log; + size_t global; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -499,11 +471,9 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(log == 0, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 0); /* set size we'll truncate file to */ off_t truncsize = 5*bufsize + 42; @@ -514,11 +484,9 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, (int)truncsize, rc, strerror(errno)); /* file should be of size 5MB + 42 at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(log == 0, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 0); /* this kind of tests that the ftruncate above implied an fsync, * can't really since the writes may have gone to disk on their @@ -528,11 +496,9 @@ int truncate_empty_read(char* unifyfs_root, off_t seekpos) __FILE__, __LINE__, rc, strerror(errno)); /* file should still be 5MB + 42 bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(log == 0, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 0); close(fd); @@ -610,7 +576,7 @@ int truncate_ftrunc_before_sync(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, log; + size_t global; size_t bufsize = 1024; char* buf = (char*) malloc(bufsize); @@ -623,11 +589,9 @@ int truncate_ftrunc_before_sync(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(log == 0, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 0); /* write a small amount, intended to be small enough that * the write itself does not cause an implicit fsync */ @@ -651,11 +615,9 @@ int truncate_ftrunc_before_sync(char* unifyfs_root) /* finally, check that the file is 0 bytes, * i.e., check that the writes happened before the truncate * and not at the fsync */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(log == bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, bufsize); close(fd); @@ -669,7 +631,7 @@ int truncate_trunc_before_sync(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, log; + size_t global; size_t bufsize = 1024; char* buf = (char*) malloc(bufsize); @@ -682,11 +644,9 @@ int truncate_trunc_before_sync(char* unifyfs_root) __FILE__, __LINE__, path, fd, strerror(errno)); /* file should be 0 bytes at this point */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, 0); - ok(log == 0, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, 0); /* write a small amount, intended to be small enough that * the write itself does not cause an implicit fsync */ @@ -710,11 +670,9 @@ int truncate_trunc_before_sync(char* unifyfs_root) /* finally, check that the file is 0 bytes, * i.e., check that the writes happened before the truncate * and not at the fsync */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == truncsize, "%s:%d global size is %d expected %d", __FILE__, __LINE__, global, (int)truncsize); - ok(log == bufsize, "%s:%d log size is %d expected %d", - __FILE__, __LINE__, log, bufsize); close(fd); diff --git a/t/sys/write-read-hole.c b/t/sys/write-read-hole.c index d1b2831ed..dc64eb0cf 100644 --- a/t/sys/write-read-hole.c +++ b/t/sys/write-read-hole.c @@ -40,7 +40,7 @@ int write_read_hole_test(char* unifyfs_root) char path[64]; int rc; int fd; - size_t global, log; + size_t global; size_t bufsize = 1024*1024; char* buf = (char*) malloc(bufsize); @@ -78,11 +78,9 @@ int write_read_hole_test(char* unifyfs_root) __FILE__, __LINE__, rc, strerror(errno)); /* Check global size on our un-laminated file */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(log == 2*bufsize, "%s:%d log size is %d: %s", - __FILE__, __LINE__, log, strerror(errno)); /* flush writes */ rc = fsync(fd); @@ -90,11 +88,9 @@ int write_read_hole_test(char* unifyfs_root) __FILE__, __LINE__, rc, strerror(errno)); /* Check global size on our un-laminated file */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 3*bufsize, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(log == 2*bufsize, "%s:%d log size is %d: %s", - __FILE__, __LINE__, log, strerror(errno)); /* truncate file at 4MB, extends file so that * [3MB, 4MB) is implied "0" */ @@ -108,11 +104,9 @@ int write_read_hole_test(char* unifyfs_root) __FILE__, __LINE__, rc, strerror(errno)); /* Check global size on our un-laminated file */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 4*bufsize, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(log == 2*bufsize, "%s:%d log size is %d: %s", - __FILE__, __LINE__, log, strerror(errno)); close(fd); diff --git a/t/sys/write-read.c b/t/sys/write-read.c index 8245a4737..f1bbdf00c 100644 --- a/t/sys/write-read.c +++ b/t/sys/write-read.c @@ -31,7 +31,7 @@ int write_read_test(char* unifyfs_root) char path[64]; char buf[64] = {0}; int fd = -1; - size_t global, log; + size_t global; errno = 0; @@ -65,21 +65,17 @@ int write_read_test(char* unifyfs_root) __FILE__, __LINE__, strerror(errno)); /* Check global size on our un-laminated and un-synced file */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 0, "%s:%d global size before fsync is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(log == 21, "%s:%d log size before fsync is %d: %s", - __FILE__, __LINE__, log, strerror(errno)); ok(fsync(fd) == 0, "%s:%d fsync() worked: %s", __FILE__, __LINE__, strerror(errno)); /* Check global size on our un-laminated file */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 15, "%s:%d global size after fsync is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(log == 21, "%s:%d log size after fsync is %d: %s", - __FILE__, __LINE__, log, strerror(errno)); /* read from file open as write-only should fail with errno=EBADF */ ok(lseek(fd, 0, SEEK_SET) == 0, "%s:%d lseek(0): %s", @@ -111,22 +107,18 @@ int write_read_test(char* unifyfs_root) __FILE__, __LINE__, strerror(errno)); /* Check global size on our un-laminated file */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 21, "%s:%d global size before laminate is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(log == 27, "%s:%d log size before laminate is %d: %s", - __FILE__, __LINE__, log, strerror(errno)); /* Laminate */ ok(chmod(path, 0444) == 0, "%s:%d chmod(0444): %s", __FILE__, __LINE__, strerror(errno)); /* Verify we're getting the correct file size */ - testutil_get_size(path, &global, &log); + testutil_get_size(path, &global); ok(global == 21, "%s:%d global size after laminate is %d: %s", __FILE__, __LINE__, global, strerror(errno)); - ok(log == 27, "%s:%d log size after laminate is %d: %s", - __FILE__, __LINE__, log, strerror(errno)); /* open laminated file for write should fail with errno=EROFS */ fd = open(path, O_WRONLY | O_CREAT, 0222); From c7a4fbf34a51d2254aabdf7b4a70210b717e312a Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 17 Apr 2020 17:44:25 -0700 Subject: [PATCH 100/168] client: drop chunks field from filemeta The chunks field in the unifyfs_filemeta_t struct tracked the number of fixed-size storage chunks allocated to a file. This field is no longer used now that storage space has moved to logio. --- client/src/unifyfs-dirops.c | 1 - client/src/unifyfs-internal.h | 1 - client/src/unifyfs.c | 25 ++++--------------------- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/client/src/unifyfs-dirops.c b/client/src/unifyfs-dirops.c index 8d450d664..ba443cb81 100644 --- a/client/src/unifyfs-dirops.c +++ b/client/src/unifyfs-dirops.c @@ -143,7 +143,6 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) } meta->global_size = sb.st_size; - meta->chunks = sb.st_blocks; unifyfs_dirstream_t* dirp = unifyfs_dirstream_alloc(fid); diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index df9315308..00ed60b2f 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -269,7 +269,6 @@ typedef struct { int gfid; /* global file id for this file */ int needs_sync; /* have unsynced writes */ - off_t chunks; /* number of chunks allocated to file */ int is_laminated; /* Is this file laminated */ uint32_t mode; /* st_mode bits. This has file * permission info and will tell you if this diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index b8e117fd6..5f414dca8 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -864,7 +864,6 @@ int unifyfs_fid_create_file(const char* path) meta->storage = FILE_STORAGE_NULL; meta->gfid = unifyfs_generate_gfid(path); meta->needs_sync = 0; - meta->chunks = 0; meta->is_laminated = 0; meta->mode = UNIFYFS_STAT_DEFAULT_FILE_MODE; @@ -1630,7 +1629,7 @@ int unifyfs_fid_write( /* determine storage type to write file data */ if (meta->storage == FILE_STORAGE_LOGIO) { - /* file stored in fixed-size chunks */ + /* file stored in logged i/o */ rc = unifyfs_fid_logio_write(fid, meta, pos, buf, count, bytes); if (rc == UNIFYFS_SUCCESS) { /* write succeeded, remember that we have new data @@ -1900,11 +1899,8 @@ int unifyfs_fid_unlink(int fid) * ------------- */ /* The super block is a region of shared memory that is used to - * persist file system data. It contains both room for data - * structures used to track file names, meta data, the list of - * storage blocks used for each file, and optional blocks. - * It also contains a fixed-size region for keeping log - * index entries for each file. + * persist file system meta data. It also contains a fixed-size + * region for keeping log index entries for each file. * * - stack of free local file ids of length max_files, * the local file id is used to index into other data @@ -1915,20 +1911,7 @@ int unifyfs_fid_unlink(int fid) * slot is in use and if so, the current file name * * - array of unifyfs_filemeta structs, indexed by local - * file id, records list of storage blocks used to - * store data for the file - * - * - array of unifyfs_chunkmeta structs, indexed by local - * file id and then by chunk id for recording metadata - * of each chunk allocated to a file, including host - * storage and id of that chunk within its storage - * - * - stack to track free list of memory chunks - * - * - stack to track free list of spillover chunks - * - * - array of storage chunks of length unifyfs_max_chunks, - * if storing data in memory + * file id * * - count of number of active index entries * - array of index metadata to track physical offset From 2f0e5d486b3e9d4a1223aa356dc13e90ab491612 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 20 Apr 2020 17:26:14 -0700 Subject: [PATCH 101/168] client: rename bytes to nread/nwritten in fd_read/write and fid_write Rename function parameter bytes to be nread/nwritten to be more descriptive of its purpose. Fix some mistakes in comments. --- client/src/unifyfs-fixed.c | 28 ++++++++++++++-------------- client/src/unifyfs-fixed.h | 2 +- client/src/unifyfs-internal.h | 4 ++-- client/src/unifyfs-sysio.c | 16 ++++++++-------- client/src/unifyfs-sysio.h | 4 ++-- client/src/unifyfs.c | 14 +++++++------- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index d704c42ef..f049595d8 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -358,11 +358,12 @@ int unifyfs_sync(int gfid) /** * Write data to file using log-based I/O * - * @param fid file id to write to - * @param meta metadata for file - * @param pos file position to start writing at - * @param buf user buffer holding data - * @param count number of bytes to write + * @param fid file id to write to + * @param meta metadata for file + * @param pos file position to start writing at + * @param buf user buffer holding data + * @param count number of bytes to write + * @param nwritten number of bytes written * @return UNIFYFS_SUCCESS, or error code */ int unifyfs_fid_logio_write(int fid, @@ -370,8 +371,11 @@ int unifyfs_fid_logio_write(int fid, off_t pos, const void* buf, size_t count, - size_t* bytes) + size_t* nwritten) { + /* assume we'll fail to write anything */ + *nwritten = 0; + assert(meta != NULL); if (meta->storage != FILE_STORAGE_LOGIO) { LOGERR("file (fid=%d) storage mode != FILE_STORAGE_LOGIO", fid); @@ -387,25 +391,21 @@ int unifyfs_fid_logio_write(int fid, } /* do the write */ - size_t nwritten = 0; - rc = unifyfs_logio_write(logio_ctx, log_off, count, buf, &nwritten); + rc = unifyfs_logio_write(logio_ctx, log_off, count, buf, nwritten); if (rc != UNIFYFS_SUCCESS) { LOGERR("logio_write(%zu, %zu) failed", log_off, count); return rc; } - if (nwritten < count) { + if (*nwritten < count) { LOGWARN("partial logio_write() @ offset=%zu (%zu of %zu bytes)", - (size_t)log_off, nwritten, count); + (size_t)log_off, *nwritten, count); } else { LOGDBG("successful logio_write() @ log offset=%zu (%zu bytes)", (size_t)log_off, count); } - /* return number of bytes written */ - *bytes = nwritten; - /* update our write metadata for this write */ - rc = add_write_meta_to_index(meta, pos, log_off, nwritten); + rc = add_write_meta_to_index(meta, pos, log_off, *nwritten); return rc; } diff --git a/client/src/unifyfs-fixed.h b/client/src/unifyfs-fixed.h index f05abf1dc..d5884bbc8 100644 --- a/client/src/unifyfs-fixed.h +++ b/client/src/unifyfs-fixed.h @@ -58,7 +58,7 @@ int unifyfs_fid_logio_write( off_t pos, /* file position to start writing at */ const void* buf, /* user buffer holding data */ size_t count, /* number of bytes to write */ - size_t* bytes /* returns number of bytes written */ + size_t* nwritten /* returns number of bytes written */ ); #endif /* UNIFYFS_FIXED_H */ diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 00ed60b2f..58ba9d0ee 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -512,11 +512,11 @@ int unifyfs_fid_read_reqs(read_req_t* in_reqs, int in_count); /* write count bytes from buf into file starting at offset pos */ int unifyfs_fid_write( - int fid, /* global file id to write to */ + int fid, /* local file id to write to */ off_t pos, /* starting offset within file */ const void* buf, /* buffer of data to be written */ size_t count, /* number of bytes to write */ - size_t* bytes /* returns number of bytes written */ + size_t* nwritten /* returns number of bytes written */ ); /* truncate file id to given length, frees resources if length is diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 5039d94f9..21370f5f8 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -509,10 +509,10 @@ int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) * Returns number of bytes actually read, or -1 on error, in which * case errno will be set. */ -int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* bytes) +int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* nread) { - /* assume we'll fail, set bytes to 0 as a clue */ - *bytes = 0; + /* assume we'll fail, set bytes read to 0 as a clue */ + *nread = 0; /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); @@ -564,7 +564,7 @@ int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* bytes) } /* success, get number of bytes read from read request field */ - *bytes = req.nread; + *nread = req.nread; return UNIFYFS_SUCCESS; } @@ -576,10 +576,10 @@ int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* bytes) * is ignored. Fills any gaps with zeros */ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count, - size_t* bytes) + size_t* nwritten) { - /* assume we'll fail, set bytes to 0 as a clue */ - *bytes = 0; + /* assume we'll fail, set bytes written to 0 as a clue */ + *nwritten = 0; /* get the file id for this file descriptor */ int fid = unifyfs_get_fid_from_fd(fd); @@ -606,7 +606,7 @@ int unifyfs_fd_write(int fd, off_t pos, const void* buf, size_t count, } /* finally write specified data to file */ - int write_rc = unifyfs_fid_write(fid, pos, buf, count, bytes); + int write_rc = unifyfs_fid_write(fid, pos, buf, count, nwritten); return write_rc; } diff --git a/client/src/unifyfs-sysio.h b/client/src/unifyfs-sysio.h index 17879c053..18716f807 100644 --- a/client/src/unifyfs-sysio.h +++ b/client/src/unifyfs-sysio.h @@ -113,7 +113,7 @@ int unifyfs_fd_read( off_t pos, /* offset within file to read from */ void* buf, /* buffer to hold data */ size_t count, /* number of bytes to read */ - size_t* bytes /* number of bytes read */ + size_t* nread /* number of bytes read */ ); /* @@ -126,7 +126,7 @@ int unifyfs_fd_write( off_t pos, /* offset within file to write to */ const void* buf, /* buffer holding data to write */ size_t count, /* number of bytes to write */ - size_t* bytes /* number of bytes written */ + size_t* nwritten /* number of bytes written */ ); #include "unifyfs-dirops.h" diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 5f414dca8..51fdd80d7 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -1608,16 +1608,16 @@ int unifyfs_fid_read_reqs(read_req_t* in_reqs, int in_count) * Returns UNIFYFS_SUCCESS, or an error code */ int unifyfs_fid_write( - int fid, /* global file id to write to */ - off_t pos, /* starting position in file */ - const void* buf, /* buffer to be written */ - size_t count, /* number of bytes to write */ - size_t* bytes) /* returns number of bytes written */ + int fid, /* local file id to write to */ + off_t pos, /* starting position in file */ + const void* buf, /* buffer to be written */ + size_t count, /* number of bytes to write */ + size_t* nwritten) /* returns number of bytes written */ { int rc; /* assume we won't write anything */ - *bytes = 0; + *nwritten = 0; /* short-circuit a 0-byte write */ if (count == 0) { @@ -1630,7 +1630,7 @@ int unifyfs_fid_write( /* determine storage type to write file data */ if (meta->storage == FILE_STORAGE_LOGIO) { /* file stored in logged i/o */ - rc = unifyfs_fid_logio_write(fid, meta, pos, buf, count, bytes); + rc = unifyfs_fid_logio_write(fid, meta, pos, buf, count, nwritten); if (rc == UNIFYFS_SUCCESS) { /* write succeeded, remember that we have new data * that needs to be synced with the server */ From 751b6a6ed67593033745870b91b3924f58e7549b Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 20 Apr 2020 15:37:07 -0700 Subject: [PATCH 102/168] client: sync file if needed before read For a file that was opened as read/write, in order to read back data that was just written, it is necessary to sync updates to the server before executing the read. Originally this required the user to explicitly call fsync in between the write and the read. This adds support so that read now implies an fsync if there are pending writes that have not been sync'd. It also updates stat to imply an fsync and it changes truncate and ftruncate to call a newly added unifyfs_fid_sync function that wraps the logic. --- client/src/unifyfs-fixed.c | 7 +- client/src/unifyfs-fixed.h | 2 +- client/src/unifyfs-internal.h | 14 +- client/src/unifyfs-stdio.c | 3 +- client/src/unifyfs-sysio.c | 103 +- client/src/unifyfs.c | 1793 +++++++++++++++++---------------- t/sys/write-read-hole.c | 2 +- t/sys/write-read.c | 2 +- 8 files changed, 983 insertions(+), 943 deletions(-) diff --git a/client/src/unifyfs-fixed.c b/client/src/unifyfs-fixed.c index f049595d8..2a1eb0cbe 100644 --- a/client/src/unifyfs-fixed.c +++ b/client/src/unifyfs-fixed.c @@ -166,7 +166,7 @@ static void add_index_entry_to_seg_tree(unifyfs_filemeta_t* meta, if (unifyfs_segment_count >= (unifyfs_max_index_entries - 2)) { /* this will flush our segments, sync them, and set the running * segment count back to 0 */ - unifyfs_sync(meta->gfid); + unifyfs_sync(); } else { /* increase the running global segment count by the number of * new entries we added to this tree */ @@ -222,7 +222,7 @@ static int add_write_meta_to_index(unifyfs_filemeta_t* meta, /* if we have filled the key/value buffer, flush it to server */ if (0 == remaining_entries) { /* index buffer is full, flush it */ - int ret = unifyfs_sync(cur_idx.gfid); + int ret = unifyfs_sync(); if (ret != UNIFYFS_SUCCESS) { /* something went wrong when trying to flush key/values */ LOGERR("failed to flush key/value index to server"); @@ -311,7 +311,7 @@ void unifyfs_rewrite_index_from_seg_tree(void) * * Returns 0 on success, nonzero otherwise. */ -int unifyfs_sync(int gfid) +int unifyfs_sync(void) { /* NOTE: we currently ignore gfid and sync extents for all files in * the index. If we ever switch to storing extents in a per-file index, @@ -350,7 +350,6 @@ int unifyfs_sync(int gfid) return UNIFYFS_SUCCESS; } - /* --------------------------------------- * Operations on file storage * --------------------------------------- */ diff --git a/client/src/unifyfs-fixed.h b/client/src/unifyfs-fixed.h index d5884bbc8..3898cfcd3 100644 --- a/client/src/unifyfs-fixed.h +++ b/client/src/unifyfs-fixed.h @@ -49,7 +49,7 @@ void unifyfs_rewrite_index_from_seg_tree(void); /* sync all writes from client's index with local server */ -int unifyfs_sync(int gfid); +int unifyfs_sync(void); /* write data to file using log-based I/O */ int unifyfs_fid_logio_write( diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 58ba9d0ee..fdc70cfae 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -404,9 +404,9 @@ int unifyfs_would_overflow_offt(off_t a, off_t b); * added together */ int unifyfs_would_overflow_long(long a, long b); -int unifyfs_stack_lock(); +int unifyfs_stack_lock(void); -int unifyfs_stack_unlock(); +int unifyfs_stack_unlock(void); /* sets flag if the path is a special path */ int unifyfs_intercept_path(const char* path); @@ -495,7 +495,7 @@ int unifyfs_fid_update_file_meta(int fid, unifyfs_file_attr_t* gfattr); /* allocate a file id slot for a new file * return the fid or -1 on error */ -int unifyfs_fid_alloc(); +int unifyfs_fid_alloc(void); /* return the file id back to the free pool */ int unifyfs_fid_free(int fid); @@ -508,8 +508,6 @@ int unifyfs_fid_create_file(const char* path); * returns the new fid, or a negative value on error */ int unifyfs_fid_create_directory(const char* path); -int unifyfs_fid_read_reqs(read_req_t* in_reqs, int in_count); - /* write count bytes from buf into file starting at offset pos */ int unifyfs_fid_write( int fid, /* local file id to write to */ @@ -524,6 +522,9 @@ int unifyfs_fid_write( * is more than size */ int unifyfs_fid_truncate(int fid, off_t length); +/* sync data for file id to server if needed */ +int unifyfs_fid_sync(int fid); + /* opens a new file id with specified path, access flags, and permissions, * fills outfid with file id and outpos with position for current file pointer, * returns UNIFYFS error code */ @@ -538,6 +539,9 @@ int unifyfs_fid_unlink(int fid); /* functions used in UnifyFS */ +/* issue a set of read requests */ +int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count); + int unifyfs_set_global_file_meta_from_fid( int fid, int create); diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index a2d99f4dd..2cccf8b75 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -501,8 +501,7 @@ static int unifyfs_stream_flush(FILE* stream) } /* invoke fsync rpc to register index metadata with server */ - int gfid = unifyfs_gfid_from_fid(fid); - int ret = unifyfs_sync(gfid); + int ret = unifyfs_fid_sync(fid); if (ret != UNIFYFS_SUCCESS) { /* sync failed for some reason, set errno and return error */ s->err = 1; diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 21370f5f8..3157772b8 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -223,20 +223,17 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) { /* determine whether we should intercept this path or not */ if (unifyfs_intercept_path(path)) { - /* get global file id for path */ - int gfid = unifyfs_generate_gfid(path); - - /* before we truncate, sync any data cached this file id */ - int ret = unifyfs_sync(gfid); - if (ret != UNIFYFS_SUCCESS) { - /* sync failed for some reason, set errno and return error */ - errno = unifyfs_rc_errno(ret); - return -1; - } - /* get file id for path name */ int fid = unifyfs_get_fid_from_path(path); if (fid >= 0) { + /* before we truncate, sync any data cached this file id */ + int ret = unifyfs_fid_sync(fid); + if (ret != UNIFYFS_SUCCESS) { + /* sync failed for some reason, set errno and return error */ + errno = unifyfs_rc_errno(ret); + return -1; + } + /* got the file locally, use fid_truncate the file */ int rc = unifyfs_fid_truncate(fid, length); if (rc != UNIFYFS_SUCCESS) { @@ -245,6 +242,7 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) } } else { /* invoke truncate rpc */ + int gfid = unifyfs_generate_gfid(path); int rc = invoke_client_truncate_rpc(gfid, length); if (rc != UNIFYFS_SUCCESS) { LOGDBG("truncate rpc failed %s in UNIFYFS", path); @@ -379,6 +377,16 @@ static int __stat(const char* path, struct stat* buf) return -1; } + /* flush any pending writes if needed */ + int fid = unifyfs_get_fid_from_path(path); + if (fid != -1) { + int sync_rc = unifyfs_fid_sync(fid); + if (sync_rc != UNIFYFS_SUCCESS) { + errno = EIO; + return -1; + } + } + /* clear the user buffer */ memset(buf, 0, sizeof(*buf)); @@ -394,7 +402,6 @@ static int __stat(const char* path, struct stat* buf) } /* update local file metadata (if applicable) */ - int fid = unifyfs_get_fid_from_path(path); if (fid != -1) { unifyfs_fid_update_file_meta(fid, &fattr); } @@ -409,10 +416,12 @@ int UNIFYFS_WRAP(stat)(const char* path, struct stat* buf) { LOGDBG("stat was called for %s", path); if (unifyfs_intercept_path(path)) { - return __stat(path, buf); + int ret = __stat(path, buf); + return ret; } else { MAP_OR_FAIL(stat); - return UNIFYFS_REAL(stat)(path, buf); + int ret = UNIFYFS_REAL(stat)(path, buf); + return ret; } } @@ -424,10 +433,12 @@ int UNIFYFS_WRAP(fstat)(int fd, struct stat* buf) if (unifyfs_intercept_fd(&fd)) { int fid = unifyfs_get_fid_from_fd(fd); const char* path = unifyfs_path_from_fid(fid); - return __stat(path, buf); + int ret = __stat(path, buf); + return ret; } else { MAP_OR_FAIL(fstat); - return UNIFYFS_REAL(fstat)(fd, buf); + int ret = UNIFYFS_REAL(fstat)(fd, buf); + return ret; } } @@ -450,7 +461,8 @@ int UNIFYFS_WRAP(__xstat)(int vers, const char* path, struct stat* buf) errno = EINVAL; return -1; } - return __stat(path, buf); + int ret = __stat(path, buf); + return ret; } else { MAP_OR_FAIL(__xstat); int ret = UNIFYFS_REAL(__xstat)(vers, path, buf); @@ -469,10 +481,12 @@ int UNIFYFS_WRAP(__lxstat)(int vers, const char* path, struct stat* buf) errno = EINVAL; return -1; } - return __stat(path, buf); + int ret = __stat(path, buf); + return ret; } else { MAP_OR_FAIL(__lxstat); - return UNIFYFS_REAL(__lxstat)(vers, path, buf); + int ret = UNIFYFS_REAL(__lxstat)(vers, path, buf); + return ret; } } #endif @@ -491,10 +505,12 @@ int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) int fid = unifyfs_get_fid_from_fd(fd); const char* path = unifyfs_path_from_fid(fid); - return __stat(path, buf); + int ret = __stat(path, buf); + return ret; } else { MAP_OR_FAIL(__fxstat); - return UNIFYFS_REAL(__fxstat)(vers, fd, buf); + int ret = UNIFYFS_REAL(__fxstat)(vers, fd, buf); + return ret; } } #endif @@ -544,6 +560,10 @@ int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* nread) return UNIFYFS_SUCCESS; } + /* TODO: handle error if sync fails? */ + /* sync data for file before reading, if needed */ + unifyfs_fid_sync(fid); + /* fill in read request */ read_req_t req; req.gfid = unifyfs_gfid_from_fid(fid); @@ -554,7 +574,7 @@ int unifyfs_fd_read(int fd, off_t pos, void* buf, size_t count, size_t* nread) req.buf = buf; /* execute read operation */ - int ret = unifyfs_fid_read_reqs(&req, 1); + int ret = unifyfs_gfid_read_reqs(&req, 1); if (ret != UNIFYFS_SUCCESS) { /* failed to issue read operation */ return EIO; @@ -1156,6 +1176,11 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], if (fid < 0) { AIOCB_ERROR_CODE(cbp) = EINVAL; } else { + /* TODO: handle error if sync fails? */ + /* sync data for file before reading, if needed */ + unifyfs_fid_sync(fid); + + /* define read request for this file */ reqs[reqcnt].gfid = unifyfs_gfid_from_fid(fid); reqs[reqcnt].offset = (size_t)(cbp->aio_offset); reqs[reqcnt].length = cbp->aio_nbytes; @@ -1183,11 +1208,12 @@ int UNIFYFS_WRAP(lio_listio)(int mode, struct aiocb* const aiocb_list[], } if (reqcnt) { - rc = unifyfs_fid_read_reqs(reqs, reqcnt); + rc = unifyfs_gfid_read_reqs(reqs, reqcnt); if (rc != UNIFYFS_SUCCESS) { /* error reading data */ ret = -1; } + /* update aiocb fields to record error status and return value */ ndx = 0; for (i = 0; i < reqcnt; i++) { @@ -1229,6 +1255,10 @@ ssize_t UNIFYFS_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) return (ssize_t)(-1); } + /* TODO: handle error if sync fails? */ + /* sync data for file before reading, if needed */ + unifyfs_fid_sync(fid); + /* fill in read request */ read_req_t req; req.gfid = unifyfs_gfid_from_fid(fid); @@ -1240,7 +1270,7 @@ ssize_t UNIFYFS_WRAP(pread)(int fd, void* buf, size_t count, off_t offset) /* execute read operation */ ssize_t retcount; - int ret = unifyfs_fid_read_reqs(&req, 1); + int ret = unifyfs_gfid_read_reqs(&req, 1); if (ret != UNIFYFS_SUCCESS) { /* error reading data */ errno = EIO; @@ -1341,11 +1371,8 @@ int UNIFYFS_WRAP(ftruncate)(int fd, off_t length) return -1; } - /* get global file id for fid */ - int gfid = unifyfs_gfid_from_fid(fid); - /* before we truncate, sync any data cached this file id */ - int ret = unifyfs_sync(gfid); + int ret = unifyfs_fid_sync(fid); if (ret != UNIFYFS_SUCCESS) { /* sync failed for some reason, set errno and return error */ errno = unifyfs_rc_errno(ret); @@ -1379,23 +1406,14 @@ int UNIFYFS_WRAP(fsync)(int fd) return -1; } - /* skip this file if no new data has been written to it */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (!meta->needs_sync) { - return 0; - } - /* invoke fsync rpc to register index metadata with server */ - int gfid = unifyfs_gfid_from_fid(fid); - int ret = unifyfs_sync(gfid); + int ret = unifyfs_fid_sync(fid); if (ret != UNIFYFS_SUCCESS) { /* sync failed for some reason, set errno and return error */ errno = unifyfs_rc_errno(ret); return -1; } - /* update metadata to indicate that data has been synced */ - meta->needs_sync = 0; return 0; } else { MAP_OR_FAIL(fsync); @@ -1581,7 +1599,6 @@ void* UNIFYFS_WRAP(mmap64)(void* addr, size_t length, int prot, int flags, int UNIFYFS_WRAP(close)(int fd) { /* check whether we should intercept this file descriptor */ - int origfd = fd; if (unifyfs_intercept_fd(&fd)) { LOGDBG("closing fd %d", fd); @@ -1601,9 +1618,13 @@ int UNIFYFS_WRAP(close)(int fd) return -1; } - /* if file was opened for writing, fsync it */ + /* if file was opened for writing, sync it */ if (filedesc->write) { - UNIFYFS_WRAP(fsync)(origfd); + int sync_rc = unifyfs_fid_sync(fid); + if (sync_rc != UNIFYFS_SUCCESS) { + errno = unifyfs_rc_errno(sync_rc); + return -1; + } } /* close the file id */ diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 51fdd80d7..28dbcd578 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -259,7 +259,7 @@ inline int unifyfs_would_overflow_long(long a, long b) } /* lock access to shared data structures in superblock */ -inline int unifyfs_stack_lock() +inline int unifyfs_stack_lock(void) { if (unifyfs_use_single_shm) { return pthread_mutex_lock(&unifyfs_stack_mutex); @@ -268,7 +268,7 @@ inline int unifyfs_stack_lock() } /* unlock access to shared data structures in superblock */ -inline int unifyfs_stack_unlock() +inline int unifyfs_stack_unlock(void) { if (unifyfs_use_single_shm) { return pthread_mutex_unlock(&unifyfs_stack_mutex); @@ -519,589 +519,326 @@ static int fid_store_free(int fid) } /* ======================================= - * Operations on file ids + * Operations on global file ids * ======================================= */ -/* checks to see if fid is a directory - * returns 1 for yes - * returns 0 for no */ -int unifyfs_fid_is_dir(int fid) -{ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (meta && meta->mode & S_IFDIR) { - return 1; - } else { - /* if it doesn't exist, then it's not a directory? */ - return 0; - } -} - -int unifyfs_gfid_from_fid(const int fid) +/* order by file id then by offset */ +static int compare_read_req(const void* a, const void* b) { - /* check that local file id is in range */ - if (fid < 0 || fid >= unifyfs_max_files) { - return -1; - } - - /* return global file id, cached in file meta struct */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - return meta->gfid; -} + const read_req_t* ptr_a = a; + const read_req_t* ptr_b = b; -/* scan list of files and return fid corresponding to target gfid, - * returns -1 if not found */ -int unifyfs_fid_from_gfid(int gfid) -{ - int i; - for (i = 0; i < unifyfs_max_files; i++) { - if (unifyfs_filelist[i].in_use && - unifyfs_filemetas[i].gfid == gfid) { - /* found a file id that's in use and it matches - * the target fid, this is the one */ - return i; + if (ptr_a->gfid != ptr_b->gfid) { + if (ptr_a->gfid < ptr_b->gfid) { + return -1; + } else { + return 1; } } - return -1; -} -/* Given a fid, return the path. */ -const char* unifyfs_path_from_fid(int fid) -{ - unifyfs_filename_t* fname = &unifyfs_filelist[fid]; - if (fname->in_use) { - return fname->filename; + if (ptr_a->offset == ptr_b->offset) { + return 0; + } else if (ptr_a->offset < ptr_b->offset) { + return -1; + } else { + return 1; } - return NULL; } -/* checks to see if a directory is empty - * assumes that check for is_dir has already been made - * only checks for full path matches, does not check relative paths, - * e.g. ../dirname will not work - * returns 1 for yes it is empty - * returns 0 for no */ -int unifyfs_fid_is_dir_empty(const char* path) +/* notify our delegator that the shared memory buffer + * is now clear and ready to hold more read data */ +static void delegator_signal(void) { - int i = 0; - while (i < unifyfs_max_files) { - /* only check this element if it's active */ - if (unifyfs_filelist[i].in_use) { - /* if the file starts with the path, it is inside of that directory - * also check to make sure that it's not the directory entry itself */ - char* strptr = strstr(path, unifyfs_filelist[i].filename); - if (strptr == unifyfs_filelist[i].filename && - strcmp(path, unifyfs_filelist[i].filename) != 0) { - /* found a child item in path */ - LOGDBG("File found: unifyfs_filelist[%d].filename = %s", - i, (char*)&unifyfs_filelist[i].filename); - return 0; - } - } - - /* go on to next file */ - i++; - } + LOGDBG("receive buffer now empty"); - /* couldn't find any files with this prefix, dir must be empty */ - return 1; -} + /* set shm flag to signal delegator we're done */ + shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); + hdr->state = SHMEM_REGION_EMPTY; -/* Return the global (laminated) size of the file */ -off_t unifyfs_fid_global_size(int fid) -{ - /* get meta data for this file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (NULL != meta) { - return meta->global_size; - } - return (off_t)-1; + /* TODO: MEM_FLUSH */ } -/* - * Return the size of the file. If the file is laminated, return the - * laminated size. If the file is not laminated, return the local - * size. - */ -off_t unifyfs_fid_logical_size(int fid) +/* wait for delegator to inform us that shared memory buffer + * is filled with read data */ +static int delegator_wait(void) { - /* get meta data for this file */ - if (unifyfs_fid_is_laminated(fid)) { - return unifyfs_fid_global_size(fid); - } else { - /* invoke an rpc to ask the server what the file size is */ - - /* get gfid for this file */ - int gfid = unifyfs_gfid_from_fid(fid); + int rc = (int)UNIFYFS_SUCCESS; - /* sync any writes to disk before requesting file size */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (meta->needs_sync) { - /* we have some changes to sync for this file */ - unifyfs_sync(gfid); + /* specify time to sleep between checking flag in shared + * memory indicating server has produced */ + struct timespec shm_wait_tm; + shm_wait_tm.tv_sec = 0; + shm_wait_tm.tv_nsec = SHM_WAIT_INTERVAL; - /* just synced writes for this file */ - meta->needs_sync = 0; - } + /* get pointer to flag in shared memory */ + shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); - /* get file size for this file */ - size_t filesize; - int ret = invoke_client_filesize_rpc(gfid, &filesize); - if (ret != UNIFYFS_SUCCESS) { - /* failed to get file size */ - return (off_t)-1; + /* wait for server to set flag to non-zero */ + int max_sleep = 5000000; // 5s + volatile int* vip = (volatile int*)&(hdr->state); + while (*vip == SHMEM_REGION_EMPTY) { + /* not there yet, sleep for a while */ + nanosleep(&shm_wait_tm, NULL); + /* TODO: MEM_FETCH */ + max_sleep--; + if (0 == max_sleep) { + LOGERR("timed out waiting for non-empty"); + rc = (int)UNIFYFS_ERROR_SHMEM; + break; } - return (off_t)filesize; } + + return rc; } -/* if we have a local fid structure corresponding to the gfid - * in question, we attempt the file lookup with the fid method - * otherwise call back to the rpc */ -off_t unifyfs_gfid_filesize(int gfid) +/* copy read data from shared memory buffer to user buffers from read + * calls, sets done=1 on return when delegator informs us it has no + * more data */ +static int process_read_data(read_req_t* read_reqs, int count, int* done) { - off_t filesize = (off_t)-1; + /* assume we'll succeed */ + int rc = UNIFYFS_SUCCESS; - /* see if we have a fid for this gfid */ - int fid = unifyfs_fid_from_gfid(gfid); - if (fid >= 0) { - /* got a fid, look up file size through that - * method, since it may avoid a server rpc call */ - filesize = unifyfs_fid_logical_size(fid); - } else { - /* no fid for this gfid, - * look it up with server rpc */ - size_t size; - int ret = invoke_client_filesize_rpc(gfid, &size); - if (ret == UNIFYFS_SUCCESS) { - /* got the file size successfully */ - filesize = size; - } - } + /* get pointer to start of shared memory buffer */ + shm_data_header* shm_hdr = (shm_data_header*)(shm_recv_ctx->addr); + char* shmptr = ((char*)shm_hdr) + sizeof(shm_data_header); - return filesize; -} + /* get number of read replies in shared memory */ + size_t num = shm_hdr->meta_cnt; -/* Update local metadata for file from global metadata */ -int unifyfs_fid_update_file_meta(int fid, unifyfs_file_attr_t* gfattr) -{ - if (NULL == gfattr) { - return UNIFYFS_FAILURE; - } + /* process each of our read replies */ + size_t i; + for (i = 0; i < num; i++) { + /* get pointer to current read reply header */ + shm_data_meta* rep = (shm_data_meta*)shmptr; + shmptr += sizeof(shm_data_meta); - /* lookup local metadata for file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - if (NULL != meta) { - /* update lamination state */ - meta->is_laminated = gfattr->is_laminated; - if (meta->is_laminated) { - /* update file size */ - meta->global_size = (off_t)gfattr->size; - LOGDBG("laminated file size is %zu bytes", - (size_t)meta->global_size); - } - return UNIFYFS_SUCCESS; - } - /* else, bad fid */ - return UNIFYFS_FAILURE; -} + /* get pointer to data */ + char* rep_buf = shmptr; + shmptr += rep->length; -/* - * Set the metadata values for a file (after optionally creating it). - * The gfid for the file is in f_meta->gfid. - * - * gfid: The global file id on which to set metadata. - * - * create: If set to 1, attempt to create the file first. If the file - * already exists, then update its metadata with the values in - * gfattr. If set to 0, and the file does not exist, then - * the server will return an error. - * - * gfattr: The metadata values to store. - */ -int unifyfs_set_global_file_meta( - int gfid, /* file id to set meta data for */ - int create, /* whether to set size/laminated fields (1) or not (0) */ - unifyfs_file_attr_t* gfattr) /* meta data to store for file */ -{ - /* check that we have an input buffer */ - if (NULL == gfattr) { - return UNIFYFS_FAILURE; - } + /* get start and end offset of reply */ + size_t rep_start = rep->offset; + size_t rep_end = rep->offset + rep->length; - /* force the gfid field value to match the gfid we're - * submitting this under */ - gfattr->gfid = gfid; + /* iterate over each of our read requests */ + size_t j; + for (j = 0; j < count; j++) { + /* get pointer to read request */ + read_req_t* req = &read_reqs[j]; - /* submit file attributes to global key/value store */ - int ret = invoke_client_metaset_rpc(create, gfattr); - return ret; -} - -int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) -{ - /* check that we have an output buffer to write to */ - if (NULL == gfattr) { - return UNIFYFS_FAILURE; - } - - /* attempt to lookup file attributes in key/value store */ - unifyfs_file_attr_t fmeta; - int ret = invoke_client_metaget_rpc(gfid, &fmeta); - if (ret == UNIFYFS_SUCCESS) { - /* found it, copy attributes to output struct */ - *gfattr = fmeta; - } - return ret; -} - -/* - * Set the metadata values for a file (after optionally creating it), - * using metadata associated with a given local file id. - * - * fid: The local file id on which to base global metadata values. - * - * create: If set to 1, attempt to create the file first. If the file - * already exists, then update its metadata with the values in - * gfattr. If set to 0, and the file does not exist, then - * the server will return an error. - */ -int unifyfs_set_global_file_meta_from_fid(int fid, int create) -{ - /* initialize an empty file attributes structure */ - unifyfs_file_attr_t fattr = {0}; - - /* lookup local metadata for file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - - /* copy our file name */ - const char* path = unifyfs_path_from_fid(fid); - sprintf(fattr.filename, "%s", path); - - /* set global file id */ - fattr.gfid = meta->gfid; - - /* use current time for atime/mtime/ctime */ - struct timespec tp = {0}; - clock_gettime(CLOCK_REALTIME, &tp); - fattr.atime = tp; - fattr.mtime = tp; - fattr.ctime = tp; - - /* copy file mode bits and lamination flag */ - fattr.mode = meta->mode; + /* skip if this request if not the same file */ + if (rep->gfid != req->gfid) { + /* request and reply are for different files */ + continue; + } - /* these fields are set by server, except when we're creating a - * new file in which case, we should initialize them both to 0 */ - fattr.is_laminated = 0; - fattr.size = 0; + /* same file, now get start and end offsets + * of this read request */ + size_t req_start = req->offset; + size_t req_end = req->offset + req->length; - /* capture current uid and gid */ - fattr.uid = getuid(); - fattr.gid = getgid(); + /* test whether reply overlaps with request, + * overlap if: + * start of reply comes before the end of request + * AND + * end of reply comes after the start of request */ + int overlap = (rep_start < req_end && rep_end > req_start); + if (!overlap) { + /* reply does not overlap with this request */ + continue; + } - /* submit file attributes to global key/value store */ - int ret = unifyfs_set_global_file_meta(meta->gfid, create, &fattr); - return ret; -} + /* this reply overlaps with the request, check that + * we didn't get an error */ + if (rep->errcode != UNIFYFS_SUCCESS) { + /* TODO: should we look for the reply with an errcode + * with the lowest start offset? */ -/* allocate a file id slot for a new file - * return the fid or -1 on error */ -int unifyfs_fid_alloc() -{ - unifyfs_stack_lock(); - int fid = unifyfs_stack_pop(free_fid_stack); - unifyfs_stack_unlock(); - LOGDBG("unifyfs_stack_pop() gave %d", fid); - if (fid < 0) { - /* need to create a new file, but we can't */ - LOGERR("unifyfs_stack_pop() failed (%d)", fid); - return -1; - } - return fid; -} + /* read reply has an error, mark the read request + * as also having an error, then quit processing */ + req->errcode = rep->errcode; + continue; + } -/* return the file id back to the free pool */ -int unifyfs_fid_free(int fid) -{ - unifyfs_stack_lock(); - unifyfs_stack_push(free_fid_stack, fid); - unifyfs_stack_unlock(); - return UNIFYFS_SUCCESS; -} + /* otherwise, we have an error-free, overlapping reply + * for this request, copy data into request buffer */ -/* add a new file and initialize metadata - * returns the new fid, or negative value on error */ -int unifyfs_fid_create_file(const char* path) -{ - int rc; + /* start of overlapping segment is the maximum of + * reply and request start offsets */ + size_t start = rep_start; + if (req_start > start) { + start = req_start; + } - /* check that pathname is within bounds */ - size_t pathlen = strlen(path) + 1; - if (pathlen > UNIFYFS_MAX_FILENAME) { - return ENAMETOOLONG; - } + /* end of overlapping segment is the mimimum of + * reply and request end offsets */ + size_t end = rep_end; + if (req_end < end) { + end = req_end; + } - /* allocate an id for this file */ - int fid = unifyfs_fid_alloc(); - if (fid < 0) { - /* was there an error? if so, return it */ - errno = ENOSPC; - return fid; - } + /* compute length of overlapping segment */ + size_t length = end - start; - /* mark this slot as in use */ - unifyfs_filelist[fid].in_use = 1; + /* get number of bytes from start of reply and request + * buffers to the start of the overlap region */ + size_t rep_offset = start - rep_start; + size_t req_offset = start - req_start; - /* copy file name into slot */ - strcpy((void*)&unifyfs_filelist[fid].filename, path); - LOGDBG("Filename %s got unifyfs fd %d", - unifyfs_filelist[fid].filename, fid); + /* if we have a gap, fill with zeros */ + size_t gap_start = req_start + req->nread; + if (start > gap_start) { + size_t gap_length = start - gap_start; + char* req_ptr = req->buf + req->nread; + memset(req_ptr, 0, gap_length); + } - /* initialize meta data */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - meta->global_size = 0; - meta->flock_status = UNLOCKED; - meta->storage = FILE_STORAGE_NULL; - meta->gfid = unifyfs_generate_gfid(path); - meta->needs_sync = 0; - meta->is_laminated = 0; - meta->mode = UNIFYFS_STAT_DEFAULT_FILE_MODE; + /* copy data from reply buffer into request buffer */ + char* req_ptr = req->buf + req_offset; + char* rep_ptr = rep_buf + rep_offset; + memcpy(req_ptr, rep_ptr, length); - if (unifyfs_flatten_writes) { - /* Initialize our segment tree that will record our writes */ - rc = seg_tree_init(&meta->extents_sync); - if (rc != 0) { - errno = rc; - fid = -1; + /* update max number of bytes we have written to in the + * request buffer */ + size_t nread = end - req_start; + if (nread > req->nread) { + req->nread = nread; + } } } - /* Initialize our segment tree to track extents for all writes - * by this process, can be used to read back local data */ - if (unifyfs_local_extents) { - rc = seg_tree_init(&meta->extents); - if (rc != 0) { - errno = rc; - fid = -1; - } + /* set done flag if there is no more data */ + if (shm_hdr->state == SHMEM_REGION_DATA_COMPLETE) { + *done = 1; } - /* PTHREAD_PROCESS_SHARED allows Process-Shared Synchronization */ - pthread_spin_init(&meta->fspinlock, PTHREAD_PROCESS_SHARED); - - return fid; + return rc; } -int unifyfs_fid_create_directory(const char* path) +/* This uses information in the extent map for a file on the client to + * complete any read requests. It only complets a request if it contains + * all of the data. Otherwise the request is copied to the list of + * requests to be handled by the server. */ +static void service_local_reqs( + read_req_t* read_reqs, /* list of input read requests */ + int count, /* number of input read requests */ + read_req_t* local_reqs, /* list to copy requests completed by client */ + read_req_t* server_reqs, /* list to copy requests to be handled by server */ + int* out_count) /* number of items copied to server list */ { - /* check that pathname is within bounds */ - size_t pathlen = strlen(path) + 1; - if (pathlen > UNIFYFS_MAX_FILENAME) { - return (int) ENAMETOOLONG; - } - - /* get local and global file ids */ - int fid = unifyfs_get_fid_from_path(path); - int gfid = unifyfs_generate_gfid(path); - - /* test whether we have info for file in our local file list */ - int found_local = (fid >= 0); - - /* test whether we have metadata for file in global key/value store */ - int found_global = 0; - unifyfs_file_attr_t gfattr = { 0, }; - if (unifyfs_get_global_file_meta(gfid, &gfattr) == UNIFYFS_SUCCESS) { - found_global = 1; - } - - /* can't create if it already exists */ - if (found_global) { - return (int) EEXIST; - } - - if (found_local) { - /* exists locally, but not globally - * - * FIXME: so, we have detected the cache inconsistency here. - * we cannot simply unlink or remove the entry because then we also - * need to check whether any subdirectories or files exist. - * - * this can happen when - * - a process created a directory. this process (A) has opened it at - * least once. - * - then, the directory has been deleted by another process (B). it - * deletes the global entry without checking any local used entries - * in other processes. - * - * we currently return EIO, and this needs to be addressed according to - * a consistency model this fs intance assumes. - */ - return EIO; - } - - /* now, we need to create a new directory. */ - fid = unifyfs_fid_create_file(path); - if (fid < 0) { - /* FIXME: ENOSPC or EIO? */ - return EIO; - } - - /* Set as directory */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; - - /* insert global meta data for directory */ - int ret = unifyfs_set_global_file_meta_from_fid(fid, 1); - if (ret != UNIFYFS_SUCCESS) { - LOGERR("Failed to populate the global meta entry for %s (fid:%d)", - path, fid); - return EIO; - } - - return UNIFYFS_SUCCESS; -} + /* this will track the total number of requests we're passing + * on to the server */ + int local_count = 0; + int server_count = 0; -/* order by file id then by offset */ -static int compare_read_req(const void* a, const void* b) -{ - const read_req_t* ptr_a = a; - const read_req_t* ptr_b = b; + /* iterate over each input read request, satisfy it locally if we can + * otherwise copy request into output list that the server will handle + * for us */ + int i; + for (i = 0; i < count; i++) { + /* get current read request */ + read_req_t* req = &read_reqs[i]; - if (ptr_a->gfid != ptr_b->gfid) { - if (ptr_a->gfid < ptr_b->gfid) { - return -1; - } else { - return 1; + /* skip any request that's already completed or errored out, + * we pass those requests on to server */ + if (req->nread >= req->length || req->errcode != UNIFYFS_SUCCESS) { + /* copy current request into list of requests + * that we'll ask server for */ + memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); + server_count++; + continue; } - } - if (ptr_a->offset == ptr_b->offset) { - return 0; - } else if (ptr_a->offset < ptr_b->offset) { - return -1; - } else { - return 1; - } -} - -/* notify our delegator that the shared memory buffer - * is now clear and ready to hold more read data */ -static void delegator_signal(void) -{ - LOGDBG("receive buffer now empty"); - - /* set shm flag to signal delegator we're done */ - shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); - hdr->state = SHMEM_REGION_EMPTY; - - /* TODO: MEM_FLUSH */ -} - -/* wait for delegator to inform us that shared memory buffer - * is filled with read data */ -static int delegator_wait(void) -{ - int rc = (int)UNIFYFS_SUCCESS; - - /* specify time to sleep between checking flag in shared - * memory indicating server has produced */ - struct timespec shm_wait_tm; - shm_wait_tm.tv_sec = 0; - shm_wait_tm.tv_nsec = SHM_WAIT_INTERVAL; + /* get gfid, start, and length of this request */ + int gfid = req->gfid; + size_t req_start = req->offset; + size_t req_end = req->offset + req->length; - /* get pointer to flag in shared memory */ - shm_data_header* hdr = (shm_data_header*)(shm_recv_ctx->addr); + /* lookup local extents if we have them */ + int fid = unifyfs_fid_from_gfid(gfid); - /* wait for server to set flag to non-zero */ - int max_sleep = 5000000; // 5s - volatile int* vip = (volatile int*)&(hdr->state); - while (*vip == SHMEM_REGION_EMPTY) { - /* not there yet, sleep for a while */ - nanosleep(&shm_wait_tm, NULL); - /* TODO: MEM_FETCH */ - max_sleep--; - if (0 == max_sleep) { - LOGERR("timed out waiting for non-empty"); - rc = (int)UNIFYFS_ERROR_SHMEM; - break; + /* move to next request if we can't find the matching fid */ + if (fid < 0) { + /* copy current request into list of requests + * that we'll ask server for */ + memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); + server_count++; + continue; } - } - return rc; -} - -/* copy read data from shared memory buffer to user buffers from read - * calls, sets done=1 on return when delegator informs us it has no - * more data */ -static int process_read_data(read_req_t* read_reqs, int count, int* done) -{ - /* assume we'll succeed */ - int rc = UNIFYFS_SUCCESS; - - /* get pointer to start of shared memory buffer */ - shm_data_header* shm_hdr = (shm_data_header*)(shm_recv_ctx->addr); - char* shmptr = ((char*)shm_hdr) + sizeof(shm_data_header); - - /* get number of read replies in shared memory */ - size_t num = shm_hdr->meta_cnt; - - /* process each of our read replies */ - size_t i; - for (i = 0; i < num; i++) { - /* get pointer to current read reply header */ - shm_data_meta* rep = (shm_data_meta*)shmptr; - shmptr += sizeof(shm_data_meta); + /* get pointer to extents for this file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + struct seg_tree* extents = &meta->extents; - /* get pointer to data */ - char* rep_buf = shmptr; - shmptr += rep->length; + /* lock the extent tree for reading */ + seg_tree_rdlock(extents); - /* get start and end offset of reply */ - size_t rep_start = rep->offset; - size_t rep_end = rep->offset + rep->length; + /* identify whether we can satisfy this full request + * or not, assume we can */ + int have_local = 1; - /* iterate over each of our read requests */ - size_t j; - for (j = 0; j < count; j++) { - /* get pointer to read request */ - read_req_t* req = &read_reqs[j]; + /* this will point to the offset of the next byte we + * need to account for */ + size_t expected_start = req_start; - /* skip if this request if not the same file */ - if (rep->gfid != req->gfid) { - /* request and reply are for different files */ - continue; + /* iterate over extents we have for this file, + * and check that there are no holes in coverage, + * we search for a starting extent using a range + * of just the very first byte that we need */ + struct seg_tree_node* first; + first = seg_tree_find_nolock(extents, req_start, req_start); + struct seg_tree_node* next = first; + while (next != NULL && next->start < req_end) { + if (expected_start >= next->start) { + /* this extent has the next byte we expect, + * bump up to the first byte past the end + * of this extent */ + expected_start = next->end + 1; + } else { + /* there is a gap between extents so we're missing + * some bytes */ + have_local = 0; + break; } - /* same file, now get start and end offsets - * of this read request */ - size_t req_start = req->offset; - size_t req_end = req->offset + req->length; + /* get the next element in the tree */ + next = seg_tree_iter(extents, next); + } - /* test whether reply overlaps with request, - * overlap if: - * start of reply comes before the end of request - * AND - * end of reply comes after the start of request */ - int overlap = (rep_start < req_end && rep_end > req_start); - if (!overlap) { - /* reply does not overlap with this request */ - continue; - } + /* check that we account for the full request + * up until the last byte */ + if (expected_start < req_end) { + /* missing some bytes at the end of the request */ + have_local = 0; + } - /* this reply overlaps with the request, check that - * we didn't get an error */ - if (rep->errcode != UNIFYFS_SUCCESS) { - /* TODO: should we look for the reply with an errcode - * with the lowest start offset? */ + /* if we can't fully satisfy the request, copy request to + * output array, so it can be passed on to server */ + if (!have_local) { + /* copy current request into list of requests + * that we'll ask server for */ + memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); + server_count++; - /* read reply has an error, mark the read request - * as also having an error, then quit processing */ - req->errcode = rep->errcode; - continue; - } + /* release lock before we go to next request */ + seg_tree_unlock(extents); + + continue; + } - /* otherwise, we have an error-free, overlapping reply - * for this request, copy data into request buffer */ + /* otherwise we can copy the data locally, iterate + * over the extents and copy data into request buffer, + * again search for a starting extent using a range + * of just the very first byte that we need */ + next = first; + while ((next != NULL) && (next->start < req_end)) { + /* get start and end of this extent (reply) */ + size_t rep_start = next->start; + size_t rep_end = next->end + 1; + + /* get the offset into the log */ + size_t rep_log_pos = next->ptr; /* start of overlapping segment is the maximum of * reply and request start offsets */ @@ -1133,474 +870,732 @@ static int process_read_data(read_req_t* read_reqs, int count, int* done) memset(req_ptr, 0, gap_length); } - /* copy data from reply buffer into request buffer */ + /* copy data from local write log into request buffer */ char* req_ptr = req->buf + req_offset; - char* rep_ptr = rep_buf + rep_offset; - memcpy(req_ptr, rep_ptr, length); - - /* update max number of bytes we have written to in the - * request buffer */ - size_t nread = end - req_start; - if (nread > req->nread) { - req->nread = nread; + off_t log_offset = rep_log_pos + rep_offset; + size_t nread = 0; + int rc = unifyfs_logio_read(logio_ctx, log_offset, length, + req_ptr, &nread); + if (rc == UNIFYFS_SUCCESS) { + if (nread < length) { + /* account for short read by updating end offset */ + end -= (length - nread); + } + /* update max number of bytes we have filled in the req buf */ + size_t req_nread = end - req_start; + if (req_nread > req->nread) { + req->nread = req_nread; + } + } else { + LOGERR("local log read failed for offset=%zu size=%zu", + (size_t)log_offset, length); + req->errcode = EIO; } + + /* get the next element in the tree */ + next = seg_tree_iter(extents, next); } - } - /* set done flag if there is no more data */ - if (shm_hdr->state == SHMEM_REGION_DATA_COMPLETE) { - *done = 1; + /* copy request data to list we completed locally */ + memcpy(&local_reqs[local_count], req, sizeof(read_req_t)); + local_count++; + + /* done reading the tree */ + seg_tree_unlock(extents); } - return rc; + /* return to user the number of key/values we set */ + *out_count = server_count; + + return; } -/* This uses information in the extent map for a file on the client to - * complete any read requests. It only complets a request if it contains - * all of the data. Otherwise the request is copied to the list of - * requests to be handled by the server. */ -static void service_local_reqs( - read_req_t* read_reqs, /* list of input read requests */ - int count, /* number of input read requests */ - read_req_t* local_reqs, /* list to copy requests completed by client */ - read_req_t* server_reqs, /* list to copy requests to be handled by server */ - int* out_count) /* number of items copied to server list */ +/* + * get data for a list of read requests from the + * delegator + * + * @param read_reqs: a list of read requests + * @param count: number of read requests + * @return error code + * */ +int unifyfs_gfid_read_reqs(read_req_t* in_reqs, int in_count) { - /* this will track the total number of requests we're passing - * on to the server */ - int local_count = 0; - int server_count = 0; - - /* iterate over each input read request, satisfy it locally if we can - * otherwise copy request into output list that the server will handle - * for us */ int i; - for (i = 0; i < count; i++) { - /* get current read request */ - read_req_t* req = &read_reqs[i]; + int read_rc; - /* skip any request that's already completed or errored out, - * we pass those requests on to server */ - if (req->nread >= req->length || req->errcode != UNIFYFS_SUCCESS) { - /* copy current request into list of requests - * that we'll ask server for */ - memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); - server_count++; + /* assume we'll succeed */ + int rc = UNIFYFS_SUCCESS; + + /* assume we'll service all requests from the server */ + int count = in_count; + read_req_t* read_reqs = in_reqs; + + /* TODO: if the file is laminated so that we know the file size, + * we can adjust read requests to not read past the EOF */ + + /* if the option is enabled to service requests locally, try it, + * in this case we'll allocate a large array which we split into + * two, the first half will record requests we completed locally + * and the second half will store requests to be sent to the server */ + + /* this records the pointer to the temp request array if + * we allocate one, we should free this later if not NULL */ + read_req_t* reqs = NULL; + + /* this will point to the start of the array of requests we + * complete locally */ + read_req_t* local_reqs = NULL; + + /* attempt to complete requests locally if enabled */ + if (unifyfs_local_extents) { + /* allocate space to make local and server copies of the requests, + * each list will be at most in_count long */ + size_t reqs_size = 2 * in_count * sizeof(read_req_t); + reqs = (read_req_t*) malloc(reqs_size); + if (reqs == NULL) { + return ENOMEM; + } + + /* define pointers to space where we can build our list + * of requests handled on the client and those left + * for the server */ + local_reqs = &reqs[0]; + read_reqs = &reqs[in_count]; + + /* service reads from local extent info if we can, this copies + * completed requests from in_reqs into local_reqs, and it copies + * any requests that can't be completed locally into the read_reqs + * to be processed by the server */ + service_local_reqs(in_reqs, in_count, local_reqs, read_reqs, &count); + + /* bail early if we satisfied all requests locally */ + if (count == 0) { + /* copy completed requests back into user's array */ + memcpy(in_reqs, local_reqs, in_count * sizeof(read_req_t)); + + /* free the temporary array */ + free(reqs); + return rc; + } + } + + /* TODO: When the number of read requests exceed the + * request buffer, split list io into multiple bulk + * sends and transfer in bulks */ + + /* check that we have enough slots for all read requests */ + if (count > UNIFYFS_MAX_READ_CNT) { + LOGERR("Too many requests to pass to server"); + if (reqs != NULL) { + free(reqs); + } + return ENOSPC; + } + + /* order read request by increasing file id, then increasing offset */ + qsort(read_reqs, count, sizeof(read_req_t), compare_read_req); + + /* prepare our shared memory buffer for delegator */ + delegator_signal(); + + /* we select different rpcs depending on the number of + * read requests */ + if (count > 1) { + /* got multiple read requests, + * build up a flat buffer to include them all */ + flatcc_builder_t builder; + flatcc_builder_init(&builder); + + /* create request vector */ + unifyfs_Extent_vec_start(&builder); + + /* fill in values for each request entry */ + for (i = 0; i < count; i++) { + unifyfs_Extent_vec_push_create(&builder, + read_reqs[i].gfid, read_reqs[i].offset, read_reqs[i].length); + } + + /* complete the array */ + unifyfs_Extent_vec_ref_t extents = unifyfs_Extent_vec_end(&builder); + unifyfs_ReadRequest_create_as_root(&builder, extents); + //unifyfs_ReadRequest_end_as_root(&builder); + + /* allocate our buffer to be sent */ + size_t size = 0; + void* buffer = flatcc_builder_finalize_buffer(&builder, &size); + assert(buffer); + + LOGDBG("mread: n_reqs:%d, flatcc buffer (%p) sz:%zu", + count, buffer, size); + + /* invoke multi-read rpc */ + read_rc = invoke_client_mread_rpc(count, size, buffer); + + /* free flat buffer resources */ + flatcc_builder_clear(&builder); + free(buffer); + } else { + /* got a single read request */ + int gfid = read_reqs[0].gfid; + size_t offset = read_reqs[0].offset; + size_t length = read_reqs[0].length; + + LOGDBG("read: offset:%zu, len:%zu", offset, length); + + /* invoke single read rpc */ + read_rc = invoke_client_read_rpc(gfid, offset, length); + } + + /* bail out with error if we failed to even start the read */ + if (read_rc != UNIFYFS_SUCCESS) { + LOGERR("Failed to issue read RPC to server"); + if (reqs != NULL) { + free(reqs); + } + return read_rc; + } + + /* + * ToDo: Exception handling when some of the requests + * are missed + * */ + + /* spin waiting for read data to come back from the server, + * we process it in batches as it comes in, eventually the + * server will tell us it's sent us everything it can */ + int done = 0; + while (!done) { + int tmp_rc = delegator_wait(); + if (tmp_rc != UNIFYFS_SUCCESS) { + rc = UNIFYFS_FAILURE; + done = 1; + } else { + tmp_rc = process_read_data(read_reqs, count, &done); + if (tmp_rc != UNIFYFS_SUCCESS) { + rc = UNIFYFS_FAILURE; + } + delegator_signal(); + } + } + + /* got all of the data we'll get from the server, + * check for short reads and whether those short + * reads are from errors, holes, or the end of the file */ + for (i = 0; i < count; i++) { + /* get pointer to next read request */ + read_req_t* req = &read_reqs[i]; + + /* if we hit an error on our read, nothing else to do */ + if (req->errcode != UNIFYFS_SUCCESS) { continue; } - /* get gfid, start, and length of this request */ - int gfid = req->gfid; - size_t req_start = req->offset; - size_t req_end = req->offset + req->length; - - /* lookup local extents if we have them */ - int fid = unifyfs_fid_from_gfid(gfid); - - /* move to next request if we can't find the matching fid */ - if (fid < 0) { - /* copy current request into list of requests - * that we'll ask server for */ - memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); - server_count++; + /* if we read all of the bytes, we're done */ + if (req->nread == req->length) { continue; } - /* get pointer to extents for this file */ - unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - struct seg_tree* extents = &meta->extents; + /* otherwise, we have a short read, check whether there + * would be a hole after us, in which case we fill the + * request buffer with zeros */ - /* lock the extent tree for reading */ - seg_tree_rdlock(extents); + /* get file size for this file */ + off_t filesize_offt = unifyfs_gfid_filesize(req->gfid); + if (filesize_offt == (off_t)-1) { + /* failed to get file size */ + req->errcode = ENOENT; + continue; + } + size_t filesize = (size_t)filesize_offt; - /* identify whether we can satisfy this full request - * or not, assume we can */ - int have_local = 1; + /* get offset of where hole starts */ + size_t gap_start = req->offset + req->nread; - /* this will point to the offset of the next byte we - * need to account for */ - size_t expected_start = req_start; + /* get last offset of the read request */ + size_t req_end = req->offset + req->length; - /* iterate over extents we have for this file, - * and check that there are no holes in coverage, - * we search for a starting extent using a range - * of just the very first byte that we need */ - struct seg_tree_node* first; - first = seg_tree_find_nolock(extents, req_start, req_start); - struct seg_tree_node* next = first; - while (next != NULL && next->start < req_end) { - if (expected_start >= next->start) { - /* this extent has the next byte we expect, - * bump up to the first byte past the end - * of this extent */ - expected_start = next->end + 1; - } else { - /* there is a gap between extents so we're missing - * some bytes */ - have_local = 0; - break; + /* if file size is larger than last offset we wrote to in + * read request, then there is a hole we can fill */ + if (filesize > gap_start) { + /* assume we can fill the full request with zero */ + size_t gap_length = req_end - gap_start; + if (req_end > filesize) { + /* request is trying to read past end of file, + * so only fill zeros up to end of file */ + gap_length = filesize - gap_start; } - /* get the next element in the tree */ - next = seg_tree_iter(extents, next); - } + /* copy zeros into request buffer */ + char* req_ptr = req->buf + req->nread; + memset(req_ptr, 0, gap_length); - /* check that we account for the full request - * up until the last byte */ - if (expected_start < req_end) { - /* missing some bytes at the end of the request */ - have_local = 0; + /* update number of bytes read */ + req->nread += gap_length; } + } - /* if we can't fully satisfy the request, copy request to - * output array, so it can be passed on to server */ - if (!have_local) { - /* copy current request into list of requests - * that we'll ask server for */ - memcpy(&server_reqs[server_count], req, sizeof(read_req_t)); - server_count++; - - /* release lock before we go to next request */ - seg_tree_unlock(extents); + /* if we attempted to service requests from our local extent map, + * then we need to copy the resulting read requests from the local + * and server arrays back into the user's original array */ + if (unifyfs_local_extents) { + /* TODO: would be nice to copy these back into the same order + * in which we received them. */ - continue; + /* copy locally completed requests back into user's array */ + int local_count = in_count - count; + if (local_count > 0) { + memcpy(in_reqs, local_reqs, local_count * sizeof(read_req_t)); } - /* otherwise we can copy the data locally, iterate - * over the extents and copy data into request buffer, - * again search for a starting extent using a range - * of just the very first byte that we need */ - next = first; - while ((next != NULL) && (next->start < req_end)) { - /* get start and end of this extent (reply) */ - size_t rep_start = next->start; - size_t rep_end = next->end + 1; - - /* get the offset into the log */ - size_t rep_log_pos = next->ptr; + /* copy sever completed requests back into user's array */ + if (count > 0) { + /* skip past any items we copied in from the local requests */ + read_req_t* in_ptr = in_reqs + local_count; + memcpy(in_ptr, read_reqs, count * sizeof(read_req_t)); + } - /* start of overlapping segment is the maximum of - * reply and request start offsets */ - size_t start = rep_start; - if (req_start > start) { - start = req_start; - } + /* free storage we used for copies of requests */ + if (reqs != NULL) { + free(reqs); + reqs = NULL; + } + } - /* end of overlapping segment is the mimimum of - * reply and request end offsets */ - size_t end = rep_end; - if (req_end < end) { - end = req_end; - } + return rc; +} - /* compute length of overlapping segment */ - size_t length = end - start; +/* ======================================= + * Operations on file ids + * ======================================= */ - /* get number of bytes from start of reply and request - * buffers to the start of the overlap region */ - size_t rep_offset = start - rep_start; - size_t req_offset = start - req_start; +/* checks to see if fid is a directory + * returns 1 for yes + * returns 0 for no */ +int unifyfs_fid_is_dir(int fid) +{ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if (meta && meta->mode & S_IFDIR) { + return 1; + } else { + /* if it doesn't exist, then it's not a directory? */ + return 0; + } +} - /* if we have a gap, fill with zeros */ - size_t gap_start = req_start + req->nread; - if (start > gap_start) { - size_t gap_length = start - gap_start; - char* req_ptr = req->buf + req->nread; - memset(req_ptr, 0, gap_length); - } +int unifyfs_gfid_from_fid(const int fid) +{ + /* check that local file id is in range */ + if (fid < 0 || fid >= unifyfs_max_files) { + return -1; + } - /* copy data from local write log into request buffer */ - char* req_ptr = req->buf + req_offset; - off_t log_offset = rep_log_pos + rep_offset; - size_t nread = 0; - int rc = unifyfs_logio_read(logio_ctx, log_offset, length, - req_ptr, &nread); - if (rc == UNIFYFS_SUCCESS) { - if (nread < length) { - /* account for short read by updating end offset */ - end -= (length - nread); - } - /* update max number of bytes we have filled in the req buf */ - size_t req_nread = end - req_start; - if (req_nread > req->nread) { - req->nread = req_nread; - } - } else { - LOGERR("local log read failed for offset=%zu size=%zu", - (size_t)log_offset, length); - req->errcode = EIO; - } + /* return global file id, cached in file meta struct */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + return meta->gfid; +} - /* get the next element in the tree */ - next = seg_tree_iter(extents, next); +/* scan list of files and return fid corresponding to target gfid, + * returns -1 if not found */ +int unifyfs_fid_from_gfid(int gfid) +{ + int i; + for (i = 0; i < unifyfs_max_files; i++) { + if (unifyfs_filelist[i].in_use && + unifyfs_filemetas[i].gfid == gfid) { + /* found a file id that's in use and it matches + * the target fid, this is the one */ + return i; } + } + return -1; +} - /* copy request data to list we completed locally */ - memcpy(&local_reqs[local_count], req, sizeof(read_req_t)); - local_count++; +/* Given a fid, return the path. */ +const char* unifyfs_path_from_fid(int fid) +{ + unifyfs_filename_t* fname = &unifyfs_filelist[fid]; + if (fname->in_use) { + return fname->filename; + } + return NULL; +} + +/* checks to see if a directory is empty + * assumes that check for is_dir has already been made + * only checks for full path matches, does not check relative paths, + * e.g. ../dirname will not work + * returns 1 for yes it is empty + * returns 0 for no */ +int unifyfs_fid_is_dir_empty(const char* path) +{ + int i = 0; + while (i < unifyfs_max_files) { + /* only check this element if it's active */ + if (unifyfs_filelist[i].in_use) { + /* if the file starts with the path, it is inside of that directory + * also check that it's not the directory entry itself */ + char* strptr = strstr(path, unifyfs_filelist[i].filename); + if (strptr == unifyfs_filelist[i].filename && + strcmp(path, unifyfs_filelist[i].filename) != 0) { + /* found a child item in path */ + LOGDBG("File found: unifyfs_filelist[%d].filename = %s", + i, (char*)&unifyfs_filelist[i].filename); + return 0; + } + } - /* done reading the tree */ - seg_tree_unlock(extents); + /* go on to next file */ + i++; } - /* return to user the number of key/values we set */ - *out_count = server_count; + /* couldn't find any files with this prefix, dir must be empty */ + return 1; +} - return; +/* Return the global (laminated) size of the file */ +off_t unifyfs_fid_global_size(int fid) +{ + /* get meta data for this file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if (NULL != meta) { + return meta->global_size; + } + return (off_t)-1; } /* - * get data for a list of read requests from the - * delegator - * - * @param read_reqs: a list of read requests - * @param count: number of read requests - * @return error code - * */ -int unifyfs_fid_read_reqs(read_req_t* in_reqs, int in_count) + * Return the size of the file. If the file is laminated, return the + * laminated size. If the file is not laminated, return the local + * size. + */ +off_t unifyfs_fid_logical_size(int fid) { - int i; - int read_rc; + /* get meta data for this file */ + if (unifyfs_fid_is_laminated(fid)) { + return unifyfs_fid_global_size(fid); + } else { + /* invoke an rpc to ask the server what the file size is */ - /* assume we'll succeed */ - int rc = UNIFYFS_SUCCESS; + /* sync any writes to disk before requesting file size */ + unifyfs_fid_sync(fid); - /* assume we'll service all requests from the server */ - int count = in_count; - read_req_t* read_reqs = in_reqs; + /* get file size for this file */ + size_t filesize; + int gfid = unifyfs_gfid_from_fid(fid); + int ret = invoke_client_filesize_rpc(gfid, &filesize); + if (ret != UNIFYFS_SUCCESS) { + /* failed to get file size */ + return (off_t)-1; + } + return (off_t)filesize; + } +} - /* TODO: if the file is laminated so that we know the file size, - * we can adjust read requests to not read past the EOF */ +/* if we have a local fid structure corresponding to the gfid + * in question, we attempt the file lookup with the fid method + * otherwise call back to the rpc */ +off_t unifyfs_gfid_filesize(int gfid) +{ + off_t filesize = (off_t)-1; - /* if the option is enabled to service requests locally, try it, - * in this case we'll allocate a large array which we split into - * two, the first half will record requests we completed locally - * and the second half will store requests to be sent to the server */ + /* see if we have a fid for this gfid */ + int fid = unifyfs_fid_from_gfid(gfid); + if (fid >= 0) { + /* got a fid, look up file size through that + * method, since it may avoid a server rpc call */ + filesize = unifyfs_fid_logical_size(fid); + } else { + /* no fid for this gfid, + * look it up with server rpc */ + size_t size; + int ret = invoke_client_filesize_rpc(gfid, &size); + if (ret == UNIFYFS_SUCCESS) { + /* got the file size successfully */ + filesize = size; + } + } - /* this records the pointer to the temp request array if - * we allocate one, we should free this later if not NULL */ - read_req_t* reqs = NULL; + return filesize; +} - /* this will point to the start of the array of requests we - * complete locally */ - read_req_t* local_reqs = NULL; +/* Update local metadata for file from global metadata */ +int unifyfs_fid_update_file_meta(int fid, unifyfs_file_attr_t* gfattr) +{ + if (NULL == gfattr) { + return UNIFYFS_FAILURE; + } - /* attempt to complete requests locally if enabled */ - if (unifyfs_local_extents) { - /* allocate space to make local and server copies of the requests, - * each list will be at most in_count long */ - size_t reqs_size = 2 * in_count * sizeof(read_req_t); - reqs = (read_req_t*) malloc(reqs_size); - if (reqs == NULL) { - return ENOMEM; + /* lookup local metadata for file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if (NULL != meta) { + /* update lamination state */ + meta->is_laminated = gfattr->is_laminated; + if (meta->is_laminated) { + /* update file size */ + meta->global_size = (off_t)gfattr->size; + LOGDBG("laminated file size is %zu bytes", + (size_t)meta->global_size); } + return UNIFYFS_SUCCESS; + } + /* else, bad fid */ + return UNIFYFS_FAILURE; +} - /* define pointers to space where we can build our list - * of requests handled on the client and those left - * for the server */ - local_reqs = &reqs[0]; - read_reqs = &reqs[in_count]; +/* + * Set the metadata values for a file (after optionally creating it). + * The gfid for the file is in f_meta->gfid. + * + * gfid: The global file id on which to set metadata. + * + * create: If set to 1, attempt to create the file first. If the file + * already exists, then update its metadata with the values in + * gfattr. If set to 0, and the file does not exist, then + * the server will return an error. + * + * gfattr: The metadata values to store. + */ +int unifyfs_set_global_file_meta( + int gfid, /* file id to set meta data for */ + int create, /* whether to set size/laminated fields (1) or not (0) */ + unifyfs_file_attr_t* gfattr) /* meta data to store for file */ +{ + /* check that we have an input buffer */ + if (NULL == gfattr) { + return UNIFYFS_FAILURE; + } - /* service reads from local extent info if we can, this copies - * completed requests from in_reqs into local_reqs, and it copies - * any requests that can't be completed locally into the read_reqs - * to be processed by the server */ - service_local_reqs(in_reqs, in_count, local_reqs, read_reqs, &count); + /* force the gfid field value to match the gfid we're + * submitting this under */ + gfattr->gfid = gfid; - /* bail early if we satisfied all requests locally */ - if (count == 0) { - /* copy completed requests back into user's array */ - memcpy(in_reqs, local_reqs, in_count * sizeof(read_req_t)); + /* submit file attributes to global key/value store */ + int ret = invoke_client_metaset_rpc(create, gfattr); + return ret; +} - /* free the temporary array */ - free(reqs); - return rc; - } +int unifyfs_get_global_file_meta(int gfid, unifyfs_file_attr_t* gfattr) +{ + /* check that we have an output buffer to write to */ + if (NULL == gfattr) { + return UNIFYFS_FAILURE; } - /* TODO: When the number of read requests exceed the - * request buffer, split list io into multiple bulk - * sends and transfer in bulks */ - - /* check that we have enough slots for all read requests */ - if (count > UNIFYFS_MAX_READ_CNT) { - LOGERR("Too many requests to pass to server"); - if (reqs != NULL) { - free(reqs); - } - return ENOSPC; + /* attempt to lookup file attributes in key/value store */ + unifyfs_file_attr_t fmeta; + int ret = invoke_client_metaget_rpc(gfid, &fmeta); + if (ret == UNIFYFS_SUCCESS) { + /* found it, copy attributes to output struct */ + *gfattr = fmeta; } + return ret; +} - /* order read request by increasing file id, then increasing offset */ - qsort(read_reqs, count, sizeof(read_req_t), compare_read_req); +/* + * Set the metadata values for a file (after optionally creating it), + * using metadata associated with a given local file id. + * + * fid: The local file id on which to base global metadata values. + * + * create: If set to 1, attempt to create the file first. If the file + * already exists, then update its metadata with the values in + * gfattr. If set to 0, and the file does not exist, then + * the server will return an error. + */ +int unifyfs_set_global_file_meta_from_fid(int fid, int create) +{ + /* initialize an empty file attributes structure */ + unifyfs_file_attr_t fattr = {0}; - /* prepare our shared memory buffer for delegator */ - delegator_signal(); + /* lookup local metadata for file */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); - /* we select different rpcs depending on the number of - * read requests */ - if (count > 1) { - /* got multiple read requests, - * build up a flat buffer to include them all */ - flatcc_builder_t builder; - flatcc_builder_init(&builder); + /* copy our file name */ + const char* path = unifyfs_path_from_fid(fid); + sprintf(fattr.filename, "%s", path); - /* create request vector */ - unifyfs_Extent_vec_start(&builder); + /* set global file id */ + fattr.gfid = meta->gfid; - /* fill in values for each request entry */ - for (i = 0; i < count; i++) { - unifyfs_Extent_vec_push_create(&builder, - read_reqs[i].gfid, read_reqs[i].offset, read_reqs[i].length); - } + /* use current time for atime/mtime/ctime */ + struct timespec tp = {0}; + clock_gettime(CLOCK_REALTIME, &tp); + fattr.atime = tp; + fattr.mtime = tp; + fattr.ctime = tp; - /* complete the array */ - unifyfs_Extent_vec_ref_t extents = unifyfs_Extent_vec_end(&builder); - unifyfs_ReadRequest_create_as_root(&builder, extents); - //unifyfs_ReadRequest_end_as_root(&builder); + /* copy file mode bits and lamination flag */ + fattr.mode = meta->mode; - /* allocate our buffer to be sent */ - size_t size = 0; - void* buffer = flatcc_builder_finalize_buffer(&builder, &size); - assert(buffer); + /* these fields are set by server, except when we're creating a + * new file in which case, we should initialize them both to 0 */ + fattr.is_laminated = 0; + fattr.size = 0; + + /* capture current uid and gid */ + fattr.uid = getuid(); + fattr.gid = getgid(); - LOGDBG("mread: n_reqs:%d, flatcc buffer (%p) sz:%zu", - count, buffer, size); + /* submit file attributes to global key/value store */ + int ret = unifyfs_set_global_file_meta(meta->gfid, create, &fattr); + return ret; +} - /* invoke multi-read rpc */ - read_rc = invoke_client_mread_rpc(count, size, buffer); +/* allocate a file id slot for a new file + * return the fid or -1 on error */ +int unifyfs_fid_alloc(void) +{ + unifyfs_stack_lock(); + int fid = unifyfs_stack_pop(free_fid_stack); + unifyfs_stack_unlock(); + LOGDBG("unifyfs_stack_pop() gave %d", fid); + if (fid < 0) { + /* need to create a new file, but we can't */ + LOGERR("unifyfs_stack_pop() failed (%d)", fid); + return -1; + } + return fid; +} - /* free flat buffer resources */ - flatcc_builder_clear(&builder); - free(buffer); - } else { - /* got a single read request */ - int gfid = read_reqs[0].gfid; - size_t offset = read_reqs[0].offset; - size_t length = read_reqs[0].length; +/* return the file id back to the free pool */ +int unifyfs_fid_free(int fid) +{ + unifyfs_stack_lock(); + unifyfs_stack_push(free_fid_stack, fid); + unifyfs_stack_unlock(); + return UNIFYFS_SUCCESS; +} - LOGDBG("read: offset:%zu, len:%zu", offset, length); +/* add a new file and initialize metadata + * returns the new fid, or negative value on error */ +int unifyfs_fid_create_file(const char* path) +{ + int rc; - /* invoke single read rpc */ - read_rc = invoke_client_read_rpc(gfid, offset, length); + /* check that pathname is within bounds */ + size_t pathlen = strlen(path) + 1; + if (pathlen > UNIFYFS_MAX_FILENAME) { + return ENAMETOOLONG; } - /* bail out with error if we failed to even start the read */ - if (read_rc != UNIFYFS_SUCCESS) { - LOGERR("Failed to issue read RPC to server"); - if (reqs != NULL) { - free(reqs); - } - return read_rc; + /* allocate an id for this file */ + int fid = unifyfs_fid_alloc(); + if (fid < 0) { + /* was there an error? if so, return it */ + errno = ENOSPC; + return fid; } - /* - * ToDo: Exception handling when some of the requests - * are missed - * */ + /* mark this slot as in use */ + unifyfs_filelist[fid].in_use = 1; - /* spin waiting for read data to come back from the server, - * we process it in batches as it comes in, eventually the - * server will tell us it's sent us everything it can */ - int done = 0; - while (!done) { - int tmp_rc = delegator_wait(); - if (tmp_rc != UNIFYFS_SUCCESS) { - rc = UNIFYFS_FAILURE; - done = 1; - } else { - tmp_rc = process_read_data(read_reqs, count, &done); - if (tmp_rc != UNIFYFS_SUCCESS) { - rc = UNIFYFS_FAILURE; - } - delegator_signal(); - } - } + /* copy file name into slot */ + strcpy((void*)&unifyfs_filelist[fid].filename, path); + LOGDBG("Filename %s got unifyfs fd %d", + unifyfs_filelist[fid].filename, fid); - /* got all of the data we'll get from the server, - * check for short reads and whether those short - * reads are from errors, holes, or the end of the file */ - for (i = 0; i < count; i++) { - /* get pointer to next read request */ - read_req_t* req = &read_reqs[i]; + /* initialize meta data */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + meta->global_size = 0; + meta->flock_status = UNLOCKED; + meta->storage = FILE_STORAGE_NULL; + meta->gfid = unifyfs_generate_gfid(path); + meta->needs_sync = 0; + meta->is_laminated = 0; + meta->mode = UNIFYFS_STAT_DEFAULT_FILE_MODE; - /* if we hit an error on our read, nothing else to do */ - if (req->errcode != UNIFYFS_SUCCESS) { - continue; + if (unifyfs_flatten_writes) { + /* Initialize our segment tree that will record our writes */ + rc = seg_tree_init(&meta->extents_sync); + if (rc != 0) { + errno = rc; + fid = -1; } + } - /* if we read all of the bytes, we're done */ - if (req->nread == req->length) { - continue; + /* Initialize our segment tree to track extents for all writes + * by this process, can be used to read back local data */ + if (unifyfs_local_extents) { + rc = seg_tree_init(&meta->extents); + if (rc != 0) { + errno = rc; + fid = -1; } + } - /* otherwise, we have a short read, check whether there - * would be a hole after us, in which case we fill the - * request buffer with zeros */ + /* PTHREAD_PROCESS_SHARED allows Process-Shared Synchronization */ + pthread_spin_init(&meta->fspinlock, PTHREAD_PROCESS_SHARED); - /* get file size for this file */ - off_t filesize_offt = unifyfs_gfid_filesize(req->gfid); - if (filesize_offt == (off_t)-1) { - /* failed to get file size */ - req->errcode = ENOENT; - continue; - } - size_t filesize = (size_t)filesize_offt; + return fid; +} - /* get offset of where hole starts */ - size_t gap_start = req->offset + req->nread; +int unifyfs_fid_create_directory(const char* path) +{ + /* check that pathname is within bounds */ + size_t pathlen = strlen(path) + 1; + if (pathlen > UNIFYFS_MAX_FILENAME) { + return (int) ENAMETOOLONG; + } - /* get last offset of the read request */ - size_t req_end = req->offset + req->length; + /* get local and global file ids */ + int fid = unifyfs_get_fid_from_path(path); + int gfid = unifyfs_generate_gfid(path); - /* if file size is larger than last offset we wrote to in - * read request, then there is a hole we can fill */ - if (filesize > gap_start) { - /* assume we can fill the full request with zero */ - size_t gap_length = req_end - gap_start; - if (req_end > filesize) { - /* request is trying to read past end of file, - * so only fill zeros up to end of file */ - gap_length = filesize - gap_start; - } + /* test whether we have info for file in our local file list */ + int found_local = (fid >= 0); - /* copy zeros into request buffer */ - char* req_ptr = req->buf + req->nread; - memset(req_ptr, 0, gap_length); + /* test whether we have metadata for file in global key/value store */ + int found_global = 0; + unifyfs_file_attr_t gfattr = { 0, }; + if (unifyfs_get_global_file_meta(gfid, &gfattr) == UNIFYFS_SUCCESS) { + found_global = 1; + } - /* update number of bytes read */ - req->nread += gap_length; - } + /* can't create if it already exists */ + if (found_global) { + return (int) EEXIST; } - /* if we attempted to service requests from our local extent map, - * then we need to copy the resulting read requests from the local - * and server arrays back into the user's original array */ - if (unifyfs_local_extents) { - /* TODO: would be nice to copy these back into the same order - * in which we received them. */ + if (found_local) { + /* exists locally, but not globally + * + * FIXME: so, we have detected the cache inconsistency here. + * we cannot simply unlink or remove the entry because then we also + * need to check whether any subdirectories or files exist. + * + * this can happen when + * - a process created a directory. this process (A) has opened it at + * least once. + * - then, the directory has been deleted by another process (B). it + * deletes the global entry without checking any local used entries + * in other processes. + * + * we currently return EIO, and this needs to be addressed according to + * a consistency model this fs intance assumes. + */ + return EIO; + } - /* copy locally completed requests back into user's array */ - int local_count = in_count - count; - if (local_count > 0) { - memcpy(in_reqs, local_reqs, local_count * sizeof(read_req_t)); - } + /* now, we need to create a new directory. */ + fid = unifyfs_fid_create_file(path); + if (fid < 0) { + /* FIXME: ENOSPC or EIO? */ + return EIO; + } - /* copy sever completed requests back into user's array */ - if (count > 0) { - /* skip past any items we copied in from the local requests */ - read_req_t* in_ptr = in_reqs + local_count; - memcpy(in_ptr, read_reqs, count * sizeof(read_req_t)); - } + /* Set as directory */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + meta->mode = (meta->mode & ~S_IFREG) | S_IFDIR; - /* free storage we used for copies of requests */ - if (reqs != NULL) { - free(reqs); - reqs = NULL; - } + /* insert global meta data for directory */ + int ret = unifyfs_set_global_file_meta_from_fid(fid, 1); + if (ret != UNIFYFS_SUCCESS) { + LOGERR("Failed to populate the global meta entry for %s (fid:%d)", + path, fid); + return EIO; } - return rc; + return UNIFYFS_SUCCESS; } /* Write count bytes from buf into file starting at offset pos. @@ -1676,6 +1671,29 @@ int unifyfs_fid_truncate(int fid, off_t length) return UNIFYFS_SUCCESS; } +/* sync data for file id to server if needed */ +int unifyfs_fid_sync(int fid) +{ + /* assume we'll succeed */ + int ret = UNIFYFS_SUCCESS; + + /* sync any writes to disk */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(fid); + if (meta->needs_sync) { + /* TODO: no way to just sync data for this file, + * so we sync every file for now */ + /* sync data with server */ + ret = unifyfs_sync(); + + /* just synced writes for this file */ + if (ret == UNIFYFS_SUCCESS) { + meta->needs_sync = 0; + } + } + + return ret; +} + /* opens a new file id with specified path, access flags, and permissions, * fills outfid with file id and outpos with position for current file pointer, * returns UNIFYFS error code @@ -2971,4 +2989,3 @@ int unifyfs_transfer_file(const char* src, const char* dst, int parallel) } return local_return_val; } - diff --git a/t/sys/write-read-hole.c b/t/sys/write-read-hole.c index dc64eb0cf..9a0c06f02 100644 --- a/t/sys/write-read-hole.c +++ b/t/sys/write-read-hole.c @@ -79,7 +79,7 @@ int write_read_hole_test(char* unifyfs_root) /* Check global size on our un-laminated file */ testutil_get_size(path, &global); - ok(global == 0, "%s:%d global size is %d: %s", + ok(global == 3*bufsize, "%s:%d global size is %d: %s", __FILE__, __LINE__, global, strerror(errno)); /* flush writes */ diff --git a/t/sys/write-read.c b/t/sys/write-read.c index f1bbdf00c..d98d54c8a 100644 --- a/t/sys/write-read.c +++ b/t/sys/write-read.c @@ -66,7 +66,7 @@ int write_read_test(char* unifyfs_root) /* Check global size on our un-laminated and un-synced file */ testutil_get_size(path, &global); - ok(global == 0, "%s:%d global size before fsync is %d: %s", + ok(global == 15, "%s:%d global size before fsync is %d: %s", __FILE__, __LINE__, global, strerror(errno)); ok(fsync(fd) == 0, "%s:%d fsync() worked: %s", From c637c1d801b2b8907f97829cd899b56d5646ea30 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 20 Apr 2020 21:48:53 -0700 Subject: [PATCH 103/168] client: reset client extent trees on init If a process reattaches to the superblock created by a process from a previous run, it will use the filemeta structures defined in that block. However, these structures record pointers to segment trees that were malloc'd by the original process and the calling process cannot access that memory, which leads to a segfault on write. A more proper fix would be to record the segment trees in a way such that the second process can access the data structure, such as allocating the tree and its nodes within the superblock itself or reconstructing the trees from extent data provided by the server on mount or open. For now, this patch resets those segment trees to effectively make them empty. This will force any reads for existing data to go through the server, though the newly allocated trees will be valid for any newly written data. --- client/src/unifyfs.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 28dbcd578..4be0ba8d5 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -2065,6 +2065,44 @@ static int init_superblock_shm(size_t super_sz) /* superblock structure has been initialized, * so set flag to indicate that fact */ *(uint32_t*)addr = (uint32_t)0xDEADBEEF; + } else { + /* In this case, we have reattached to an existing superblock from + * an earlier run. We need to reset the segtree pointers to + * newly allocated segtrees, because they point to structures + * allocated in the last run whose memory addresses are no longer + * valid. */ + + /* TODO: what to do if a process calls unifyfs_init multiple times + * in a run? */ + + /* Clear any index entries from the cache. We do this to ensure + * the newly allocated seg trees are consistent with the extents + * in the index. It would be nice to call unifyfs_sync to flush + * any entries to the server, but we can't do that since that will + * try to rewrite the index using the trees, which point to invalid + * memory at this point. */ + /* initialize count of key/value entries */ + *(unifyfs_indices.ptr_num_entries) = 0; + + int i; + for (i = 0; i < unifyfs_max_files; i++) { + /* if the file entry is active, reset its segment trees */ + if (unifyfs_filelist[i].in_use) { + /* got a live file, get pointer to its metadata */ + unifyfs_filemeta_t* meta = unifyfs_get_meta_from_fid(i); + + /* Reset our segment tree that will record our writes */ + if (unifyfs_flatten_writes) { + seg_tree_init(&meta->extents_sync); + } + + /* Reset our segment tree to track extents for all writes + * by this process, can be used to read back local data */ + if (unifyfs_local_extents) { + seg_tree_init(&meta->extents); + } + } + } } /* return starting memory address of super block */ From b722ce2d9ed780294ccb0ec2aca05df040fb9ec2 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Mon, 27 Apr 2020 22:24:26 -0700 Subject: [PATCH 104/168] drop checkpoint-restart label from config output --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 46d1c84bb..73aa2eabc 100755 --- a/configure.ac +++ b/configure.ac @@ -390,7 +390,7 @@ AC_OUTPUT AC_MSG_RESULT([ ========================== - UNIFYFS Checkpoint-Restart + UNIFYFS ========================== prefix ${prefix} compiler ${CC} From 971847a49950e5e28919109d004a68d45041b856 Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Tue, 28 Apr 2020 10:38:34 -0400 Subject: [PATCH 105/168] possible uninitialized variable --- client/src/unifyfs-stdio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index 2cccf8b75..ce6627c2b 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -2632,6 +2632,7 @@ __svfscanf(unifyfs_stream_t* fp, const char* fmt0, va_list ap) char ccltab[256]; /* character class table for %[...] */ char buf[BUF]; /* buffer for numeric conversions */ + base = 0; nassigned = 0; nconversions = 0; nread = 0; From 9cacacf80b415aedf46f03d3d6cbcf07cb8de944 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 28 Apr 2020 16:06:09 -0700 Subject: [PATCH 106/168] client: drop function that counts ranks local to a node --- client/src/unifyfs-internal.h | 4 - client/src/unifyfs.c | 174 ---------------------------------- 2 files changed, 178 deletions(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index fdc70cfae..ba60a9cb8 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -320,10 +320,6 @@ extern unsigned long unifyfs_max_index_entries; /* tracks total number of unsync'd segments for all files */ extern unsigned long unifyfs_segment_count; -extern int local_rank_cnt; -extern int local_rank_idx; -extern int local_del_cnt; - /* shmem context for read-request replies data region */ extern shm_context* shm_recv_ctx; diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 4be0ba8d5..32314afc0 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -45,7 +45,6 @@ #include "unifyfs_runstate.h" #include -#include #ifdef UNIFYFS_GOTCHA #include "gotcha/gotcha_types.h" @@ -75,12 +74,8 @@ unsigned long unifyfs_max_index_entries; /* max metadata log entries */ unsigned long unifyfs_segment_count; int global_rank_cnt; /* count of world ranks */ -int local_rank_cnt; /* count of app ranks on local host */ -int local_rank_idx; /* index within local ranks */ int client_rank; /* client-provided rank (for debugging) */ -int local_del_cnt = 1; /* count of local servers */ - /* shared memory buffer to transfer read replies * from server to client */ shm_context* shm_recv_ctx; // = NULL @@ -2141,167 +2136,6 @@ static int init_recv_shm(void) return UNIFYFS_SUCCESS; } -/** - * calculate the number of ranks per node - * - * sets global variables local_rank_cnt & local_rank_idx - * - * @param numTasks: number of tasks in the application - * @return success/error code - */ -static int CountTasksPerNode(int rank, int numTasks) -{ - char hostname[UNIFYFS_MAX_HOSTNAME]; - char localhost[UNIFYFS_MAX_HOSTNAME]; - int resultsLen = UNIFYFS_MAX_HOSTNAME; - MPI_Status status; - int i, j, rc; - int* local_rank_lst = NULL; - - if (numTasks <= 0) { - LOGERR("invalid number of tasks"); - return -1; - } - - rc = MPI_Get_processor_name(localhost, &resultsLen); - if (rc != 0) { - LOGERR("failed to get the processor's name"); - } - - if (rank == 0) { - /* a container of (rank, host) mappings*/ - name_rank_pair_t* host_set = - (name_rank_pair_t*)calloc(numTasks, - sizeof(name_rank_pair_t)); - - strcpy(host_set[0].hostname, localhost); - host_set[0].rank = 0; - - /* - * MPI_Recv all hostnames, and compare to local hostname - */ - for (i = 1; i < numTasks; i++) { - rc = MPI_Recv(hostname, UNIFYFS_MAX_HOSTNAME, - MPI_CHAR, MPI_ANY_SOURCE, - MPI_ANY_TAG, MPI_COMM_WORLD, - &status); - if (rc != 0) { - LOGERR("cannot receive hostnames"); - return -1; - } - strcpy(host_set[i].hostname, hostname); - host_set[i].rank = status.MPI_SOURCE; - } - - /* sort by hostname */ - qsort(host_set, numTasks, sizeof(name_rank_pair_t), - compare_name_rank_pair); - - /* - * rank_cnt: records the number of processes on each node - * rank_set: the list of ranks for each node - */ - int** rank_set = (int**)calloc(numTasks, sizeof(int*)); - int* rank_cnt = (int*)calloc(numTasks, sizeof(int)); - int cursor = 0; - int set_counter = 0; - - for (i = 1; i < numTasks; i++) { - if (strcmp(host_set[i].hostname, - host_set[i - 1].hostname) != 0) { - // found a different host, so switch to a new set - rank_set[set_counter] = - (int*)calloc((i - cursor), sizeof(int)); - rank_cnt[set_counter] = i - cursor; - int hiter, riter = 0; - for (hiter = cursor; hiter < i; hiter++, riter++) { - rank_set[set_counter][riter] = host_set[hiter].rank; - } - - set_counter++; - cursor = i; - } - } - - /* fill rank_cnt and rank_set entry for the last node */ - rank_set[set_counter] = (int*)calloc((i - cursor), sizeof(int)); - rank_cnt[set_counter] = numTasks - cursor; - j = 0; - for (i = cursor; i < numTasks; i++, j++) { - rank_set[set_counter][j] = host_set[i].rank; - } - set_counter++; - - /* broadcast the rank_cnt and rank_set information to each rank */ - for (i = 0; i < set_counter; i++) { - /* send each rank set to all of its ranks */ - for (j = 0; j < rank_cnt[i]; j++) { - if (rank_set[i][j] != 0) { - rc = MPI_Send(&rank_cnt[i], 1, MPI_INT, rank_set[i][j], - 0, MPI_COMM_WORLD); - if (rc != 0) { - LOGERR("cannot send local rank cnt"); - return -1; - } - rc = MPI_Send(rank_set[i], rank_cnt[i], MPI_INT, - rank_set[i][j], 0, MPI_COMM_WORLD); - if (rc != 0) { - LOGERR("cannot send local rank list"); - return -1; - } - } else { - local_rank_cnt = rank_cnt[i]; - local_rank_lst = (int*)calloc(rank_cnt[i], sizeof(int)); - memcpy(local_rank_lst, rank_set[i], - (local_rank_cnt * sizeof(int))); - } - } - } - - for (i = 0; i < set_counter; i++) { - free(rank_set[i]); - } - free(rank_cnt); - free(host_set); - free(rank_set); - } else { - /* non-root process - MPI_Send hostname to root node */ - rc = MPI_Send(localhost, UNIFYFS_MAX_HOSTNAME, MPI_CHAR, - 0, 0, MPI_COMM_WORLD); - if (rc != 0) { - LOGERR("cannot send host name"); - return -1; - } - /* receive the local rank set count */ - rc = MPI_Recv(&local_rank_cnt, 1, MPI_INT, - 0, 0, MPI_COMM_WORLD, &status); - if (rc != 0) { - LOGERR("cannot receive local rank cnt"); - return -1; - } - /* receive the the local rank set */ - local_rank_lst = (int*)calloc(local_rank_cnt, sizeof(int)); - rc = MPI_Recv(local_rank_lst, local_rank_cnt, MPI_INT, - 0, 0, MPI_COMM_WORLD, &status); - if (rc != 0) { - free(local_rank_lst); - LOGERR("cannot receive local rank list"); - return -1; - } - } - - /* sort local ranks by rank */ - qsort(local_rank_lst, local_rank_cnt, sizeof(int), compare_int); - for (i = 0; i < local_rank_cnt; i++) { - if (local_rank_lst[i] == rank) { - local_rank_idx = i; - break; - } - } - free(local_rank_lst); - return 0; -} - static int unifyfs_init(void) { int rc; @@ -2637,14 +2471,6 @@ int unifyfs_mount(const char prefix[], int rank, size_t size, LOGDBG("mismatch on mount vs kvstore rank/size"); } - /* compute our local rank on the node, - * the following call initializes local_rank_{cnt,ndx} */ - rc = CountTasksPerNode(client_rank, size); - if (rc < 0) { - LOGERR("cannot get the local rank list."); - return -1; - } - /* open rpc connection to server */ rc = unifyfs_client_rpc_init(); if (rc != UNIFYFS_SUCCESS) { From b0baeb50a3e291b6648d3f7c992d1e09fd34352d Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 28 Apr 2020 15:18:17 -0700 Subject: [PATCH 107/168] client: add UNIFYFS_CLIENT_CWD for current working dir Some applications cd into a working directory before starting the job and then use relative path names for files within the run. Since one cannot cd into a UnifyFS directory, this defines a UNIFYFS_CLIENT_CWD configuration option. When set, UnifyFS will prepend this string to any relative file name to act as a current working directory when determining whether to intercept a given path. TEST_CHECKPATCH_ALLOW_FAILURE=yes --- client/src/unifyfs-internal.h | 3 +++ client/src/unifyfs.c | 34 ++++++++++++++++++++++++++++++- common/src/unifyfs_configurator.h | 1 + docs/configuration.rst | 9 ++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index ba60a9cb8..79049129b 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -355,6 +355,9 @@ extern unifyfs_filename_t* unifyfs_filelist; extern char* unifyfs_mount_prefix; extern size_t unifyfs_mount_prefixlen; +/* tracks current working directory within unifyfs directory namespace */ +extern char* unifyfs_cwd; + /* array of file descriptors */ extern unifyfs_fd_t unifyfs_fds[UNIFYFS_MAX_FILEDESCS]; extern rlim_t unifyfs_fd_limit; diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 32314afc0..4e79d54a8 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -148,6 +148,9 @@ void* unifyfs_dirstream_stack; char* unifyfs_mount_prefix; size_t unifyfs_mount_prefixlen = 0; +/* to track current working directory within unifyfs namespace */ +char* unifyfs_cwd; + /* mutex to lock stack operations */ pthread_mutex_t unifyfs_stack_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -271,6 +274,20 @@ inline int unifyfs_stack_unlock(void) return 0; } +inline void unifyfs_normalize_path(const char* path, char* normalized) +{ + /* if we have a relative path, prepend the current working directory */ + if (path[0] != '/' && unifyfs_cwd != NULL) { + /* got a relative path, add our cwd */ + snprintf(normalized, UNIFYFS_MAX_FILENAME, "%s/%s", unifyfs_cwd, path); + } else { + snprintf(normalized, UNIFYFS_MAX_FILENAME, "%s", path); + } + + /* TODO: normalize path to handle '.', '..', + * and extra or trailing '/' characters */ +} + /* sets flag if the path is a special path */ inline int unifyfs_intercept_path(const char* path) { @@ -279,8 +296,12 @@ inline int unifyfs_intercept_path(const char* path) return 0; } + /* if we have a relative path, prepend the current working directory */ + char target[UNIFYFS_MAX_FILENAME]; + unifyfs_normalize_path(path, target); + /* if the path starts with our mount point, intercept it */ - if (strncmp(path, unifyfs_mount_prefix, unifyfs_mount_prefixlen) == 0) { + if (strncmp(target, unifyfs_mount_prefix, unifyfs_mount_prefixlen) == 0) { return 1; } return 0; @@ -2185,6 +2206,12 @@ static int unifyfs_init(void) unifyfs_max_long = LONG_MAX; unifyfs_min_long = LONG_MIN; + /* set our current working directory if user gave us one */ + cfgval = client_cfg.client_cwd; + if (cfgval != NULL) { + unifyfs_cwd = strdup(cfgval); + } + /* determine max number of files to store in file system */ unifyfs_max_files = UNIFYFS_MAX_FILES; cfgval = client_cfg.client_max_files; @@ -2581,6 +2608,11 @@ int unifyfs_unmount(void) * free configuration values ************************/ + /* free global holding current working directory */ + if (unifyfs_cwd != NULL) { + free(unifyfs_cwd); + } + /* clean up configuration */ rc = unifyfs_config_fini(&client_cfg); if (rc) { diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index 07c547ff8..0523c553d 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -74,6 +74,7 @@ UNIFYFS_CFG(client, local_extents, BOOL, off, "track extents to service reads of local data", NULL) \ UNIFYFS_CFG(client, recv_data_size, INT, UNIFYFS_DATA_RECV_SIZE, "shared memory segment size in bytes for receiving data from server", NULL) \ UNIFYFS_CFG(client, write_index_size, INT, UNIFYFS_INDEX_BUF_SIZE, "write metadata index buffer size", NULL) \ + UNIFYFS_CFG(client, cwd, STRING, NULLSTRING, "current working directory", NULL) \ UNIFYFS_CFG_CLI(log, verbosity, INT, 0, "log verbosity level", NULL, 'v', "specify logging verbosity level") \ UNIFYFS_CFG_CLI(log, file, STRING, unifyfsd.log, "log file name", NULL, 'l', "specify log file name") \ UNIFYFS_CFG_CLI(log, dir, STRING, LOGDIR, "log file directory", configurator_directory_check, 'L', "specify full path to directory to contain log file") \ diff --git a/docs/configuration.rst b/docs/configuration.rst index 0f5104afc..d8d4f1e29 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -66,6 +66,7 @@ a given section and key. ================ ====== ================================================================= Key Type Description ================ ====== ================================================================= + cwd STRING effective starting current working directory max_files INT maximum number of open files per client process (default: 128) flatten_writes BOOL enable flattening writes (optimization for overwrite-heavy codes) local_extents BOOL service reads from local data if possible (default: off) @@ -73,6 +74,14 @@ a given section and key. write_index_size INT maximum size (B) of memory buffer for storing write log metadata ================ ====== ================================================================= +The ``cwd`` setting is used to emulate the behavior one +expects when changing into a working directory before starting a job +and then using relative file names within the application. +If set, the value specified in ``cwd`` is prepended to any +relative path name when determining whether UnifyFS will intercept +a path. The value specified in ``cwd`` must match the UnifyFS mount point. +It does not modify the job's current working directory. + Enabling the ``local_extents`` optimization may significantly improve read performance. However, it should not be used by applications in which different processes write to a given byte offset within From 14fe5d5f35389d87974a725ae132b907901dc04b Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 5 May 2020 10:48:19 -0700 Subject: [PATCH 108/168] check that cwd is in mount point --- client/src/unifyfs.c | 9 ++++++++- docs/configuration.rst | 5 +++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 4e79d54a8..467df1c61 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -274,7 +274,7 @@ inline int unifyfs_stack_unlock(void) return 0; } -inline void unifyfs_normalize_path(const char* path, char* normalized) +static void unifyfs_normalize_path(const char* path, char* normalized) { /* if we have a relative path, prepend the current working directory */ if (path[0] != '/' && unifyfs_cwd != NULL) { @@ -2210,6 +2210,13 @@ static int unifyfs_init(void) cfgval = client_cfg.client_cwd; if (cfgval != NULL) { unifyfs_cwd = strdup(cfgval); + + /* check that cwd falls somewhere under the mount point */ + if (strncmp(unifyfs_cwd, unifyfs_mount_prefix, + unifyfs_mount_prefixlen) != 0) { + LOGERR("UNIFYFS_CLIENT_CWD '%s' must be within the mount '%s'", + unifyfs_cwd, unifyfs_mount_prefix); + } } /* determine max number of files to store in file system */ diff --git a/docs/configuration.rst b/docs/configuration.rst index d8d4f1e29..b11293c8d 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -79,8 +79,9 @@ expects when changing into a working directory before starting a job and then using relative file names within the application. If set, the value specified in ``cwd`` is prepended to any relative path name when determining whether UnifyFS will intercept -a path. The value specified in ``cwd`` must match the UnifyFS mount point. -It does not modify the job's current working directory. +a path. The value specified in ``cwd`` must be within the directory space +of the UnifyFS mount point. +Setting ``cwd`` does not modify the job's actual current working directory. Enabling the ``local_extents`` optimization may significantly improve read performance. However, it should not be used by applications From c158d2a76ee47f203a6657377a8ee24c40e50a72 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Tue, 5 May 2020 11:14:12 -0700 Subject: [PATCH 109/168] client: fix for potential trailing chars in intercept_path check --- client/src/unifyfs.c | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 467df1c61..7ef4c1116 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -301,10 +301,19 @@ inline int unifyfs_intercept_path(const char* path) unifyfs_normalize_path(path, target); /* if the path starts with our mount point, intercept it */ + int intercept = 0; if (strncmp(target, unifyfs_mount_prefix, unifyfs_mount_prefixlen) == 0) { - return 1; + /* characters in target up through mount point match, + * assume we match */ + intercept = 1; + + /* if we have another character, it must be '/' */ + if (strlen(target) > unifyfs_mount_prefixlen && + target[unifyfs_mount_prefixlen] != '/') { + intercept = 0; + } } - return 0; + return intercept; } /* given an fd, return 1 if we should intercept this file, 0 otherwise, @@ -2212,10 +2221,27 @@ static int unifyfs_init(void) unifyfs_cwd = strdup(cfgval); /* check that cwd falls somewhere under the mount point */ + int cwd_within_mount = 0; if (strncmp(unifyfs_cwd, unifyfs_mount_prefix, - unifyfs_mount_prefixlen) != 0) { + unifyfs_mount_prefixlen) == 0) { + /* characters in target up through mount point match, + * assume we match */ + cwd_within_mount = 1; + + /* if we have another character, it must be '/' */ + if (strlen(unifyfs_cwd) > unifyfs_mount_prefixlen && + unifyfs_cwd[unifyfs_mount_prefixlen] != '/') { + cwd_within_mount = 0; + } + } + if (!cwd_within_mount) { + /* path given in CWD is outside of the UnifyFS mount point */ LOGERR("UNIFYFS_CLIENT_CWD '%s' must be within the mount '%s'", unifyfs_cwd, unifyfs_mount_prefix); + + /* ignore setting and set back to NULL */ + free(unifyfs_cwd); + unifyfs_cwd = NULL; } } From 46c9de68537fec10081b6c54d9c9892dc1058b10 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 6 May 2020 17:52:43 -0700 Subject: [PATCH 110/168] return the normalized path from intercept_path --- client/src/unifyfs-dirops.c | 12 ++-- client/src/unifyfs-internal.h | 6 +- client/src/unifyfs-stdio.c | 5 +- client/src/unifyfs-sysio.c | 122 +++++++++++++++++++--------------- client/src/unifyfs.c | 14 +++- 5 files changed, 95 insertions(+), 64 deletions(-) diff --git a/client/src/unifyfs-dirops.c b/client/src/unifyfs-dirops.c index ba443cb81..5d9a37f47 100644 --- a/client/src/unifyfs-dirops.c +++ b/client/src/unifyfs-dirops.c @@ -89,7 +89,8 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) { /* call real opendir and return early if this is * not one of our paths */ - if (!unifyfs_intercept_path(name)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (!unifyfs_intercept_path(name, upath)) { MAP_OR_FAIL(opendir); return UNIFYFS_REAL(opendir)(name); } @@ -99,8 +100,8 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) * if valid, populate the local file meta cache accordingly. */ - int fid = unifyfs_get_fid_from_path(name); - int gfid = unifyfs_generate_gfid(name); + int fid = unifyfs_get_fid_from_path(upath); + int gfid = unifyfs_generate_gfid(upath); unifyfs_file_attr_t gfattr = { 0, }; int ret = unifyfs_get_global_file_meta(gfid, &gfattr); @@ -132,7 +133,7 @@ DIR* UNIFYFS_WRAP(opendir)(const char* name) return NULL; } } else { - fid = unifyfs_fid_create_file(name); + fid = unifyfs_fid_create_file(upath); if (fid < 0) { errno = EIO; return NULL; @@ -235,7 +236,8 @@ int UNIFYFS_WRAP(scandir)(const char* path, struct dirent** namelist, int (*compar)(const struct dirent**, const struct dirent**)) { - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); errno = ENOSYS; diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 79049129b..6b472332e 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -407,8 +407,10 @@ int unifyfs_stack_lock(void); int unifyfs_stack_unlock(void); -/* sets flag if the path is a special path */ -int unifyfs_intercept_path(const char* path); +/* sets flag if the path should be intercept as a unifyfs path, + * and if so, writes normalized path in upath, which should + * be a buffer of size UNIFYFS_MAX_FILENAME */ +int unifyfs_intercept_path(const char* path, char* upath); /* given an fd, return 1 if we should intercept this file, 0 otherwise, * convert fd to new fd value if needed */ diff --git a/client/src/unifyfs-stdio.c b/client/src/unifyfs-stdio.c index ce6627c2b..d1fc63208 100644 --- a/client/src/unifyfs-stdio.c +++ b/client/src/unifyfs-stdio.c @@ -980,9 +980,10 @@ static int unifyfs_fseek(FILE* stream, off_t offset, int whence) FILE* UNIFYFS_WRAP(fopen)(const char* path, const char* mode) { /* check whether we should intercept this path */ - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { FILE* stream; - int rc = unifyfs_fopen(path, mode, &stream); + int rc = unifyfs_fopen(upath, mode, &stream); if (rc != UNIFYFS_SUCCESS) { errno = unifyfs_rc_errno(rc); return NULL; diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 3157772b8..945308ba7 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -51,17 +51,18 @@ int UNIFYFS_WRAP(access)(const char* path, int mode) { /* determine whether we should intercept this path */ - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* check if path exists */ - if (unifyfs_get_fid_from_path(path) < 0) { + if (unifyfs_get_fid_from_path(upath) < 0) { LOGDBG("access: unifyfs_get_id_from path failed, returning -1, %s", - path); + upath); errno = ENOENT; return -1; } /* currently a no-op */ - LOGDBG("access: path intercepted, returning 0, %s", path); + LOGDBG("access: path intercepted, returning 0, %s", upath); return 0; } else { LOGDBG("access: calling MAP_OR_FAIL, %s", path); @@ -80,15 +81,16 @@ int UNIFYFS_WRAP(mkdir)(const char* path, mode_t mode) * It doesn't check to see if parent directory exists */ /* determine whether we should intercept this path */ - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* check if it already exists */ - if (unifyfs_get_fid_from_path(path) >= 0) { + if (unifyfs_get_fid_from_path(upath) >= 0) { errno = EEXIST; return -1; } /* add directory to file list */ - int ret = unifyfs_fid_create_directory(path); + int ret = unifyfs_fid_create_directory(upath); if (ret != UNIFYFS_SUCCESS) { /* failed to create the directory, * set errno and return */ @@ -108,15 +110,16 @@ int UNIFYFS_WRAP(mkdir)(const char* path, mode_t mode) int UNIFYFS_WRAP(rmdir)(const char* path) { /* determine whether we should intercept this path */ - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* check if the mount point itself is being deleted */ - if (!strcmp(path, unifyfs_mount_prefix)) { + if (!strcmp(upath, unifyfs_mount_prefix)) { errno = EBUSY; return -1; } /* check if path exists */ - int fid = unifyfs_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(upath); if (fid < 0) { errno = ENOENT; return -1; @@ -129,7 +132,7 @@ int UNIFYFS_WRAP(rmdir)(const char* path) } /* is it empty? */ - if (!unifyfs_fid_is_dir_empty(path)) { + if (!unifyfs_fid_is_dir_empty(upath)) { errno = ENOTEMPTY; return -1; } @@ -158,26 +161,28 @@ int UNIFYFS_WRAP(rename)(const char* oldpath, const char* newpath) * linux fs, which means we'll need to do a read / write */ /* check whether the old path is in our file system */ - if (unifyfs_intercept_path(oldpath)) { + char old_upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(oldpath, old_upath)) { /* for now, we can only rename within our file system */ - if (!unifyfs_intercept_path(newpath)) { + char new_upath[UNIFYFS_MAX_FILENAME]; + if (!unifyfs_intercept_path(newpath, new_upath)) { /* ERROR: can't yet rename across file systems */ errno = EXDEV; return -1; } /* verify that we really have a file by the old name */ - int fid = unifyfs_get_fid_from_path(oldpath); + int fid = unifyfs_get_fid_from_path(old_upath); if (fid < 0) { /* ERROR: oldname does not exist */ - LOGDBG("Couldn't find entry for %s in UNIFYFS", oldpath); + LOGDBG("Couldn't find entry for %s in UNIFYFS", old_upath); errno = ENOENT; return -1; } LOGDBG("orig file in position %d", fid); /* check that new name is within bounds */ - size_t newpathlen = strlen(newpath) + 1; + size_t newpathlen = strlen(new_upath) + 1; if (newpathlen > UNIFYFS_MAX_FILENAME) { errno = ENAMETOOLONG; return -1; @@ -186,7 +191,7 @@ int UNIFYFS_WRAP(rename)(const char* oldpath, const char* newpath) /* TODO: rename should replace existing file atomically */ /* verify that we don't already have a file by the new name */ - int newfid = unifyfs_get_fid_from_path(newpath); + int newfid = unifyfs_get_fid_from_path(new_upath); if (newfid >= 0) { /* something exists in newpath, need to delete it */ int ret = UNIFYFS_WRAP(unlink)(newpath); @@ -199,14 +204,15 @@ int UNIFYFS_WRAP(rename)(const char* oldpath, const char* newpath) /* finally overwrite the old name with the new name */ LOGDBG("Changing %s to %s", - (char*)&unifyfs_filelist[fid].filename, newpath); - strcpy((void*)&unifyfs_filelist[fid].filename, newpath); + (char*)&unifyfs_filelist[fid].filename, new_upath); + strcpy((void*)&unifyfs_filelist[fid].filename, new_upath); /* success */ return 0; } else { /* for now, we can only rename within our file system */ - if (unifyfs_intercept_path(newpath)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(newpath, upath)) { /* ERROR: can't yet rename across file systems */ errno = EXDEV; return -1; @@ -222,9 +228,10 @@ int UNIFYFS_WRAP(rename)(const char* oldpath, const char* newpath) int UNIFYFS_WRAP(truncate)(const char* path, off_t length) { /* determine whether we should intercept this path or not */ - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* get file id for path name */ - int fid = unifyfs_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(upath); if (fid >= 0) { /* before we truncate, sync any data cached this file id */ int ret = unifyfs_fid_sync(fid); @@ -242,10 +249,10 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) } } else { /* invoke truncate rpc */ - int gfid = unifyfs_generate_gfid(path); + int gfid = unifyfs_generate_gfid(upath); int rc = invoke_client_truncate_rpc(gfid, length); if (rc != UNIFYFS_SUCCESS) { - LOGDBG("truncate rpc failed %s in UNIFYFS", path); + LOGDBG("truncate rpc failed %s in UNIFYFS", upath); errno = EIO; return -1; } @@ -263,12 +270,13 @@ int UNIFYFS_WRAP(truncate)(const char* path, off_t length) int UNIFYFS_WRAP(unlink)(const char* path) { /* determine whether we should intercept this path or not */ - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* get file id for path name */ - int fid = unifyfs_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(upath); if (fid < 0) { /* ERROR: file does not exist */ - LOGDBG("Couldn't find entry for %s in UNIFYFS", path); + LOGDBG("Couldn't find entry for %s in UNIFYFS", upath); errno = ENOENT; return -1; } @@ -276,7 +284,7 @@ int UNIFYFS_WRAP(unlink)(const char* path) /* check that it's not a directory */ if (unifyfs_fid_is_dir(fid)) { /* ERROR: is a directory */ - LOGDBG("Attempting to unlink a directory %s in UNIFYFS", path); + LOGDBG("Attempting to unlink a directory %s in UNIFYFS", upath); errno = EISDIR; return -1; } @@ -300,12 +308,13 @@ int UNIFYFS_WRAP(unlink)(const char* path) int UNIFYFS_WRAP(remove)(const char* path) { /* determine whether we should intercept this path or not */ - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* get file id for path name */ - int fid = unifyfs_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(upath); if (fid < 0) { /* ERROR: file does not exist */ - LOGDBG("Couldn't find entry for %s in UNIFYFS", path); + LOGDBG("Couldn't find entry for %s in UNIFYFS", upath); errno = ENOENT; return -1; } @@ -314,7 +323,7 @@ int UNIFYFS_WRAP(remove)(const char* path) if (unifyfs_fid_is_dir(fid)) { /* TODO: shall be equivalent to rmdir(path) */ /* ERROR: is a directory */ - LOGDBG("Attempting to remove a directory %s in UNIFYFS", path); + LOGDBG("Attempting to remove a directory %s in UNIFYFS", upath); errno = EISDIR; return -1; } @@ -415,8 +424,9 @@ static int __stat(const char* path, struct stat* buf) int UNIFYFS_WRAP(stat)(const char* path, struct stat* buf) { LOGDBG("stat was called for %s", path); - if (unifyfs_intercept_path(path)) { - int ret = __stat(path, buf); + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { + int ret = __stat(upath, buf); return ret; } else { MAP_OR_FAIL(stat); @@ -456,12 +466,13 @@ int UNIFYFS_WRAP(__xstat)(int vers, const char* path, struct stat* buf) { LOGDBG("xstat was called for %s", path); - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { if (vers != _STAT_VER) { errno = EINVAL; return -1; } - int ret = __stat(path, buf); + int ret = __stat(upath, buf); return ret; } else { MAP_OR_FAIL(__xstat); @@ -476,12 +487,13 @@ int UNIFYFS_WRAP(__lxstat)(int vers, const char* path, struct stat* buf) { LOGDBG("lxstat was called for %s", path); - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { if (vers != _STAT_VER) { errno = EINVAL; return -1; } - int ret = __stat(path, buf); + int ret = __stat(upath, buf); return ret; } else { MAP_OR_FAIL(__lxstat); @@ -635,13 +647,14 @@ int UNIFYFS_WRAP(creat)(const char* path, mode_t mode) /* equivalent to open(path, O_WRONLY|O_CREAT|O_TRUNC, mode) */ /* check whether we should intercept this path */ - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* TODO: handle relative paths using current working directory */ /* create the file */ int fid; off_t pos; - int rc = unifyfs_fid_open(path, O_WRONLY | O_CREAT | O_TRUNC, mode, &fid, &pos); + int rc = unifyfs_fid_open(upath, O_WRONLY | O_CREAT | O_TRUNC, mode, &fid, &pos); if (rc != UNIFYFS_SUCCESS) { errno = unifyfs_rc_errno(rc); return -1; @@ -661,7 +674,7 @@ int UNIFYFS_WRAP(creat)(const char* path, mode_t mode) filedesc->pos = pos; filedesc->read = 0; filedesc->write = 1; - LOGDBG("UNIFYFS_open generated fd %d for file %s", fd, path); + LOGDBG("UNIFYFS_open generated fd %d for file %s", fd, upath); /* don't conflict with active system fds that range from 0 - (fd_limit) */ int ret = fd + unifyfs_fd_limit; @@ -676,7 +689,8 @@ int UNIFYFS_WRAP(creat)(const char* path, mode_t mode) int UNIFYFS_WRAP(creat64)(const char* path, mode_t mode) { /* check whether we should intercept this path */ - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* ERROR: fn not yet supported */ fprintf(stderr, "Function not yet supported @ %s:%d\n", __FILE__, __LINE__); @@ -702,13 +716,14 @@ int UNIFYFS_WRAP(open)(const char* path, int flags, ...) /* determine whether we should intercept this path */ int ret; - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* TODO: handle relative paths using current working directory */ /* create the file */ int fid; off_t pos; - int rc = unifyfs_fid_open(path, flags, mode, &fid, &pos); + int rc = unifyfs_fid_open(upath, flags, mode, &fid, &pos); if (rc != UNIFYFS_SUCCESS) { errno = unifyfs_rc_errno(rc); return -1; @@ -731,7 +746,7 @@ int UNIFYFS_WRAP(open)(const char* path, int flags, ...) filedesc->write = ((flags & O_WRONLY) == O_WRONLY) || ((flags & O_RDWR) == O_RDWR); filedesc->append = ((flags & O_APPEND)); - LOGDBG("UNIFYFS_open generated fd %d for file %s", fd, path); + LOGDBG("UNIFYFS_open generated fd %d for file %s", fd, upath); /* don't conflict with active system fds that range from 0 - (fd_limit) */ ret = fd + unifyfs_fd_limit; @@ -761,7 +776,8 @@ int UNIFYFS_WRAP(open64)(const char* path, int flags, ...) /* check whether we should intercept this path */ int ret; - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* Call open wrapper with LARGEFILE flag set*/ if (flags & O_CREAT) { ret = UNIFYFS_WRAP(open)(path, flags | O_LARGEFILE, mode); @@ -797,8 +813,9 @@ int UNIFYFS_WRAP(__open_2)(const char* path, int flags, ...) } /* check whether we should intercept this path */ - if (unifyfs_intercept_path(path)) { - LOGDBG("__open_2 was intercepted for path %s", path); + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { + LOGDBG("__open_2 was intercepted for path %s", upath); /* Call open wrapper */ if (flags & O_CREAT) { @@ -1751,17 +1768,18 @@ int UNIFYFS_WRAP(fchmod)(int fd, mode_t mode) int UNIFYFS_WRAP(chmod)(const char* path, mode_t mode) { /* determine whether we should intercept this path */ - if (unifyfs_intercept_path(path)) { + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { /* check if path exists */ - int fid = unifyfs_get_fid_from_path(path); + int fid = unifyfs_get_fid_from_path(upath); if (fid < 0) { LOGDBG("chmod: unifyfs_get_id_from path failed, returning -1, %s", - path); + upath); errno = ENOENT; return -1; } - LOGDBG("chmod: setting %s to %o", path, mode); + LOGDBG("chmod: setting %s to %o", upath, mode); return __chmod(fid, mode); } else { MAP_OR_FAIL(chmod); diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 7ef4c1116..62e3a78ce 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -289,7 +289,7 @@ static void unifyfs_normalize_path(const char* path, char* normalized) } /* sets flag if the path is a special path */ -inline int unifyfs_intercept_path(const char* path) +inline int unifyfs_intercept_path(const char* path, char* upath) { /* don't intecept anything until we're initialized */ if (!unifyfs_initialized) { @@ -313,6 +313,12 @@ inline int unifyfs_intercept_path(const char* path) intercept = 0; } } + + /* copy normalized path into upath */ + if (intercept) { + strncpy(upath, target, UNIFYFS_MAX_FILENAME); + } + return intercept; } @@ -2864,7 +2870,8 @@ int unifyfs_transfer_file(const char* src, const char* dst, int parallel) return -ENOMEM; } - if (unifyfs_intercept_path(src)) { + char src_upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(src, src_upath)) { dir = UNIFYFS_TX_STAGE_OUT; unify_src = 1; } @@ -2876,7 +2883,8 @@ int unifyfs_transfer_file(const char* src, const char* dst, int parallel) pos += sprintf(pos, "%s", dst); - if (unifyfs_intercept_path(dst)) { + char dst_upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(dst, dst_upath)) { dir = UNIFYFS_TX_STAGE_IN; unify_dst = 1; } From dbb70a179a1a60c8ceb55d91bd1d2a069b803f7f Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Thu, 7 May 2020 13:09:56 -0700 Subject: [PATCH 111/168] util: fix overrun in parsing LSF host list --- util/unifyfs/src/unifyfs-rm.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/util/unifyfs/src/unifyfs-rm.c b/util/unifyfs/src/unifyfs-rm.c index da804ed38..850f84ec2 100644 --- a/util/unifyfs/src/unifyfs-rm.c +++ b/util/unifyfs/src/unifyfs-rm.c @@ -218,6 +218,12 @@ static int lsf_read_resource(unifyfs_resource_t* resource) lsb_hosts = strdup(val); + // get length of host string + size_t hosts_len = strlen(lsb_hosts) + 1; + + // pointer to character just past terminating NULL + char* hosts_end = lsb_hosts + hosts_len; + // replace spaces with zeroes for (pos = lsb_hosts; *pos; pos++) { if (isspace(*pos)) { @@ -232,7 +238,7 @@ static int lsf_read_resource(unifyfs_resource_t* resource) } else { pos += (strlen(pos) + 1); // skip launch node slot count } - for (n_nodes = 0; *pos;) { + for (n_nodes = 0; pos < hosts_end;) { node = pos; if (!mcpu) { if (strcmp(last_node, node) != 0) { @@ -259,7 +265,7 @@ static int lsf_read_resource(unifyfs_resource_t* resource) } else { pos += (strlen(pos) + 1); // skip launch node slot count } - for (i = 0; *pos && i < n_nodes;) { + for (i = 0; pos < hosts_end && i < n_nodes;) { node = pos; if (!mcpu) { if (strcmp(last_node, node) != 0) { From 4387f1f80d5feef1cdb688fbcf08fb35fd462795 Mon Sep 17 00:00:00 2001 From: CamStan Date: Wed, 13 May 2020 10:23:30 -0700 Subject: [PATCH 112/168] Remove Spack bootstrap command from CI build The spack bootstrap command is no longer a valid command. This removes the command from our Travis CI build. Spack is now set up to make `spack load` work without using modules. As a result, the `spack module` command is no longer recognized. This removes that command and replaces it with the individual `spack load` commands. --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 271c42023..f07bc6820 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,14 +52,12 @@ before_install: EOF install: - - $HOME/spack/bin/spack bootstrap - . $HOME/spack/share/spack/setup-env.sh - - spack install leveldb - - spack install gotcha@0.0.2 - - spack install flatcc - - spack install margo^mercury+bmi~boostsys + - spack install leveldb && spack load leveldb + - spack install gotcha@0.0.2 && spack load gotcha@0.0.2 + - spack install flatcc && spack load flatcc + - spack install margo^mercury+bmi~boostsys && spack load argobots && spack load mercury && spack load margo # prepare build environment - - source <(spack module tcl loads leveldb gotcha@0.0.2 flatcc mercury argobots margo) - eval $(./scripts/git_log_test_env.sh) - export TEST_CHECKPATCH_SKIP_FILES From 2a3474d8b85d8b105743f33e521deab04ad097f0 Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Tue, 12 May 2020 14:05:57 -0400 Subject: [PATCH 113/168] unifyfs utility to wait until all servers become ready. - unifyfsd publishes a server pid file (unifyfsd.pids) under the shared state state directory, indicating that all servers are ready - unifyfs utility blocks until the pid file appears - update the document about new server config option (init_timeout) - checkpatch generates 2 warnings on function prototype and long line TEST_CHECKPATCH_ALLOW_FAILURE=yes --- common/src/unifyfs_configurator.h | 1 + common/src/unifyfs_const.h | 3 + common/src/unifyfs_server_rpcs.h | 10 ++ docs/configuration.rst | 11 +- server/src/Makefile.am | 3 +- server/src/margo_server.c | 5 + server/src/margo_server.h | 1 + server/src/unifyfs_global.h | 2 +- server/src/unifyfs_server.c | 18 +++ server/src/unifyfs_server_pid.c | 240 ++++++++++++++++++++++++++++++ util/unifyfs/src/unifyfs-rm.c | 116 ++++++++++++++- util/unifyfs/src/unifyfs.c | 10 +- util/unifyfs/src/unifyfs.h | 1 + 13 files changed, 409 insertions(+), 12 deletions(-) create mode 100644 server/src/unifyfs_server_pid.c diff --git a/common/src/unifyfs_configurator.h b/common/src/unifyfs_configurator.h index 0523c553d..27787706b 100644 --- a/common/src/unifyfs_configurator.h +++ b/common/src/unifyfs_configurator.h @@ -90,6 +90,7 @@ UNIFYFS_CFG_CLI(runstate, dir, STRING, RUNDIR, "runstate file directory", configurator_directory_check, 'R', "specify full path to directory to contain server runstate file") \ UNIFYFS_CFG_CLI(server, hostfile, STRING, NULLSTRING, "server hostfile name", NULL, 'H', "specify full path to server hostfile") \ UNIFYFS_CFG_CLI(sharedfs, dir, STRING, NULLSTRING, "shared file system directory", configurator_directory_check, 'S', "specify full path to directory to contain server shared files") \ + UNIFYFS_CFG_CLI(server, init_timeout, INT, UNIFYFS_DEFAULT_INIT_TIMEOUT, "timeout of waiting for server initialization", NULL, 't', "timeout in seconds to wait for servers to be ready for clients") \ #ifdef __cplusplus diff --git a/common/src/unifyfs_const.h b/common/src/unifyfs_const.h index f4ad09401..d3890ba82 100644 --- a/common/src/unifyfs_const.h +++ b/common/src/unifyfs_const.h @@ -65,6 +65,9 @@ // Server - General #define MAX_NUM_APPS 64 /* max # apps supported by a single server */ #define MAX_APP_CLIENTS 64 /* app processes per server */ +/* timeout (in seconds) of waiting for initialization of all servers */ +#define UNIFYFS_DEFAULT_INIT_TIMEOUT 120 +#define UNIFYFSD_PID_FILENAME "unifyfsd.pids" // Client #define UNIFYFS_MAX_FILES 128 diff --git a/common/src/unifyfs_server_rpcs.h b/common/src/unifyfs_server_rpcs.h index fd86f674d..10b3244e9 100644 --- a/common/src/unifyfs_server_rpcs.h +++ b/common/src/unifyfs_server_rpcs.h @@ -24,6 +24,16 @@ MERCURY_GEN_PROC(server_hello_out_t, ((int32_t)(ret))) DECLARE_MARGO_RPC_HANDLER(server_hello_rpc) +/* server_pid_rpc (server => server, n:1) + * + * notify readiness with pid to the master server (rank:0) */ +MERCURY_GEN_PROC(server_pid_in_t, + ((int32_t)(rank)) + ((int32_t)(pid))) +MERCURY_GEN_PROC(server_pid_out_t, + ((int32_t)(ret))) +DECLARE_MARGO_RPC_HANDLER(server_pid_handle_rpc) + /* server_request_rpc (server => server) * * request from one server to another */ diff --git a/docs/configuration.rst b/docs/configuration.rst index b11293c8d..07e2cc414 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -136,11 +136,12 @@ files. .. table:: ``[server]`` section - server settings :widths: auto - ========== ====== ======================== - Key Type Description - ========== ====== ======================== - hostfile STRING path to server hostfile - ========== ====== ======================== + ============ ====== ============================================================================= + Key Type Description + ============ ====== ============================================================================= + hostfile STRING path to server hostfile + init_timeout INT timeout in seconds to wait for servers to be ready for clients (default: 120) + ============ ====== ============================================================================= .. table:: ``[sharedfs]`` section - server shared files settings :widths: auto diff --git a/server/src/Makefile.am b/server/src/Makefile.am index d3a113a91..ca8217b3d 100644 --- a/server/src/Makefile.am +++ b/server/src/Makefile.am @@ -12,7 +12,8 @@ libunifyfsd_a_SOURCES = \ unifyfs_request_manager.c \ unifyfs_request_manager.h \ unifyfs_service_manager.c \ - unifyfs_service_manager.h + unifyfs_service_manager.h \ + unifyfs_server_pid.c bin_PROGRAMS = unifyfsd diff --git a/server/src/margo_server.c b/server/src/margo_server.c index 79325b1c6..ead8a20f5 100644 --- a/server/src/margo_server.c +++ b/server/src/margo_server.c @@ -87,6 +87,11 @@ static void register_server_server_rpcs(margo_instance_id mid) server_hello_in_t, server_hello_out_t, server_hello_rpc); + unifyfsd_rpc_context->rpcs.server_pid_id = + MARGO_REGISTER(mid, "server_pid_rpc", + server_pid_in_t, server_pid_out_t, + server_pid_handle_rpc); + unifyfsd_rpc_context->rpcs.request_id = MARGO_REGISTER(mid, "server_request_rpc", server_request_in_t, server_request_out_t, diff --git a/server/src/margo_server.h b/server/src/margo_server.h index 2a9f1a298..7e3ad7c81 100644 --- a/server/src/margo_server.h +++ b/server/src/margo_server.h @@ -17,6 +17,7 @@ typedef struct ServerRpcIds { hg_id_t hello_id; + hg_id_t server_pid_id; hg_id_t request_id; hg_id_t chunk_read_request_id; hg_id_t chunk_read_response_id; diff --git a/server/src/unifyfs_global.h b/server/src/unifyfs_global.h index f75df1972..db5157336 100644 --- a/server/src/unifyfs_global.h +++ b/server/src/unifyfs_global.h @@ -64,6 +64,7 @@ /* PMI server rank and server count */ extern int glb_pmi_rank; extern int glb_pmi_size; +extern int server_pid; /* hostname for this server */ extern char glb_host[UNIFYFS_MAX_HOSTNAME]; @@ -197,5 +198,4 @@ unifyfs_rc disconnect_app_client(app_client* clnt); unifyfs_rc cleanup_app_client(app_client* clnt); - #endif // UNIFYFS_GLOBAL_H diff --git a/server/src/unifyfs_server.c b/server/src/unifyfs_server.c index 10412867e..049507435 100644 --- a/server/src/unifyfs_server.c +++ b/server/src/unifyfs_server.c @@ -48,6 +48,7 @@ /* PMI information */ int glb_pmi_rank; /* = 0 */ int glb_pmi_size = 1; // for standalone server tests +int server_pid; char glb_host[UNIFYFS_MAX_HOSTNAME]; size_t glb_host_ndx; // index of localhost in glb_servers @@ -59,6 +60,14 @@ unifyfs_cfg_t server_cfg; app_config* app_configs[MAX_NUM_APPS]; /* list of apps */ +/** + * @brief create a ready status file to notify that all servers are ready for + * accepting client requests. + * + * @return 0 on success, error otherwise + */ +int unifyfs_publish_server_pids(void); + static int unifyfs_exit(void); #if defined(UNIFYFS_MULTIPLE_DELEGATORS) @@ -265,6 +274,8 @@ int main(int argc, char* argv[]) daemonize(); } + server_pid = getpid(); + /* unifyfs default log level is LOG_ERR */ if (server_cfg.log_verbosity != NULL) { long l; @@ -371,6 +382,13 @@ int main(int argc, char* argv[]) LOGDBG("finished service initialization"); + rc = unifyfs_publish_server_pids(); + if (rc != 0) { + LOGERR("failed to publish server pid file: %s", + unifyfs_rc_enum_description(rc)); + exit(1); + } + while (1) { sleep(1); if (time_to_exit) { diff --git a/server/src/unifyfs_server_pid.c b/server/src/unifyfs_server_pid.c new file mode 100644 index 000000000..5a0cfa60c --- /dev/null +++ b/server/src/unifyfs_server_pid.c @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ +#include +#include +#include +#include +#include + +#include "unifyfs_configurator.h" +#include "unifyfs_global.h" +#include "margo_server.h" +#include "unifyfs_server_rpcs.h" + +extern unifyfs_cfg_t server_cfg; + +static pthread_cond_t server_pid_cond = PTHREAD_COND_INITIALIZER; +static pthread_mutex_t server_pid_mutex = PTHREAD_MUTEX_INITIALIZER; +static struct timespec server_pid_timeout; + +static int* server_pids; +static pthread_mutex_t server_pid_alloc_lock = PTHREAD_MUTEX_INITIALIZER; + +static int alloc_server_pid(void) +{ + int ret = 0; + + pthread_mutex_lock(&server_pid_alloc_lock); + { + if (!server_pids) { + server_pids = calloc(glb_pmi_size, sizeof(*server_pids)); + if (!server_pids) { + LOGERR("failed to allocate memory (%s)", strerror(errno)); + ret = ENOMEM; + } + } + } + pthread_mutex_unlock(&server_pid_alloc_lock); + + return ret; +} + +static int server_pid_invoke_rpc(void) +{ + int ret = 0; + hg_return_t hret = 0; + hg_handle_t handle; + server_pid_in_t in; + server_pid_out_t out; + + in.rank = glb_pmi_rank; + in.pid = server_pid; + + hret = margo_create(unifyfsd_rpc_context->svr_mid, + glb_servers[0].margo_svr_addr, + unifyfsd_rpc_context->rpcs.server_pid_id, + &handle); + if (hret != HG_SUCCESS) { + LOGERR("failed to create rpc handle (ret=%d)", hret); + return UNIFYFS_ERROR_MARGO; + } + + hret = margo_forward(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("failed to forward rpc (ret=%d)", hret); + return UNIFYFS_ERROR_MARGO; + } + + hret = margo_get_output(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("failed to get rpc result (ret=%d)", hret); + return UNIFYFS_ERROR_MARGO; + } + + ret = out.ret; + + return ret; +} + +static void server_pid_handle_rpc(hg_handle_t handle) +{ + int ret = 0; + int i = 0; + int count = 0; + hg_return_t hret = 0; + server_pid_in_t in; + server_pid_out_t out; + + hret = margo_get_input(handle, &in); + if (hret != HG_SUCCESS) { + LOGERR("failed to get input (ret=%d)", hret); + return; + } + + if (!server_pids) { + ret = alloc_server_pid(); + if (ret) { + LOGERR("failed to allocate pid array"); + return; + } + } + + server_pids[in.rank] = in.pid; + + out.ret = 0; + hret = margo_respond(handle, &out); + if (hret != HG_SUCCESS) { + LOGERR("failed to respond rpc (ret=%d)", hret); + return; + } + + margo_free_input(handle, &in); + margo_destroy(handle); + + for (i = 0; i < glb_pmi_size; i++) { + if (server_pids[i] > 0) { + count++; + } + } + + if (count == glb_pmi_size) { + ret = pthread_cond_signal(&server_pid_cond); + if (ret) { + LOGERR("failed to signal condition (%s)", strerror(ret)); + } + } +} +DEFINE_MARGO_RPC_HANDLER(server_pid_handle_rpc); + +static inline int set_timeout(void) +{ + int ret = 0; + long timeout_sec = 0; + + if (server_cfg.server_init_timeout) { + ret = configurator_int_val(server_cfg.server_init_timeout, + &timeout_sec); + if (ret) { + LOGERR("failed to read configuration"); + return ret; + } + } + + clock_gettime(CLOCK_REALTIME, &server_pid_timeout); + server_pid_timeout.tv_sec += timeout_sec; + + return 0; +} + +static int create_server_pid_file(void) +{ + int i = 0; + int ret = 0; + char filename[PATH_MAX] = { 0, }; + FILE* fp = NULL; + + if (!server_pids) { + LOGERR("cannot access the server pids"); + return EINVAL; + } + + sprintf(filename, "%s/%s", server_cfg.sharedfs_dir, UNIFYFSD_PID_FILENAME); + + fp = fopen(filename, "w"); + if (!fp) { + LOGERR("failed to create file %s (%s)", filename, strerror(errno)); + return errno; + } + + for (i = 0; i < glb_pmi_size; i++) { + fprintf(fp, "[%d] %d\n", i, server_pids[i]); + } + + fclose(fp); + + return ret; +} + +int unifyfs_publish_server_pids(void) +{ + int ret = UNIFYFS_SUCCESS; + + if (glb_pmi_rank > 0) { + ret = server_pid_invoke_rpc(); + if (ret) { + LOGERR("failed to invoke pid rpc (%s)", strerror(ret)); + } + } else { + ret = set_timeout(); + if (ret) { + return ret; + } + + if (!server_pids) { + ret = alloc_server_pid(); + if (ret) { + return ret; + } + } + + server_pids[0] = server_pid; + + if (glb_pmi_size > 1) { + ret = pthread_cond_timedwait(&server_pid_cond, + &server_pid_mutex, + &server_pid_timeout); + if (ETIMEDOUT == ret) { + LOGERR("some servers failed to initialize within timeout"); + goto out; + } else if (ret) { + LOGERR("failed to wait on condition (err=%d, %s)", + errno, strerror(errno)); + goto out; + } + } + + ret = create_server_pid_file(); + if (UNIFYFS_SUCCESS == ret) { + LOGDBG("all servers are ready to accept client connection"); + } + } +out: + if (server_pids) { + free(server_pids); + server_pids = NULL; + } + + return ret; +} + diff --git a/util/unifyfs/src/unifyfs-rm.c b/util/unifyfs/src/unifyfs-rm.c index 850f84ec2..1bad9797b 100644 --- a/util/unifyfs/src/unifyfs-rm.c +++ b/util/unifyfs/src/unifyfs-rm.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include "unifyfs.h" @@ -184,6 +185,92 @@ static int write_hostfile(unifyfs_resource_t* resource, return ret; } +/** + * @brief wait until servers become ready for client connections + * + * @param resource The job resource record + * @param args The command-line options + * + * @return 0 on success, negative errno otherwise + */ +static int wait_server_initialization(unifyfs_resource_t* resource, + unifyfs_args_t* args) +{ + int ret = UNIFYFS_SUCCESS; + int count = 0; + unsigned int interval = 3; + unsigned int wait_time = 0; + FILE* fp = NULL; + char linebuf[32] = { 0, }; + char filename[PATH_MAX] = { 0, }; + + sprintf(filename, "%s/%s", args->share_dir, UNIFYFSD_PID_FILENAME); + + while (1) { + fp = fopen(filename, "r"); + if (fp) { + while (fgets(linebuf, 31, fp) != NULL) { + count++; + } + + if (count != resource->n_nodes) { + fprintf(stderr, + "incorrect server initialization: " + "expected %lu processes but only %u processes found\n", + resource->n_nodes, count); + ret = UNIFYFS_FAILURE; + } + + fclose(fp); + break; + } + + if (errno != ENOENT) { + fprintf(stderr, "failed to open file %s (%s)\n", + filename, strerror(errno)); + ret = -errno; + break; + } + + wait_time += interval; + sleep(interval); + + if (wait_time > args->timeout) { + ret = UNIFYFS_FAILURE; + break; + } + } + + return ret; +} + +/** + * @brief remove server pid file if exists (possibly from previous run). + * returns 0 (success) if the pid file does not exist. + * + * @return 0 on success, negative errno otherwise + */ +static int remove_server_pid_file(unifyfs_args_t* args) +{ + int ret = 0; + char filename[PATH_MAX] = { 0, }; + + sprintf(filename, "%s/%s", args->share_dir, UNIFYFSD_PID_FILENAME); + + ret = unlink(filename); + if (ret) { + if (ENOENT == errno) { + ret = 0; + } else { + fprintf(stderr, "failed to unlink existing pid file %s (%s)\n", + filename, strerror(errno)); + ret = -errno; + } + } + + return ret; +} + /** * @brief Get node list from $LSB_HOSTS or $LSB_MCPU_HOSTS. * @@ -796,6 +883,7 @@ int unifyfs_start_servers(unifyfs_resource_t* resource, unifyfs_args_t* args) { int rc; + pid_t pid; if ((resource == NULL) || (args == NULL)) { return -EINVAL; @@ -807,11 +895,31 @@ int unifyfs_start_servers(unifyfs_resource_t* resource, return rc; } - if (args->script != NULL) { - return script_launch(resource, args); - } else { - return resource_managers[resource->rm].launch(resource, args); + rc = remove_server_pid_file(args); + if (rc) { + fprintf(stderr, "ERROR: failed to remove server pid file\n"); + return rc; } + + pid = fork(); + if (pid < 0) { + fprintf(stderr, "failed to create server launch server process (%s)\n", + strerror(errno)); + return -errno; + } else if (pid == 0) { + if (args->script != NULL) { + return script_launch(resource, args); + } else { + return resource_managers[resource->rm].launch(resource, args); + } + } + + rc = wait_server_initialization(resource, args); + if (rc) { + fprintf(stderr, "ERROR: failed to wait for server initialization\n"); + } + + return rc; } int unifyfs_stop_servers(unifyfs_resource_t* resource, diff --git a/util/unifyfs/src/unifyfs.c b/util/unifyfs/src/unifyfs.c index f77801157..be54a52f7 100644 --- a/util/unifyfs/src/unifyfs.c +++ b/util/unifyfs/src/unifyfs.c @@ -75,11 +75,12 @@ static struct option const long_opts[] = { { "share-dir", required_argument, NULL, 'S' }, { "stage-in", required_argument, NULL, 'i' }, { "stage-out", required_argument, NULL, 'o' }, + { "timeout", required_argument, NULL, 't' }, { 0, 0, 0, 0 }, }; static char* program; -static char* short_opts = ":cC:de:hi:m:o:s:S:"; +static char* short_opts = ":cC:de:hi:m:o:s:S:t:"; static char* usage_str = "\n" "Usage: %s [options...]\n" @@ -97,6 +98,7 @@ static char* usage_str = " -e, --exe= [OPTIONAL] where unifyfsd is installed\n" " -m, --mount= [OPTIONAL] mount UnifyFS at \n" " -s, --script= [OPTIONAL] to custom launch script\n" + " -t, --timeout= [OPTIONAL] wait until all servers become ready\n" " -S, --share-dir= [REQUIRED] shared file system for use by servers\n" " -c, --cleanup [OPTIONAL] clean up the UnifyFS storage upon server exit\n" " -i, --stage-in= [OPTIONAL, NOT YET SUPPORTED] stage in file(s) at \n" @@ -119,6 +121,7 @@ static void parse_cmd_arguments(int argc, char** argv) int ch = 0; int optidx = 2; int cleanup = 0; + int timeout = UNIFYFS_DEFAULT_INIT_TIMEOUT; unifyfs_cm_e consistency = UNIFYFS_CM_LAMINATED; char* mountpoint = NULL; char* script = NULL; @@ -162,6 +165,10 @@ static void parse_cmd_arguments(int argc, char** argv) share_dir = strdup(optarg); break; + case 't': + timeout = atoi(optarg); + break; + case 'i': printf("WARNING: stage-in not yet supported!\n"); stage_in = strdup(optarg); @@ -188,6 +195,7 @@ static void parse_cmd_arguments(int argc, char** argv) cli_args.share_dir = share_dir; cli_args.stage_in = stage_in; cli_args.stage_out = stage_out; + cli_args.timeout = timeout; } int main(int argc, char** argv) diff --git a/util/unifyfs/src/unifyfs.h b/util/unifyfs/src/unifyfs.h index 51917d366..28f46001c 100644 --- a/util/unifyfs/src/unifyfs.h +++ b/util/unifyfs/src/unifyfs.h @@ -57,6 +57,7 @@ struct _unifyfs_args { int debug; /* enable debug output */ int cleanup; /* cleanup on termination? (0 or 1) */ + int timeout; /* timeout of server initialization */ unifyfs_cm_e consistency; /* consistency model */ char* mountpoint; /* mountpoint */ char* server_path; /* full path to installed unifyfsd */ From 9b02626d634ae5c9af2a4e4b06ae15bfe63061e5 Mon Sep 17 00:00:00 2001 From: Hyogi Sim Date: Fri, 15 May 2020 13:07:52 -0400 Subject: [PATCH 114/168] Correctly count the LSB_MCPU_HOSTS in unifyfs utility: `LSB_MCPU_HOSTS` includes an extra space character in the string, which causes the node count to be incorrectly calculated in `lsf_read_resource()`. --- util/unifyfs/src/unifyfs-rm.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/util/unifyfs/src/unifyfs-rm.c b/util/unifyfs/src/unifyfs-rm.c index 1bad9797b..c59790c76 100644 --- a/util/unifyfs/src/unifyfs-rm.c +++ b/util/unifyfs/src/unifyfs-rm.c @@ -271,6 +271,20 @@ static int remove_server_pid_file(unifyfs_args_t* args) return ret; } +static inline char* str_rtrim(char* str) +{ + if (str) { + char* pos = &str[strlen(str) - 1]; + + while (pos >= str && isspace(*pos)) { + *pos = '\0'; + pos--; + } + } + + return str; +} + /** * @brief Get node list from $LSB_HOSTS or $LSB_MCPU_HOSTS. * @@ -303,7 +317,9 @@ static int lsf_read_resource(unifyfs_resource_t* resource) } } - lsb_hosts = strdup(val); + // LSB_MCPU_HOSTS string includes a space at the end, which causes extra + // node count (n_nodes). + lsb_hosts = str_rtrim(strdup(val)); // get length of host string size_t hosts_len = strlen(lsb_hosts) + 1; From 1ba037294821fd5713d45a96ccb7d68bc4051790 Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Wed, 29 Apr 2020 16:54:33 -0400 Subject: [PATCH 115/168] improve MPI-IO support * added wrappers for statfs() and fstatfs() * migrated gotcha support to latest API * travis update for new gotcha@1.0.3 spack package * fixed examples to not break when Unify configured with --enable-mpi-mount. previously, they ended up calling unifyfs_mount() twice and the second call failed. * added MPI-IO support to testutil, so all of the newer examples based on testutil can seamlessly use MPI-IO as the I/O mechanism TEST_CHECKPATCH_SKIP_FILES="client/src/gotcha_map_unifyfs_list.c" TEST_CHECKPATCH_SKIP_FILES+=",client/src/unifyfs-internal.h" --- .travis.yml | 2 +- client/src/Makefile.am | 2 +- client/src/gotcha_map_unifyfs_list.c | 435 +++++++++++++++++++++++++++ client/src/gotcha_map_unifyfs_list.h | 219 -------------- client/src/pmpi_wrappers.c | 5 +- client/src/unifyfs-dirops.c | 1 - client/src/unifyfs-dirops.h | 6 +- client/src/unifyfs-internal.h | 40 ++- client/src/unifyfs-sysio.c | 71 +++++ client/src/unifyfs-sysio.h | 7 +- client/src/unifyfs.c | 24 +- common/src/unifyfs_rc.h | 1 + configure.ac | 21 +- examples/src/Makefile.am | 4 + examples/src/checkpoint-restart.c | 2 + examples/src/multi-write.c | 2 +- examples/src/read.c | 2 + examples/src/testutil.h | 210 +++++++++---- examples/src/testutil_rdwr.h | 34 ++- examples/src/write.c | 2 + examples/src/writeread.c | 2 + 21 files changed, 758 insertions(+), 334 deletions(-) create mode 100644 client/src/gotcha_map_unifyfs_list.c delete mode 100644 client/src/gotcha_map_unifyfs_list.h diff --git a/.travis.yml b/.travis.yml index f07bc6820..b8f972809 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ before_install: install: - . $HOME/spack/share/spack/setup-env.sh - spack install leveldb && spack load leveldb - - spack install gotcha@0.0.2 && spack load gotcha@0.0.2 + - spack install gotcha@1.0.3 && spack load gotcha@1.0.3 - spack install flatcc && spack load flatcc - spack install margo^mercury+bmi~boostsys && spack load argobots && spack load mercury && spack load margo # prepare build environment diff --git a/client/src/Makefile.am b/client/src/Makefile.am index 23428522a..0325df935 100644 --- a/client/src/Makefile.am +++ b/client/src/Makefile.am @@ -76,7 +76,7 @@ libunifyfs_la_LIBADD = $(CLIENT_COMMON_LIBADD) if HAVE_GOTCHA -libunifyfs_gotcha_la_SOURCES = $(CLIENT_COMMON_SOURCES) gotcha_map_unifyfs_list.h +libunifyfs_gotcha_la_SOURCES = $(CLIENT_COMMON_SOURCES) gotcha_map_unifyfs_list.c libunifyfs_gotcha_la_CPPFLAGS = $(CLIENT_COMMON_CPPFLAGS) -DUNIFYFS_GOTCHA libunifyfs_gotcha_la_CFLAGS = $(CLIENT_COMMON_CFLAGS) $(GOTCHA_CFLAGS) libunifyfs_gotcha_la_LDFLAGS = $(CLIENT_COMMON_LDFLAGS) $(GOTCHA_LDFLAGS) diff --git a/client/src/gotcha_map_unifyfs_list.c b/client/src/gotcha_map_unifyfs_list.c new file mode 100644 index 000000000..8f0fe8605 --- /dev/null +++ b/client/src/gotcha_map_unifyfs_list.c @@ -0,0 +1,435 @@ +/* + * Copyright (c) 2020, Lawrence Livermore National Security, LLC. + * Produced at the Lawrence Livermore National Laboratory. + * + * Copyright 2020, UT-Battelle, LLC. + * + * LLNL-CODE-741539 + * All rights reserved. + * + * This is the license for UnifyFS. + * For details, see https://github.com/LLNL/UnifyFS. + * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. + */ + +#include "unifyfs-internal.h" +#include + +/* define gotcha-specific state to use with our wrapper */ +#define UNIFYFS_DEF(name, ret, args, argnames) \ +gotcha_wrappee_handle_t wrappee_handle_ ## name; \ +ret (*__real_ ## name) args = NULL; + +UNIFYFS_DEF(access, int, + (const char* path, int mode), + (path, mode)) +UNIFYFS_DEF(chmod, int, + (const char* path, mode_t mode), + (path, mode)) +UNIFYFS_DEF(mkdir, int, + (const char* path, mode_t mode), + (path, mode)) +UNIFYFS_DEF(rmdir, int, + (const char* path), + (path)) +UNIFYFS_DEF(rename, int, + (const char* oldpath, const char* newpath), + (oldpath, newpath)) +UNIFYFS_DEF(truncate, int, + (const char* path, off_t length), + (path, length)) +UNIFYFS_DEF(unlink, int, + (const char* path), + (path)) +UNIFYFS_DEF(remove, int, + (const char* path), + (path)) + +UNIFYFS_DEF(stat, int, + (const char* path, struct stat* buf), + (path, buf)) +UNIFYFS_DEF(fstat, int, + (int fd, struct stat* buf), + (fd, buf)) +UNIFYFS_DEF(__xstat, int, + (int vers, const char* path, struct stat* buf), + (vers, path, buf)) +UNIFYFS_DEF(__fxstat, int, + (int vers, int fd, struct stat* buf), + (vers, fd, buf)) +UNIFYFS_DEF(__lxstat, int, + (int vers, const char* path, struct stat* buf), + (vers, path, buf)) +UNIFYFS_DEF(statfs, int, + (const char* path, struct statfs* fsbuf), + (path, fsbuf)) +UNIFYFS_DEF(fstatfs, int, + (int fd, struct statfs* fsbuf), + (fd, fsbuf)) + +UNIFYFS_DEF(creat, int, + (const char* path, mode_t mode), + (path, mode)) +UNIFYFS_DEF(creat64, int, + (const char* path, mode_t mode), + (path, mode)) +UNIFYFS_DEF(open, int, + (const char* path, int flags, ...), + (path, flags)) +UNIFYFS_DEF(open64, int, + (const char* path, int flags, ...), + (path, flags)) +UNIFYFS_DEF(__open_2, int, + (const char* path, int flags, ...), + (path, flags)) + +UNIFYFS_DEF(lio_listio, int, + (int m, struct aiocb* const cblist[], int n, struct sigevent* sep), + (m, cblist, n, sep)) +UNIFYFS_DEF(lseek, off_t, + (int fd, off_t offset, int whence), + (fd, offset, whence)) +UNIFYFS_DEF(lseek64, off64_t, + (int fd, off64_t offset, int whence), + (fd, offset, whence)) + +UNIFYFS_DEF(posix_fadvise, int, + (int fd, off_t offset, off_t len, int advice), + (fd, offset, len, advice)) + +UNIFYFS_DEF(read, ssize_t, + (int fd, void* buf, size_t count), + (fd, buf, count)) +UNIFYFS_DEF(write, ssize_t, + (int fd, const void* buf, size_t count), + (fd, buf, count)) +UNIFYFS_DEF(readv, ssize_t, + (int fd, const struct iovec* iov, int iovcnt), + (fd, iov, iovcnt)) +UNIFYFS_DEF(writev, ssize_t, + (int fd, const struct iovec* iov, int iovcnt), + (fd, iov, iovcnt)) +UNIFYFS_DEF(pread, ssize_t, + (int fd, void* buf, size_t count, off_t off), + (fd, buf, count, off)) +UNIFYFS_DEF(pread64, ssize_t, + (int fd, void* buf, size_t count, off64_t off), + (fd, buf, count, off)) +UNIFYFS_DEF(pwrite, ssize_t, + (int fd, const void* buf, size_t count, off_t off), + (fd, buf, count, off)) +UNIFYFS_DEF(pwrite64, ssize_t, + (int fd, const void* buf, size_t count, off64_t off), + (fd, buf, count, off)) +UNIFYFS_DEF(close, int, + (int fd), + (fd)) +UNIFYFS_DEF(ftruncate, int, + (int fd, off_t length), + (fd, length)) +UNIFYFS_DEF(fsync, int, + (int fd), + (fd)) +UNIFYFS_DEF(fdatasync, int, + (int fd), + (fd)) +UNIFYFS_DEF(flock, int, + (int fd, int operation), + (fd, operation)) +UNIFYFS_DEF(fchmod, int, + (int fd, mode_t mode), + (fd, mode)) + +UNIFYFS_DEF(mmap, void*, + (void* addr, size_t len, int prot, int fl, int fd, off_t off), + (addr, len, prot, fl, fd, off)) +UNIFYFS_DEF(msync, int, + (void* addr, size_t length, int flags), + (addr, length, flags)) +UNIFYFS_DEF(munmap, int, + (void* addr, size_t length), + (addr, length)) +UNIFYFS_DEF(mmap64, void*, + (void* addr, size_t len, int prot, int fl, int fd, off_t off), + (addr, len, prot, fl, fd, off)) + +UNIFYFS_DEF(opendir, DIR*, + (const char* name), + (name)) +UNIFYFS_DEF(fdopendir, DIR*, + (int fd), + (fd)) +UNIFYFS_DEF(closedir, int, + (DIR* dirp), + (dirp)) +UNIFYFS_DEF(readdir, struct dirent*, + (DIR* dirp), + (dirp)) +UNIFYFS_DEF(rewinddir, void, + (DIR* dirp), + (dirp)) +UNIFYFS_DEF(dirfd, int, + (DIR* dirp), + (dirp)) +UNIFYFS_DEF(telldir, long, + (DIR* dirp), + (dirp)) +UNIFYFS_DEF(scandir, int, + (const char* dirp, struct dirent** namelist, + int (*filter)(const struct dirent*), + int (*compar)(const struct dirent**, const struct dirent**)), + (dirp, namelist, filter, compar)) +UNIFYFS_DEF(seekdir, void, + (DIR* dirp, long loc), + (dirp, loc)) + +UNIFYFS_DEF(fopen, FILE*, + (const char* path, const char* mode), + (path, mode)) +UNIFYFS_DEF(freopen, FILE*, + (const char* path, const char* mode, FILE* stream), + (path, mode, stream)) +UNIFYFS_DEF(setvbuf, int, + (FILE* stream, char* buf, int type, size_t size), + (stream, buf, type, size)) +UNIFYFS_DEF(setbuf, void, + (FILE* stream, char* buf), + (stream, buf)) +UNIFYFS_DEF(ungetc, int, + (int c, FILE* stream), + (c, stream)) +UNIFYFS_DEF(fgetc, int, + (FILE* stream), + (stream)) +UNIFYFS_DEF(fputc, int, + (int c, FILE* stream), + (c, stream)) +UNIFYFS_DEF(getc, int, + (FILE* stream), + (stream)) +UNIFYFS_DEF(putc, int, + (int c, FILE* stream), + (c, stream)) +UNIFYFS_DEF(fgets, char*, + (char* s, int n, FILE* stream), + (s, n, stream)) +UNIFYFS_DEF(fputs, int, + (const char* s, FILE* stream), + (s, stream)) +UNIFYFS_DEF(fread, size_t, + (void* ptr, size_t size, size_t nitems, FILE* stream), + (ptr, size, nitems, stream)) +UNIFYFS_DEF(fwrite, size_t, + (const void* ptr, size_t size, size_t nitems, FILE* stream), + (ptr, size, nitems, stream)) +UNIFYFS_DEF(fprintf, int, + (FILE* stream, const char* format, ...), + (stream, format)) +UNIFYFS_DEF(fscanf, int, + (FILE* stream, const char* format, ...), + (stream, format)) +UNIFYFS_DEF(vfprintf, int, + (FILE* stream, const char* format, va_list args), + (stream, format, args)) +UNIFYFS_DEF(vfscanf, int, + (FILE* stream, const char* format, va_list args), + (stream, format, args)) +UNIFYFS_DEF(fseek, int, + (FILE* stream, long offset, int whence), + (stream, offset, whence)) +UNIFYFS_DEF(fseeko, int, + (FILE* stream, off_t offset, int whence), + (stream, offset, whence)) +UNIFYFS_DEF(ftell, long, + (FILE* stream), + (stream)) +UNIFYFS_DEF(ftello, off_t, + (FILE* stream), + (stream)) +UNIFYFS_DEF(rewind, void, + (FILE* stream), + (stream)) +UNIFYFS_DEF(fgetpos, int, + (FILE* stream, fpos_t* pos), + (stream, pos)) +UNIFYFS_DEF(fsetpos, int, + (FILE* stream, const fpos_t* pos), + (stream, pos)) +UNIFYFS_DEF(fflush, int, + (FILE* stream), + (stream)) +UNIFYFS_DEF(feof, int, + (FILE* stream), + (stream)) +UNIFYFS_DEF(ferror, int, + (FILE* stream), + (stream)) +UNIFYFS_DEF(clearerr, void, + (FILE* stream), + (stream)) +UNIFYFS_DEF(fileno, int, + (FILE* stream), + (stream)) +UNIFYFS_DEF(fclose, int, + (FILE* stream), + (stream)) +UNIFYFS_DEF(fwprintf, int, + (FILE* stream, const wchar_t* format, ...), + (stream, format)) +UNIFYFS_DEF(fwscanf, int, + (FILE* stream, const wchar_t* format, ...), + (stream, format)) +UNIFYFS_DEF(vfwprintf, int, + (FILE* stream, const wchar_t* format, va_list args), + (stream, format, args)) +UNIFYFS_DEF(vfwscanf, int, + (FILE* stream, const wchar_t* format, va_list args), + (stream, format, args)) +UNIFYFS_DEF(fgetwc, wint_t, + (FILE* stream), + (stream)) +UNIFYFS_DEF(fgetws, wchar_t*, + (wchar_t* s, int n, FILE* stream), + (s, n, stream)) +UNIFYFS_DEF(fputwc, wint_t, + (wchar_t wc, FILE* stream), + (wc, stream)) +UNIFYFS_DEF(fputws, int, + (const wchar_t* s, FILE* stream), + (s, stream)) +UNIFYFS_DEF(fwide, int, + (FILE* stream, int mode), + (stream, mode)) +UNIFYFS_DEF(getwc, wint_t, + (FILE* stream), + (stream)) +UNIFYFS_DEF(putwc, wint_t, + (wchar_t c, FILE* stream), + (c, stream)) +UNIFYFS_DEF(ungetwc, wint_t, + (wint_t c, FILE* stream), + (c, stream)) + +struct gotcha_binding_t unifyfs_wrappers[] = { + { "access", UNIFYFS_WRAP(access), &wrappee_handle_access }, + { "chmod", UNIFYFS_WRAP(chmod), &wrappee_handle_chmod }, + { "mkdir", UNIFYFS_WRAP(mkdir), &wrappee_handle_mkdir }, + { "rmdir", UNIFYFS_WRAP(rmdir), &wrappee_handle_rmdir }, + { "rename", UNIFYFS_WRAP(rename), &wrappee_handle_rename }, + { "truncate", UNIFYFS_WRAP(truncate), &wrappee_handle_truncate }, + { "unlink", UNIFYFS_WRAP(unlink), &wrappee_handle_unlink }, + { "remove", UNIFYFS_WRAP(remove), &wrappee_handle_remove }, + { "stat", UNIFYFS_WRAP(stat), &wrappee_handle_stat }, + { "fstat", UNIFYFS_WRAP(fstat), &wrappee_handle_fstat }, + { "__xstat", UNIFYFS_WRAP(__xstat), &wrappee_handle___xstat }, + { "__fxstat", UNIFYFS_WRAP(__fxstat), &wrappee_handle___fxstat }, + { "__lxstat", UNIFYFS_WRAP(__lxstat), &wrappee_handle___lxstat }, + { "statfs", UNIFYFS_WRAP(statfs), &wrappee_handle_statfs }, + { "fstatfs", UNIFYFS_WRAP(fstatfs), &wrappee_handle_fstatfs }, + { "creat", UNIFYFS_WRAP(creat), &wrappee_handle_creat }, + { "creat64", UNIFYFS_WRAP(creat64), &wrappee_handle_creat64 }, + { "open", UNIFYFS_WRAP(open), &wrappee_handle_open }, + { "open64", UNIFYFS_WRAP(open64), &wrappee_handle_open64 }, + { "__open_2", UNIFYFS_WRAP(__open_2), &wrappee_handle___open_2 }, + { "lio_listio", UNIFYFS_WRAP(lio_listio), &wrappee_handle_lio_listio }, + { "lseek", UNIFYFS_WRAP(lseek), &wrappee_handle_lseek }, + { "lseek64", UNIFYFS_WRAP(lseek64), &wrappee_handle_lseek64 }, + { "posix_fadvise", UNIFYFS_WRAP(posix_fadvise), &wrappee_handle_posix_fadvise }, + { "read", UNIFYFS_WRAP(read), &wrappee_handle_read }, + { "write", UNIFYFS_WRAP(write), &wrappee_handle_write }, + { "readv", UNIFYFS_WRAP(readv), &wrappee_handle_readv }, + { "writev", UNIFYFS_WRAP(writev), &wrappee_handle_writev }, + { "pread", UNIFYFS_WRAP(pread), &wrappee_handle_pread }, + { "pread64", UNIFYFS_WRAP(pread64), &wrappee_handle_pread64 }, + { "pwrite", UNIFYFS_WRAP(pwrite), &wrappee_handle_pwrite }, + { "pwrite64", UNIFYFS_WRAP(pwrite64), &wrappee_handle_pwrite64 }, + { "ftruncate", UNIFYFS_WRAP(ftruncate), &wrappee_handle_ftruncate }, + { "fsync", UNIFYFS_WRAP(fsync), &wrappee_handle_fsync }, + { "fdatasync", UNIFYFS_WRAP(fdatasync), &wrappee_handle_fdatasync }, + { "flock", UNIFYFS_WRAP(flock), &wrappee_handle_flock }, + { "fchmod", UNIFYFS_WRAP(fchmod), &wrappee_handle_fchmod }, + { "mmap", UNIFYFS_WRAP(mmap), &wrappee_handle_mmap }, + { "msync", UNIFYFS_WRAP(msync), &wrappee_handle_msync }, + { "munmap", UNIFYFS_WRAP(munmap), &wrappee_handle_munmap }, + { "mmap64", UNIFYFS_WRAP(mmap64), &wrappee_handle_mmap64 }, + { "close", UNIFYFS_WRAP(close), &wrappee_handle_close }, + { "opendir", UNIFYFS_WRAP(opendir), &wrappee_handle_opendir }, + { "fdopendir", UNIFYFS_WRAP(fdopendir), &wrappee_handle_fdopendir }, + { "closedir", UNIFYFS_WRAP(closedir), &wrappee_handle_closedir }, + { "readdir", UNIFYFS_WRAP(readdir), &wrappee_handle_readdir }, + { "rewinddir", UNIFYFS_WRAP(rewinddir), &wrappee_handle_rewinddir }, + { "dirfd", UNIFYFS_WRAP(dirfd), &wrappee_handle_dirfd }, + { "telldir", UNIFYFS_WRAP(telldir), &wrappee_handle_telldir }, + { "scandir", UNIFYFS_WRAP(scandir), &wrappee_handle_scandir }, + { "seekdir", UNIFYFS_WRAP(seekdir), &wrappee_handle_seekdir }, + { "fopen", UNIFYFS_WRAP(fopen), &wrappee_handle_fopen }, + { "freopen", UNIFYFS_WRAP(freopen), &wrappee_handle_freopen }, + { "setvbuf", UNIFYFS_WRAP(setvbuf), &wrappee_handle_setvbuf }, + { "setbuf", UNIFYFS_WRAP(setbuf), &wrappee_handle_setbuf }, + { "ungetc", UNIFYFS_WRAP(ungetc), &wrappee_handle_ungetc }, + { "fgetc", UNIFYFS_WRAP(fgetc), &wrappee_handle_fgetc }, + { "fputc", UNIFYFS_WRAP(fputc), &wrappee_handle_fputc }, + { "getc", UNIFYFS_WRAP(getc), &wrappee_handle_getc }, + { "putc", UNIFYFS_WRAP(putc), &wrappee_handle_putc }, + { "fgets", UNIFYFS_WRAP(fgets), &wrappee_handle_fgets }, + { "fputs", UNIFYFS_WRAP(fputs), &wrappee_handle_fputs }, + { "fread", UNIFYFS_WRAP(fread), &wrappee_handle_fread }, + { "fwrite", UNIFYFS_WRAP(fwrite), &wrappee_handle_fwrite }, + { "fprintf", UNIFYFS_WRAP(fprintf), &wrappee_handle_fprintf }, + { "vfprintf", UNIFYFS_WRAP(vfprintf), &wrappee_handle_vfprintf }, + { "fscanf", UNIFYFS_WRAP(fscanf), &wrappee_handle_fscanf }, + { "vfscanf", UNIFYFS_WRAP(vfscanf), &wrappee_handle_vfscanf }, + { "fseek", UNIFYFS_WRAP(fseek), &wrappee_handle_fseek }, + { "fseeko", UNIFYFS_WRAP(fseeko), &wrappee_handle_fseeko }, + { "ftell", UNIFYFS_WRAP(ftell), &wrappee_handle_ftell }, + { "ftello", UNIFYFS_WRAP(ftello), &wrappee_handle_ftello }, + { "rewind", UNIFYFS_WRAP(rewind), &wrappee_handle_rewind }, + { "fgetpos", UNIFYFS_WRAP(fgetpos), &wrappee_handle_fgetpos }, + { "fsetpos", UNIFYFS_WRAP(fsetpos), &wrappee_handle_fsetpos }, + { "fflush", UNIFYFS_WRAP(fflush), &wrappee_handle_fflush }, + { "feof", UNIFYFS_WRAP(feof), &wrappee_handle_feof }, + { "ferror", UNIFYFS_WRAP(ferror), &wrappee_handle_ferror }, + { "clearerr", UNIFYFS_WRAP(clearerr), &wrappee_handle_clearerr }, + { "fileno", UNIFYFS_WRAP(fileno), &wrappee_handle_fileno }, + { "fclose", UNIFYFS_WRAP(fclose), &wrappee_handle_fclose }, + { "fwprintf", UNIFYFS_WRAP(fwprintf), &wrappee_handle_fwprintf }, + { "fwscanf", UNIFYFS_WRAP(fwscanf), &wrappee_handle_fwscanf }, + { "vfwprintf", UNIFYFS_WRAP(vfwprintf), &wrappee_handle_vfwprintf }, + { "vfwscanf", UNIFYFS_WRAP(vfwscanf), &wrappee_handle_vfwscanf }, + { "fgetwc", UNIFYFS_WRAP(fgetwc), &wrappee_handle_fgetwc }, + { "fgetws", UNIFYFS_WRAP(fgetws), &wrappee_handle_fgetws }, + { "fputwc", UNIFYFS_WRAP(fputwc), &wrappee_handle_fputwc }, + { "fputws", UNIFYFS_WRAP(fputws), &wrappee_handle_fputws }, + { "fwide", UNIFYFS_WRAP(fwide), &wrappee_handle_fwide }, + { "getwc", UNIFYFS_WRAP(getwc), &wrappee_handle_getwc }, + { "putwc", UNIFYFS_WRAP(putwc), &wrappee_handle_putwc }, + { "ungetwc", UNIFYFS_WRAP(ungetwc), &wrappee_handle_ungetwc }, +}; + +#define GOTCHA_NFUNCS (sizeof(unifyfs_wrappers) / sizeof(gotcha_binding_t)) + +int setup_gotcha_wrappers(void) +{ + /* insert our I/O wrappers using gotcha */ + enum gotcha_error_t result; + result = gotcha_wrap(unifyfs_wrappers, GOTCHA_NFUNCS, "unifyfs"); + if (result != GOTCHA_SUCCESS) { + LOGERR("gotcha_wrap() returned %d", (int) result); + if (result == GOTCHA_FUNCTION_NOT_FOUND) { + /* one or more functions were not found */ + void* fn; + gotcha_wrappee_handle_t* hdlptr; + for (int i = 0; i < GOTCHA_NFUNCS; i++) { + hdlptr = unifyfs_wrappers[i].function_handle; + fn = gotcha_get_wrappee(*hdlptr); + if (NULL == fn) { + LOGWARN("Gotcha failed to wrap function '%s'", + unifyfs_wrappers[i].name); + } + } + } else { + return UNIFYFS_ERROR_GOTCHA; + } + } + return UNIFYFS_SUCCESS; +} diff --git a/client/src/gotcha_map_unifyfs_list.h b/client/src/gotcha_map_unifyfs_list.h deleted file mode 100644 index 97a9c2317..000000000 --- a/client/src/gotcha_map_unifyfs_list.h +++ /dev/null @@ -1,219 +0,0 @@ -//Generated by translate.py - -#include "config.h" -#include -#include - -UNIFYFS_DEF(access, int, (const char* path, int mode)); -UNIFYFS_DEF(chmod, int, (const char* path, mode_t mode)); -UNIFYFS_DEF(fchmod, int, (int fd, mode_t mode)); -UNIFYFS_DEF(mkdir, int, (const char* path, mode_t mode)); -UNIFYFS_DEF(rmdir, int, (const char* path)); -UNIFYFS_DEF(rename, int, (const char* oldpath, const char* newpath)); -UNIFYFS_DEF(truncate, int, (const char* path, off_t length)); -UNIFYFS_DEF(unlink, int, (const char* path)); -UNIFYFS_DEF(remove, int, (const char* path)); -UNIFYFS_DEF(stat, int, (const char* path, struct stat* buf)); -UNIFYFS_DEF(fstat, int, (int fd, struct stat* buf)); -#ifdef HAVE___XSTAT -UNIFYFS_DEF(__xstat, int, (int vers, const char* path, struct stat* buf)); -#endif -#ifdef HAVE___LXSTAT -UNIFYFS_DEF(__lxstat, int, (int vers, const char* path, struct stat* buf)); -#endif -UNIFYFS_DEF(creat, int, (const char* path, mode_t mode)); -UNIFYFS_DEF(creat64, int, (const char* path, mode_t mode)); -UNIFYFS_DEF(open, int, (const char* path, int flags, ...)); -#ifdef HAVE_OPEN64 -UNIFYFS_DEF(open64, int, (const char* path, int flags, ...)); -#endif -UNIFYFS_DEF(__open_2, int, (const char* path, int flags, ...)); - -#ifdef HAVE_LIO_LISTIO -UNIFYFS_DEF(lio_listio, int, (int mode, struct aiocb* const aiocb_list[], - int nitems, struct sigevent* sevp)); -#endif -UNIFYFS_DEF(lseek, off_t, (int fd, off_t offset, int whence)); -UNIFYFS_DEF(lseek64, off64_t, (int fd, off64_t offset, int whence)); - -#ifdef HAVE_POSIX_FADVISE -UNIFYFS_DEF(posix_fadvise, int, (int fd, off_t offset, off_t len, int advice)); -#endif -UNIFYFS_DEF(read, ssize_t, (int fd, void* buf, size_t count)); -UNIFYFS_DEF(write, ssize_t, (int fd, const void* buf, size_t count)); -UNIFYFS_DEF(readv, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); -UNIFYFS_DEF(writev, ssize_t, (int fd, const struct iovec* iov, int iovcnt)); -UNIFYFS_DEF(pread, ssize_t, (int fd, void* buf, size_t count, off_t offset)); -UNIFYFS_DEF(pread64, ssize_t, (int fd, void* buf, size_t count, - off64_t offset)); -UNIFYFS_DEF(pwrite, ssize_t, (int fd, const void* buf, size_t count, - off_t offset)); -UNIFYFS_DEF(pwrite64, ssize_t, (int fd, const void* buf, size_t count, - off64_t offset)); -UNIFYFS_DEF(ftruncate, int, (int fd, off_t length)); -UNIFYFS_DEF(fsync, int, (int fd)); -UNIFYFS_DEF(fdatasync, int, (int fd)); -UNIFYFS_DEF(flock, int, (int fd, int operation)); -UNIFYFS_DEF(mmap, void*, (void* addr, size_t length, int prot, int flags, - int fd, off_t offset)); -UNIFYFS_DEF(msync, int, (void* addr, size_t length, int flags)); -UNIFYFS_DEF(munmap, int, (void* addr, size_t length)); -UNIFYFS_DEF(mmap64, void*, (void* addr, size_t length, int prot, int flags, - int fd, off_t offset)); -#ifdef HAVE___FXSTAT -UNIFYFS_DEF(__fxstat, int, (int vers, int fd, struct stat* buf)); -#endif -UNIFYFS_DEF(close, int, (int fd)); -UNIFYFS_DEF(opendir, DIR*, (const char* name)); -UNIFYFS_DEF(fdopendir, DIR*, (int fd)); -UNIFYFS_DEF(closedir, int, (DIR* dirp)); -UNIFYFS_DEF(readdir, struct dirent*, (DIR* dirp)); -UNIFYFS_DEF(rewinddir, void, (DIR* dirp)); -UNIFYFS_DEF(dirfd, int, (DIR* dirp)); -UNIFYFS_DEF(telldir, long, (DIR* dirp)); -UNIFYFS_DEF(scandir, int, (const char* dirp, struct dirent** namelist, - int (*filter)(const struct dirent*), - int (*compar)(const struct dirent**, - const struct dirent**))); -UNIFYFS_DEF(seekdir, void, (DIR* dirp, long loc)); -UNIFYFS_DEF(fopen, FILE*, (const char* path, const char* mode)); -UNIFYFS_DEF(freopen, FILE*, (const char* path, const char* mode, - FILE* stream)); -UNIFYFS_DEF(setvbuf, int, (FILE* stream, char* buf, int type, size_t size)); -UNIFYFS_DEF(setbuf, void, (FILE* stream, char* buf)); -UNIFYFS_DEF(ungetc, int, (int c, FILE* stream)); -UNIFYFS_DEF(fgetc, int, (FILE* stream)); -UNIFYFS_DEF(fputc, int, (int c, FILE* stream)); -UNIFYFS_DEF(getc, int, (FILE* stream)); -UNIFYFS_DEF(putc, int, (int c, FILE* stream)); -UNIFYFS_DEF(fgets, char*, (char* s, int n, FILE* stream)); -UNIFYFS_DEF(fputs, int, (const char* s, FILE* stream)); -UNIFYFS_DEF(fread, size_t, (void* ptr, size_t size, size_t nitems, - FILE* stream)); -UNIFYFS_DEF(fwrite, size_t, (const void* ptr, size_t size, size_t nitems, - FILE* stream)); -UNIFYFS_DEF(fprintf, int, (FILE* stream, const char* format, ...)); -UNIFYFS_DEF(vfprintf, int, (FILE* stream, const char* format, va_list ap)); -UNIFYFS_DEF(fscanf, int, (FILE* stream, const char* format, ...)); -UNIFYFS_DEF(vfscanf, int, (FILE* stream, const char* format, va_list ap)); -UNIFYFS_DEF(fseek, int, (FILE* stream, long offset, int whence)); -UNIFYFS_DEF(fseeko, int, (FILE* stream, off_t offset, int whence)); -UNIFYFS_DEF(ftell, long, (FILE* stream)); -UNIFYFS_DEF(ftello, off_t, (FILE* stream)); -UNIFYFS_DEF(rewind, void, (FILE* stream)); -UNIFYFS_DEF(fgetpos, int, (FILE* stream, fpos_t* pos)); -UNIFYFS_DEF(fsetpos, int, (FILE* stream, const fpos_t* pos)); -UNIFYFS_DEF(fflush, int, (FILE* stream)); -UNIFYFS_DEF(feof, int, (FILE* stream)); -UNIFYFS_DEF(ferror, int, (FILE* stream)); -UNIFYFS_DEF(clearerr, void, (FILE* stream)); -UNIFYFS_DEF(fileno, int, (FILE* stream)); -UNIFYFS_DEF(fclose, int, (FILE* stream)); -UNIFYFS_DEF(fwprintf, int, (FILE* stream, const wchar_t* format, ...)); -UNIFYFS_DEF(fwscanf, int, (FILE* stream, const wchar_t* format, ...)); -UNIFYFS_DEF(vfwprintf, int, (FILE* stream, const wchar_t* format, va_list arg)); -UNIFYFS_DEF(vfwscanf, int, (FILE* stream, const wchar_t* format, va_list arg)); -UNIFYFS_DEF(fgetwc, wint_t, (FILE* stream)); -UNIFYFS_DEF(fgetws, wchar_t*, (wchar_t* s, int n, FILE* stream)); -UNIFYFS_DEF(fputwc, wint_t, (wchar_t wc, FILE* stream)); -UNIFYFS_DEF(fputws, int, (const wchar_t* s, FILE* stream)); -UNIFYFS_DEF(fwide, int, (FILE* stream, int mode)); -UNIFYFS_DEF(getwc, wint_t, (FILE* stream)); -UNIFYFS_DEF(putwc, wint_t, (wchar_t c, FILE* stream)); -UNIFYFS_DEF(ungetwc, wint_t, (wint_t c, FILE* stream)); - -struct gotcha_binding_t wrap_unifyfs_list[] = { - { "access", UNIFYFS_WRAP(access), &UNIFYFS_REAL(access) }, - { "chmod", UNIFYFS_WRAP(chmod), &UNIFYFS_REAL(chmod) }, - { "fchmod", UNIFYFS_WRAP(fchmod), &UNIFYFS_REAL(fchmod) }, - { "mkdir", UNIFYFS_WRAP(mkdir), &UNIFYFS_REAL(mkdir) }, - { "rmdir", UNIFYFS_WRAP(rmdir), &UNIFYFS_REAL(rmdir) }, - { "rename", UNIFYFS_WRAP(rename), &UNIFYFS_REAL(rename) }, - { "truncate", UNIFYFS_WRAP(truncate), &UNIFYFS_REAL(truncate) }, - { "unlink", UNIFYFS_WRAP(unlink), &UNIFYFS_REAL(unlink) }, - { "remove", UNIFYFS_WRAP(remove), &UNIFYFS_REAL(remove) }, - { "stat", UNIFYFS_WRAP(stat), &UNIFYFS_REAL(stat) }, - { "fstat", UNIFYFS_WRAP(fstat), &UNIFYFS_REAL(fstat) }, - { "__xstat", UNIFYFS_WRAP(__xstat), &UNIFYFS_REAL(__xstat) }, - { "__lxstat", UNIFYFS_WRAP(__lxstat), &UNIFYFS_REAL(__lxstat) }, - { "creat", UNIFYFS_WRAP(creat), &UNIFYFS_REAL(creat) }, - { "creat64", UNIFYFS_WRAP(creat64), &UNIFYFS_REAL(creat64) }, - { "open", UNIFYFS_WRAP(open), &UNIFYFS_REAL(open) }, - { "open64", UNIFYFS_WRAP(open64), &UNIFYFS_REAL(open64) }, - { "__open_2", UNIFYFS_WRAP(__open_2), &UNIFYFS_REAL(__open_2) }, - { "lio_listio", UNIFYFS_WRAP(lio_listio), &UNIFYFS_REAL(lio_listio) }, - { "lseek", UNIFYFS_WRAP(lseek), &UNIFYFS_REAL(lseek) }, - { "lseek64", UNIFYFS_WRAP(lseek64), &UNIFYFS_REAL(lseek64) }, - { "posix_fadvise", UNIFYFS_WRAP(posix_fadvise), &UNIFYFS_REAL(posix_fadvise) }, - { "read", UNIFYFS_WRAP(read), &UNIFYFS_REAL(read) }, - { "write", UNIFYFS_WRAP(write), &UNIFYFS_REAL(write) }, - { "readv", UNIFYFS_WRAP(readv), &UNIFYFS_REAL(readv) }, - { "writev", UNIFYFS_WRAP(writev), &UNIFYFS_REAL(writev) }, - { "pread", UNIFYFS_WRAP(pread), &UNIFYFS_REAL(pread) }, - { "pread64", UNIFYFS_WRAP(pread64), &UNIFYFS_REAL(pread64) }, - { "pwrite", UNIFYFS_WRAP(pwrite), &UNIFYFS_REAL(pwrite) }, - { "pwrite64", UNIFYFS_WRAP(pwrite64), &UNIFYFS_REAL(pwrite64) }, - { "ftruncate", UNIFYFS_WRAP(ftruncate), &UNIFYFS_REAL(ftruncate) }, - { "fsync", UNIFYFS_WRAP(fsync), &UNIFYFS_REAL(fsync) }, - { "fdatasync", UNIFYFS_WRAP(fdatasync), &UNIFYFS_REAL(fdatasync) }, - { "flock", UNIFYFS_WRAP(flock), &UNIFYFS_REAL(flock) }, - { "mmap", UNIFYFS_WRAP(mmap), &UNIFYFS_REAL(mmap) }, - { "msync", UNIFYFS_WRAP(msync), &UNIFYFS_REAL(msync) }, - { "munmap", UNIFYFS_WRAP(munmap), &UNIFYFS_REAL(munmap) }, - { "mmap64", UNIFYFS_WRAP(mmap64), &UNIFYFS_REAL(mmap64) }, - { "__fxstat", UNIFYFS_WRAP(__fxstat), &UNIFYFS_REAL(__fxstat) }, - { "close", UNIFYFS_WRAP(close), &UNIFYFS_REAL(close) }, - { "opendir", UNIFYFS_WRAP(opendir), &UNIFYFS_REAL(opendir) }, - { "fdopendir", UNIFYFS_WRAP(fdopendir), &UNIFYFS_REAL(fdopendir) }, - { "closedir", UNIFYFS_WRAP(closedir), &UNIFYFS_REAL(closedir) }, - { "readdir", UNIFYFS_WRAP(readdir), &UNIFYFS_REAL(readdir) }, - { "rewinddir", UNIFYFS_WRAP(rewinddir), &UNIFYFS_REAL(rewinddir) }, - { "dirfd", UNIFYFS_WRAP(dirfd), &UNIFYFS_REAL(dirfd) }, - { "telldir", UNIFYFS_WRAP(telldir), &UNIFYFS_REAL(telldir) }, - { "scandir", UNIFYFS_WRAP(scandir), &UNIFYFS_REAL(scandir) }, - { "seekdir", UNIFYFS_WRAP(seekdir), &UNIFYFS_REAL(seekdir) }, - { "fopen", UNIFYFS_WRAP(fopen), &UNIFYFS_REAL(fopen) }, - { "freopen", UNIFYFS_WRAP(freopen), &UNIFYFS_REAL(freopen) }, - { "setvbuf", UNIFYFS_WRAP(setvbuf), &UNIFYFS_REAL(setvbuf) }, - { "setbuf", UNIFYFS_WRAP(setbuf), &UNIFYFS_REAL(setbuf) }, - { "ungetc", UNIFYFS_WRAP(ungetc), &UNIFYFS_REAL(ungetc) }, - { "fgetc", UNIFYFS_WRAP(fgetc), &UNIFYFS_REAL(fgetc) }, - { "fputc", UNIFYFS_WRAP(fputc), &UNIFYFS_REAL(fputc) }, - { "getc", UNIFYFS_WRAP(getc), &UNIFYFS_REAL(getc) }, - { "putc", UNIFYFS_WRAP(putc), &UNIFYFS_REAL(putc) }, - { "fgets", UNIFYFS_WRAP(fgets), &UNIFYFS_REAL(fgets) }, - { "fputs", UNIFYFS_WRAP(fputs), &UNIFYFS_REAL(fputs) }, - { "fread", UNIFYFS_WRAP(fread), &UNIFYFS_REAL(fread) }, - { "fwrite", UNIFYFS_WRAP(fwrite), &UNIFYFS_REAL(fwrite) }, - { "fprintf", UNIFYFS_WRAP(fprintf), &UNIFYFS_REAL(fprintf) }, - { "vfprintf", UNIFYFS_WRAP(vfprintf), &UNIFYFS_REAL(vfprintf) }, - { "fscanf", UNIFYFS_WRAP(fscanf), &UNIFYFS_REAL(fscanf) }, - { "vfscanf", UNIFYFS_WRAP(vfscanf), &UNIFYFS_REAL(vfscanf) }, - { "fseek", UNIFYFS_WRAP(fseek), &UNIFYFS_REAL(fseek) }, - { "fseeko", UNIFYFS_WRAP(fseeko), &UNIFYFS_REAL(fseeko) }, - { "ftell", UNIFYFS_WRAP(ftell), &UNIFYFS_REAL(ftell) }, - { "ftello", UNIFYFS_WRAP(ftello), &UNIFYFS_REAL(ftello) }, - { "rewind", UNIFYFS_WRAP(rewind), &UNIFYFS_REAL(rewind) }, - { "fgetpos", UNIFYFS_WRAP(fgetpos), &UNIFYFS_REAL(fgetpos) }, - { "fsetpos", UNIFYFS_WRAP(fsetpos), &UNIFYFS_REAL(fsetpos) }, - { "fflush", UNIFYFS_WRAP(fflush), &UNIFYFS_REAL(fflush) }, - { "feof", UNIFYFS_WRAP(feof), &UNIFYFS_REAL(feof) }, - { "ferror", UNIFYFS_WRAP(ferror), &UNIFYFS_REAL(ferror) }, - { "clearerr", UNIFYFS_WRAP(clearerr), &UNIFYFS_REAL(clearerr) }, - { "fileno", UNIFYFS_WRAP(fileno), &UNIFYFS_REAL(fileno) }, - { "fclose", UNIFYFS_WRAP(fclose), &UNIFYFS_REAL(fclose) }, - { "fwprintf", UNIFYFS_WRAP(fwprintf), &UNIFYFS_REAL(fwprintf) }, - { "fwscanf", UNIFYFS_WRAP(fwscanf), &UNIFYFS_REAL(fwscanf) }, - { "vfwprintf", UNIFYFS_WRAP(vfwprintf), &UNIFYFS_REAL(vfwprintf) }, - { "vfwscanf", UNIFYFS_WRAP(vfwscanf), &UNIFYFS_REAL(vfwscanf) }, - { "fgetwc", UNIFYFS_WRAP(fgetwc), &UNIFYFS_REAL(fgetwc) }, - { "fgetws", UNIFYFS_WRAP(fgetws), &UNIFYFS_REAL(fgetws) }, - { "fputwc", UNIFYFS_WRAP(fputwc), &UNIFYFS_REAL(fputwc) }, - { "fputws", UNIFYFS_WRAP(fputws), &UNIFYFS_REAL(fputws) }, - { "fwide", UNIFYFS_WRAP(fwide), &UNIFYFS_REAL(fwide) }, - { "getwc", UNIFYFS_WRAP(getwc), &UNIFYFS_REAL(getwc) }, - { "putwc", UNIFYFS_WRAP(putwc), &UNIFYFS_REAL(putwc) }, - { "ungetwc", UNIFYFS_WRAP(ungetwc), &UNIFYFS_REAL(ungetwc) }, -}; - -#define GOTCHA_NFUNCS (sizeof(wrap_unifyfs_list) / sizeof(wrap_unifyfs_list[0])) diff --git a/client/src/pmpi_wrappers.c b/client/src/pmpi_wrappers.c index b94958709..e9f137434 100644 --- a/client/src/pmpi_wrappers.c +++ b/client/src/pmpi_wrappers.c @@ -12,9 +12,12 @@ * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ + +#include + #include "pmpi_wrappers.h" #include "unifyfs.h" -#include +#include "unifyfs_rc.h" int unifyfs_mpi_init(int* argc, char*** argv) { diff --git a/client/src/unifyfs-dirops.c b/client/src/unifyfs-dirops.c index 5d9a37f47..dcb7ca710 100644 --- a/client/src/unifyfs-dirops.c +++ b/client/src/unifyfs-dirops.c @@ -11,7 +11,6 @@ * For details, see https://github.com/LLNL/UnifyFS. * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text. */ -#include #include "unifyfs-sysio.h" diff --git a/client/src/unifyfs-dirops.h b/client/src/unifyfs-dirops.h index 7694ed492..8f239461c 100644 --- a/client/src/unifyfs-dirops.h +++ b/client/src/unifyfs-dirops.h @@ -14,11 +14,7 @@ #ifndef __UNIFYFS_DIROPS_H #define __UNIFYFS_DIROPS_H -#include - -#include -#include -#include +#include "unifyfs-internal.h" /* * FIXME: is this portable to use the linux dirent structure? diff --git a/client/src/unifyfs-internal.h b/client/src/unifyfs-internal.h index 6b472332e..d6dca5260 100644 --- a/client/src/unifyfs-internal.h +++ b/client/src/unifyfs-internal.h @@ -92,6 +92,10 @@ #include #include +#ifdef HAVE_SYS_STATFS_H +#include +#endif + // common headers #include "unifyfs_configurator.h" #include "unifyfs_const.h" @@ -119,26 +123,32 @@ unifyfs_unsupported(__func__, __FILE__, __LINE__, fmt, ##args) #ifdef UNIFYFS_GOTCHA +#include -/* gotcha fills in address of original/real function - * and we need to declare function prototype for each - * wrapper */ -#define UNIFYFS_DECL(name,ret,args) \ - extern ret(*__real_ ## name)args; \ - ret __wrap_ ## name args; - -/* define each DECL function in a .c file */ -#define UNIFYFS_DEF(name,ret,args) \ - ret(*__real_ ## name)args = NULL; - -/* we define our wrapper function as __wrap_ instead of */ +/* the name of our wrapper - we use __wrap_ instead of */ #define UNIFYFS_WRAP(name) __wrap_ ## name -/* gotcha maps the call to __real_() */ +/* the name of the real function pointer */ #define UNIFYFS_REAL(name) __real_ ## name -/* no need to look up the address of the real function (gotcha does that) */ -#define MAP_OR_FAIL(func) +/* declare anything that will be used externally */ +#define UNIFYFS_DECL(name, ret, args) \ + extern gotcha_wrappee_handle_t wrappee_handle_ ## name; \ + extern ret (*__real_ ## name) args; \ + ret __wrap_ ## name args + +/* ask gotcha for the address of the real function */ +#define MAP_OR_FAIL(name) \ +do { \ + if (NULL == __real_ ## name) { \ + __real_ ## name = gotcha_get_wrappee(wrappee_handle_ ## name); \ + if (NULL == __real_ ## name) { \ + assert(!"missing Gotcha wrappee for " #name); \ + } \ + } \ +} while (0) + +int setup_gotcha_wrappers(void); #elif UNIFYFS_PRELOAD diff --git a/client/src/unifyfs-sysio.c b/client/src/unifyfs-sysio.c index 945308ba7..94c80c7ce 100644 --- a/client/src/unifyfs-sysio.c +++ b/client/src/unifyfs-sysio.c @@ -527,6 +527,77 @@ int UNIFYFS_WRAP(__fxstat)(int vers, int fd, struct stat* buf) } #endif + +#ifdef HAVE_SYS_STATFS_H + +/* tmpfs seems like a safe choice for something like UnifyFS */ +#ifndef TMPFS_MAGIC +#define TMPFS_MAGIC 0x01021994 +#endif + +static int unifyfs_statfs(struct statfs* fsbuf) +{ + if (NULL != fsbuf) { + memset(fsbuf, 0, sizeof(*fsbuf)); + + fsbuf->f_type = TMPFS_MAGIC; /* File system type */ + fsbuf->f_bsize = UNIFYFS_LOGIO_CHUNK_SIZE; /* Optimal block size */ + //fsbuf->f_blocks = ??; /* Total data blocks in filesystem */ + //fsbuf->f_bfree = ??; /* Free blocks in filesystem */ + //fsbuf->f_bavail = ??; /* Free blocks available */ + fsbuf->f_files = unifyfs_max_files; /* Total file nodes */ + //fsbuf->f_ffree = ??; /* Free file nodes in filesystem */ + fsbuf->f_namelen = UNIFYFS_MAX_FILENAME; /* Max filename length */ + return 0; + } else { + return EFAULT; + } +} + +#ifdef HAVE_STATFS +int UNIFYFS_WRAP(statfs)(const char* path, struct statfs* fsbuf) +{ + LOGDBG("statfs() was called for %s", path); + + int ret; + char upath[UNIFYFS_MAX_FILENAME]; + if (unifyfs_intercept_path(path, upath)) { + ret = unifyfs_statfs(fsbuf); + if (ret) { + errno = ret; + ret = -1; + } + } else { + MAP_OR_FAIL(statfs); + ret = UNIFYFS_REAL(statfs)(path, fsbuf); + } + return ret; +} +#endif + +#ifdef HAVE_FSTATFS +int UNIFYFS_WRAP(fstatfs)(int fd, struct statfs* fsbuf) +{ + LOGDBG("fstatfs() was called for fd: %d", fd); + + /* check whether we should intercept this file descriptor */ + int ret; + if (unifyfs_intercept_fd(&fd)) { + ret = unifyfs_statfs(fsbuf); + if (ret) { + errno = ret; + ret = -1; + } + } else { + MAP_OR_FAIL(fstatfs); + ret = UNIFYFS_REAL(fstatfs)(fd, fsbuf); + } + return ret; +} +#endif + +#endif /* HAVE_SYS_STATFS_H */ + /* --------------------------------------- * POSIX wrappers: file descriptors * --------------------------------------- */ diff --git a/client/src/unifyfs-sysio.h b/client/src/unifyfs-sysio.h index 18716f807..45ba6f5cb 100644 --- a/client/src/unifyfs-sysio.h +++ b/client/src/unifyfs-sysio.h @@ -60,10 +60,10 @@ UNIFYFS_DECL(remove, int, (const char* path)); UNIFYFS_DECL(rename, int, (const char* oldpath, const char* newpath)); UNIFYFS_DECL(truncate, int, (const char* path, off_t length)); UNIFYFS_DECL(stat, int, (const char* path, struct stat* buf)); -UNIFYFS_DECL(fstat, int, (int fd, struct stat* buf)); UNIFYFS_DECL(__xstat, int, (int vers, const char* path, struct stat* buf)); UNIFYFS_DECL(__lxstat, int, (int vers, const char* path, struct stat* buf)); -UNIFYFS_DECL(__fxstat, int, (int vers, int fd, struct stat* buf)); +UNIFYFS_DECL(statfs, int, (const char* path, struct statfs* fsbuf)); + /* --------------------------------------- * POSIX wrappers: file descriptors @@ -89,6 +89,9 @@ UNIFYFS_DECL(posix_fadvise, int, (int fd, off_t offset, off_t len, int advice)); UNIFYFS_DECL(lseek, off_t, (int fd, off_t offset, int whence)); UNIFYFS_DECL(lseek64, off64_t, (int fd, off64_t offset, int whence)); UNIFYFS_DECL(ftruncate, int, (int fd, off_t length)); +UNIFYFS_DECL(fstat, int, (int fd, struct stat* buf)); +UNIFYFS_DECL(__fxstat, int, (int vers, int fd, struct stat* buf)); +UNIFYFS_DECL(fstatfs, int, (int fd, struct statfs* fsbuf)); UNIFYFS_DECL(fsync, int, (int fd)); UNIFYFS_DECL(fdatasync, int, (int fd)); UNIFYFS_DECL(flock, int, (int fd, int operation)); diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index 62e3a78ce..e472c7821 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -46,12 +46,6 @@ #include -#ifdef UNIFYFS_GOTCHA -#include "gotcha/gotcha_types.h" -#include "gotcha/gotcha.h" -#include "gotcha_map_unifyfs_list.h" -#endif - #include "unifyfs_client_rpcs.h" #include "unifyfs_server_rpcs.h" #include "unifyfs_rpc_util.h" @@ -2184,20 +2178,10 @@ static int unifyfs_init(void) if (!unifyfs_initialized) { #ifdef UNIFYFS_GOTCHA - /* insert our I/O wrappers using gotcha */ - enum gotcha_error_t result; - result = gotcha_wrap(wrap_unifyfs_list, GOTCHA_NFUNCS, "unifyfs"); - if (result != GOTCHA_SUCCESS) { - LOGERR("gotcha_wrap returned %d", (int) result); - } - - /* check for an errors when registering functions with gotcha */ - for (i = 0; i < GOTCHA_NFUNCS; i++) { - if (*(void**)(wrap_unifyfs_list[i].function_address_pointer) == 0) { - LOGERR("This function name failed to be wrapped: %s", - wrap_unifyfs_list[i].name); - - } + rc = setup_gotcha_wrappers(); + if (rc != UNIFYFS_SUCCESS) { + LOGERR("failed to setup gotcha wrappers"); + return rc; } #endif diff --git a/common/src/unifyfs_rc.h b/common/src/unifyfs_rc.h index fb96635c4..ed14d46e1 100644 --- a/common/src/unifyfs_rc.h +++ b/common/src/unifyfs_rc.h @@ -40,6 +40,7 @@ */ #define UNIFYFS_ERROR_ENUMERATOR \ ENUMITEM(BADCONFIG, "Configuration has invalid setting") \ + ENUMITEM(GOTCHA, "Gotcha operation error") \ ENUMITEM(KEYVAL, "Key-value store operation error") \ ENUMITEM(MARGO, "Mercury/Argobots operation error") \ ENUMITEM(MDHIM, "MDHIM operation error") \ diff --git a/configure.ac b/configure.ac index 73aa2eabc..ebf9f3439 100755 --- a/configure.ac +++ b/configure.ac @@ -60,10 +60,12 @@ AC_CHECK_MEMBERS([struct stat.st_rdev]) AC_CHECK_TYPES([ptrdiff_t]) # Checks for header files. -AC_CHECK_HEADERS([fcntl.h limits.h stdlib.h string.h sys/socket.h sys/time.h]) -AC_CHECK_HEADERS([unistd.h arpa/inet.h inttypes.h netdb.h netinet/in.h]) -AC_CHECK_HEADERS([stddef.h stdint.h libgen.h strings.h syslog.h]) -AC_CHECK_HEADERS([inttypes.h wchar.h wctype.h]) +AC_CHECK_HEADERS([stddef.h stdint.h stdlib.h string.h unistd.h]) +AC_CHECK_HEADERS([fcntl.h inttypes.h libgen.h limits.h mntent.h strings.h syslog.h]) +AC_CHECK_HEADERS([wchar.h wctype.h]) +AC_CHECK_HEADERS([sys/mount.h sys/socket.h sys/statfs.h sys/time.h]) +AC_CHECK_HEADERS([arpa/inet.h netdb.h netinet/in.h]) + AC_CHECK_HEADER([openssl/md5.h], [], [AC_MSG_FAILURE([*** openssl/md5.h missing, openssl-devel package required])]) @@ -198,7 +200,6 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]],[[void* __wrap_malloc(si ]) LDFLAGS=$OLD_LDFLAGS -AC_CHECK_HEADERS(mntent.h sys/mount.h) # HDF found? AX_LIB_HDF5 @@ -216,6 +217,7 @@ AC_CHECK_FUNCS(lio_listio,[ CP_WRAPPERS+=",-wrap,lio_listio" ], []) LIBS=$OLD_LIBS + CP_WRAPPERS+=",-wrap,mkdir" CP_WRAPPERS+=",-wrap,rmdir" CP_WRAPPERS+=",-wrap,unlink" @@ -225,13 +227,20 @@ CP_WRAPPERS+=",-wrap,truncate" CP_WRAPPERS+=",-wrap,stat" CP_WRAPPERS+=",-wrap,fstat" + +AC_CHECK_FUNCS(statfs,[ + CP_WRAPPERS+=",-wrap,statfs" +],[]) +AC_CHECK_FUNCS(fstatfs,[ + CP_WRAPPERS+=",-wrap,fstatfs" +],[]) + AC_CHECK_FUNCS(__lxstat,[ CP_WRAPPERS+=",-wrap,__lxstat" ],[]) AC_CHECK_FUNCS(__xstat,[ CP_WRAPPERS+=",-wrap,__xstat" ],[]) - AC_CHECK_FUNCS(__fxstat,[ CP_WRAPPERS+=",-wrap,__fxstat" ],[]) diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am index 7721fc921..5161dae80 100644 --- a/examples/src/Makefile.am +++ b/examples/src/Makefile.am @@ -74,6 +74,10 @@ noinst_HEADERS = \ test_cppflags = $(AM_CPPFLAGS) $(MPI_CFLAGS) \ -I$(top_srcdir)/client/src -I$(top_srcdir)/common/src +if USE_PMPI_WRAPPERS +test_cppflags += -DENABLE_MPI_MOUNT +endif + if HAVE_FORTRAN test_ftn_flags = $(AM_FCFLAGS) $(MPI_FFLAGS) \ -I$(top_srcdir)/client/src -I$(top_srcdir)/common/src diff --git a/examples/src/checkpoint-restart.c b/examples/src/checkpoint-restart.c index 101d89c7f..3c4ef7206 100644 --- a/examples/src/checkpoint-restart.c +++ b/examples/src/checkpoint-restart.c @@ -177,6 +177,8 @@ int verify_restart_data(test_cfg* cfg, char* data, uint64_t last_ckpt_id) * cfg.use_mapio - support is not yet implemented. When enabled, * direct memory loads and stores will be used for reads and writes. * + * cfg.use_mpiio - when enabled, MPI-IO will be used. + * * cfg.use_prdwr - when enabled, pread(2) and pwrite(2) will be used. * * cfg.use_stdio - when enabled, fread(2) and fwrite(2) will be used. diff --git a/examples/src/multi-write.c b/examples/src/multi-write.c index 8ff9402bb..4c95cdb18 100644 --- a/examples/src/multi-write.c +++ b/examples/src/multi-write.c @@ -130,7 +130,7 @@ int do_test(test_cfg* cfg) /* Write our files */ for (i = 0; i < NUM_WRITES; i++) { - /* Randomly pick on of our files to write to */ + /* Randomly pick one of our files to write to */ rnd = rand() % NUM_FILES; fd = fds[rnd]; diff --git a/examples/src/read.c b/examples/src/read.c index 5616d74cb..ae5630ad3 100644 --- a/examples/src/read.c +++ b/examples/src/read.c @@ -89,6 +89,8 @@ size_t generate_read_reqs(test_cfg* cfg, char* dstbuf, * cfg.use_mapio - support is not yet implemented. When enabled, * direct memory loads will be used for reads. * + * cfg.use_mpiio - when enabled, MPI-IO will be used. + * * cfg.use_prdwr - when enabled, pread(2) will be used. * * cfg.use_stdio - when enabled, fread(3) will be used. diff --git a/examples/src/testutil.h b/examples/src/testutil.h index 6bc82506b..80677cc18 100644 --- a/examples/src/testutil.h +++ b/examples/src/testutil.h @@ -118,6 +118,7 @@ typedef struct { int verbose; /* print verbose information to stderr */ int use_mpi; int use_unifyfs; + int enable_mpi_mount; /* automount during MPI_Init() */ /* I/O behavior options */ int io_pattern; /* N1 or NN */ @@ -126,6 +127,7 @@ typedef struct { int use_aio; /* use asynchronous IO */ int use_lio; /* use lio_listio instead of read/write */ int use_mapio; /* use mmap instead of read/write */ + int use_mpiio; /* use MPI-IO instead of POSIX I/O */ int use_prdwr; /* use pread/pwrite instead of read/write */ int use_stdio; /* use fread/fwrite instead of read/write */ int use_vecio; /* use readv/writev instead of read/write */ @@ -139,16 +141,19 @@ typedef struct { char* filename; char* mountpt; FILE* fp; + int fd; + int fd_access; /* access flags for cfg.fd */ void* mapped; /* address of mapped extent of cfg.fd */ off_t mapped_off; /* start offset for mapped extent */ size_t mapped_sz; /* size of mapped extent */ - int fd; - int fd_access; /* access flags for cfg.fd */ + MPI_File mpifh; /* MPI file handle (when use_mpiio) */ /* MPI info */ - int app_id; int rank; int n_ranks; + + /* UnifyFS info */ + int app_id; } test_cfg; static inline @@ -168,6 +173,10 @@ void test_config_init(test_cfg* cfg) cfg->use_unifyfs = 1; cfg->io_pattern = IO_PATTERN_N1; +#if defined(ENABLE_MPI_MOUNT) + cfg->enable_mpi_mount = 1; +#endif + // invalidate file descriptor cfg->fd = -1; @@ -190,6 +199,7 @@ void test_config_print(test_cfg* cfg) fprintf(stderr, "\t verbose = %d\n", cfg->verbose); fprintf(stderr, "\t use_mpi = %d\n", cfg->use_mpi); fprintf(stderr, "\t use_unifyfs = %d\n", cfg->use_unifyfs); + fprintf(stderr, "\t mpi_mount = %d\n", cfg->enable_mpi_mount); fprintf(stderr, "\n-- IO Behavior --\n"); fprintf(stderr, "\t io_pattern = %s\n", io_pattern_str(cfg->io_pattern)); @@ -198,6 +208,7 @@ void test_config_print(test_cfg* cfg) fprintf(stderr, "\t use_aio = %d\n", cfg->use_aio); fprintf(stderr, "\t use_lio = %d\n", cfg->use_lio); fprintf(stderr, "\t use_mapio = %d\n", cfg->use_mapio); + fprintf(stderr, "\t use_mpiio = %d\n", cfg->use_mpiio); fprintf(stderr, "\t use_prdwr = %d\n", cfg->use_prdwr); fprintf(stderr, "\t use_stdio = %d\n", cfg->use_stdio); fprintf(stderr, "\t use_vecio = %d\n", cfg->use_vecio); @@ -447,7 +458,7 @@ int test_is_static(const char* program) // common options for all tests -static const char* test_short_opts = "a:Ab:c:df:hkLm:Mn:p:PSUvVx"; +static const char* test_short_opts = "a:Ab:c:df:hkLm:MNn:p:PSUvVx"; static const struct option test_long_opts[] = { { "appid", 1, 0, 'a' }, @@ -460,8 +471,9 @@ static const struct option test_long_opts[] = { { "check", 0, 0, 'k' }, { "listio", 0, 0, 'L' }, { "mount", 1, 0, 'm' }, - { "mapio", 0, 0, 'M' }, + { "mpiio", 0, 0, 'M' }, { "nblocks", 1, 0, 'n' }, + { "mapio", 0, 0, 'N' }, { "pattern", 1, 0, 'p' }, { "prdwr", 0, 0, 'P' }, { "stdio", 0, 0, 'S' }, @@ -495,10 +507,12 @@ static const char* test_usage_str = " (default: off)\n" " -m, --mount= use for unifyfs\n" " (default: /unifyfs)\n" - " -M, --mapio use mmap instead of read|write\n" + " -M, --mpiio use MPI-IO instead of POSIX I/O\n" " (default: off)\n" " -n, --nblocks= count of blocks each process will read|write\n" " (default: 32)\n" + " -N, --mapio use mmap instead of read|write\n" + " (default: off)\n" " -p, --pattern= 'n1' (N-to-1 shared file) or 'nn' (N-to-N file per process)\n" " (default: 'n1')\n" " -P, --prdwr use pread|pwrite instead of read|write\n" @@ -573,13 +587,17 @@ int test_process_argv(test_cfg* cfg, break; case 'M': - cfg->use_mapio = 1; + cfg->use_mpiio = 1; break; case 'n': cfg->n_blocks = (uint64_t) strtoul(optarg, NULL, 0); break; + case 'N': + cfg->use_mapio = 1; + break; + case 'p': cfg->io_pattern = check_io_pattern(optarg); break; @@ -652,40 +670,49 @@ int test_process_argv(test_cfg* cfg, // exhaustive check of incompatible I/O modes if (cfg->use_aio && - (cfg->use_mapio || cfg->use_prdwr || cfg->use_stdio || cfg->use_vecio)) { - test_print_once(cfg, "USAGE ERROR: --aio incompatible with " - "[--mapio, --prdwr, --stdio, --vecio]"); + (cfg->use_mapio || cfg->use_mpiio || cfg->use_prdwr + || cfg->use_stdio || cfg->use_vecio)) { + test_print_once(cfg, + "USAGE ERROR: --aio incompatible with " + "[--mapio, --mpiio, --prdwr, --stdio, --vecio]"); exit(-1); } - if (cfg->use_lio && - (cfg->use_mapio || cfg->use_prdwr || cfg->use_stdio || cfg->use_vecio)) { - test_print_once(cfg, "USAGE ERROR: --listio incompatible with " - "[--mapio, --prdwr, --stdio, --vecio]"); + (cfg->use_mapio || cfg->use_mpiio || cfg->use_prdwr + || cfg->use_stdio || cfg->use_vecio)) { + test_print_once(cfg, + "USAGE ERROR: --listio incompatible with " + "[--mapio, --mpiio, --prdwr, --stdio, --vecio]"); exit(-1); } - if (cfg->use_mapio && - (cfg->use_prdwr || cfg->use_stdio || cfg->use_vecio)) { - test_print_once(cfg, "USAGE ERROR: --mapio incompatible with " - "[--aio, --listio, --prdwr, --stdio, --vecio]"); + (cfg->use_mpiio || cfg->use_prdwr || cfg->use_stdio + || cfg->use_vecio)) { + test_print_once(cfg, + "USAGE ERROR: --mapio incompatible with " + "[--aio, --listio, --mpiio, --prdwr, --stdio, --vecio]"); exit(-1); } - - if (cfg->use_prdwr && - (cfg->use_stdio || cfg->use_vecio)) { - test_print_once(cfg, "USAGE ERROR: --prdwr incompatible with " - "[--aio, --listio, --mapio, --stdio, --vecio]"); + if (cfg->use_mpiio && + (cfg->use_prdwr || cfg->use_stdio || cfg->use_vecio)) { + test_print_once(cfg, + "USAGE ERROR: --mpiio incompatible with " + "[--aio, --listio, --mpiio, --prdwr, --stdio, --vecio]"); + exit(-1); + } + if (cfg->use_prdwr && (cfg->use_stdio || cfg->use_vecio)) { + test_print_once(cfg, + "USAGE ERROR: --prdwr incompatible with " + "[--aio, --listio, --mapio, --mpiio, --stdio, --vecio]"); exit(-1); } - if (cfg->use_stdio && cfg->use_vecio) { - test_print_once(cfg, "USAGE ERROR: --stdio incompatible with " - "[--aio, --listio, --mapio, --prdwr, --vecio]"); + test_print_once(cfg, + "USAGE ERROR: --stdio incompatible with " + "[--aio, --listio, --mapio, --mpiio, --prdwr, --vecio]"); exit(-1); } - if (NULL == cfg->filename) { // set filename default cfg->filename = strdup("testfile"); @@ -750,13 +777,37 @@ int lipsum_check(const char* buf, uint64_t len, uint64_t offset, /* ---------- MPI Utilities ---------- */ +/* MPI checker + * from: https://stackoverflow.com/questions/22859269/ + */ +#define MPI_CHECK(cfgp, fncall) \ + do { \ + mpi_error = 0; \ + int _merr = fncall; \ + if (_merr != MPI_SUCCESS) { \ + mpi_error = _merr; \ + handle_mpi_error(cfgp, #fncall); \ + } \ + } while (0) + +static int mpi_error; + +static inline +void handle_mpi_error(test_cfg* cfg, char* context) +{ + char errstr[MPI_MAX_ERROR_STRING]; + int len = 0; + MPI_Error_string(mpi_error, errstr, &len); + test_print(cfg, "MPI ERROR: %s returned %s", context, errstr); +} + static inline void test_barrier(test_cfg* cfg) { assert(NULL != cfg); if (cfg->use_mpi) { - MPI_Barrier(MPI_COMM_WORLD); + MPI_CHECK(cfg, (MPI_Barrier(MPI_COMM_WORLD))); } } @@ -768,8 +819,9 @@ double test_reduce_double_sum(test_cfg* cfg, double local_val) assert(NULL != cfg); if (cfg->use_mpi) { - MPI_Reduce(&local_val, &aggr_val, - 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); + MPI_CHECK(cfg, (MPI_Reduce(&local_val, &aggr_val, + 1, MPI_DOUBLE, MPI_SUM, + 0, MPI_COMM_WORLD))); } else { aggr_val = local_val; } @@ -784,8 +836,9 @@ double test_reduce_double_max(test_cfg* cfg, double local_val) assert(NULL != cfg); if (cfg->use_mpi) { - MPI_Reduce(&local_val, &aggr_val, - 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); + MPI_CHECK(cfg, (MPI_Reduce(&local_val, &aggr_val, + 1, MPI_DOUBLE, MPI_MAX, + 0, MPI_COMM_WORLD))); } else { aggr_val = local_val; } @@ -800,8 +853,9 @@ double test_reduce_double_min(test_cfg* cfg, double local_val) assert(NULL != cfg); if (cfg->use_mpi) { - MPI_Reduce(&local_val, &aggr_val, - 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); + MPI_CHECK(cfg, (MPI_Reduce(&local_val, &aggr_val, + 1, MPI_DOUBLE, MPI_MIN, + 0, MPI_COMM_WORLD))); } else { aggr_val = local_val; } @@ -841,6 +895,21 @@ const char* test_access_to_stdio_mode(int access) return NULL; } +static int test_access_to_mpiio_mode(int access) +{ + switch (access) { + case O_RDWR: + return MPI_MODE_RDWR; + case O_RDONLY: + return MPI_MODE_RDONLY; + case O_WRONLY: + return MPI_MODE_WRONLY; + default: + break; + } + return 0; +} + /* * open the given file */ @@ -853,7 +922,19 @@ int test_open_file(test_cfg* cfg, const char* filepath, int access) assert(NULL != cfg); - if (cfg->use_stdio) { + if (cfg->use_mpiio) { + int amode = test_access_to_mpiio_mode(access); + if (cfg->io_pattern == IO_PATTERN_N1) { + MPI_CHECK(cfg, (MPI_File_open(MPI_COMM_WORLD, filepath, amode, + MPI_INFO_NULL, &cfg->mpifh))); + } else { + MPI_CHECK(cfg, (MPI_File_open(MPI_COMM_SELF, filepath, amode, + MPI_INFO_NULL, &cfg->mpifh))); + } + if (mpi_error) { + return -1; + } + } else if (cfg->use_stdio) { fmode = test_access_to_stdio_mode(access); fp = fopen(filepath, fmode); if (NULL == fp) { @@ -861,16 +942,15 @@ int test_open_file(test_cfg* cfg, const char* filepath, int access) return -1; } cfg->fp = fp; - return 0; - } - - fd = open(filepath, access); - if (-1 == fd) { - test_print(cfg, "ERROR: open(%s) failed", filepath); - return -1; + } else { + fd = open(filepath, access); + if (-1 == fd) { + test_print(cfg, "ERROR: open(%s) failed", filepath); + return -1; + } + cfg->fd = fd; + cfg->fd_access = access; } - cfg->fd = fd; - cfg->fd_access = access; return 0; } @@ -882,6 +962,10 @@ int test_close_file(test_cfg* cfg) { assert(NULL != cfg); + if (cfg->use_mpiio) { + MPI_CHECK(cfg, (MPI_File_close(&cfg->mpifh))); + } + if (NULL != cfg->fp) { fclose(cfg->fp); } @@ -910,13 +994,29 @@ int test_create_file(test_cfg* cfg, const char* filepath, int access) assert(NULL != cfg); - if (cfg->use_stdio) { - fmode = test_access_to_stdio_mode(access); + if (cfg->use_mpiio) { + create_mode = test_access_to_mpiio_mode(access); + create_mode |= MPI_MODE_CREATE; + if (cfg->io_pattern == IO_PATTERN_N1) { + MPI_CHECK(cfg, (MPI_File_open(MPI_COMM_WORLD, filepath, create_mode, + MPI_INFO_NULL, &cfg->mpifh))); + } else { + create_mode |= MPI_MODE_EXCL; + MPI_CHECK(cfg, (MPI_File_open(MPI_COMM_SELF, filepath, create_mode, + MPI_INFO_NULL, &cfg->mpifh))); + } + if (mpi_error) { + return -1; + } + return 0; } - // rank 0 creates or all ranks create if using file-per-process + /* POSIX I/O + * N-to-1 - rank 0 creates shared files + * N-to-N - all ranks create per-process files */ if (cfg->rank == 0 || cfg->io_pattern == IO_PATTERN_NN) { if (cfg->use_stdio) { + fmode = test_access_to_stdio_mode(access); fp = fopen(filepath, fmode); if (NULL == fp) { test_print(cfg, "ERROR: fopen(%s) failed", filepath); @@ -1036,7 +1136,7 @@ int test_init(int argc, char** argv, } if (cfg->use_mpi) { - MPI_Init(&argc, &argv); + MPI_CHECK(cfg, (MPI_Init(&argc, &argv))); MPI_Comm_size(MPI_COMM_WORLD, &(cfg->n_ranks)); MPI_Comm_rank(MPI_COMM_WORLD, &(cfg->rank)); } else { @@ -1048,7 +1148,7 @@ int test_init(int argc, char** argv, test_config_print(cfg); } - if (cfg->use_unifyfs) { + if (cfg->use_unifyfs && !cfg->enable_mpi_mount) { #ifndef DISABLE_UNIFYFS if (cfg->debug) { test_pause(cfg, "Before unifyfs_mount()"); @@ -1060,10 +1160,10 @@ int test_init(int argc, char** argv, } #endif test_barrier(cfg); - } else { - if (cfg->debug) { - test_pause(cfg, "Finished test initialization"); - } + } + + if (cfg->debug) { + test_pause(cfg, "Finished test initialization"); } return 0; @@ -1082,7 +1182,7 @@ void test_fini(test_cfg* cfg) test_close_file(cfg); - if (cfg->use_unifyfs) { + if (cfg->use_unifyfs && !cfg->enable_mpi_mount) { #ifndef DISABLE_UNIFYFS int rc = unifyfs_unmount(); if (rc) { @@ -1092,7 +1192,7 @@ void test_fini(test_cfg* cfg) } if (cfg->use_mpi) { - MPI_Finalize(); + MPI_CHECK(cfg, (MPI_Finalize())); } if (NULL != cfg->filename) { diff --git a/examples/src/testutil_rdwr.h b/examples/src/testutil_rdwr.h index d2c0db012..367f8d2c8 100644 --- a/examples/src/testutil_rdwr.h +++ b/examples/src/testutil_rdwr.h @@ -24,14 +24,16 @@ int issue_write_req(test_cfg* cfg, struct aiocb* req) { int rc, err; ssize_t ss; - size_t written, remaining; off_t off; void* src; assert(NULL != cfg); - errno = 0; + size_t written = 0; + size_t remaining = req->aio_nbytes; + if (cfg->use_aio) { // aio_write(2) + errno = 0; rc = aio_write(req); if (-1 == rc) { test_print(cfg, "aio_write() failed"); @@ -39,11 +41,17 @@ int issue_write_req(test_cfg* cfg, struct aiocb* req) return rc; } else if (cfg->use_mapio) { // mmap(2) return ENOTSUP; + } else if (cfg->use_mpiio) { // MPI-IO + MPI_Status mst; + MPI_Offset off = (MPI_Offset) req->aio_offset; + void* src_buf = (void*) req->aio_buf; + int count = (int) remaining; + MPI_CHECK(cfg, (MPI_File_write_at(cfg->mpifh, off, src_buf, + count, MPI_CHAR, &mst))); } else if (cfg->use_prdwr) { // pwrite(2) - written = 0; - remaining = req->aio_nbytes; do { src = (void*)((char*)req->aio_buf + written); + errno = 0; ss = pwrite(req->aio_fildes, src, remaining, (req->aio_offset + written)); if (-1 == ss) { @@ -62,8 +70,7 @@ int issue_write_req(test_cfg* cfg, struct aiocb* req) } else if (cfg->use_vecio) { // writev(2) return EINVAL; } else { // write(2) - written = 0; - remaining = req->aio_nbytes; + errno = 0; off = lseek(req->aio_fildes, req->aio_offset, SEEK_SET); if (-1 == off) { test_print(cfg, "lseek() failed"); @@ -71,6 +78,7 @@ int issue_write_req(test_cfg* cfg, struct aiocb* req) } do { src = (void*)((char*)req->aio_buf + written); + errno = 0; ss = write(req->aio_fildes, src, remaining); if (-1 == ss) { err = errno; @@ -204,6 +212,8 @@ int write_sync(test_cfg* cfg) test_print(cfg, "fsync() failed"); return -1; } + } else if (cfg->use_mpiio) { + MPI_CHECK(cfg, (MPI_File_sync(cfg->mpifh))); } return 0; } @@ -267,8 +277,8 @@ int issue_read_req(test_cfg* cfg, struct aiocb* req) assert(NULL != cfg); - errno = 0; if (cfg->use_aio) { // aio_read(2) + errno = 0; rc = aio_read(req); if (-1 == rc) { test_print(cfg, "aio_read() failed"); @@ -276,11 +286,19 @@ int issue_read_req(test_cfg* cfg, struct aiocb* req) return rc; } else if (cfg->use_mapio) { // mmap(2) return ENOTSUP; + } else if (cfg->use_mpiio) { // MPI-IO + MPI_Status mst; + MPI_Offset off = (MPI_Offset) req->aio_offset; + void* dst_buf = (void*) req->aio_buf; + int count = (int) req->aio_nbytes; + MPI_CHECK(cfg, (MPI_File_read_at(cfg->mpifh, off, dst_buf, + count, MPI_CHAR, &mst))); } else if (cfg->use_prdwr) { // pread(2) nread = 0; remaining = req->aio_nbytes; do { dst = (void*)((char*)req->aio_buf + nread); + errno = 0; ss = pread(req->aio_fildes, dst, remaining, (req->aio_offset + nread)); if (-1 == ss) { @@ -304,6 +322,7 @@ int issue_read_req(test_cfg* cfg, struct aiocb* req) } else { // read(2) nread = 0; remaining = req->aio_nbytes; + errno = 0; off = lseek(req->aio_fildes, req->aio_offset, SEEK_SET); if (-1 == off) { test_print(cfg, "lseek() failed"); @@ -311,6 +330,7 @@ int issue_read_req(test_cfg* cfg, struct aiocb* req) } do { dst = (void*)((char*)req->aio_buf + nread); + errno = 0; ss = read(req->aio_fildes, dst, remaining); if (-1 == ss) { err = errno; diff --git a/examples/src/write.c b/examples/src/write.c index 4405f10f2..c89cd9177 100644 --- a/examples/src/write.c +++ b/examples/src/write.c @@ -102,6 +102,8 @@ size_t generate_write_reqs(test_cfg* cfg, char* srcbuf, * cfg.use_mapio - support is not yet implemented. When enabled, * direct memory loads and stores will be used for writes. * + * cfg.use_mpiio - when enabled, MPI-IO will be used. + * * cfg.use_prdwr - when enabled, pwrite(2) will be used. * * cfg.use_stdio - when enabled, fwrite(2) will be used. diff --git a/examples/src/writeread.c b/examples/src/writeread.c index a101990a4..8a63d791b 100644 --- a/examples/src/writeread.c +++ b/examples/src/writeread.c @@ -155,6 +155,8 @@ size_t generate_read_reqs(test_cfg* cfg, char* dstbuf, * cfg.use_mapio - support is not yet implemented. When enabled, * direct memory loads and stores will be used for reads and writes. * + * cfg.use_mpiio - when enabled, MPI-IO will be used. + * * cfg.use_prdwr - when enabled, pread(2) and pwrite(2) will be used. * * cfg.use_stdio - when enabled, fread(2) and fwrite(2) will be used. From 23c8a325f476715b7be8a0e23422f91a46a3433a Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Mon, 18 May 2020 16:32:06 -0400 Subject: [PATCH 116/168] update gotcha dependency to 1.0.3 --- docs/dependencies.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dependencies.rst b/docs/dependencies.rst index aebb50112..62466206f 100644 --- a/docs/dependencies.rst +++ b/docs/dependencies.rst @@ -2,7 +2,7 @@ UnifyFS Dependencies ==================== -- `GOTCHA `_ version 0.0.2 (compatibility with latest release in progress) +- `GOTCHA `_ version 1.0.3 - `leveldb `_ version 1.22 From 8a280c9b3c917ae3b071133261e5443c8bcf8c35 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Mon, 18 May 2020 15:52:14 -0700 Subject: [PATCH 117/168] Update bootstrap.sh to use GOTCHA 1.0.3 --- bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index 563f23041..cf05fb15e 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -54,7 +54,7 @@ cd .. echo "### building GOTCHA ###" cd GOTCHA # Unify won't build against latest GOTCHA, so use a known compatible version. -git checkout 0.0.2 +git checkout 1.0.3 mkdir -p build && cd build cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" .. make -j $(nproc) && make install From 0c9eb9ab10a9008ddb1789efebe62e10ab7a6eac Mon Sep 17 00:00:00 2001 From: "Michael J. Brim" Date: Tue, 19 May 2020 11:12:57 -0400 Subject: [PATCH 118/168] fixes for logio shutdown errors * reset logio context values in logio_close() * reset logio_ctx to NULL after logio_close() * avoid redundant client disconnect * safer resource manager thread cleanup --- client/src/unifyfs.c | 9 ++++++--- common/src/unifyfs_logio.c | 2 ++ server/src/unifyfs_request_manager.c | 17 +++++++++-------- server/src/unifyfs_request_manager.h | 2 +- server/src/unifyfs_server.c | 12 +++++++++--- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/client/src/unifyfs.c b/client/src/unifyfs.c index e472c7821..ab9966a4a 100644 --- a/client/src/unifyfs.c +++ b/client/src/unifyfs.c @@ -298,7 +298,7 @@ inline int unifyfs_intercept_path(const char* path, char* upath) int intercept = 0; if (strncmp(target, unifyfs_mount_prefix, unifyfs_mount_prefixlen) == 0) { /* characters in target up through mount point match, - * assume we match */ + * assume we match */ intercept = 1; /* if we have another character, it must be '/' */ @@ -2215,7 +2215,7 @@ static int unifyfs_init(void) if (strncmp(unifyfs_cwd, unifyfs_mount_prefix, unifyfs_mount_prefixlen) == 0) { /* characters in target up through mount point match, - * assume we match */ + * assume we match */ cwd_within_mount = 1; /* if we have another character, it must be '/' */ @@ -2371,7 +2371,10 @@ static int unifyfs_finalize(void) } /* close spillover files */ - unifyfs_logio_close(logio_ctx); + if (NULL != logio_ctx) { + unifyfs_logio_close(logio_ctx); + logio_ctx = NULL; + } if (unifyfs_spillmetablock != -1) { close(unifyfs_spillmetablock); unifyfs_spillmetablock = -1; diff --git a/common/src/unifyfs_logio.c b/common/src/unifyfs_logio.c index 6ebe0af91..b6d38482d 100644 --- a/common/src/unifyfs_logio.c +++ b/common/src/unifyfs_logio.c @@ -396,6 +396,7 @@ int unifyfs_logio_close(logio_context* ctx) LOGERR("Failed to unmap logio spill file header (errno=%s)", strerror(err)); } + ctx->spill_hdr = NULL; } if (-1 != ctx->spill_fd) { /* close spill file */ @@ -405,6 +406,7 @@ int unifyfs_logio_close(logio_context* ctx) LOGERR("Failed to close logio spill file (errno=%s)", strerror(err)); } + ctx->spill_fd = -1; } } diff --git a/server/src/unifyfs_request_manager.c b/server/src/unifyfs_request_manager.c index da8e41e8a..7315bd70f 100644 --- a/server/src/unifyfs_request_manager.c +++ b/server/src/unifyfs_request_manager.c @@ -1430,15 +1430,14 @@ int rm_cmd_mread( * returns UNIFYFS_SUCCESS on success */ int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl) { - /* grab the lock */ - RM_LOCK(thrd_ctrl); - if (thrd_ctrl->exited) { /* already done */ - RM_UNLOCK(thrd_ctrl); return UNIFYFS_SUCCESS; } + /* grab the lock */ + RM_LOCK(thrd_ctrl); + /* if delegator thread is not waiting in critical * section, let's wait on it to come back */ if (!thrd_ctrl->has_waiting_delegator) { @@ -1461,10 +1460,12 @@ int rm_cmd_exit(reqmgr_thrd_t* thrd_ctrl) RM_UNLOCK(thrd_ctrl); /* wait for delegator thread to exit */ - void* status; - pthread_join(thrd_ctrl->thrd, &status); - thrd_ctrl->exited = 1; - + int rc = pthread_join(thrd_ctrl->thrd, NULL); + if (0 == rc) { + pthread_cond_destroy(&(thrd_ctrl->thrd_cond)); + pthread_mutex_destroy(&(thrd_ctrl->thrd_lock)); + thrd_ctrl->exited = 1; + } return UNIFYFS_SUCCESS; } diff --git a/server/src/unifyfs_request_manager.h b/server/src/unifyfs_request_manager.h index 2f1caead7..4c4310e89 100644 --- a/server/src/unifyfs_request_manager.h +++ b/server/src/unifyfs_request_manager.h @@ -53,7 +53,7 @@ typedef struct reqmgr_thrd { /* condition variable to synchronize request manager thread * and main thread delivering work */ - pthread_cond_t thrd_cond; + pthread_cond_t thrd_cond; /* lock for shared data structures (variables below) */ pthread_mutex_t thrd_lock; diff --git a/server/src/unifyfs_server.c b/server/src/unifyfs_server.c index 049507435..86813c073 100644 --- a/server/src/unifyfs_server.c +++ b/server/src/unifyfs_server.c @@ -817,7 +817,6 @@ unifyfs_rc attach_app_client(app_client* client, if (failure) { LOGERR("failed to attach application client"); - cleanup_app_client(client); return UNIFYFS_FAILURE; } @@ -838,13 +837,16 @@ unifyfs_rc disconnect_app_client(app_client* client) return EINVAL; } + if (!client->connected) { + /* already done */ + return UNIFYFS_SUCCESS; + } + client->connected = 0; /* stop client request manager thread */ if (NULL != client->reqmgr) { rm_cmd_exit(client->reqmgr); - free(client->reqmgr); - client->reqmgr = NULL; } /* free margo client address */ @@ -888,6 +890,7 @@ unifyfs_rc cleanup_app_client(app_client* client) /* close client logio context */ if (NULL != client->logio) { unifyfs_logio_close(client->logio); + client->logio = NULL; } /* reset app->clients array index if set */ @@ -900,6 +903,9 @@ unifyfs_rc cleanup_app_client(app_client* client) } /* free client structure */ + if (NULL != client->reqmgr) { + free(client->reqmgr); + } free(client); return UNIFYFS_SUCCESS; From 1d12d24a7710dfd5ba7bc096993fffb1891f3ff2 Mon Sep 17 00:00:00 2001 From: CamStan Date: Tue, 19 May 2020 22:47:59 -0700 Subject: [PATCH 119/168] Update bootstrap and docs for recent usage changes This updates bootstrap.sh to a more recent and stable version of Argobots. This also applies a variety of documentation changes to bring the docs more up-to-date. * build-intercept.rst - reorganizes the build documentation to make it easier to follow by fully separating the process of building with spack vs building without - update process for building with Spack - remove some redundency * configuration.rst - updates the unifyfsd command line options * dependencies.rst - updates the preferred version of Argobots - adds a link for the OpenSSL dependency * examples.rst - update example program options - fix section names * start-stop.rst - update example script for starting unifyfs servers - update unifyfs utilty command line options --- bootstrap.sh | 2 +- docs/build-intercept.rst | 139 +++++++++++++++++++++------------------ docs/configuration.rst | 5 +- docs/dependencies.rst | 4 +- docs/examples.rst | 12 ++-- docs/start-stop.rst | 52 +++++++-------- 6 files changed, 116 insertions(+), 98 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index cf05fb15e..73913c61e 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -63,7 +63,7 @@ cd .. echo "### building argobots ###" cd argobots -git checkout v1.0rc2 +git checkout v1.0 ./autogen.sh && CC=gcc ./configure --prefix="$INSTALL_DIR" make -j $(nproc) && make install cd .. diff --git a/docs/build-intercept.rst b/docs/build-intercept.rst index e8dd48f8f..dfcfb254b 100644 --- a/docs/build-intercept.rst +++ b/docs/build-intercept.rst @@ -14,7 +14,7 @@ In this section, we describe how to build UnifyFS with I/O interception. as specified in the project `github `_. -.. _build-label: +--------------------------- --------------------------------------- UnifyFS Build Configuration Options @@ -25,7 +25,11 @@ Fortran To enable UnifyFS use with Fortran applications, pass the ``--enable-fortran`` option to configure. Note that only GCC Fortran (i.e., gfortran) is known to -work with UnifyFS. +work with UnifyFS. There is an open +`ifort_issue `_ with the Intel +Fortran compiler as well as an +`xlf_issue <://github.com/LLNL/UnifyFS/issues/304>`_ with the IBM Fortran +compiler. GOTCHA ****** @@ -62,41 +66,45 @@ your target system, it can be omitted during UnifyFS configuration by using the ``--without-hdf5`` configure option. --------------------------- -How to Build UnifyFS + +--------------------------- +Building UnifyFS with Spack --------------------------- +Full Build +********** + To install all dependencies and set up your build environment, we recommend using the `Spack package manager `_. If you already have Spack, make sure you have the latest release or if using a clone of their develop branch, ensure you have pulled the latest changes. -Building with Spack -******************** - -These instructions assume that you do not already have a module system installed -such as LMod or Environment Modules. If your system already has -LMod installed then installing the environment-modules package with Spack -is unnecessary (so you can safely skip that step). +.. _build-label: -First, install Spack if you don't already have it: +Install Spack +^^^^^^^^^^^^^ .. code-block:: Bash $ git clone https://github.com/spack/spack - $ ./spack/bin/spack install environment-modules + $ # optionally create a packages.yaml specific to your machine $ . spack/share/spack/setup-env.sh Make use of Spack's `shell support `_ to automatically add Spack to your ``PATH`` and allow the use of the ``spack`` command. -Then install UnifyFS: +Install UnifyFS +^^^^^^^^^^^^^^^ .. code-block:: Bash $ spack install unifyfs $ spack load unifyfs +If the most recent changes on the development branch ('dev') of UnifyFS are +desired, then do ``spack install unifyfs@develop``. + .. Edit the following admonition if the default of variants are changed or when new variants are added. @@ -106,17 +114,17 @@ build is desired. Type ``spack info unifyfs`` for more info. .. table:: UnifyFS Build Variants :widths: auto - ======= ======================================== =========================== - Variant Command Description - ======= ======================================== =========================== - HDF5 ``spack install unifyfs+hdf5`` Build with parallel HDF5 + ========== ======================================== =========================== + Variant Command Description + ========== ======================================== =========================== + Auto-mount ``spack install unifyfs+auto-mount`` Enable transparent mounting + HDF5 ``spack install unifyfs+hdf5`` Build with parallel HDF5 - ``spack install unifyfs+hdf5 ^hdf5~mpi`` Build with serial HDF5 - Fortran ``spack install unifyfs+fortran`` Enable Fortran support - PMI ``spack install unifyfs+pmi`` Enable PMI2 support - PMIx ``spack install unifyfs+pmix`` Enable PMIx support - PMPI ``spack install unifyfs+pmpi`` Enable transparent mounting - ======= ======================================== =========================== + ``spack install unifyfs+hdf5 ^hdf5~mpi`` Build with serial HDF5 + Fortran ``spack install unifyfs+fortran`` Enable Fortran support + PMI ``spack install unifyfs+pmi`` Enable PMI2 support + PMIx ``spack install unifyfs+pmix`` Enable PMIx support + ========== ======================================== =========================== .. attention:: @@ -131,32 +139,30 @@ build is desired. Type ``spack info unifyfs`` for more info. --------------------------- -Building with Autotools -************************ - -Download the latest UnifyFS release from the `Releases -`_ page. - -Building the Dependencies -^^^^^^^^^^^^^^^^^^^^^^^^^^ +Manual Build +************ -UnifyFS requires MPI, LevelDB, GOTCHA(version 0.0.2), FlatCC, and Margo. -References to these dependencies can be found :doc:`here `. +Optionally, you can install the dependencies with Spack and still build UnifyFS +manually. This is useful if wanting to be able to edit the UnifyFS source code +between builds, but still letting Spack take care of the dependencies. Take +advantage of +`Spack Environments `_ +to streamline this process. .. _spack-build-label: -Build the Dependencies with Spack -"""""""""""""""""""""""""""""""""" +Build the Dependencies +^^^^^^^^^^^^^^^^^^^^^^ -Once Spack is installed on your system (see :ref:`above `), you -can install just the dependencies for an easier manual installation of UnifyFS. +Once Spack is installed on your system (see :ref:`above `), the +UnifyFS dependencies can then be installed. .. code-block:: Bash - $ spack install leveldb - $ spack install gotcha@0.0.2 $ spack install flatcc - $ spack install margo + $ spack install gotcha + $ spack install leveldb + $ spack install margo ^mercury+bmi .. tip:: @@ -165,15 +171,20 @@ can install just the dependencies for an easier manual installation of UnifyFS. Keep in mind this will also install all the build dependencies and dependencies of dependencies if you haven't already installed them through - Spack or told Spack where they are locally installed on your system. + Spack or told Spack where they are locally installed on your system via a + `packages.yaml `_. + +Build UnifyFS +^^^^^^^^^^^^^ -Then to manually build UnifyFS: +Once the dependencies are installed, load them into your environment and then +manually build UnifyFS. .. code-block:: Bash - $ spack load leveldb - $ spack load gotcha@0.0.2 $ spack load flatcc + $ spack load gotcha + $ spack load leveldb $ spack load mercury $ spack load argobots $ spack load margo @@ -183,36 +194,35 @@ Then to manually build UnifyFS: $ make $ make install -.. note:: **Fortran Compatibility** - - To build with gfortran compatibility, include the ``--enable-fortran`` - configure option: +To see all available build configuration options, type ``./configure --help`` +after ``./autogen.sh`` has been run. - ``./configure --prefix=/path/to/install/ --enable-fortran`` +--------------------------- - There is a known `ifort_issue `_ - with the Intel Fortran compiler as well as an `xlf_issue <://github.com/LLNL/UnifyFS/issues/304>`_ - with the IBM Fortran compiler. Other Fortran compilers are currently - unknown. +------------------------------- +Building UnifyFS with Autotools +------------------------------- -To see all available build configuration options, type ``./configure --help`` -after ``./autogen.sh`` has been run. +Download the latest UnifyFS release from the `Releases +`_ page or clone the develop branch +from the `UnifyFS repository `_. -.. TODO: Add a section in build docs that shows all the build config options +Build the Dependencies +********************** -Build the Dependencies without Spack -""""""""""""""""""""""""""""""""""""" +UnifyFS requires MPI, LevelDB, GOTCHA, FlatCC, Margo and OpenSSL. +References to these dependencies can be found on our :doc:`` page. -For users who cannot use Spack, a `bootstrap.sh `_ -script has been provided in order to make manual build and installation of -dependencies easier. Simply run the script in the top level directory of the source code. +A `bootstrap.sh `_ script +has been provided in order to make manual build and installation of dependencies +easier. Simply run the script in the top level directory of the source code. .. code-block:: Bash $ ./bootstrap.sh -References to the UnifyFS dependencies can be found :doc:`here `. - +Build UnifyFS +************* After bootstrap.sh is finished building the dependencies, it will print out the commands you need to run to build UnifyFS. The commands look something like @@ -226,6 +236,9 @@ this: $ make $ make install +To see all available build configuration options, type ``./configure --help`` +after ``./autogen.sh`` has been run. + --------------------------- --------------------------- diff --git a/docs/configuration.rst b/docs/configuration.rst index 07e2cc414..005fb5e0b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -188,11 +188,12 @@ is used, the value must immediately follow the option character (e.g., ``-Cyes`` --unifyfs-consistency -c --unifyfs-daemonize -D --unifyfs-mountpoint -m - --log-dir -L - --log-file -l --log-verbosity -v + --log-file -l + --log-dir -L --runstate-dir -R --server-hostfile -H --sharedfs-dir -S + --server-init_timeout -t ====================== ======== diff --git a/docs/dependencies.rst b/docs/dependencies.rst index 62466206f..6c8799a57 100644 --- a/docs/dependencies.rst +++ b/docs/dependencies.rst @@ -10,11 +10,13 @@ UnifyFS Dependencies - `Margo `_ version 0.4.3 and its dependencies: - - `Argobots `_ version 1.0rc1 + - `Argobots `_ version 1.0 - `Mercury `_ version 1.0.1 - `bmi `_ +- `OpenSSL `_ + .. important:: Margo uses pkg-config to ensure it compiles and links correctly with all of diff --git a/docs/examples.rst b/docs/examples.rst index 1874d67a4..f7859f337 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -31,8 +31,8 @@ To easily navigate to this location and find the examples, do: $ spack cd -i unifyfs $ cd libexec -Installed without Spack -^^^^^^^^^^^^^^^^^^^^^^^ +Installed with Autotools +^^^^^^^^^^^^^^^^^^^^^^^^ The autotools installation of UnifyFS will place the example programs in the *libexec/* directory of the path provided to ``--prefix=/path/to/install`` during @@ -67,8 +67,8 @@ The GOTCHA examples are one directory deeper in ``spack cd unifyfs+hdf5 ^hdf5~mpi`` -Built without Spack -^^^^^^^^^^^^^^^^^^^ +Built with Autotools +^^^^^^^^^^^^^^^^^^^^ The autotools build of UnifyFS will place the static and POSIX example programs in the *examples/src* directory and the GOTCHA example programs in the @@ -113,10 +113,12 @@ to aid in this process. (default: off) -m, --mount= use for unifyfs (default: /unifyfs) - -M, --mapio use mmap instead of read|write + -M, --mpiio use MPI-IO instead of POSIX I/O (default: off) -n, --nblocks= count of blocks each process will read|write (default: 32) + -N, --mapio use mmap instead of read|write + (default: off) -p, --pattern= 'n1' (N-to-1 shared file) or 'nn' (N-to-N file per process) (default: 'n1') -P, --prdwr use pread|pwrite instead of read|write diff --git a/docs/start-stop.rst b/docs/start-stop.rst index 648f81dd9..8792d54b2 100644 --- a/docs/start-stop.rst +++ b/docs/start-stop.rst @@ -40,16 +40,15 @@ for further details on customizing the UnifyFS runtime configuration. .. code-block:: Bash :linenos: - #!/bin/bash + #!/bin/bash - # spillover checkpoint data to node-local ssd storage - export UNIFYFS_SPILLOVER_DATA_DIR=/mnt/ssd/$USER/data - export UNIFYFS_SPILLOVER_META_DIR=/mnt/ssd/$USER/meta + # spillover data to node-local ssd storage + export UNIFYFS_LOGIO_SPILL_DIR=/mnt/ssd/$USER/data - # store server logs in job-specific scratch area - export UNIFYFS_LOG_DIR=$JOBSCRATCH/logs + # store server logs in job-specific scratch area + export UNIFYFS_LOG_DIR=$JOBSCRATCH/logs - unifyfs start --share-dir=/path/to/shared/file/system & + unifyfs start --share-dir=/path/to/shared/file/system ``unifyfs`` provides command-line options to choose the client mountpoint, @@ -59,30 +58,31 @@ The full usage for ``unifyfs`` is as follows: .. code-block:: Bash :linenos: - [prompt]$ unifyfs --help + [prompt]$ unifyfs --help - Usage: unifyfs [options...] + Usage: unifyfs [options...] - should be one of the following: - start start the UnifyFS server daemons - terminate terminate the UnifyFS server daemons + should be one of the following: + start start the UnifyFS server daemons + terminate terminate the UnifyFS server daemons - Common options: - -d, --debug enable debug output - -h, --help print usage + Common options: + -d, --debug enable debug output + -h, --help print usage - Command options for "start": - -c, --cleanup [OPTIONAL] clean up the UnifyFS storage upon server exit - -C, --consistency= [OPTIONAL] consistency model (NONE | LAMINATED | POSIX) - -e, --exe= [OPTIONAL] where unifyfsd is installed - -m, --mount= [OPTIONAL] mount UnifyFS at - -s, --script= [OPTIONAL] to custom launch script - -S, --share-dir= [REQUIRED] shared file system for use by servers - -i, --stage-in= [OPTIONAL] stage in file(s) at - -o, --stage-out= [OPTIONAL] stage out file(s) to on termination + Command options for "start": + -C, --consistency= [OPTIONAL] consistency model (NONE | LAMINATED | POSIX) + -e, --exe= [OPTIONAL] where unifyfsd is installed + -m, --mount= [OPTIONAL] mount UnifyFS at + -s, --script= [OPTIONAL] to custom launch script + -t, --timeout= [OPTIONAL] wait until all servers become ready + -S, --share-dir= [REQUIRED] shared file system for use by servers + -c, --cleanup [OPTIONAL] clean up the UnifyFS storage upon server exit + -i, --stage-in= [OPTIONAL] stage in manifest file(s) at - Command options for "terminate": - -s, --script= to custom termination script + Command options for "terminate": + -s, --script= [OPTIONAL] to custom termination script + -o, --stage-out= [OPTIONAL] stage out manifest file(s) at After UnifyFS servers have been successfully started, you may run your From a85f9c8d7b1045034fb936a3dce3f909b1086837 Mon Sep 17 00:00:00 2001 From: CamStan Date: Tue, 2 Jun 2020 16:56:29 -0700 Subject: [PATCH 120/168] Add UnifyFS Developer Doc slides to ReadTheDocs This adds the UnifyFS Developrers Documentation slides as a downloadable PDF to our docs. Also adds a clarifying step to the build docs when building manually with Spack. --- docs/build-intercept.rst | 6 +++++- docs/conf.py | 4 ++-- docs/contribute-ways.rst | 15 +++++++++++++++ .../UnifyFS-developers-documentation.png | Bin 0 -> 565002 bytes .../UnifyFS-developers-documentation.pdf | Bin 0 -> 424703 bytes 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 docs/images/UnifyFS-developers-documentation.png create mode 100644 docs/slides/UnifyFS-developers-documentation.pdf diff --git a/docs/build-intercept.rst b/docs/build-intercept.rst index dfcfb254b..822911016 100644 --- a/docs/build-intercept.rst +++ b/docs/build-intercept.rst @@ -177,8 +177,12 @@ UnifyFS dependencies can then be installed. Build UnifyFS ^^^^^^^^^^^^^ +Download the latest UnifyFS release from the `Releases +`_ page or clone the develop branch +from the `UnifyFS repository `_. + Once the dependencies are installed, load them into your environment and then -manually build UnifyFS. +manually build UnifyFS from inside the source code directory. .. code-block:: Bash diff --git a/docs/conf.py b/docs/conf.py index 9066c3847..87684ba88 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,9 +54,9 @@ # built documents. # # The short X.Y version. -version = u'0.1' +version = u'0.9.0' # The full version, including alpha/beta/rc tags. -release = u'0.1' +release = u'0.9.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/contribute-ways.rst b/docs/contribute-ways.rst index 1deac78f2..c67279bf2 100644 --- a/docs/contribute-ways.rst +++ b/docs/contribute-ways.rst @@ -102,6 +102,21 @@ able to quickly identify and resolve issues. Documentation ============= +Here is our current documentation of how the internals of UnifyFS function for +several basic operations. + +.. rubric:: UnifyFS Developer's Documentation + +.. image:: images/UnifyFS-developers-documentation.png + :target: slides/UnifyFS-developers-documentation.pdf + :height: 72px + :align: left + :alt: UnifyFS Developer's Documentation + +:download:`Download slides `. + +| + As UnifyFS is continually improved and updated, it is easy for documentation to become out-of-date. Any contributions to the documentation, no matter how small, is always greatly appreciated. If you are not in a position to update diff --git a/docs/images/UnifyFS-developers-documentation.png b/docs/images/UnifyFS-developers-documentation.png new file mode 100644 index 0000000000000000000000000000000000000000..557a4c4ab24bf228ab90715541bb21f6db5fbbd6 GIT binary patch literal 565002 zcma&M2Ut_>vNntbQ53Nt3QDmc9aMS^ii&{rUV{SCL3$@*L#2r{sX=;2dM6@61ccB7 zp+pQ2AQVGSUH{^J-@VWIuK(KGBbOlw>nStOJTv#)GjDaZ)mWH0nHd-uSkxcf)n{Nh zCeOgYD0K7)_)YHxvp5EZ6AzqJRdv->Re5#2Jf1kY+A}abc$;X#WNOgOo@t@4?ihab z!Lio2R~{svxf99DYxMc_2bDW-FJFE$9%p~fviZ%krtFd{@5&iZDJ+|x?_4wIepgsp z8p(K3$U|Sn1hG4_+rQh_3hPhbgnww9<8gaM==2*thQ;@nPyUPyIQZr&FLzv;g4NqkTu&bW@Ax-)(w9`z6|EOe8Kbk$ zc)c+d>znPPLMDj_J{h%}7dlR!Ontz9x}cO}XVrLT>j7W>?WwEku`%JMJiq)Ohxc$l zP7u<+d#!j?&!8>+Q1FqDr<$1qF2|ipyL0J!_uH7E#bJqWk1XHJGT%PlNt!wwy?*7B zqNH_)+0``e^Ac|)>@L-H)ra#NbQ)pc`gZ(PQq=mpch-0T!WF$cRq_r|=DgbPUYlNg zoUf>$tnc<>g6*yB;0x(n_tI~{F1`tU_2IXT6hvA|m7TqyGI_p-iO|XZ_S=*rOL@Pz7@8cAO8AY*LQW?{?j!*1M#uTeXKFVdu<5|-<(%G zi*N8-7S%r=H?%%gLF_!VG5g$b6E`?S9Y5@w84$$oVOc8A({k$Wfs;oMh-5vx%Jq=R zOl4|pc;pbQ$l)sk>vxswrw*Qvell>?OzLVlIzaa@{rQ8VbE;wO#b%cUo{@D_&9_Gx z?Fye9H{_o9dGlK%@0qzTSmwlJI|1`E0f(i2t!HmODwlgIfNw)re!8xaqjzGX%IApa z+fUz4JwALhYVl3l&jSrS+>8~spEk_SG@xwipex5>kFCCpcu9D9^6RS8ug4QI+n>J# z3>>)n(#csHXnHbrvMT6jdjn)HVoLU`%mJ~z3m4u-GkqQB*6kg%D=(#6tieWp8=&)Y{Fc44gyz!6rz($_^bh=~C(1#6%^B9?u;z~=KZlNUtQ__` zsSqV~fZgweOPAPVwS)l2b4vPU+3k*=#UWLfWA~Z^6ty1O^Us%Wmrs}TjCzy4-kGPI zewwf8(DN-NJB0mW8|jru`Adk`;oiQ{zHf0c{X8lK!9=HM$Yt5@t6CbIb4~VzgQ0(KKbJ8Pno9&ERUeu zxf+6+zDNulpLrH``1XUNT#tCW-Y`Cnl)dWyM&``3msj*2T>9`l_j1ac!+N~cajzc8 zRh=oiBYoNTd&T$LbBDc7I;pu|Wqt2H$G35yQ;8(^M!ay!&&bDTi_R=wlYs z<5YOXV=hzH9d7<#C(>Uve>rz4oc~vdA}c&nyuIZ2YQWVnX2d(lmtZ35uzY*M&xgPH zo*p}UEafG~OZ_{HkA!C#W+g*Xn~r+CxqBtvQD^J9jxF=eNZH7SmkpT>X#wfu+AG&~ z;vQVH{UFf|XH)$Zx1pYw>zkXAYnv;aD>kH1#Xj+T<^IX51t0k49(yqcGLZyG%%o$D z@NCV>ygmKL0tb5@_T1`W*El|ZCNFYGXEX<6Kr1>nBC>qYnXfiHC^aaVtf8sdtI@0G z@;u4ZrOh~RjJvGboV8|Vc*{v!@}%1>t=kqFllK+0o3(=P6%4N~-rh0{QVtbtjS2e@ z2B&MMeNxg-D-j*YEU+q@E)2a#X*_ukX&g}|VN~#-;En^lpD%;k>Row;m|Ajyf=Rvk zZq}OCw6fCc8OPQOt(L8PK?+;_OG=~sOOs0x%g47gw`2(Lk##$p6D~(0+8spfTpy)B z@!>edvcl4E#O%oYk>(Sa6G1H7EXWJjIoLUkFLK{pv3dMZ;d*jPhh^7PZ%aG%#t+vE z!5^QF|9ZsIQGO!WMRH2u`vXVACpD9cO@mCfNuGtCbwp;|bzJcEhPUzEhvPY}Uwo*Y zwsk`0!kSDvq{Ve_W)}7_J6}8BGr#(wOq#Xis)U5hahdqkG6@BzG24Dg)?27sP`9v= zmdWRjPTO3rGj;R)@k=M^a%5l6Bcm$q${BeLrgBGKePWae!GA5>d~tq_aEFJiD_86 ze@2Y;@sjM zbTI%KBu-Q&sQx~-lY;ji$G27#c(kosV-Yk9jPVIPf5pO}W4dNio*mHx(^e);D6n{%gS8lPAy*T+mM^t()@dlp=AiX+Au z#j)n3XWQwC;s%Mht_n_=3`W}g%rx7f#|<}29pZc&bCCDzdUvj1<)ahl-Tn(2KPvFfKb8D!) zOzZX5WYlcMJml=*W5&muUeUiMe&y$&a|lahJ|R1l_pGopz7x8u^f;zLgL*~yii_>o zNPX+@`hp*u`N=8tyK>ujODJtuXyy! zJiE9P$;{Nr`i_b50OKg*`jcyY<0geB0)j2F!%hpoQgq&ZeHWJU3iS)+oZ%)H+`hu( zJ?<^tcW)sd<(A=~RAFb0rv4^Pp4UBZC%)G@UK?OfB)sZAZlUo*V*Lsv^ps}j>mJr1LZmi{Nx8@@ax zca6{`FG~E;AIC(>1DmZs(|cZLv9Mh1lTPxW{;uA`TcmxJks8B{8oOS17aOMrXbvSw zAOZmW`cXR<(wNTVwMqFpq#MKtZ)8&6;e)ISs$_cq`0lagtWbj9>dS+y~( z=&g`8(KS$ymFTI-CR8xABW#i;#dLwGhcm7#CAw8BJ3nS2VPWS5?qprvUhO4W$&G=! zj--f^*bCfdehEP}bBkXpvu#6>dB|XGyUaz`jIkj6+_!kNwwCOupZvzw1omr~Lg4Gr zg>f^WIs!$?R0x=%gbXvHjz^zY664;R#_b+*%XEuQZV{7$?levfDc8{!6Tb-am%p~8 zt#a;SUJ(Y-lOES*8hpJs`eH1eTj+S;r^1M8?L0L91uWl`Xmdeu#jmy{8U`juckz#K zK*lj$W))M8f&~=BV@WG<)IntV2jmv=oQ_E5h>~w`P=i7#Wdj~hUkSP%I)*hMC(l5o zP+XbofWqQ>Wb4`BX6x_PCt55wdiF-^(6-j~JDoezuy{&7;5MOv^Pj{9BOsfXNgeTY z4jy$LVTfFqGwBkEw5GE5)?;j9BQ36sA4o(wx*9-TkjIIHVEdiFWP@|v=wd3kvi zz3iUI>)*Zq&v5XQ(iKM^pQrNT;{N{rV*V0h9$pUOH|6By#BbaZzjaF#d_vSa(B0=z zfT+9o)xRhC*F1Oay=}dmp87a>xbyDM`{=QUuaDA|EBg!m``_RDv=4ClpOxIb{}Brm zP<;O{;y1-^i2r+TaH!(`xAMA90rsw@cb(kq-Mzszly6B$-%|W*!2j=0|Fh)(8fx}G zLvKpl`hSN0uRr}WR8f3ig#RVd-}?INTTo%j%!=awR=zT`SMn_`@H{R#-PJJ!e;?dG z9Pn2n_#*K4Z}2^1=kvoa5O*0ER2bCn-Zl(4urkf`jBTETBr>hty?yWQ-FuAZR?bDg zQ{r6Vv{PI)DgGkBe``QYPwhPe^|es`_!MsxlCPw*>Bj9gt` z%lu~lkN!2~z!~k5iblVY?)+3Uen5O9?K@!NV37TNnK~gzKTq3Pcv_=P`D=*q$Ke$`qp-d)l8D0HH?20+f9hQwXMdot!Dj1 zVC-TAxxlBVM!}E=4axfywqAnCy+>QmmF}di&slkKZCB3_GE7a*lTRUeb}<=-@QYU6`{maCJ8(+lS_?Qw&&tW!q&n{*=r@52UQS zy?0tfkoFv?s(>ww2T2yBx1_mDHvqmfuuXc}E_6GxHol_36Fwz}OVZo%Vbk2eQk`7x>6pZ5+5rmGUEwJE9456j&DwLoC4MDDT8h zL1uu~A3xzzbPV7{9fgUj0!y$3x^Fj;zwijGb&;^N2VbGNhEVOvdW(CgOE6C02Yvlj zKxRR`-Vb{VBk!R}{Aie02XTEeVp@I%cmdRG!k4JYQ9zA%ydSC*heF=L!%JzXxOWtX z7#NoC$My4afIZ2f3#)T&m|Q`digfHj5J~(tOAZ)z-IqRkGj#%>?P-$GBx1#;jx2DUJ_CaQS|^dPaf(s!!d~w(VpqXH zYm6i_4b0JygdaO-ij^P*E}xzEiT;|1S`kP=`?w!lhlpVV00Tf`J*S%jT?DSlxCQ`e zQ@`mK$?KVcyAY4(@akJgM5@6L2tYqiHNZnhq3FOC1Yp0+iU&$*;wX|h91-wgmnQ?Z z(E>1ndkD_xCP89hffQKBo&hYH8W{~+rXhD>R=}&x3hd~%S_oY|29lBdl>#O1La}AF zo068mxe^$L65D@Zcn9%oo1X#=-v!<+qn&t1fv$B#SZUHY@Ph)K0)7McqsusN51ME)3*iPM!hFf&o zX9zH^v2G%GG;lxKJ!heGHrghn`Qlyx<7 zmOQ!c5K4FSp-p}U_~4M14qzZ%t95RFk?-OD+ zePz8U8Uj4onujX`cqQN(9OAWu*uskgbYasa3M7;+5Jbc7QPWz0SFhPI!0QJY(sbO*g5iWZ_y^osC$>SO*|PO>EPgvbb&SaFbtq3%K_3M zK#Dvtx0+D41Wd`RB$L@MvN!^DcxDX4I(uB7OCq4$# zCbx(Y5DE{H2M(bhBMg6sh>*(y zbvq1!je__!O3)*VvUOej!U4GLP~7%*r{b|U0nJL8 z;oXGmsM0(K5|xXKqKY!^>_x#&nPjkRM^M^chY1kHA1hGCv~jD0Xd{ExlL$hmcKWC4 z(LfqmC4=-q1CqJNL>^!Vh(8c7SwkNz?}}5l@?f$oy$^UoMgc)-RR_p1N<>f5SLZ_L zxqC}=N^(oF>@dtjuXR_Z9ymXmf6P2J@wqp)ZM;^SU0hcZ1((ry))Ux78!)>mj~R=IY_1TY1qz>2|(gW?*d zz;_A)fa72gyx0Y$p&KOk2u-}q6wMCed1^HUJAs7A(ZB9F@Br<)pvT-Js8#&XGo0Hq zq0OX4A(mZ2s4mzPHXgv&*%yZc-F^kV#3%F^UUe;WqryX<0>jb7y{X_nZ_!zBa3oA) zjA&v*4ZT#exF^1XeM-g4(SOmzci?_hdulDRlNL+ei=;{5dARP)=&UkX!*Vt~F2E`j zcK0+hV7_6QY;f2z^^p3nifJp`(@d;;bhaJfH4d@@OQg6=`A{S11u?)^+HTl2S}Y3j zXkCNqOHaA1OmC&mQ3(nUWPu}K$f3kW5lIE{>&R|WeGyHZKsEVEO-lzNS7AlFJlN2D zE`S9O?u@czysprW^{9=1p^qw2Ic|uF#aI zGF_6Pn}nssZ^Ak#upBTT^N8!L!zL)uvq)GHP(!Yc0q)Zq!9YiWwUd?330$IiA|C&GYa791H9NvK1LG`*+yV;T zIHBRQWSMmFAv6yQ9(YPm^P`>b2UyUMXe`WEDk%&YMDvDa?%wjDcZULKk`CG=Wf$?9 zfSB8ippf!0(n|;gKr(`M&}CL>Iso#`k8K?)olx6@#Uk#|gt`blp@2>~tb@_H10O3u zp{b#+ooFZv1vZBF7+tz8 zGaKP<1)Ey~)M+ee#4NdDh`<9`s8R-&d##j#%(b0 zWd*SvF7b94q!TvPE`SI}Cug0IVln1mG0LduW z7zxq}ILrVWv9L8f{51+j+zWpHaDzU%54Gh13bKA2iu))CqVScq&24P>3MXBVafb=xVz0 zIx&b53M-zcAJGE*rcdsX>-X?H<&b<7Qeil7`2Y42UMHZEWngjX9Ofm%!K1v-{!cGv2M^2Jd;b1EFLK~8CYvE#+s)I^ z@$CP3k)w9UPAw*MwjcUGN&-sk7zu8ywo~tu*8f$KqrC6=bQ4ty)DbjS9E{v0(j%0{ z^sm7xeTU^oM=k0b$!2(trjgQ<%=iBwa(La<{n;wG5#4RUv}5b%;s@pVWTpaPgVNt*M)76CNFqe(Jz@kwneE#;9m9UXS9o z$Aw*RqRC7mkGNJswFucy|D-O*iwBLEeK1ppGG0$)*?LqriAmH(XAUy|Gw!21s*6>eZYyJTzGS*@7^5627iQU*BQC2gAV|WLOM%A7l z`Ga{4cTJe9l?z#XSSCKE7}GyjNAS&ITX+gQ;=p zJX0x%ky+Tf@YiJt_Y((iLNm`D`9E9&l?ia0iCSGp{yB&N%>K4_j_N}Gu%PJv0=;Pl zSO0lV(g57RlY>=qe;Bu_3qI#3>nO?b_eNvHz>$cO%Iwpkg+ zPc4r0GreE9`rjqg4F@mlxva{6%=TzaIrzDZptJff%li-3TJw7Uxlh^||5A*|frA;T zoye{~jnf35!_0pkbo|=}58 zW##FArmxB0-*=JD^q=+_v;Q1UyX;@`o!FNz9GU;8aR&SHEqxhu{YSp4efa{nmK6TG zd}`-F`Ep_uY5&N_wJ%=`Q*8d9LsRoBtk z!!_$vGNEGDV`aS37z*X-S5gYzsp-#9@RyBs_PH?ZERbY!BwTwwuu}TJ%Fb|b_yxEb zHybt{7}Ya)r4s4dlOS04{YvC`X7JjCP?@vvs$*h*vYwNLCwUJSyV^!stMJpnhN~{xwWo~ z9yQe{<;gbka>@N@s7*Md>-gtK(j6xxB!kMv&&s2OhEwWFB!v^|01_Uq`b5FL)}`!( z{rp;d?Jd!T3`**AyTnf=Q};Lyr}g~IQ|W<*kvbDdnlO3v4_RE4G-aha0^x;C?sc=R z+&<{kePfODe793mn7uEmKnIfVX(u+sJOi} zjHJ_d5fNEMQ+L>o)}pP0CsbLL*sQ?~%&)$_=zg?auaPkJizq$mYFm#D3l=bkaF?u? zRn{c>)!fJS7FCzYW&|x4`HkqQ@oAa7E;w-L$hFu>a>khQ8m37{6yr~tPo4x-d_G^n zkR*>4XwZ-BSCGdFjTfdbNgya|3eBL#IwapQYh&D-SA{UFJOMus1QR>YtJ*jNbeh}u z_m@9eiIQ~qD0ZR`dzY#)A_`nUs&d81y#tyT-D=`Q>&71k%8*v;u`3;{ked3LAmPKE z(U;_I*#Y`ChOyjAuCuSs*L9wUPDvW~+%4Q)O0ovSlauGN>VHQMS=)>l*Q{aiOkG?` zEAL~JrRV31bcJK;MGWd~EnEWNdY*IRe*Y@VYQt=MUbxB3K> zKGfz3`5iv(H|Z95=+5zNQk)uhC5!YpZy+f zL$=1qu&K4l`idncThSg<;6i*s#yju-2QzNOc-2U zk7>aPWun(yoEoN^AD%Ado6`R(HK{cH-o#7k3A=c^_)Qc9ds6o2vpLTqe`9Fq_9?@h z?L@8z!@0Fv`fI_r%Rpn;1whL^<;v!flQpByURJO>VjHU@?c%GIK}Nyo!{x0z=EDgA z=4=DGE6|?tyc2IZFESbnrX{d5%C>@5SNH9V=1ZjLHfy-HD4&+nA1+i47Z^7_S{%)*?wz-i3^OIJZm}(1d+D{)ZJv@OaiQ^cd(T=fBA=ac%bByoN2TKV$9|>V zsZ)z=OgEjJn{U53@+#CGJWFXUCmH9F!uK2xQ{a;JGkZi_$4JG}4{NPaXf*nmPx0$3 z_J0CrZTsNtg1DANK~Y*5m8fQ67kAIKym7&~GR{>vTmtlZqDP>Q?zwe31%VIt;mW_x zdeo;0_71%|Z+!uS&C^UiHBw}gydXAgk!2u9u>+5RBYsMyK%b@?;9sbx*^y`ycn8(AdT`ZRe1e1QU{{^G|<4br9 z2pv$IPZg}g|q9I%A`4XYe+hZ1!T>ew1d(v%VPlS%NaEyDi`OgDY@53NDuC z`SF(hJEvkenH)qIDKaq2OFAQa_FkNm`DOdZMWgx}3K~VhV5pV`1y8DSE-5!~QYe`> zbjBaAtu_zWe$^vh8@jaZbD~hu9yVAxa7NQRrZpc^u%f(dAR618kIAxy=5M&hEP(;r zBUYL0Sm-_zh_9XPh>$VzRj;TYRVhfi&6q&Q?wH2JSv->B}qb6nKU8>QrLJn`zFgJC8n?}soJ2DH+^pgz{@??D*M zQERXQgI`sT)7pzWtB%D}V2Gap58WYk?ZL@Zx<|hpzGmV{nkdzYQ>9h`y8$AB$&!jT zXJ(DUl9J=YNBSjDiX>yVt@P)RE!CD=?;M%S^OCJG2}St>L+O6`y`>|9R`)9@xUOi4 zRf`<8*po2M;J{*!rqKrBkL5!u1$H!+k; zwHTQ6K-5;+B468TPI%R@{*Wh>&R!ZW(1AZM7g9T5POQvB0((fvN}`GN;Pef#-6?OK z1|A>>=iAMf9L19qGRYaPU3;ndF~u$~Up^lSqivu1Y`hkzy?%&OZ8aI=2L@&4bv+`3 zk;H&tsJj8ldGky7AzU$LN)$w0?hhPK{Q;gyod=IJd@sR3yAo6;CV}hz0g2sdf9UNH z#?2PeLYg1i%75axc+07SXV?~Vcm)}!1Z?iK>ywg{cE&q)rv0&{$wLq^VcEOw=a{pG zd{kJK_p(|kYt1`!%6eLuh)I^_ekg97PpZDG`ue5oMKZCOQ?t#FP1vMe-2*|Zn-0Wh z?2K7>y!v^Hkyqe%p+&uWKErt$ZI%`GS+suY)?vNx{H+3oHZNGDZ!jtfzPGKP_N%@c zt9Ywbqn$z~!ca=PC#-vTeLByXjDxxzcBDDiNLV>GYA?B^HC0pspf|T=g-}0-Qi(`u z5Llg3$>mMpOEUeMWcGw(@=*kr>wKa~op-++GDd#6F!w=O#j}h%nzdX$r7gs+i#s4B zO?&=?EhP8LC8dfylM3rktvx~2t96s+Ae7^jDoe*{Uo_bqS6Ir8|9Z0`R6oVl$1>)^ z&99kZA8LYkRQGlVggKY=^9Acy5*`d%C&I5|_HRGkE8DYz_Pulad8u1^|E3@l~6I9avXuOKc+ z*&5cV?>MO-vQAGd9eL*u`m%c=#eX10vT6@gnYNyYqqtkmwtkC6s>Sn1x%ioR9-v=E zgiYwsV<7asy}S<|O1yj%*7kwW`R#O^!)qb)BqsU0+;&5SEgJCeZfccYiophR z?KXoLmx_$hF0&^oe5O9D7%2itOJ-V3qAs0jFRYoqgW`olxzvkK4yzrJTo0cC>xxO z;K|%>VUj;rn$3PvylwAsO&6!jHMfBa0bQTIOL*POqzP;=w9HAde*8XJ=Ot1Sx{^nj zae#+jA#r(qzZ~cAPy@rYp1d^a%;SIa2WaH%;9apDTuAenar|LVIACVB`k{j9{VVTh zu3qQ2`KQ zjXI!sL3$&rH9Is4s`&fMq4-tB2SPz0R>mcNb(5MA#vQC^UTew~i|zznD^K89%ig@S zzrwFZ%GX*X(jPjqfVuzTPfSpj^g%nH4XAf*LTa8GM^fFSd-Hpadnq;vH1&zZgvJLP z-U*)`aL6WvcSndMwEq5jd=d-?JZml%_LnupmztDx1?&)deu7#$`9*7rdq~Bn0wYCB zr3oo~)(Nvn*rM3S+d|eA36nkwzkcYG>I@JdjNnvQ_-MP5=FxVnq%by8r0RiVNmc&| z9%V(ly|`ETGsD?YH$mJSU)J#J-K-}S1>)!nnXx>PYo#YQ^0v?`o2Sp&l?ZxcMrd~`_<41bALEbFgVtI^t@cy)}Y$u>d5>X@r_Iu z`8tD7-Jgllu6?STe>FO-OCu5^=Qxn1i^w-Nut=QNH25MYJi0UK;h`FFQVYY06+~8H zkk(D^tYXXt!&bqowU>o0G`an&1}-d(%kBeRoCW@QC%c8nL!E*mo?4WPMsol}cZXjT z!z$J|_u)P=D*yBpm{fgDa4-j<{LFDm%$jhC?wV)eLr?C-?Brf^^Q$=qqDV0-zqP>6 za``P0xnrh&$?JQXPkdXMLUP|qe<4ozGG-KKfy#IC%#|2=UqY6b7Ho4WxrOj z1-F~iqEg>&>2nhz1 z_zf!4;^LkxUw_X6YI9=q%llFGGKLA4J!9t}-6EAsLZwnJFK)H5YFj3fQ&*JIJfCE)QgYy5S* zV+ATv3AOw)P&~*f4DiF-ZjixHe3doeC`U<%bJ6z~Pd{w`+V(ARl^`XV%T9UqimG3% zsSt{c)q#98YUPHI7jxot&=;)0yw$9tlU1X(w6ij~+=8RfZk!XDA)?4DvSIW6o8iQ) z&m~95uFcmk^X*ULm~`)hp~zwj38CNIut~NHwKp~xE1#N=6mNSI64d>>zDKC!8bSDw z6*VNJ1XueZuU10dD0pRva<6UECNICIEty*)1nitu0ug5(R7d^dLvs_&A+9^-6 z*02ah_LSy2s=va7$q$=i9~#zyWF!=PxunHo>_|Kz+})J=d=1QjMQWUlv*Ix&JSj>g z58A(+V@z>m&S~&SRW9Y!6g-C)86uWdUR zICPsEj4vMX!}4HK9j2a=dVM{^f2Lcgnte4&SfEqTx;|l2J^uPVg_X~$Lp&UU{!6uai8ME4iGrNYtRzR^KLFv{Zy9qkxh;E)6zWCy44q~E zNa=;rKln3iwX(uN_G}PDu;-oR+s5{pfYsAkVSmVj&8@Sh56gE#24wJcjp5q(41ep5 zoa!(Jlh=uNB6U{JjgF4;5hw}wUK#f>`{*+y>g1d|M?P)-s&Ixc+47d)WI@9_HiqG; zyab-T(Bg&%kO3qvi|1FGd4k>pYvMk&qm!dG6fR+|(|-09JL9uG!*KOLGEqrJLZb{* zB%S!nuyB=>xmz@SCHm1y^^k_}x#@btE*6=oQy_W6KXHF@VPE5_m@fr2LY#{U6)2+j zlGi)N#F1a)nrN#f&t&lnqH7jK^~zSJa|Wlc+b$X0x)0)K7cXzd3;EU0%07Q(j(fhq zEINO9o=xZ~cC(CY(Qvg=bNzhhj=EWbOAv^;_;IDSJDjL-6Gx)cMs^sD%txpIS%^TB zJ{BMS*Gl$!F3aLj_Lhc&t-xY>>5At@G)jwDwlgTvFMD;PNJ0~1g7Yc~)yy)ey(h@} zA5*=*`Sd#R0r!I$Y?8Sm_vfkebd^akyYylI1fsp{#YLgt4^m$L;(4v3v_0~iPyjPU zOEYUbIqQAawXAcOsx|RW_EBJ_eSuBbC) zywI1cVw&|J`uEK7bs5z*GJI!{bL3p$FA!h!Eyq3&(lBHD{@(BV`9(wFh<$`rWP!}> z$~Aqj^@*O~Y!+!NZsnhcw5s&h_@siEdPB?(OSc?y3T{wdRck| z^H|IaY7b=;EBP(avIhch)Cr=uz&H_^kRr&v4th6)&HJqU&?xk9*{#NT3$H^NLz8UH z*fNXykWN07V3x73d&BJO%Q$PrGUuj;jP^4DxM;n+VeadRMgb8j1YjB!F$gzsxg_D!MXsZs|6ilN?!9UGFTdp%knY=jX1m{GtTloymdGh&dI7R4K zIN;O0wFPDh#wAeq?UYB&<40+$9h0Yp zmAur}b;X}3qQ7uJadrvJDEmMj*Nre)?=j|F>s%lt*eO_YQ%g~n`nzKl^bHTkHdJH? zSOjdNVStM@vpOe6UMO(lP=@Q5X9v1%r;8F_Fx8KP0LcXb(1QzgkP83}3%tVlgfp_= zU^VY`R{O-1XFm38T)1}4XVN=XVKqmY%Pc6dYJDA~x)(UGU;X6aQJ3U~YJlXbJvRS^ z6yFaur|=D!nS+QEdr09w9BKg6nNljd5bUPS}>U=OBCPgOP= zMijv8t*MA+u6PvL6~krmbQna9a`q;Vi+8135l(s#rM+&SA9G?$v^)6reI`XQ(qhf| zQ|VT4o~o3hBzd#SBmVc5VTe}m&eLPC%}}>zK;<7Oq-S5Y{<@(dJv{qlU=QbM#>Z128`v!k zI)f%Unc$Xe<#%j;6pfy|=+W{hQ;$y&5{d_t!&)%m5(`}3r$^!X0)gpm2K}rAqMwPS z=PSVg0c-1kS;mW!j=I#(*^~MhA|3)vtbwPY8@9VPy$a@BVnt$%s$h^2eq@E}Kll6s zn?RyHv8%zZ~`RQcc#2ib4Ku?0XmokU5_317dqCX?g3p4 z3kJ!0-<6&TaiN<=1BCaNRnKTM#&G)#+ylQl+pgvDXv;%aZVIwDg{+9&RVg@Ue~!`68-BAN8>OtfZ^eUJ%tslE&*(5Pl^+k z-YL51OQIAvui1m=95V<~HE=0=G)R43VO@NyxF}$Ts@}ZIon`QumN$~oOayOxa>9DB z&0LDEmx&<7Q*hNZgxFcVY&1Pj53Gx8;m)+W=F$X-hOYhMaQZ-&O#I-$*=@zASZ=SN z>De<`cu=Rjsz*p2u4tjd0%Dv>>5xlZ^R z#~yhtE0Z35jr#@W@`ce5>hBj;;2aC>>^mT}_F6=I$*Jtf$xDt*+yOSkiPm}D6y`JA zKWrXhl*?N7Ig?@1{A5Ftt%>$TVrtNnrW>6(7siEr9pMM*Rp)T{I?FY zz-ui@>+mrU23vqkoWO9PfVCtGzE1{w2~tV|$xJE!{DXhG9+@gqL)yLxMFj4^xwr=g zLL`CvnXmXUjyciUU~%y-clw$Af|g#r`}9{y>(pXx1Aiyx0XEa>4|Gg$xH|DrP4oxnlBs|8Vfmvql+rlYN%4{ALomaeQBncLzuYrEd zXI=~&x@(nNBrwWE3*0HszW28W(l_A3agQWznHaEl87P8PfQ;@SH_yZZq*6AwQz@iX)=Z z3~MbI-*-YXQDC@?b7($h-zysy_URS7#=%1yMwY#PNxu>*v|eQz=&DVLPq%@X08EX7 zW(ov*G5rz*yVi21S|3M4v*lwbjk6@l7iO!Qf;cWy1V;x?K0++`e$ltbm7VO7Y!8fGo%&Sx8T0c|Rxm1#CsL|*x#IMqp|3vk z-Tz)a$?Ds0`?zsei`GykBCOnLxRFz?qhwgy?pC}Oa&Gj>&+G(00aHF&#}0VulPUuH zg7#YjeiW;=+bQ$eBmp2-ZCn#$URTXlkR9C{NNm~c?_I8VKA&pNX0tGS<%wIe_&`z8 z$-El9l=@#ZKJ3TD$7UtA6$wq=6w^rD+FZPjc9h-x@W*-Mv|S>IjJf*2UWV?vo>PV* z7Y)6idu>SbZs+QlB-f~h%C6Q7Ux6oy*503OR_3#>0gGVQCn=KfdlCjkxqksm8RQlS zosO9?;BdAG+D_zt^TYegH^RyjLO+2eE9UggrR&F84Kq#Fz@)`)YGfe<8IajmyaOT) zqnupcKV9T#uxErtb#x?l5{Hb9eUjljHm7D~?-8^;YLZjYMiv^X(S*>p3?!fY_~;0B zsKkv$7gz|}>o|g%v>*9PlPzEk;!#lBUc}yfl12H-iYSR3AEpi2L4o1;2D#ap-ZCBdYeUprE)$x#xlHcS;)a|2-ZGkAvs;Wm&RL|MO)x06Y|UR{9v(*bflV%PWGSbi-0ZYsp8cwHj*m)u)j=n$;J@uK zDAv?ATI$2i6?6sE2PAs=W2K}PEj(I{qAs@>m#Oe)n7D&=N?iv7vs}Z&;Exp(5fofn)cdLXddZi&ovm^$UODYDL*9}%+!U+Aj5S!1B|_c zU7oor0Ki$TD;Cx;k?!D?s^-Ff1KW-xL zg6`eA{e&3#HZ=TZ>W5X3M103KH84^IMq@t6p768BU=0mvqu+a+JihcGL3}{n*JSecM6EWM)Gxl} z&pdkDo%Ec{Cg2H{kL|YAhvVnP{+)cKxp+<$QMYQMICWM5kOK0Pbf0SQ*}nWt-$2gV zchty?qu=MD!0+7gq4ccu$Mefl_6IRwa#Gy|Vkq=U{OwibQkWx5k2ikD#QxHLLRgh? z@KP;Qs`hW!+n8@+9cVw~A5Hn7DVCzh4dx^22dYPzeTLeUz+RraU`FG4G+>Bij$fU! zk&4bcD1ZX%RfTPB%1-NNG)43zRjP|_md*Jd%n;JEz4)gKQm?`dZ-Z_d!I-?)_zl-NWNIjMjWIi&4E+hm`f?BW_xk)Dwsu8 z_eoEVgG?QF=EjQ9!tFy#$sM7rmQ`SNZAdZRAP4OQ|IHIsrUO^58@O{kg~{oo!`kO_ z#-{CxvV8tV!g*^`(d7tw&?M-tJ`})%*W@ffEyg1grvz)cGD#grSh-!f3r`j6!NES8 zp{(Rju-=aEskfEczgB>1_D<~@sPqa zo!)b6lp0gNZ>QDRrme-t_H9FI;>!p#_hCC_T5{kiy#QlMEYn~Llfp+GohQmHri-x- zmNzH9n^1=LNy&t4;)NSFDjTwG25wBKTeg|q!Yh-LZuJ>uV7HYt@{;#BnBlvEU2W1R zXZ9Tyl&l?3=$4-2Yn_rO^7+0GbCdN&T|8G3&HbBNYh~9A+hSNzmQT3Z5)2Dyu#&Pm z<|pzVOQu7C-G_94RLi6=9e?E;yu0Sc?QcY__sl$Y#xB1waAJc2fMh|3KuqR(1p(aiA2L^IpJ@5}^yD1c5S ze)aft(OWA*nI0XQ*u2^x$9IDMn+Y|J`cSiX0~F& za8DU_#fXl2B=wRflImou@fJ~z=8~^!anap0!Sz;W4VC9%#1DA&;!koirU2Xf>FYHS z!BSjajxm!W5hG9J0KE~M)}@rn%;o9N5@g9LJd7lBIQAjx#o)s5wzZ#zrRY+)Ok8^* z7+y5aw`!3QUh?=rJcoN?14Md@W0GlXT2`0?AqRm`ysRy$lwsO;YTNyU!jP+9Jq3y$ zQZ>!bPqK5I%9`09lhF6|x6NP6!nqRPw$X z@7;2SJXSq+xu54+0)iGxHN2f-%4t_>(19=J;44M&@Kxlyn1us0r|%xJoo@%bKGG|Z zRlGfM@`xY)fcS#K=Bv{TtkP>yei)LYyqU0xu%DJu9FZD`EwOC87LV6nI^}Y@^_1YF zhQl(Ynne@tZjS1X)qJVYCt<*h-G=mgo;iiib80-VDPAJVv6JfAZZAi*yMFB|Ga9r4 z8lCc3z}AyR%bj_klLC0^Nc&mYfSG=UG$mWF((~pxpxzh>n-^7yoJCjMJ{a}kJ!*3@ zqU8o0kCcj%!@(NCAdHGquUGEiBcS>37U$9L`Xh@DL-|l)^;{Ngq~_6Nt7M9#fsyXU zb3eTtfDEj3nwm2NRap~{mN8W>k?%%JydL=73n1ZZ$b`hH@CJE{MvcP=0D*}gUeNe| z=)5Y$z@>ws9h!GA!9B5;*wS;U?CWcwIO|748CGXUT9Sdu*@LHwEp25@% z{qig$j7p1cr#&Cbnx;Y`GVo(T8l&wNxr%~{`pLZZSQ4hUp5KU}&9A5x>+x_d*?Afi z$QY;g1(lie30#|izsxsZ9qx|I;zezctM|OXsZW~zI3J-|s}(9U6E)bY4Xb+YYcJ(<)MS)2RKG+7ZIJnt9Q}Yk;gHF$+}{ zmY*CH`#hYf?(CtYRfa&H_!NEL{u0kc{k2*JIw+3>aR4=v5vNQ^=e?#pYiou~xbvbWHd z1%ud_FF+6lQh^rC$lj_qo0$C{h|=*%DNy-nhTw!4MpX|Z4cl$9Gn`SKwhZq|U>F#; z7KXQIz-$(YqCp%!Ug@#os>NMOou)=AS{_dpketYSxxLKYSddcRKa5t$v;|jv@J$llY50>fH>QYd%4-hf}WnG29aSzFpTk?DN`{A`0)( zZ9OB&6j~(HTj;y!!@~pW{|1`w>wihkR237Xkhw}8h-{CBNr}UT4CScMq{b9llhCE< zv-1TDjS>aYULtMWa}x-^(xQQfd<)L%4l*FhmsK!<$Z#N&8dgwJXN%o0WS=1G&S1|C z^ZQFwSHlAut}i`3Va5P*l~i_wG+_j>QLPVC*C_R4EWC-W+F7)@^<;bYgrj=>k**?& zV^0s$clef!rq%PEuIWZ-I2amKKb+zPln1-r*7U*1CpyIqXXhGq`}vRXE9g?+L%ESc5n>*Bje%lC4-{nP<#{)iR1GOlr$EC8_#s_^ zs-jRrBiidVnS9fPa4! z_12IJ&e!t);;Ajp*3T;pXmN}oUmnC{5 z;lJq4EuTewXcM;7VTAGp;jUu=Vp(5w{eY4t@ubqM3o!tdY=NY?`BsIgASjop-0XfD zdH$bFn)@>3__>X21H1$?$Oh-k=a59Oklk>(KO?l`VcP?EJ++k7Az1x_Kh}To#Wn@9VJWz|_R*}EJ zA?v8%_dX9xMlzF0N*w~tmA6(J5P|xPodA{WMAe&}vZGo5F(5vnPj{CBUa)v@Som<` zjDryuB^#@8*t*1l(zu zK^=bsdRriS*!R!L%RkSAwLIARTq|Gu%>OkWxCcFeTwWy3CqX~yEEt63e;+*D)jboDtHy5DmVKe9 z%=AFyQY&_~O7hZZ5A)Zbsw!yhiRBIVe+xBvZ;DZX#4%ch>&(9BR)#?2aw0GWKOZ9R6vH z{V{NNCorUW*bL{tt!IMgjWa&8q;;p^eRik*u8tM^;Ejp3W-HqQPM#v{KRX0G%K_-H z|E=&WhPnI1yG#YuZGG-Oxq7}(yiwFHuPDTJTw3_*&ZVZB*G`~uRMqZ{dN{)-JB$|hXTnfTpMgJL;2#7y) z9*q+J{N#UsSHH&?65X-fi(OiRv=E(~WOliv5_`aPB1*Kqeh0kZ{r`NyCg=tK;?o23 zmd!DdX2Dj8)wyAYWuS=wkn%Dht{Zt!)UE^EhDMr!X(c}fLM#gQ4hCz#HO{|n-LH6G zI~J@*e8*Jk>>Xd_$uCe8a(%p^_<1FYFhn2>b$U1@y&hyQ=5-e*S*D;=9ISJ_Oj_L` z{~0cFD8VIQ{0+wRJ9_+c&i?tXyA@%ea{=d#PquV{u#Fk+@E&-p_IlhzMPH#)L_X{F^_vdim5XOl-68y*M6F54ZcGsZ2M}G z{r`UE|E#Y71ESIU4q7v6@UVOrri{1>AjARnk~H3br;0ssIp zzMKZ47#m3B(~$?dDr!S+P|LZm-EDl%C(4VRv1z*`5tB8*q4b$bCe8$aZ0_qoa*}HP zHR|U;iW2~Z!SGH>;a|t4hLx_so>uPq(#DUHu1?_qj!f#*4Zv)9N1)%Aa_0m{M;ZWY z8>ytCaNq=|pom=q!b-?WF^sG9Le5@~_+{H>-Q}b-5EL-)Pw|7cQPIH4#Pzf*dV|Vc zT8gOPIj3Z9lTqM@=Knra=j0t{MO*aVq7yP9>6i0Pn; zzNvgJg4?5}=t8k?yP)Dh8%eG`KTwOdXeP)3K19gOtO{Vo#!_&wjqmusrx1%m07To0 z-ucOX5vYms$YLH`$`+;)OxROUdtrbJ-RjO7I4f z(X}7a0sq=4iA7cU&)hfxO3lST>hS*CC8x!OHT?Qe!elRUO30j~(ug$L83;j_M}{r1 zO^fQ(&FaqiK38`1H(jvqZO)VuegqT~4AOvpLQriECM89Xv7K^$!wqy9f?JYoD+W0a zn*)jPJ>KG+C21w&YuJ5;9J97oI0<_Iz8O}3GocsndSM?qDXn66vbT)wVGV5I$Ya3Q zM2NITlP;bM72zn#l>lsO+_P(COhp%HWiDwUaU5o>TD$Gx3Ot-75D+DFCA5&2_NyTf zdD%|8BaZ^=s3(1bPP1LPZ-ex9CUN zZUAzwfFkHI`QiMQ@@GL*X|Bt&UDK-dQZyT4*RyT1qi20*hVO+U7}Xn`kxYSwM{UOC zEEplVzYNm-ADx3Loh-Qz;P~di-t4^&YDqF>`dD1r$xWk~B~QMeL?(EbQM8jrJ;QlJVsNc*HXN`MEGWz87%UNtbLQ>2B@LG91_vgdg-HB$09(0UYO%kZ!m2K$+GMrJY+<9G3$ zX4A?_e16H=@d!o(yJ~`J2NQ@vNZLW-8fZ;M!Daf{b0I{tz7JOTU&-lz{ipRk_xR?3 zz`?2YB!l^!&VbU9%gm*;_!h;Q!?-5e(bc7BL1N=gJD0uXA%6LsAnG_f@*U0EoeTRd z@qrb?91lp~nPLt(=|?)Yv_5`Kt3lJVdkwP&rXDepY#e(n*dyDu;jf;CNdTzk+|f$- z>e)yhPso%Nq@bumX|n3h+yf;Tu^^!IPwy<^3v;53#qEdw99&W7D&bRk6GkErxihBTU5_LTtn<6*OVmZ*t z6diNKOWKM;=3;YTa3PpL?SJ9{HUJBsZA*lr31n zm-^xnFn6bvF9c+arrBn8(H(;J88;>gyMQ?9(N4Skd*n}ELnYECcuE=)!Q~icKMNij z1p%zR2mf}Wt{@OgPFEYpqd$3yYFoMQK9o6mO0983L%0`FBuFe4Z{@1cRtLm39}>`! zUZq^(MD|Wb0EYy}av%6t$@UEh_gMC}sj}YsCa71XE4~qHJ7ph#-H7=JyAH0_g^6>FW33$vZY}Qf=jHK?Q)y;spM{n` z0@}yN0NgR!rvr*=IX$Z%fyZJVxRTqRjp1&6G~Zqmc`sv(KM0eRt(}L&czHk7Mv1Lu z;**QL&PoFBJ>K=U0oX03xGRlVQp2b-(t#@d=~Z(sX%J%bMBPU<|w; zFf5UO)=syPA}c$ma{je1mwONu^GU>%>8a}rUH;35ohSRth7qyFu^oJBWtq$Y#2h^s ze8N&6{GOMd5*@jBIB<#CkPqD#nJ1g)H%jA`Qql%8y7if** zlcIkCE`|jaw}wi(%UiBTLgn!nEv!#I>5D&T`IJR-Abp1JoCp?}CJE~4% zeKk$FBs_Xi*YmQiJ*teWL@%YokB%Mu8-y_n}U%+%P3 zU*>7l2-Z}VnzwH;Qo%fJ!|vt_c!k2%gVI1g;UL&HQaVzxk@`k+muq% zWNII`2XURcbUe`UR8nidu<52!pygovQZ)VcN)*USN1R#`*^|m!98C(sA~LiAP_{q9-yOn413)j$)YP*FI|fO#P}=#&E9X2fP$g@ zLCR}>jAl9JKA@%QGl0eKbl{{XH>Jg}#y8~laG@gX7_OQ?j%A?E=UUx|gXfz$#46^s zjw3$@8>iIk=jx_CILwx}1$htkK78{C$87KQ!KYU_f)jW?@<42bWROT$uW)jrBS z=+5}S0ueKi7X=+QyQ~bj1K1=S?_pM|*rfN?{htN*|2GCPu-qG3S`=6G5wG~Yao_Wi z7|YezNfP7Hhk6WzY(`-W#odAjU3FpYdG*o;nTxwhIGYFZBYFWbXpQ9 zPRHPPLTQ(xP@6Lp6M&Ly{iTsm?Y!pYNd$1eoW}zThmQceR3kpqHDrupP#*N{==t|@ z!QhfCQru}{)%orqw zUWQ|PVb;s0)G~2*5xmTR5QUj{j5^*)r*$bg6>NGVBrYg~lXV+@M#7imV2~66E-a!^ zKvoz+mgcXMVH!{Jpw%n|rhIQ3t~VPo)!A;bkX~yL}{Z_R7HO6N#ed` z1XN^0I-+PUn@q?@4iR+Y3Un^xVL7Os_)GmUEY5(iH39V?1_{Vi#U4^>jo;Yc1)Vnv zKuDv`Q(@qpWq6d#pFNO47p1%9p6BVG_i#Y#IddZB$O^eo7Zt-1m-W-%7WYf!Uv8D5& zem{oM8+UVstq0>)vB=vvUx94;H(o3x@!n^`9N8c8xgx} zA)NjOux0FhlB`~)kBE~z(kERnolilsd>8KA*KfZ}JB1@lxd?ij@EWVu$O^o7jVc^~ zAA|dh6~59{*v3c%XOXY(1Bv@>IB4<=`mckEITBJjC(c*AD%qnEZ2E;H&H=21mI+OU zopNj3IDW*DG}y$5P2&%VK!>O7@~!s3C-qdZnwN;&O`z?D)828}!JY(Z*$c1!-)%sN z#S^Bgt%gsGA512mNa|x5Ft}SQrVS`9nfpI@tgj+<4DQeM#5C5=l)rCFJ=3n%KqN+x zT|&`n9i&O2~#+pj+faZdwCzcM;S(KX7&$T#u6t8{my=cOZ=KhHs}!q^|eCQ5c>M}Fna zVNi=9PStE8MDokSdTjHuh^dR)SijSjgjWczsy1@m$J0{AihM{EZUXy=jI}caRWp+j zSxWaP|Fo#1b-!?S9NE|3FtM@egHwvvw5uaxrYE@-a++?c#EPDpTHzIPvWYP{s!7qp zGnxS;@bDBsb0z@tA!DY|kRXV1INTsch%cRVSPdbdb&(ng3@cayw+s4JDb%lW^&t-* zXT-$uI*x)I?j=+d<`gs&ZoisQ432j`P#FVPyXjJQqDtPfJNzLGPNRqTT&ovE_$h z>h)fLejpA2(w>Od!TM`Z%6gdID#*#D@HO{bV5OtVS2Y3H6qpek)xfL)jufh2pECWm z@RN-Ce4xPca!z!;y!Z_3yUPbUW7zhH;Cr=L)Kunt6syG8_(=iC`hlD%m8L*Cyq-z7 zWh~rL(-1;>7GkS>Iwp(KJY$ z7h55j*u@y%*_v9k>0mBJ0$R9r@FK;abxJ?f!B4;DE;82-9LF&f{`^M=y@XysBiu$v z!bH<1N&C2gaTW^~W%-(zyi!WGNv&5|^QB z9Z)1c$SN0ak@up>mmp|(!tqlAw;wk{@c@bmhS^Ex3wqz^**Aa|!{cgp5 zG{@ZtJQR0Lvd~{(68t%dMhL+?yYA{PZAFRIiD?@K5)9)YH|C{!SNg0y^+HaoLkv^JDaN`3!V{IuU|z}70_BM z_dWO)F~ybzcX3|GCz+^X&q8Nse2}0;smJ^L6`7vibH%jhc%~dWU*2Jcr96-YYO6k) zNbx6P7T(F@`}>tEitMRQ5(N)IPJ|lPo0KuVhp#Wys`RRMF)TwIw)IWWbvx;#1)z&3 znEVlrSsA0EfrPjoDGLDE@NgcmUCBEG5O4dVnipiz;|{&S{uN(Vp?-pr!PJ*J5}dkG zjQYJP&&aSN4z8_>Z6^u9?Z&?^qE0BbCCeW3{37kiK*5FjZOiG2yH%cjubJ~STXgDJ zfb9^aqw1l`E4p$4`NNt_FU-W#-KZn-vIr!!WNwfrpS{o8nXFrH|6G#mTTTTMOa?1TJ-k4db^kX z+s4B9;z;{O`}jfh#M&f4s9jIl_n9wzjoT=$@2emR3;g&>czXlg$>92+$+&-C^iYFq z{IXigfCc5aFfJ1U`RjoCuLYVqlSuCBEG9x%_4du$ zvl&uh1$5TL9G`pll@oK&8l)EAxuLvstHKWH{x*>pGD`jHFE%6u1gQ^e*v&=?V;yw^ z8yj94<2#F8N|(>99GHzrvx)y6S$)wB$(Rb1zC}jS*CBPd((*y*`)e2(`{dK9h)dO? z$n0DNHtXGYyT+YUBM#&zZePX1{P9IYk1%y!cs)4yxIytU+#=YdFZ-1Wo5;Q&G#r%j z5H|{I!bb*}aOv2Uh7Ss1&<}rnl|dsN%ggsaJ^XeyZ#Ppn)zI}e)$X(@y1V#uhVJ94 zatGYslOUIsMma070Z|GRZ@~|LZUh0u7t&hYjhj0A-n<5S<1v$j zHoQ*GPA@ckP;q`fS9Js+Vr&NVrCgo{uY9I_hBdvC>7TbHkpk-)TzMe7JGv<^jvQ{|PDe8KpK1L0 zraIK^t@Nb5xEZApvdou3%emMIPG$X%4lf4#pa7V z9u;_$j!Vf5re-F5@8D^YjfMm>8o60{CGW=4)VDy@H2Cj5gJOs6u9x*@x|I6PC>M6u zq$6%+%usuDWm@f@-}v(ljt3Zymz%<*Ozfjoe97Z24n(A5-Oh5dcR{F3n@{^)f9+pP z_uKdL^;({c+f7-gr0P${MRO;2;p&HLnbN5u3Mb?vE|rTevRw}%IQ>%cxJs&~0WWd) zpQ$7WATB+c+C6yo&R42lL0$EG&ln?vEMM&X-_P&QmYQ}3TWY7)Se2Z&l{qWb%zo`! zwsI9V1EEsO7}K~i`F^_W?-5bd2`m|S#e8itkFdkSh5xSteQLXn5KR+_l?&&M?e&|9abhxr( z@34plb3CGY-W$m0p(fFJf3$K~sr?gnz&BYpo-WN9C26QmdQ1(@_9*d{aoncj89>%A z&R$RDRVe#sGQRZ_*fo#@+X-S2Qr9Yq5g-(P_i<(E_tLb^fCB}@JooH?X>TkkDY9Y-GL9v>gi#zvS6CRg?#eNBb^c?`k{ONb9yS$ht1%L)ci z;x6(`ovRO&H6=hA#h-oQbE>vS7)A@r^6eX=hc z1W%rdJb!iXHmfIQJb?+R*h8VuVpL9;3bc1Q)SdA<*~pex2Qp*KYq*~L-U~l=#16&1 zM3~E4>$@2l%S(=3V}n7ZeItR}v!1-dBLDoNcwl{wjFX#QmTiKeXC}Bnh-Ma7)r@Si z$gwcp$8U5S1_l|kWr@r-ZVR<9elV{5_#YE{c_SdM@QPLK1rct;ol%GMkybMJOW&-$ z9r5XSo2spa(>kqhemSCoKKV~?F8Tyd-U%>LAa8|}Wh#KH?{-3?D%1C3 zJl(rPxwYjtFRWH6$tnH`I;n!wi*@X?7mNI*Hi6F79G#psE*bq<;=&0$Si|`?{Gv76 zfj3na%^Ug#SQ_Ap7ttNM*dET$Q3GZTUZI4iu{XCCpA3J*(nAC>kIUZ}X3ckjO7%R| zFK*Nk#$M)S;2%6!|8uFm7DFfyyB#>~aC;UT*1efo^m@?^ZKB|lQ7DSt19wlBZ9Cfr zp0i5pP#Y7Yjk=q$*CLYq%uU{(7E|yN@~6ZY630CZ;Fo_>r0fwtksne|9+43cCo~Pf zi7%N7S>1_3$kxlflvOjdw6yR1{e_zRa686BT@PH|IL{ic+b9JYMf_UMazAjzO=$guxMlusXEUH`W7#OLK3dD+dHEq(RqDL+sMlMbJ^=jP9IR~f6sF*O?-B4AnCYG zb#zQyY(t=VTVazD@lKES&YC96eB+c!c_oz#_iKeMrgvMdSx8f3~QrCd$TL zx3$Hfo->^>j1@YaQr z#b&Cy;(x@_ka}$0x-Vt^ipfUuNbMy2cRsss)QxuCWXXI+E^C5dt(e?M7(1xSk>&fW z>ww<&6WyP5K>!g|XgXHrqW>!4WXCk6Ty`jf-W;N`xwF$dbrER`avgS8P)G{*8PkXJmTWbun1 z&E2sAT_4=(flP%qr?R71JJj)v8frFa?qAO)A?9{ScSelHk%z70_GRgGuKvPspBR5{ z-<(XE-_+gpkU?3!RGrA8>tSHXj z2W@=jPY%2kjSxDfOl4b0i$U_vubO-tUoA01jn;7l>Xk;QfeaES&a&*@9>7@U=(=gG zXDCC+I1*u^qSqlq-P(8sHV>cu&fM+v3N4Y1i7NbL(ZE8%G=MAL?dj>E*0WAKi0gm6 z{wo=Aw-g6ygQdd^+UUAPpX-!bRyjMoOwUkMcUP|GCPPu!d`xFq}7{bd6p@@)p@J{DRj@*!%Qkvk)p^QHV1KK$mh9E5^IK?3O+_cEi zlJ1!M;K$}^<#vxO7k**W=DP0TUFqV;!aQeuM8NqW3kGhYN%3k_WL9!4ERp~w>M8Fl zm!ZVgFVyqO!Xm=L-a?J}+QJ!%$fod^Gy=N2WwT#Y+}>XudedgMKxM51saa4fVI{^x z7=c6_j72{_OF`?19}ad6ntq)9t6Woyh=YaE$Ll{6c}#c9r_gh->p>fxw@%I^-J3?w z9xEhWR~={~lF?B5ue~5%Umlfwd9ANt@{E!5dofq!)=N@~;@5__bdn$)y@D9W-f5&s z;8W82KJmzImKl*Y;N{iyWt=~`qY}2A(V5cGNt(jD+`B=y2u8VR>IW3kd9Y}d5NlcD z)2KTZ{gzh}G`ktKOnvuVpNMlmkmQ^Q=hwA%eUza&KxxBLt!!jOhj8th`tvw$yuz2? zZqvU;MMG!XI^clTBh3KlQ(B@5jmXP0e?Ojp8pXks2-7Bt{_w%W&FxCn z68xUMJ>TiV=B8_F9~zxCmmV4Q-IP8&n3?vL^kcTJhzFkQi{unaX&xF?<111U60AXf z1Zih6xMi1KHG^xn@~pI#3vRWyT;d~rQySj2(oJ*zx#+o+_Yy;m?Hv>hs}5UcMzKQQ zhsU|kaA>$)@OB<#USzQT*&A^pT4w?ATa#%Ln78k`$pqAs7Ye@9w-Ly}y?To}M5ke( zMe0=Ic(9q-cZWO&Pb`>4C^%-g3PC` zzM%!HCTpc(pY%1WlrN(toHf_slp1Djq%J~Ydz&geEw50S6tk7xbJWVn85uDKvXsPD z2Xo|-y${bd%K=BX1whCdKEG`j|@mOoXM>+*TVPV85 z4s(oBin6&a)3MSV=gpl_lc`!_$Ah(kb$x=l#zPzJP?)V;F88YCPLb(HdZlK-V+2Y? zv%AN0)+zypa!U3K07!VKF&357Wywf@7-Rp)jd93cQ#G#$7fAJt|m7m#Nv z=J??;s1C0-?Z(It@5VZf{)~QM7*M+5QzSOH({y;L)&JH!z-Sy97FUPGKuCPq*h9gc zD*DfwK5(a`FP-U3d({}pJ(KF=Fzs=mr*FDZfI09Qi{|=OuFN0%A$Z)Dw(^akdO&X& zNV9(5L4ML&o9>TayjhgLpO&JnJhf|KwQ1$A5orkqGL6AHxDYuu6i`ljEB;j(rPsWy zC{ln_`5@lZ?uWkZ?)==Q%oQMoXaRA>#|sT#)hB#u$pD>%=MD~%T>Uh|^4IqikI(n| za-J>&B=t+ab|bU>&P%AD8yB)%Si|>QtEV=mRe`qVP;%>9O!EZ{v$*HhjxaB_~s z^%d$#?5&Mp3wfhI0TS=RU2vY?9D&X*+0*_AcPeRn3VagJX>^V{R1^9ExxfH3=@(N?IJB) z7G76KrkhUY@TO5LZP9qf_TtytS9=OL>7Y7H9^lOeQO!h2jLo_y;ya49^rK;IVIpi2 zij@on7OA?`kxsgrVj3ysEs-%OvHr>6tKZkVA?s_c!)px2lGv&EI=TmQZ*6`L9Pg*b z`Wjfj^}Z&O5ab1=_)nH34_NS4BuyQI~_3t^`}v+z9nA6 zB{HxZHon86u-rIUS_It1O~$%2%heY}mhAa_GdW<78Z{0{It-;>>4YVI|m5k@(t#Yib_Xj3NWeY#py{;JxuA4pQ>Ea*h*t1E!#Bdh3cv-9#B#AbH z?PI=^BXp5$G;U08i{A!z!eQ4Ppj#TZ>8WjB855s`|G2b15r zO1IA=A|nkZQfX$%|cN`4o5f#?%L5nd!I&bC*z4KLmd&dlk56+n$1 zu3OEm>a3Ny)4U0~RMIJ4qB-W!<~soXaoy3wgPqHLyOO(83VrIS#rK{0HU|AlUCtID zx}djS9qe%7yL0e<<}6Z~%F7I=yTDwUZtYl_4}##J^$iB4Uuw%}vXN?q3&(Z-ojB9o zP|-Q-PgvZv{{g(HU`Or`JP-JEQiH49PRM~`V=T|ON|oV)MT3vHlp-p|#+9{SQ%Jl) zVkcLzRK`X7yMUjrMZ7rC2Y5GIoA-e8h5JVNQVi9`m~x3E;&@Db)~guav4FBq-d--= zsb(6D$NeT3_w(GKviEUPbTG&L7I)uVHgs;WH}|Na z2G#8Mi8NuDeHFXk8d${g8)S`ee3%2&I&rYE!REI9lGxWV+vHz_MBenRjVtC1375-s zwD=p>PIs)Y^b?%hv5~f$lcr7VAYJI-`?5>S0bh%4iOcT-=}bOuh3td*&CPgc)W*;f z^Qp;Nxa=B`8@2O=?>;j-PawkFwQQ4hK<8gXF^;J=!Oy$V20z6td-CK-=ZpzoH9L-t zoMf1l92W6A8X6s)bawA#dIQVR7h$WNU>qi!pkJ*UP`Io-7R#7D0H@sn^x!@?**bv< zETBwS?nFK^{BYCf>1R@*6*-<}VX`UEIv_`lK)uKk8*$-5TK?iyP#2r|pA zg4lsi;MhWtaB&=*pO-^R4saSGwKm-<-3n&yw+wq zk(GSTg$_h~J>hKvFH36FzO`=)GXJ-@48vY$H+R&*3(uK! ze|0lU`$Xb07h@|@Iz91E?yvFe9xny<&;%y|_u-c$-P2~f<3tjT5g8kC#PPr%kr|H! z@1;uxfe4E*qgPaF3T4TL~lj zU1PzGcRPVi2_boMpQ%41QIcwYaSwPZ`E1SmZM#-KE7`JsATM9Q(6;{KrJof6FyS!0 z>WsPJ-K&%II#D4@5j4=|$d;wJd_I1Pj%stKH9&jdPJ?M@3@0_)@C{?78V4Itaqc`O zyCq|bNKfq-Xo>eF-|m5ruU@gqC*1jD5R?@)j(11-jrwVLK!T)IUOYTJ{6a+K0Y{nj z-jbZjMCE{YiiEhhppz314UUdoCC~+pyh=oq3nXMKEW#IC+5>%L#`0~@S49`$)xxYe zoqchwQMqU2s`;|NSb+Fa=0H!nEoL|Ft4^BFPI0!Ro6?~ z)?SVZ^ylGH(t-6BxzD^28^3j-FA-w6qlt6G!G*}^!{6(#Jo;_q1M4+rrg@XIkE88x z>vlfZcyc8l&Nsa=gx}L1FSj^4Kb(5qnHie_D1=b>a^B)r3&C+y4D!XX3%{F-Bht# z*d6 z+W`ss$r@~THpJP(zG3Ih+VFdQn!1?9h2M)t{cF+Kga!hMxd%~OEh}~A*NS#yW^Nq2 z&U}!#uyH&C1xk*9(MX}5B4&$KrS-B7(pw&;v@1dLI0sH?X1{MeAL!O=97#4h*4}pD z+1W{3xPEo(LrtIdR|bm6 z=799|W#gt?djqB&EKRzksr>oU9E#0SN(pG^#uHrupyjoETO2%fs47|nXYCYmH|4&U zw7vS9!FbIvzp={(4P$D6k0MtOq7y_afduIZ@YG@9;n|MUI^{7teFtTjactWigy48w zm*yozk+QUyjQ_mZC;{W2_F;bYeJVHOjrX`7+uL8XQErAyc!+5{W&2IGRfpAojzk2s z@eSW+8F#eCsm()AQzA4BZQ9gPx}OB zP@Rz%!D5;5J0+5DQ7KG|%Kgq*G!NaGT$J5eV6ITnl;o`2&9J1@zdNS?Qk+19k@cp_ zaHJwyJaxPe#izvd7cA&9llDlRX0W`kdf^cP=v9LSx?-Sv52-;<5)#eiH+ngoFPNt2 z$D-TUy#%E#5$QR`Uta^#adrJei!oqvyK#$mF9g@Q`#fEz$?w3_J~Q~JSLT?QP`hy~{8;fN;ZA^~ zyeT9u>&U4=j~BH-_p(8cU1H{DOtya#!O+rVxIS8908-rp+KP|S&RfHwv#7CtmJ=#A zE~fNmP*O<{WclSCmg1V2HpSGch?Mo&37ANZ)(Uj#Ay8|$w%D(Ju|#vc@IBuDxdiq5 z5OmH93ua_tV<0}?`1Hyg!}@U+mMH&tU-D^k^FO!+wYzq3P4K+2?WjURXnfV@(Md;r znE`q%8pj`TPc?7}1&{{sDl(2XTc-=nV^#l9&a#s?svjBhEp)1RzuVGkk7Y)mNmMOX znViC2aot8?zB`wO79O3ixtu=Z*zH}AHD5+L!{_V8_?Vp)9YE(Zr*o_pG^_~dyso;3 zMF8-)q_z1r*}mcp9g4-S85blE{fxYH34RmVwFcGPa;IEOcR zrR4?vS!-oSrY0iK`zx`14#ANDfV(O$w4&!>=Q)h7jT`1D`6C>v=SDf(Yq`#|*GpZ> znipJ+p5yJ+)`6_vc~0genySpu>ME%uZ{5Y@dD3m0qymm!KqL%B{tLE zLUh`inF7JMcLijZ*!T6WlRvnR5@BQHj_m!yDKncZ+_N4qU;#j8+k|%J|V=z6iI+^H2GuAD#7xk(MNpr4fLd z;o;AkVXX8f&H^9}`uy@Voc9%0Up8&&;3NaX_caL4F^&8F0W!2|?Jf1Zd-tv$JpEXP zxmg~3;;<#2Jpd36^MED3V;MAqVb9n&#k=#BV&ugR_WkvVsv$139L+is&}_3AXtBGq zRajRaG_~HoeR~VAH)^F878a#vP$_Bq*mY<2``yLTazW~Fx#87Lp5v07@^hxO5_q37 zfHqCOa&p4O?$?wMPBniAY8n6j-*XX-P>LAAkDqrv?D;p?k|qVC>!1wp#I zr5iyyq(MMfNFy3`knS#NX%V<*-|+q2nLC%iR%V=C_Iu8Ap7W`v z4W2u?N%iUX1L}65Cy;shM*pwkVYVk(^tO87jSyv} zA@t8)NC4WOY~&w@*-f)7g;}!w`3ef1q4h}l1F&##3;>4dJy^x;)xi8DSYklVIe>WDy{w)|$-j&t?En`u0$0y8ggQZ;KrURM^g}X76_60byf#++89gNnZpc|kopW4^4Y8XSIKc<>x55Xl1k>GHy(o@D7>A=v)xY011eC2 zhy`}fM1MT=9b1pBuWjf2@GT}zGF~Iw7(E`IJfUDn8I_}~U5+yj}pl`Jfmf{~|h3Fx*3hDc)QhRo$&^q1ZJz#`pRn z6~OcnUOq0;FR=bX2^hFxu>}%|Ew0KKgeiL7PnXN>9Z1*iG7-YqD)`ruT@ryBcg5(9h9i-!CzqldM)@+RUBYVJv znZa_q&09I$z7-)!_7L$p@6~(#6??6*Q|$v{f+K9GAC*lLJQh05%(1NA0u*pmr zTKk_V8#w}_ik3gRlmhMyJAVK{BlWm`7(@m{cckN;jctX9#{p;!U8m8BzbImT(^wxc z1NCT+d*AEfrt>kD-Hq7y#qEIEy>Jpq6}CoV%5ngyu`cA>(Y7J_Zu4`qU$fFE98Y+> zqQGQ0jR%*AtODQ(rd_Gf#0quXc+O`_DTFRD5K&Z^Z(}B?k7eqp=rg~xpeFMGS-q7$ z{=&#F#u0<->KuvC`C2~Xt;MV_+B?lq$6N@_C(R)&SN=Xo4Qr95t3hVZp2#56L5)d@ zSV(G~1b(o!)30^mMgg@V6?gz@Rg~(+gl4Xlg}f76a4& z6B)L7H<8U;tKY3!Wi9y5s~kZfb7}T)q4>!kPCnrsY_@M3>?kC~)cT~LZIVKP z`=lcl0}L!wr+HRe_x?j#%s&!3CD@umnSWOZkaY?BC+@o%?&fip>_U|>gZ%l|GHL;c z8In(dkd=A*xbqk5ZMmajv9p^TM3!>{9;bJNMF^QEb^xzSKBY0jY`L8HE!P(lIO+#v z1PGIo;7{6BM3dd;ug4OSl1!F@Iy*Znm%<#7br~-8as{y|P~W|DO>b5KrCUBi4;-Q( z$WMTi|F36g`U(a`$9H{wz1S{HJV}){LeouH$J;@GSD8V|{*|4bUDjNGdW<4&`}A=Rx-+ zhosRRuV+8+I<6iEn|X;e{QQlxKVkU~Ljx6m^;cZfe|VQ&fqCrn3qd^a@#%OFWg#xs z4|{K=!2b?c4F60|gR!fux;P6-!nzeV7@HsVE;oD49!#*GMfF;R} zBI~?=p1|P02=@%Wq@2$kBf{^^WGKl|dWgaT`fBmCGl$Qn^R=t>YLUZ_6d>6XLmcCX z9x=yLm<^9zA&kJp34tn}|K%eW6Wejhyu)pMb)zkm@z_0o>jl@k%%FURLXQao{_BXk z5%G1Cf?tqhsXtn}lA^c#{r8?c2UW@)bRHJ2mt=eax^;@uX%$#4Tffi=orWLqY8bZu z(`5LoP_w)IigkN96Tu?J4ufhu>sK1SFZS zE4^OipO+0q`xOjFKYI6$Lf)g60%EDsWjJ3N^1oZF?b$*=)63zCN4oBx=tpd43V(Mh zjY_o#!|f(W_i+gnyl{G@aem;@i1JE9`46y1wdw)eblH=0tfSh_?oxNL9|qGhV((4M zqzwH8LhsuFtr`xpqi{f6WFMFl@Vby3m)3csL{YT3khfhpOe|^;UXwdX+N)zO< zq(65!NKUCUT76WVWBuTU~5p+;vg>kFgk+C)N1~Je-Il}2L&aD1e$uAtoVKOfi()|KdGK~L{ok=P-_kel(N-*;XpIw8-& zf+bb-gP1$Xe|^V{eB8z}CXa2AbmAuNLHp;a$H4WDULzfQH^NbNz+4-xvrDQgnOV9v zLx3#rU)A3~FL0qaQcunft@{ixLil40m$$fkb#LvuteNY6=X93%ccFh2!b~yGFzqGU zAqrW|d||^5npREn#uDKO(QvGf5^qNrmOIDcHWFNd?d6)@K+I{MXu@_Lj02J%B}0QAtk4X3KXh{^IYZ(*8AT_v0;JA(M)TM| z?2mC*W0FWbqnt&2 zlEE6<-tgG-etmW+L*wb4Gk9Qf4;|mUx-pdi-wHevdb@^(mq{{r0D8%vO2(-SwG;pXfBET@#m`-<$ucl zFjM1pFg0qexZ<_#k+Gbs&c9pJ$N*bPU+(#i)MJH%2v-QL?G*O>*uHk3SvmZ_kE9d; z$73VtcaPC*=sPOJzG@^$`A5dDbjk5gFZtKEu1Hz_Fs7{S z*@sI*Td=;p{1B?|E$AjUF$KZ0V}^B-&XY(hE5pcdwWChsE#6i?R5kqXjh>3Y?h&Qd zk092$9!2-SAChbUxyxp4e^pnZpj|B+6WP%W*ND;x%Zl6T zqZc-!@^^C0m!_qj$@Bj?W1%0O8~Wi-w-jF~_C;So`jRsLlksz)IsaYB!2#7ifrRR_ z`J!06jRe|cYK`h*yuSUD+{J`+J9aE)SLC*iQ{d|?-^Y&tg;-*tu@GFuxqJnHQBcT(u$C*Wo2cd>SZ8Yfqjf} z>8+9=BxRlb^B=}e)SU)29q!FP`hGQAU+JGWI5IsSezcJ#Mv$Xe=dzU&(#ZCw-0i^u zFb7^$_&1tAjL)@OC%dE?^4U3C(H)i27r5OpX;UtNPdfix@`|R*2L5|I?P_p7*WP60 zM0csIHh*K|&f03#16VL+wC$MAWi1bc_ixWQy`tI~`Dh;bcfd>^PT^(de`BQON9#vo zR6m)_71I(DT2;Hgg|E+bOH}TJW6;j^3ty)dX^SW526}2ep!vGqFKa9JzI9RBRR#Vb zfU!IV5>490Mpc4r8~nSS>21}WUst;za3BCE8P~z>t_LKvogBlQu{>id7;w~v@u2+o zZ&jiJyB`lJ-PP#VZo09X`%ka4$D;(t&5IP#Kh#hEFE`ZIhxW=TLAwQg5pfgSMS8KU!|bIEJ${zj)E65@=Hh?$`k#%K4~I@3^YR!-`pkkBAA4j4 zF4*j9<{v6=QLQ4uU6L-w-1aT&^-boii503aWBZHIqr8onNUlzBI#|JryskGg#N7%s zwaKK>DDd(FsY-YN5Rj0OK?LB{`_Z{}K!t{biR(FFZZH(dZQUU_heLLqE_)XXGJ!|{ zsUpBg5(~AKUIPhmRc3~Pfx!t#lFrqtv!Ty?_nxexqC&IPi=W}OfSLeMYbh70d_GVm zw23>M17~my`HKhuh3)s^;^7(hXmDFheiN>HsyHI@;Po2-&xC&);5@XGn(H%|M(yEu zga1^qI4FojW3?Q&*FhlC0VKwKIIT|o%MS0V*y6D&JtYCrH4UfF( z?m5O`cp59Lv4mfYnUgdM@_FBZJ#XII#IQ>9FpED_3?>W#CgHDLdEhoeVI|S%+uX6S zOJ$BmNz12_Y-dFp()x|He6!BIQ2XP8iBjl!z2tnV9T$8P4_USGReiZg>EZKg$seal z;=nZ;3dD<0Ib$EdgMJ=PgWO1cokZwxp0lm=oJaKROfn^JYsx*ZY@Ci3iBRxa_!wuP zP$4uR-R1IdfdDK^KzNai0VFqG`#oeJqb~+Xch}rf0WX6>SDq{<%;{ z4{e#yT3h!H>9^H$EPj0XV3j-Jws_nwW z$2VCD|GBSmwmoc}0-dgc#c4A1?f4v)%v4hVIRgkmgJZ}94M56e-`0j(CHVK{H5m92 z0TL87``C_5b3HYbzXN(A8VZyaJAD{negV=jYQ3%KZ-orH5t(BE@`4AHmlj>q<8$VH zF=Vrq#wcKjr_>_Ixp&8N{eIsp`Kn<3gGN?7@(lXFw^x5EEP#c{%nFH2tT5MY)W%g= zz07KRu+TpDEv;g|@DeCJ77Mu%%l`1egnUm1!fpTUW^pcUr{BcR^y>7c0>Sqd;mMPy zv&DrwUw*ZVion5>dfL?byR^;q8G2chOOZaSZ|(ZKWrv$katvC8t6^H?-I9ICUVME_KK(W#nNPU9?QXdK>At(iCE?o2DQq87 zk%_MW?C&a{TSu?=-1y&!AH&&atR4vRKoF78{mPyokYAQ6IsSbRYAg8JK1VTdG`c0gWR52Bc2 z1a&6~0H;m^L@bQynK5$iv>~O2W^Dj&TraK~sSKb)u7Yais;w9B0XL~0R09MOByVGL z&O=a%K4{p}ABrTJG5%GdRp96UtI%f93Qe5qYLvovW-6b3D<~cS%f53#8TJKtn~cmc zUe|7&>PUvlAeN4D+RAYKT1GU(+xjx~FfMPqoJVK^6yw=pkAW{S4m z-ln7p3e@EfDDP(dvewQco*}4g1Bg8ZRjk@c38z!%|x4Qce zFoO?PiteOq{laZ?OR+@e{tW+(ND*1C1J&ugL9Jk@4vxSx4+p!#mkB|o|dtpd1 z?6lu=ORw6SMnTe;*+g*U{r#0OL>+-$4YPgiGCI)tw{ zcYckFHZRg3hk6oGUUj6nX)o__MKy2ybeM2;ehn8P-aZiwal(+}y_&hZd(|8Mg+X2? zu7E2D;zak8JB~IYZTG`wP{1uvRgZ&nHW%MnyEQ|czK`rjJr2a}6zJsqsp#^7P&e;Y zG8=scOm}Z@S}+}BSPa{BXMJv1($9geD~2}&JzEj({gh=_ZfOm^-zp ze`Bb@;CawHbv4d|j`t7Ep=uw|)zv7>a$9jaoJAL3;IjRyfOYq!J@5F)Q^V<94W=`qp{7 zLhE&h*nOb}GxCL{nsng2^0vD%v))%+qKJVweV6A!#!TU&%7~xRJ8b4T;^6Ddag9>0 zxavlHmYue4Gk6u;&l0FmZ4MPQ+KNfOjI=)D9Pc$m?Z~Bp-HLv+q{5do9DW$cf#f2d z9zSZh3nh>I}8*wu7%>AoW9H8^!{U| zrWI1oTP6)u31jMz8rV0~F`u6fq~q9LpEr3D^Q0fIRVkK*R>Z#3Dy4I5zM21G3u-#V z%tw#W4iw`xY!=F|wCpmcp9yQdk3Qi$3s?dCAnb3;kA-UE{#Hbe$!G85b`ITQ= z(DnZEXcXGP^zUC+8*Ux+`x?HNZXE|3WyQXI2oEs`DP2Sqc7v;!xZG1k%19T(_3cC? zucMLcHeh2g>KVjC6m{?ESUQeQv&2VEA4w6uDXZ#za(PqXJ#ssaW#nk3*N5EK`#454 zP%Fe*1KauWb=J@chTZs3GV@_hNis*E`p0-7S;%i|Rcfv0g8@N%d- z()2Wq?tTY`ZFV<}#UZ}I}1Nk{Db0B`|_U~ zz`y2TawT%xwDU&q56-cK^E+Ost(V#uW;M6malBT0^ODLFt${Dg?Qda<7pe*Nd5*g- zUvQ|uIX28E0v<}t_4p_yng#3kicA6TBOXM(SomHEifWrZ_rY2*GFMC!?|ZJL?M__eI1wQLFFNC)$( zG@x-TAaeHi_VX-TeO`F9G?*aSzJNGZvR7_@k=H~aq6lEIwEiOrbSNvySnnTXbKL_Bf1 z+%`U~0}izBpe%`MQGReGl5he^}pffWl!IiY)y}-#43(-mo!I*3NxkFgK7~&hN@{efTkO33z_EVV5&W5m&h3}>)>Z=XmXT|MTnaRHvj>T;E^L|momQt-2 zi5GxL(^$`%Phg>mRW!7;IY0+ZDCEX|)O-eEzO=*3u4L*<6fXeLhtm$ZPs7mVL0n!% z8PSIraT;VZz;pzP1wP3EEn%N?0|-lm4ol(ysDBhO_W(IG#{LupLZvn2zlL)6M|pk` z`kN_W(s~$(&hH8dN5R181G)Jj;4SA2>F?cvWakv<_M!HZYmi8Lu-1v3DSUrzc>ZH* zaQpMB1ymQ)?rfkF?47AveGU#nisTo(Y(gFVj!@}8_@6z%*J3p*SGhmSF0qOKMzVu^75ix4bbt{U_pCa;f7U_L76#LqbT+Bi zM9_Vba*Jb8(V6o}+#-kDWp}*R43lcaIGBGqL27;)f zIfQhq<7rsov+rXyTe)I&WsH)(+lFODR}Xe@RSnL9iGW~xnEnmorla6Z<`dsD0nUj& zi2Fjg4d=!1o~^K}t}5YRxSi>s?2#{hqd)MSA6$vXdyJbIkxvavy5gO^*}Mv>3*!Fe zyhpcqyQqCTO?0-)&36^`Bw7}!;?-W^f95Sp$*|Wqevr4VAt5-;8lM7{ziyMd7{zK@Ug(%IQ;wY4S&6=X4Y>k zDI2+bg2L>#utW5r;+hYJhOnkq?W|;v_|awZ5T|3%_|Q)Vi78TM7L%=dACJuXU4=1( zeRfLw-lwAQhbQ8cH^zc2D#RwiB=_-TYn0rqB}L-!rZi2AQAi4chBHF?!caMHDIl8= zcQSNW_`F_MUM?-~?kcA(zNlae0Ru)EmXSHGm$!o!L#%Hr?Dt`q7Y(zo@3+vVGI&)= zuBrOa)R&z;UkBXo@wA_popge^&7ST-$HNDA+ZS!QsJ?Sxu6Z%O)0 zH*mMkvcIhPA`QM1v-3?!dy}#?+tdeR#@&SyU`=s~qj!C2P2i7B&pycaF`5HnhAY^5 zJMf74Im4-3%SYJzvboy*awMcylE@@0F4zLH)fE4>^iFS_!-ty);g`d+tL$HHQbUQT zPaeF@=_2InpBlsxksJ)N#}n+uUEcZ|KK7wFY4Q`-Ko%brsmj61emPeWP#4P2I&8Vh z=sX@5ZWtJdYn~qbf|SH+(d7|IRC)LsK`MjBfCIHK-8Y`&iwwiZ_dj-SMcRKS*eE56!k;72dV3CPnT>#9#Z~hY=BU zo22wroWe+z+o!Hc!=-cctX}z3dwo&nhP6(chV27=M;b>17OBehX21RpKR$?Q+?zGZ zm>spyA=m(}#X@L2%cgWYPB28|_sL^udaIV}SgW@uQ~=vRe)cJomeMJA1G{Si$WpLS z;kj^^!RYwMn)P-$FM4PAyB_#EQ@}kkB6nA*+I)ge+evx4B^d}--d11{RiRHyRZKu- z!a5-g?e?VE8T+E9RE@afY8D*<@Vm1Wc`p)Fce~l$8)&z4lm=y;(sN}(4X8ra^YT>c zY?iDDn}GU(jDFeS=O)m@9z(L4WB6~~bwS@cK-$7kWrk@P^R}2ZZelw_Gu6?k;nZj$xRQ8RJ8?l^Ja{tw+OpW6sSE zWXlZbnn5$oYs)slu@i6^ZRgVzVrFKmC*@&`m?t!rFu3F;a{L0*G>FjTr@^2ufP`XO z#s?BV%V%VKd_=u329uroiE};D4u>=j$3=hJ`I@8dd6oo0RRGPhb~JB*pm0pu)uwWl z!Z-F&ECbv6`I$bON(W~~dfuTbMii(7R0BMR=*nz(lZG& z@4qYjYFjHJP&s77gLM+kd*23$mP_HjG>Od`VnNeGm8hHId-uIzFLY%lm0e;xr0J$$ zbVUKTO!~GvrJ}><_TyJe^SiQVGAnz#6`Xky;BwnDa?U_H5G5Lg)vZD^XeW&;(R)(SN*^jb! z7SqASjrLUWD8&V>k6Y@NoaC2n@NulU0ygi0 zKFhZ$Kbu5T2(pp|inqK*GDpeHR0*|Hls1wiQ>p@I4`04(;=` zX~NZTH9zbvVO_`{7x210A}n62ClF!LHHC-ZhM(l)lJPNESRAxHzVN!5RVy)Gm=BnO z!1U*zr_H`sA1wY6(<#PT(IJagnsDd@F{Jjeu%ZuQq&wZ5@_^4%ZXLWlkWMqz*d6HsWgA-{>^Lli<+>Q4QMRIFBKA!PYi0nghs>?=O zft2bggeqGZwHM>9YC8Ko?K5eU4DXH&UH7dUDqnJ~<$3DPSX+t~7=A+a)g!;7OhHIM_=(btbQQ4Fl)Z ze@lj9M7+AbjR+_hd6>afAW`Rpt&dPg-AD3PAwiM$W8*~afglWku!FRU4*Y%Hp~Vh5 zqa;qAkA+2Fh;Sq+NJz%a^SOKm$DHW^I4Vgf{gKQNf1l6rag6zmRxrfqaAC5Ft;Z=d zM=9Rm1uYiCD_Bxj$I~w2jpg$^A`&Gh#R3tZQoiYJ@djdWZDnnHTxi~lKpNTfYJDdP z3Ul#6R`&_1g+7hF*QXUslX4)7;^ZUfw`Vyrr!4aY8A-0OX@>S~KV_aZt&Zx)pBv+8>LVS6&8c%1jfhc2rFhBv8H5clM3h=g7rSd`skZR_y>CV%x%Uk<8=-Uweg5pM zX8ALS!%xmnE4fmU2hR2}nHI0;B`&kQ3j%Q`0E2=_PVS>!`}cOymij}T(gN{K0XFP@ zY0t>@CPwq=k$BVdt2>UsVy%d#+M42dDO2u0g){4TNIt13c zy4;ZCj!FxIiv7<`Bemy0G!xZF1zwO1MO7PMBWDR@FHAygIK4Ed3XywCU%@A^`d@eB zdgeJ#g}j)KvnsMrm*r%xbI(hpLS42h*>*+b9v)U-He*H8SM`xBK4Hv*bs?(r*V&e4 zrsw6G!?Lugb`T$FM60&ev#I`W{d68;0sOE#8pSb@euIc7o&h5WsAu)NqtV+{HOt6w zgag6x1OWX@J6JXrF=hK&AB4Pa8k1cWZO&aW<;6?NzJe4wZ2I1SAtJDm5J4sam1HvF zc>w*Wz1#ZL^~cZ^__E~gCED|C**74}I7&Jm{m^zuU{5P?CS|s}HTPW4Tzp-xA&nV`m}>t^AJ3 zvxSgg=wDnR_PX{_py6Y-u#dCr@>Q4t5F@~s@}hGkmKeVBG5%4HSpVTs8$DrZT8tUW zkoTuHjl=RbF|rvtpM2t9ht()sg!AJ|s?As}4soP?2%ZYFs*Mk)XZRM~L1ZHqRs;jw zMH+ZDUp!-&`FaLgLKHjWu8p)^ds6GKwimm!8*XcqZzVr|LUPL;W9)MC)tLA<3xGwu z2~>HgCOZ(FU*4%Sr_cmQ)am&V61loQFtpJGK^npnGiqN-r`gR&d!2*BV*~KkE-qS0 zK&wmQFqod5hl8S@(%c_Up9g4hbuURVya}k?284YrhL{=_ibZ5)fQHZsOv@5%id|57 zbde3Fe${)AL;^I}3nZhfpPvJJpjHRY3Wd=xFgVWNMiyQ+hJm53b;p^d?2dom@LOX? zAi{!B$gD21caf#fA()2`Y`HTaaCmIPHaFd>8=zOvj$0iDF7u%#!;eCw?x8A~1 z-h;j7{zlrCCp_tW!-9uno=|^Kii{W@wcQtUI`%XYN){;y>cx6EV}Jec62-E>8gpms6v2=tL(L{) zHEKKa&?KPj`1U&_tPu0yaw(5#>=qkAjFq7fzg}6bdOL($j9NxnX~PN8x_`eP-Jai~lZu zdQ9GKraVT#$!)!x=ONaGBcLE3?aN#G%nvr91FozvY>PRALvJ~!@9wW^rk4`gIAgRp zRz@#|j|uKWkT^Hqxk8+1@^R+2tBKWrcdsxxed6NfwL;WIuGZo%3IA2YM9oDrNHm>! zn^)`31LG4@dngc%xgSg#*pzqBmaMm>>8h4(l!79ebFdGQ>Y6fpIKdA0X7UbsA38-dFLy(HF zZG#9a%#4;thacCzogw12@@ZAbU3g?$DPZ#$Rlh@R(J8&*x_R4d`5R)E{4BCam7*-1 zwyYmz=CtinXm+wyCvT)W5T*oR@qJDkRYU!8eb{AwuXQj@rj4OJeq1dtQaoB1Jymqk zK1v23vL}7geH_fpCWHn&Y^L;x0fCDP3Ws|$ZM_=O8SN)L6fGQZ9b(Al8xdEb11IFL zs2%QA_QHABIo5;o*T)myEd`D}2mKvgr45$65k>B|K}hI}eRd zB9pDTk4pD=xH31?7dr!&BKDqF_5N0$Gg~fj`V_^jU%R>yIX!H#{sDb>JpTDt`}FT-{x$OS>qAkLGo3Z`nEuj6Hc}ira44)dn;BD#GU>u-R? z73%3ux#e>H@sxec6K)jOo@0@?hZ*;&^0!2d+%N`UfpYjLvj--%@E=VEgqg|#OKWmY zt@VNdFeUGT$Ot_0Q;!;}Ic#X8G|-CpuXTk|%;;($WoG=P>uEY_9BlOh(4O@!-_C4N zZNYldOhxnMPI2hk7IavVt1`>MoN?7h@XElIHqRMK}{ltkoCnr7Rcx(8$Sh zOUq}Hzh3a9r>j4_vbNILw|BP6Z&3CEvBAZw;k;3Mm!CF$;P{jq3|*sY225q}LUPvR zRFE@rqcQV}9$ZKqI2MapE{GkjfGz<`);>;80$=qFa5rVgwH{p6mTkoW8a{T`#SqT!lg91`ekB=txYd4iI5O4@Ur#9tnRVkd_Ss0d8-*pf3JIbHf4Zl zTfc0)guv#f7iJWJ(01pJh8>N=C!&S=ZdfV51ePa|a=)4T)Kfp${(j*D=&H8Qk_@z` zzob@NP3gQ%^?ZpoUT=d(&1N^+gSpRp+|?dx@^mSv)(1qLK*P{&TjYNvBjTp|6`*8B zEZXyqLZw>kU~XSciFUaj3>5b=jjXep3jlV~*$pT5+>Z>V1Byajj<4i)yDRtPr9zoP zl=Cn{b50svkI_R0wJHpP@rq7`=o{YdcAwaR`*v+l6&aZOTn)xa)Ipamhv@?tMv=hE zCs2VMf?zcJ!q?ZA8G?es*?DK!-=D2jFO>BYjDN`k{Rz)8aDum=Icz@w&oGNtmKJFa z&KbdFBS-AtZUvyi=YqB+Y2_q|4?iZ4Ic~px=0TlXM3OihVCJ>ZE#eKnyldvasKE;M zy6F=6hy#(Tl52`7doxa6zAUif7RmWif6W0lkT)W^fTwdLiYbhj>ccl=%3bd_TH0ov zK^#_bWp7{#HUSO9S8icAE%%*5fv@EC*>b*`4>u>}OMe`Cj>Q*nQp&RABgKArMQG8h zMM=&idTxm)V^6i`E8qpvuPi1))CTV!8gnbvgjXNtnvhQlW7s^_(=lN?GFOklkFK`d zIKCO_Du^xU?-Y~eK>ahK(B zCiqrr)^!wVBysxwAR=HYNKfEbCc~;-C}PEEI61NBoRwo|3SL$zg7NFG_3jQE-sQ}s zVcOp=*=xT2;BuXQO?RWS!C^J+8S^Xp_ehmhC^K1Oc=nh&iY+hIZ~GVy>>eNSEI;_0N*%j#z{L-a!W*eZQeN}= z;W_+xJrMbJ%aJ*N|A9`10}XCg{fbUu`&iHirTJ0kLZ4#Xg=Ps~}RvQejrEcxN&`(E%B6OHqO?N5f7rsQoFQW$ls#;a?z zn9QaE{DBwL&ncO2IXc4eGLXl_8@koc1v|d1?!Z>9DJ#htoA;s6xN%74S->lWy{4KQ z8JR?m)t}I<>M+!(kmQLXnr)vHPjG|tcY7%s200am!+O$wX;kcSe!B6}>3CUZ?wp^W zKUk0_1-~x>%DV5#=kIHHcUQN+BsQ zQMm$)g&Q^4!r-74c;S1m&8W*^&yee|-i?&N@^Oo zV{UwCXrJkDr3yAqBFJ!xKA5Wlur5jV;mnjNEKVPB?Jf&iD%T|lf3iyoi6nUTB)+)b zi*=kae*>RK{e}e*5jMc@=~F~PSQttHB3LyXt9aY(1vJgV*Nk$wC+(a13JUCU8^N15 z9U4WBEOTyhB@WKzNRq@0ycWSOVz&!+pF-7CKE8HLCASX!X^hi!VQkk1T!-bg#VpV;k$ z*_n@?d}xsi6D(n>p(py){fS3OG_#6)4{WldXZ{SgB9!eGq%}eFM>xZT$12onkJ!eTvmQB&1Tg7F)t2h!1 zeMB6oSsw;!#w$G9037BPIe(SI)%}vePn)L+GG#DMEs30ta@E#K@^5s`IuRL9Lmx3W zFZ=$+?;Bm|YRn`!xxVpiEJKq-PIvug^?(b%E1ZnSSdvrTiSul}fhd7|bWeCXnWYxH zso}kxy>P;_B)mBx70tsHK}$kkiOp|J5CU_U?zg{e0GdKaBjcUzwkClUnl2+X8ed!i zhl6#A?>)*kb`>42m_=DSz|_bFp!-v0^k32JHI|Xe6Lp4X%Y$+yUc=R;?|0r#yKT#j zC{sR_EGQfcZ}S%W)V|Q{?pUnTgkM@(ngS9WQiQtjo#l7+113OwVYfX*1eO2BQiw={ zwN+s?S80du^Nwm1tSA}*`&+dyuREYaElS>UXNdUf#@Qu3dsphPdH0JpUy{Do!O*i= z$o=P2;48JWM{+LELw$2-0Y+1$yzu1tKqDIfh)zb>R3G2CmRRJvZQ@u}FTT`s-$NAfA`VnzBKF^fC|wX-cgcLF{Scdo`5 z-xgYvcDt>+jTW!n82?(>(Dj{zg_#?lgT)hfncaLMdA3}YDCC%Zi8tP;tBeyqbgeJs zxmQG2MqtYa@1vO*_Bx|-_JtH>J$JgmVE zJ(ca5iF$mRP68aUb zS~TwcY#DU>ea7jE8{eY~%IUc=YMBSDeJO%v4o7>W zEset@GuRehX63HHt6#e*T*lELiOZg`)z!L?@)TQkuV4*#_X zHL4ry7|~32xSF3atK#Py=Spv~xiUy|A1(#VNx*yNx-9*hZO&QO^?a4%R|>vYbGfC6 zaT%H=5(7JkC$M3LFz%DKe)L6xrx$qfnO_q7Y1Lh_C5)>TBZ9%T+Vv}oG1PKU zYv#5*1DzzNYkNCqm`xv7Z9w!2e=#!?R7G1)qy;O}*m}??u3JQx%ZzYHu8yC%4vaX> zZ8Kg*}ub2^;IgGa^{1#rsR z;bBSbXa+wHbdnrM;!2dmKE}{$mCRb(KvQsfTdwD?qEcB!jiy8O6i7{wK{iluQ7xxtedBQsxuim*=?|VgwS;sEoe{ZKoy8^^lR0=DyBwA#a6zl^yL0O)$ZZTNizzLaO&FaVVEh zv>I1Sg>vvCd9^S)r1YCRm~k#T=pm()qsjA0aRags`HTp=hNVY!kd?<pLWMS!O0N}pw zT1*rl$=W2W`STtJw)lg~#>lJUId}90bZz*4`e?3LVG~2^?NEQWMl4~gM%pT1_$Vq&B(M=v8;_AF9_zIpidb_EG*Fe+8lVOFF~#Ru)v*NFkzsyx-I z?~>%Q_=EVL-#TIGaaUTp>go;0oXy(rieNv{I(UvGF|silcTV4vTJ?#~F}#={JN|+y zyQ~MrjiF{gpkYGHP~0LR`z2fU`eT^dN|Vap8Hs&%r@c7NBJam9;UfffC0r`Qld<_> z1*i^g{EB}x^)d}w8PDWG^o=MDau`_zesH-Vm3~MwTs|;oKf6*dui*$Tr0RQpi3r;( zDVn{Tz?3IQ${nI?iT=s*w7_-c%On#)N$d#P6JkzFx1=N#I>cJ7fJiu8#^~_3@}g-RhEE__LPZAaZf;s%)k~Bb9JUZ8!i#l) znYANQ)&K+!sdTf^)6;(@?D+u!S&S-1^x$%8Y*<)zIx!UsxotFbE*mG~T{`{A^rx0% zLu)y<4vzVP28kct%5dlZVd@(L@{ZfKmzHgtS2mYzUD?)h%PlS2Ty{&VwOq@#Tee%~ z`|rM==Y8K#eaSza=W!m;Qzo6IzRAfKvdfe)??>GJTga2%v_qz7xvGjJUp5YD{H6Fj|L6u>Nqqy#UVxiOf-#nr;eBI$vRuP|%~YIzuRc8>ZG@w~Cu zo)!r=Vhd-lIUun(9h;V1ZL@l7`MVlsCb^FF z4+=pm39Ry()T73kHh(!3(a9;}7gj0`Q8EI!+8!NcHGTocztzO1uJ-Wp2J&&#f1_F5cGr;-`Z!yRDea5AIR@}Qyf z^0MN6`s%gEMnbv)g@G0R;u+antwm1t`Lk6VV3%NuzD; zqo%#Q#&7ASxL?QN)-C=SX#H>6Tu~0Ng$xOegsQ~g8&hoN&q))o1}2G^NfSVB6V0is z4;wL$0n-1s2NB-)Ac792?Oo3e?ZGFvj7aO*J9pJa!2BU{{33sqwzVZ7l^7fej^z#< z6h|PQBT|kUz1I?(xOEqJiP{7k^M{}a`QDET9F&^_|0KKWeWXiibz3{W@N15*(hA$nK}NdY@`$iLNpoW*&@;@S7jZU@tCa z$Y(<55PwRV@s%*#n)dj`c|w>|$AscTYF*Adi&mm-DZ9t^_R^;dKZ%fbso@&`>vm8g zt&sPxnsA~F_5RBG885ZrR}Wr2y)a{q0Bbe`b)zJ+AWcV9t#cMnmF=6l2a4B0h?(RFJt5PFXZrbl6nYVthpGI za+YUVBEb?Ka+%uTKYerQ0rzEn{X%K1QDCCS2N(+KC7Tqt=Aa;yUl#1l-^gIUEpBRy zkYpo|ieeJLCr~9sc@GZbfRsM!=HvW9+jXL(bb9IrDH!;o4cPQ?CuJ{wtdHErFHX1I zQDLlT)dJud^ji;O-J6vzZfu1RMf5m{MW7WgI7R)3E>kJ>?DsQ9UTjQJvQDC%PN25KtVEIw? z11kJPqu32a(nIchCSyo(FFd4w(az7fs>L_MS|fVy1_B+Erz|>2L^2}#psgZQJDU)P zB^GG&{HNeHPf^=sC8gVg=>nu9t#MeMg+XkXDi~6`QI2>G+}LTyUL)6A|Hqh(KFPkp zVcqarfJ&aO-8|3<$NbR}qLW2wfv7#)I7di5a*^jTg)4(~xon@YO?~+8LFS>OU#p33 z4hCXGbmO3Sjo06%MirAGz8X6HgsRUQEnU!y9dK4E)%f<#=@kn_&{4j!%r1}4!bi;w zVgAo66G{QJMu^1P2D;rp`O8=0Et-A*(Ysj5$NUcXIdxipua( z`CBfAglPqEE3NZni#vBWCV_p8I~5tJcFJhZpwdgkwMxDxm|Q?-iM~6vL>-q@1=N(W zlf3?lQvQ;o7D5XO?jbv@kC_5H3@J-!({!^h(d!5p6zO~uNV$K@K92oVH2 zSY>`*3#WNM<(&6qo!EK#`O)}lh>pPHg&MgX`g*aoSkUQkK3hG0k|BPNJ^$W1(QyxX zwpsvYdW~0Y^&vCxD?$UinE<)p;v%Zkp zq)aQ^!$Rk)H>e$vH-wf}O>e_?{+lQIbL6mZOo$P(dW)(ua@f2$hi`qM({6^CkaGrG z9Ms+}qN+Jl8r*o)G9kQ@8y z8F3RiAxAs#*$z@R1E5nB7yk$|4>RMQ%sg(Z|rG)Z)~H)iBlt_sWj3?f_OC?1h~rj z=>@WOTIaAQ<8omlrkcXX@SbgN-N2Yjn1_%w*}W|s14XNv;)@yW5Vyzk-J7kk6m-&& zoq9dn>0(i`+6<=$W9l^NiVZPE;*62&-R6yeBNa91ST5Eog`l{22k*O=Al1m)sej)k zCodnp0AHR}mP?sBJqrf!$cw#iY*gLj9!!uGi!dan3F)+-7)4gxHK8+n-!2b@QdpOA zLQZ62J!Vo4k;)HkutiQ5x#Dx#YxRmdgxO(B*S8%eT;&6_nxaZx7+r$@u2V0%=i93g z;N-3x5a2Lbu1*Vh-~H{SOufK^z~|vZ{hxM50Ufx7Gq%clO%z&V^#^JrvgSg!qZ61^ zShZSs?bkT7OoIlm#;jPa=2Xde;g(KG3xCY0QWe1?(|;4+@N!bKry*>W?xqjl*!fDz zuO)ip@;ihV194DC%I~SW^4@0?;(qcW`&)8$`p%BYoghxNY|0>7v&i(rQ^d)2>LA`w zQj4idT7_-T!nhI`UGiH7wSBtzGx*?PKkY3w>;P}ex_NvPl3 zC9_@;yCl?JI~pFV-Uk?Hz)AZ<2_zpzS8CPtbazAOG+1g*L?W$MUSBlA4L18R1*sYR z=d(n+(BR;vhS16~Rk8|890vCM(o%#lSJbp0#rge3>fGNnbq`ylb^O@aiaPe-5*Ix) zCd;%x)3srw(sCL?w}Mi#$=By3=o6{-u?f@)!ajWe;}f^m8^Y(av0E+4u<}d7g+FlA zW1?W;D-kv&^kV;+h=^z#qEh%zdd%&CvpE$`qmPTje1w!lW4jx7q`q~=WAKdYE|0+C zIjB8h%B%m0C3~Z>pHHQsu$NPlt(ERXNl|GPBdHIouRF!VI{O;mz` zR4A+TptNWgKbo-x2k?HHp7%JfYqte;gDAc!=+>h8#QxQ|E$Ta|$&Z}DU!>^e|0qb( zU`{v_v&>%fn5p|R_Li@XJzJ5kha=W&tdJh>`FJHB{a}Tg+XmODU$x;x5=#8JKM$La zvjuLo*fnKHsVzLLTBx(swMp*q#ARFxSA;__Hlns_$L;LM%w1V!d9z;4wLW1TE;y#d zF&wig@n}6o1hx2ap}+&(&01~xprYpau2Hk|p~&gsBQ=7j@l|cf#}OTZ|BDv<&nf%Q zhj1V0kOQo{^h|{)d##1~b$vm<&4@_qdXv$vk2IKBPxs^x~O~b$n&TX2j>LF&HAUs z;mB7?>^bOb%#mat9YY zpiD1qs19=HpltiuNmL`p>r?13tjpW66amDt@%l&xpBq)>lz&l~R1u zg1;8e8koe^n$`ggkt9hbax6;&4H^VqX1sX1iw9YYU+N0$&BRtJ2?-go@fh-YODX|LSe@UJ>_I8IA*J&1 z@yQ4y_a|DA>~7~xb(F$kl9bC&`+4$AeF7vlNyc>a*o04+eP0h{XF|UIiX+`inm$g95etc_!Q4XD8FVD0IDu00 zb0-hN6#iAEMiZurOF=}gH%Tg0qluN%S$(H* z1Os+McO*VnoV(%SpBUl^p&X*|V#X8uJV$Mqg>yvi$kx!HsF%u^b6Wpc^N$@DW`O`5 zAQ#Gx@TXi6R68EZps~V0-}4vJ(q}n8n2GplFl4M@^{~qz;W`TT#lsV($XUwOLw>H{ z`-;`npH<5yqh`5!Q1?}Eao-5epFFlq3UdjnMMD!Iz%n~gRNN|bt2_$T6me;?0a8cx z>A393O*h#RDe`eY<~!10G0wR!aGpY4O8>;?awRZpZcU&jDqBW;z%=2R`C=!y)CbEz zabiZ%LxMqJFBrX7n1{8vp@mWtgjo&)NBJ}-`buOIgAb9ku?uRO`-XV^e}`tk@&boXBpLvx)uT7qYl5F%n+Rx`SHl4K zUYoldY6(iagh`T2sK~VZuoIW$tD+1V!qU1J@fe#I9MME62((=_1?~bWYNwJpug*jl zX|H;fpQ8;I$6dG2dBqXYP>VnJ6Eg?tOaib|a>mhazU?d);UPo3p}|^IyGMYXfbKir zXN28ar-qI*R&0`EnG7$laKEOVHHcAh>Bs0?KQ32}62MJZSau*%A|7T$nO zkHCjpYdIY3^(Oi-=<(R8^4Gb6+G@w7b4&Uy@y7b!7leu>GU(7%@DN~5GYbwYTG*=G zd2CtCR&Db~dx+=bbv>s&mxJ}!mdwIIi@PogSFW_8bd8Qs`^a$^OU(rxzhy0v2Hnt< zp4{I%##Ov8&ro%~|7@NcPQqI{$Ls=7-E(56gGjb@2Jt{z>SwgqnZXpU@CfRR z`6fOu8Co$hb7A5PU!8Bn0Ndt|mv9ZB3(*Q@pOy#`@;_2VZUpTFG6_j=Nj-!Pq6(lG zieDA|(7ySC8@`;!rjqta0RjTdek>MgsU^F_XcWtt4vzbj+S{$`p< zlp?8tjZ%mw`n`mwjYrY%(YAuH-5k3RNFvtsFZSb*qbpX?|AogGQHJ>Lqh`0ote1TL zBVC_)UF#FuFVQ1o{%`8OTP9Az$)!uGbGjKL9p5cK=DZrKfCBu(v>3%OB5moc z9LCbNgTBsh4drd?h}?Ob0A})+A2n};*mx<#Zc4ED>oT-5e;P0qli=8Yk+I)?vFU6B z2HbJMiDofIl9H4EWU?f5p3LRGkKPB$#2JNtIL<$%6qqq8wTG#^uT7Mu91~R)zh}_c z{&7WEU#gv79fI2}o2=N^OiDCY?DYzYPxW4n1R#BVe zBM|6IGTlZXdQnvxZ@%mEM+>p`b)V~x!2^!HFSL&Fy|+*V0>mF!_7X4x;M!1~`)nP6 zzd~MO|ETQMU6nDQ{~l4sJC+O5SDDqW51mmrKn>kwgwIe!vOYQu;|=aY49?KWO^7bo z+wdAF0;L)j+Wqff_+Nf%>*%3AG<`hM9KwD3DgzxPB1oA>$^ZNJa-odUmtcwQebSIZ z-1Um4&;xU{Z~(yg?_&Mu10%`%b9jKJoY|>Nuif0jYkGS9*~xWITfd}j*?o83HJvkn zMfMT4FngO?!$Z)_*(=jW%wD}4wTd~3IK42KxF5+rd^(gP!$;Q{L%4>STvlLK&W5AfiORri^M&jizcKyb;gImGkRJn0m{yI{RLP#PG_ex>sE+t2MwL zo){hylRGLx1PZ-MIxPy5>MSl57=;a68@9%mSxtA<68vk{X|3CyAD*U!hkLBEK zf*@{r!?ubn7vsKsbwP2n(6*$$eMfpkX`%LqPG4JS@gwwCT)XF{nr%q~$Fer8^Dy++>x*;p5hWXYVNa>IAu85-;p7$rj-ft~x zLeVnvn0O9ctsAC=jf47QLgfHzS*NW>d>(8tSG0Wpogh%_kFKkUUW(30obl$HRub4v zP};mbY91*LIJq!ihWH{&=v&)amZ1OpmWdp(kUOPLc$vX8zS@)4%GkN31;= zlzE(aj3s<(G?|6$adW6A^+KNj`dzlY?>R*b7|~+4zf{r~8&A;e z>)G`5jKg03nSBo+ub;0MZ~fLc6_b2PWKAB2^-pBu^eGPSdf!{W<3WL*?w8x&;Mpd} zQA$rxZ?Xwxu-CFzjp4}}Bd#8Rl5r&8t&rBq@MO85)0@DgszFHhk#u_{0I!fU3}WmJePh=aqBI?5fu6fBxx^ z8Im(=pXdcMo~$MW()^}Hb^JCvwDw7$AC0tx`BQkG1gHBvuQ4(DCyZ8|0dXF2V|s^7 zSjZ~lemTo3Ys+$5m}dBZ?SXc5!udmgwQe)=9Xt zFtXA6g-V7bh=El=O227!eikM+OjQA0t+G*Y2RA=7%k`^>kx3F=FUx5v{U}^ zb>}xu(Qf@iB$62qwdp7KE1bEBlPQN}m#l7q6clI(Oex^Wq7mf1BqG2}OYiF4p%!Sp z2yvu2F>3e5wi?gw2BOHjdcOk>G6et~QYKf>t4)ej0>K}>k0?!)NxxZT^F1Ee92oF2 zXQej%zg~k3^1b7Spxf)P+0K>66EcIfcW_{JRM6WQ%${CVN5Fj>dt$jft=)~JO3b3E z%b+raG8|y)(?B;ytbJJ4^~6cms9wxvtEa9`l{c=~B;fpk`d8PY(&cel+@f+i(q2ZP_SDsp5|3nF{0EN{B6BLMDR((5BTE>CiB8C97 zno)BAwQ1JiqvlngkAgG)2W8UEyVyU0*I)u06!fVZLwyf&hdwsQs@&gldhqsU z!%f8;QcWVqC>SBdf24lWd;B} z>zOSNVpX4u2EP*eh7>YR201&T7GSec2CpHiL9NrLy1#Xsxt!<1&=!ZH&218IMXQCf zCuo`rQ3&x7a4I!b+x_)Hc@D?>n=-r1&jYfUd@XVo?V5eRIuBzegJDsf{_QF93J4Se zj6ox$i|fM~&!<}^K|#TIA)i|o64EuOce7T2ww>>JxmBi;FV<|^<_H8}E&>J!wA7_{ z=j&#`%)Sf=U2EPhZ0H79Shy^PMUpR~sw9SaA`s23Z>1O`!*WCu|Mn*Ux)-vMkx}u_ zWKJ73$qqSSR+@a;XIYw*g!%mP!sf8d31nk)a4X6X|G(Eb90I6^BO$bbdOsdMt9E}8 z*#fH8(i?YCy6Xo|nm$D&M>zOnbZ#A{p3Jh5c;(NPDe($7F^mW}hNPALq^09Vi|)#6 z?_ZT)xjFM4Vk|I|(BGCsn;6LHlv5c}2Ah;3Z78~uoa`ERog>B#Vv>YtURR+*uZnoa zjHwwyy8yxztJ{_(8I`sU&d}Y+wQ@9>+M}M-kJ_AU5gfa@SE#7t0Z?llR%YRcXw5U0}44+ZBv@ zC{Lxz{3zJLEkz?Nsv9sj_$8wU)kao;qLcRe?BKqbU{Twc@=uF2DRYLPSR!NLm6=|Z zB+p1nr`N1vzuUjQMqtATjhvesZ}rbrcwtGTkNq|R3QBj;gHN3cNKY3wFT+w8K))g# zT-7FI;7)E?Ae*Ah3B72r2+UN)!bov9A9HZ&;yIXJAF7MBQSfc5iV0^V-YZrf=644= zsW_4tjdHck=ZAkrNVR}@m7Hd|>;8DfUtEyH_6I}#L~aLNL7!Ws_o#V*^+rAIro1 ze>(;LeW&0X(gbj>Tyvgs!PcyM6?*gu;{B~m9U35-`EV}m6|5x86o{&*hU3&6)2{?; zwj4r4stgAmvS^aq#GN}?^idCMj6(K80>0V~=0q>^>+!>g+=W;6^er(zuVZjj40o=S zL`w_`qq39pJb-m@|fUWOg2cn7lsWn z5gZE`?k@_2o<&@YQ(RkgTaKXh&Nl_bl-OJ#=T)b}`7gO#@+A+^)X@^HMDT-~<3-R( z^@DE?cL84_O_NgbmNf*<1gwzf6^=m|qvp=7{w?aHScBGV*S42J!Hm#~-o#CH@Rx3Q z)vr~@z~s-~NwXAR9@V6K8Phg!X8CX}o+?3GPui+#2XExRTj&RH!Sn-ik9aas$YQ1I zVmQ~m(Fj7G&v_0jP1--Lr_xa3=KY(B&4OlTN|jZA=<|(caO^jDD}>0UG3P9=13XS3 zBi&%_%asQ+w**F> z1cdyb@qoK&0N8AoK){;Byna;^;uJl&R+jpIcMRD)P_WR?Pi9#50YGE^$b`Ix6XoN) zXpGBm1&+y@aNDb(7wx9QpSm#%q74R&ogI)P_hsIq<@_nIpj}qer}@h?nva_jc4Yo|t7Vb+r6#5zH-q^sJp2U!m6I7Iiam^^Hr&r)ix2%^ZP>U6tK zOF<2dRkLCTJSHhk6hGn(%zK~-Mjr~1*^3^OT~^VSeyn=n1dQbmF?%mI`MOA(D{$-gJj%G08*ZYtDpmWoh(@b8xI@j>$VfT(0k?TD_dtB}Bmtb)vF9b0(6>Dn*l6Z@$hw=e>AT5%C9jK*Jo{cyH>I z$@Ga!d4b=5U%|Ee8|2&jdpWD`COTW)m``>U1JBNf_Y(0#p1(y>iHO@4mNsz&yoLhr za02Ckl@43P;MiEqYwS1UzDR8AzikgzPi9Y>fX1^g9Gz0*#%v%~av4c)p{{WqP)Ml! zv{se0HZ`3GJS|mgjYVs#iIRSzkO-Dl-7L!008)*FCfqDy;5tCIRg?bTW;|Tz{elqk zpwV$1uQ2LN~%E~yxoT}a$m70 zorL%HI6siJAt8t1JIpe+DZ|Y$uOcXXUvNAje|fPaKylD61v>_&IDyd>KeGH@XcZ2! ze3}i^l7UR7I|u}dwEE0{^ZFkrgvsF6XC%l$sm&|CB0On|1*w+I!FO-Gq#uhlaHL+z zZ><+@-*Z+>Hd6rT2 z!++&|8Z>!BtLF6Cc~I5=U{}Q5COcb zFRA)mq)<#K+I5Y_Y6XzC?Sq1X4$ec{4$JM$5oM_g>Kj93ivXjAlR#GK_aj&WG&+;h>mPDl zOx|3IZ)IP*$m$slLL0P5KqJ#<Q247A6ue9y3 zsP$Lyn+YAUdhTsCi=nq6c#bW!E;=4Rl@l6cB_u5dsspc>x<$ko>C9qn@V&-a&f`pvc8 z-#__zTXPcV109P3G=pAKm3ItWAkdX=0mK|9XJ^%cOEy5mHI~i>1r)j(W3GB8DWw!g zJTxn3tG^4A^Z&Ma6;`k*>C~HxR%n!K82U250s+(qYpr)yW9)%p&U*oXg=Zh|{WcB` z!b9LODQu>T26F^{sTDm|CH{r|;revz&=&AUa{jkn{RV?tE(EBTg9a!I3J6)&wAb5x zv|^GzS1nDmDCY{wMMNFDDm)9B?t)@@fFC%R&bCbpsVCug!OgirWF!tv&l zT3Z;3)>Nhcer5fWvgouPW|O!7PIX1X-9#h&iHqyf$Vrp2TE&{{Mta%{rT+Qj>$ z>)$*TBWvF0Ey+fi=yig(pyV+KAP+>?cG<&zDL~So)s32ZxX9(y6h-`wUaCni`%n)R z7`IS5-p&H}5TbQ9qrA|TdcEa+Kfv$(ES|On;2ARFw!;S5JfJ5}4u!r7@xAi}#`Ttq zLM!R2j$wwXZ;W;x*#iIsmeK=*nZsTzZ##HaCF z2m=b+XdtAQ4kKbHh=Iqx=!r-tSTtK&=aQ&u{_)Bv>Rj)CfDaglS&0 zg?w0F*`~TeB?v($zx!(YIbZh^T@DLI){nicTn83Fe8^{!|?IgQ=Ij`7s$4|E7erVG0vwk z27mZlVd%xCGo$s7Z;BqMr9+=}yB*5v&wH7WNPdZO37;}a(@{u8LOR!1j5abfMCj3&gI)yo_u!Fo_0mrxsSyMUCkM-o$5tz2m*V;uIY5nG z9nN3n-5V)g^|N=Ugi1D%u9z0QY+}c8=DIVOaKtYimhpo*DGf;SE#-CuT*npYOs~c+ zw+H69X}H+}!{mWj5+G(8nH7CtWrYwB8TL7LzLeY^z70I3d;XEgJ@0q|M!ZUzt!8p= zrWSeZ7Q=L!6f}^)hDHGj0m#m{(F#T3*ulRXU|(k9PrllXDNK6Y#TtD_(h5YT?5Ih8 z>0MX}1?s?g!_Ek=v!uYQ8xHxa#kPde0* z1OI@=MQ%3&UK^0QAxYzHuLb~J3Zr(d2t98dKi2;4`8g`MjX&VEjlz}f@B&EfR6$T~ zxp%AU+Y_8^B(M3u6F{^a#si}J?e95@wNDh~7}mA^QarqNjPfbQ^O2MQogR+}E&Pm` zP!@Z?9VRCCrr0G%Lm(A%DWW>^6;FjUMef`1nl1&#wS5ZM-tkYWUp}a*(|zfy1oz+6 z^oqXyYce`)ocnpQe^3I~uO*NGRoL>9FH1a#(8EYAQpw&97@r{OoQiA3XNaF*n0Z29 z)3)M9KYi3;$NOBLHOHFe%Y)c<^p+(>4M1eSFCd0+M7yHVjr-Wa%nc2Vsvhx6f=G4` zrV3?0{UIculXh4Swf!exXf}JOU$pp~RG3bg-zoQ}BYK{hPuFt6?HQUb$2 zZZ!qhx~VOJv>T_!OE&9(>E=x|W1ksdsBpA272e^lQmD@y?2e<*dy)FG_aDBh0WZhv zBcBF(uQ9RQpZ`=MpCw8GQhz=m(nc@P0mR$s8HC3equh)7yH%Za`nG^>E=?Ym0R^w8K391#lLLZ0=I+qH%K#=}!IoZ2Lbz`i?;QR3uS9G(8O>kc=c_())zR0cP56QNYUtPLfrHQp=_3 zu}IVWwi&>QLw3|VftIt<5arWwUc3%<2?bo&2sGrpI&HggUwd}5&BY# zfSZ&BB+mx|rs3sHKyYyvg9aw8QZJBBgtcS?5M|}oWVMWN_3X#R4G90Ao&o6m#Kr?Z z)VtfQIesD$3J3$#xyGw;F^p6VlbpPrvBFIK4vECdj(2K9v3eRo;Phxx*%#$Yn;?LY zSf|nvUsm?oT9bSH zY0|Owsn)`Hu3aDK1I9@m4)f7u&E(Lb8KDEe%hcHm)29{F!1Tm78OqH7rDIi%NZg- zC4g$GZlJ7h!uls6QhGnk-8@3$A(7b2yq6IUBYFBD8U-+{85*^Izj40>T4W8{e2TgF z)QfEq?#t=#REVdnG5{Uw-0;P0?Diipm87$?)50xH*GMyapY?UEhAROI6;Z%FiLy6$ zh`h?w?5q#i%`{|evZ2*$*Ac=39w#axxc$*gLx4xp3;d`5?B$}Bq25#?K9{mMg?V`S zPkKO3Fa^;iL{((3!(kfGd4X2=!HlNVWYPnZlFxv&A(p2gk<~;@a&r0<5OmSN)oHE* zwMKF^31(9!r_EJ_Q}0gp0S;Ib0R6sGD6yf3pB^thDry0u$I6qZz44sNL3xu1C0`=F zKZhDW+-}@zs+_od@12SxuODyhEaZU)V6jh5yDIw{007=)iY*M^4RszOWwj$PsMQ!0 zAN>ycqXshPey2wl8EM-@nC+=mf|*iqlXBfcI|ZtydwYuk3&9P*KW*VujSZW{P(NX{ znHa3x(D3w(&$q2sXDTU~XO@VSRGbd5|Am{`pg#C&m%^2jw0%J~6px@{8+YTHR@hkIDq8 zu5Di{Vy(7V@DBOL(bA9g2&?k?=OzEW*;}cIZLZD@u)00~($Wq~UECs>OxE$M_;1jG z_Yz-IhPCUW;TLWg_TTA8TO8TE=TGOMPjznFHoQyFd1)gA%@MlN*=F`9ALFLD<~>^a zO?wWv;7^(6J7VFLhy(Pi&erVsbBcUcM*6?$K1PW{pFtOQo2vQl!l3`@sEhy@RsAN% z(;D_@i#{hb{<*aYQdH2@z(jiORn}|erF0-fC)kbWY>or ziJ05HsTQ9qX15!zgGT5j8#K!&;#%3}!kr*BOAo{Yb1>oukdBIdQ*XCn_Dh=9pII-QAa>Ih$(6gs3CFqw%80BLaexa#$L~p^n%&AtX)abziw* zf)O-@fSQ0qb;8EseQph{=^tdq8qnJ1(byx+(T^R>^wxWvwr}4~7+LA?PGOmQbHZv8 zhR*N-Sdzp1%XCu$nkR8~_>VdQhNulgbM)1DF{YqWrcdFL$ACwpjJL1j$Q_*8C~jaE znt;NMO*Ei!Qms&-v2e%sz!>sv;FuT;Q(g`Nz9%_3y73bldTOVg+9)A};Rm*fU#ayq zgnuh>Y?CyL@`nXjnwoNvNsR)2+NdD4uV}LV_cyJFODCL)aOg`9>BiBq|J4^7L9S3? zvP(>(k?z9=Jrx1Z4EP5}p%H*#O|Q`Pih=Aw|N$%;_MW9ECQV6Jz2iyU-!_Z|w1mZ)3LklXYuz&Bne7lIYDZ{J> z^E+wuA%I?P?p*JNE-2_9t^BG{%P^%#RC1782s=w*x`u>;`lGXOcEY)B{ zV0d=_^b}nEtj!jI2;x7SE$NRVtx2xn?XNQ%ILdrDNm(IcHxFNK@;KGCa3~cZtGL`U z!^5Q^aQ+?e)={6MPI~=n1~~&{C#JOL0t%on4PscB(->4TAC#3RoNSd^Xn+AvC}1>R z8CSKVS)x!T`H*J1jgXKN*3z`A`M)^oyWAxk>P5PyyRP9bo;*eXtZ;@Eo8xe1b4&6Y zp_KWP2*S$UHLGWiZidwvpX~EbF)E4Rp-*FZ)IBnl-jY+l{3N4-cTastIZufnyA`4Nk*v?V@Vbu-ND1vs^NoDZL*1yT`aGt zex$~y4sQ)mW!f@m5g?PTia#Jc(!mdEzawfBS-}=CbqwHNgDT= zqEZ~XVL~Cl$Eql@CW|7FOS#R^zv8pPgE{P%?V4JCV6N~(K|1V|WT@~i zmD0C~VdS}#6XCsx3k6GEoU(wN&W0(?Cz2um^*4p)>WwuI}!A0Cbi@y_U9YUXQzA28n^0Kmk%k&msBbTNW;<@ci5JvY_Yd zkWx0N^9XAQxMmf8m?mP!@g)5zU|78avb%$U;?|F!5!hg-%t> zucjw)#jS{|sb#w*0h>nla5i_*vfg>S|7en>uOC*@gG6dR%^vr^JdoN??;sY0+1crK zSig`}H_+aH&E%!&ruxn)29Z1jb%-wRF@llYGATI z-1?h1d`7vz2FzXf=n}Rzgrd%`|txOZDNO`^V*+))5~?6SDze)-&qvV0({GF zztsQ2Lt2J^043MG@2=gUR07vmxRh?5KN=KBUNe73?)@G;-2LYMSU{;Qb;MloE?N7- z)e5n$K)1H=n*8ZpOXymfdQJ?dF!hu7F#OPsNdCU1?_?dmo!ezp@zfg;Yp`UEJSVZw zATjIHa{b94(YRLRdPtQF>9YIrYmG7j#nDkyO}0?~x=7B8~>PJy6_%dnrk3R)4#01J_47w*!Iq z?MXT5kNSE-Hq(&hTP_J&gF=3HnpZ&}F3h5^bOm?mI8QW~_46vP#o@r^5oX=>v4oV{ z7JrpLr^PUqq_*jn^4=&AF5_$}X=sN%?Z{n#t@W^UMI`L4R_JcVTP3`YX1pqDwTN!bKIL~=& zwCYS#r5xg`eb~F4xC(+#7mbw-`fuA)S@Q5RRM2F(4!xTlIs>YssYyS7_WHq7OK8V0 zU%wl_Anc+S9HQfOorz>&YL zfu5D9fQ?@srTV^if#NJxjqv+j{y>9~aTCgBwW~T>5MWLe+yM^;J@Frlce8VJV5Q?% zHezeKC~R0#{K_AmTbY>J-KUo0bHUTuony<>%*}U+2Vkul(XQ+WC8MNY^b2#eMS!mv zEC8Xza_WN@hLY)%mM0LT6zHZ|jKoOfYT?!yqX*iTBg%np9D5@M1$I*(2^wAiDwR#7+N^+vfuSc9^tWKw1YmVx z;s%EmZVPlPN5`6Q`Ks97RL2(JZjlOw2$-kkXG4!#+4NROovtaYB%HI;*zUWzo~TS` z;Ey2@($(3kiy=)qU$kT{=z;zL*p`D%fF9>=h~!+p&RGu_62hr2(frp8BJaHyxSnF2 zxl_j>?n*>1-4LrZSWq}Nb(Tn=CR8>V?twZoae9LW?TV&k zcdo5Cd0vCXRz&pq;axZ2E>-rUXw6e(!$U=v@v1uC`_)HV9gAhdq?Rrhu*$0RckM2U z&vh8y1aKLo`7};OI6SC8gCZ5~Xl&|3(Gx(sn9QEZ_Q)kg+7|gs^UbUI?VgG0t7kYk zbFKdX{K$#{39#GJuj}}Nj1lFjlJ00n*38U*k3g3iGTWsvWF+LJd`61yl5?QehaWAP zCD&3sM1u7ZfV!g_lMWGK(S>%-#1VgQX~EA)rhIUo-DX7+b>uzutKVTQ$my!qkR`Pmwt1qDdpiNh29VN}gCPEn{4@bY66Pg0 zp1qe=qwQ$6egOf?*@<1ak{}px7=hfcbd)WPqppl%>riMmJ8lQ4EiK=}WqC}LUx241aTW+-+EjhglxZqT4bAq(J92#fs*xGyK#?YW-_ma3O}kRG@=MGXJ-vlLi`axR6bZ%yY1Q zw1KOFL7oYQ$V3Rt1}ggj`HWG(LfGE9IaB~Y@weLTCJ28=c1wY{K3n6=V7LDQNYH