From a205695106c88a80cfed31f8380e4bfa9f9bd891 Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Wed, 31 Oct 2018 06:33:20 +0200 Subject: [PATCH 1/8] list archive contents: gzip list code commented --- src/fr-command-cfile.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/fr-command-cfile.c b/src/fr-command-cfile.c index 463e156c4..4025bd558 100644 --- a/src/fr-command-cfile.c +++ b/src/fr-command-cfile.c @@ -93,9 +93,14 @@ get_uncompressed_name_from_archive (FrCommand *comm, } +/** + * Parse `gzip -lq archive.gz` output: + * @param line output line in format "compressed uncompressed ratio uncompressed_name" e.g. "758185 3454395 78.1% file.txt" + * @param data FrCommand* + */ static void -list__process_line (char *line, - gpointer data) +list__process_line_gzip(char *line, + gpointer data) { FrCommand *comm = FR_COMMAND (data); FileData *fdata; @@ -141,10 +146,17 @@ fr_command_cfile_list (FrCommand *comm) FrCommandCFile *comm_cfile = FR_COMMAND_CFILE (comm); if (is_mime_type (comm->mime_type, "application/x-gzip")) { - /* gzip let us known the uncompressed size */ + /* gzip let us known the uncompressed size. To get it run: + * $ gzip -l file.txt.gz + * compressed uncompressed ratio uncompressed_name + * 758185 3454395 78.1% file.txt + * We can simplify output with --quiet (-q) option: + * $ gzip -lq file.txt.gz + * 758185 3454395 78.1% file.txt + */ fr_process_set_out_line_func (FR_COMMAND (comm)->process, - list__process_line, + list__process_line_gzip, comm); fr_process_begin_command (comm->process, "gzip"); From 9e437748a7c0f1287a835b71cd55a9145fa76020 Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Wed, 31 Oct 2018 06:45:10 +0200 Subject: [PATCH 2/8] list archive contents: extracted fr_command_cfile_list__gzip method --- src/fr-command-cfile.c | 43 +++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/fr-command-cfile.c b/src/fr-command-cfile.c index 4025bd558..13ae8b772 100644 --- a/src/fr-command-cfile.c +++ b/src/fr-command-cfile.c @@ -139,6 +139,29 @@ list__process_line_gzip(char *line, fr_command_add_file (comm, fdata); } +/* gzip let us known the uncompressed size. To get it run: + * $ gzip -l file.txt.gz + * compressed uncompressed ratio uncompressed_name + * 758185 3454395 78.1% file.txt + * We can simplify output with --quiet (-q) option: + * $ gzip -lq file.txt.gz + * 758185 3454395 78.1% file.txt + */ +static void +fr_command_cfile_list__gzip(FrCommand *comm) +{ + fr_process_set_out_line_func (FR_COMMAND (comm)->process, + list__process_line_gzip, + comm); + + fr_process_begin_command (comm->process, "gzip"); + fr_process_add_arg (comm->process, "-l"); + fr_process_add_arg (comm->process, "-q"); + fr_process_add_arg (comm->process, comm->filename); + fr_process_end_command (comm->process); + fr_process_start (comm->process); +} + static void fr_command_cfile_list (FrCommand *comm) @@ -146,25 +169,7 @@ fr_command_cfile_list (FrCommand *comm) FrCommandCFile *comm_cfile = FR_COMMAND_CFILE (comm); if (is_mime_type (comm->mime_type, "application/x-gzip")) { - /* gzip let us known the uncompressed size. To get it run: - * $ gzip -l file.txt.gz - * compressed uncompressed ratio uncompressed_name - * 758185 3454395 78.1% file.txt - * We can simplify output with --quiet (-q) option: - * $ gzip -lq file.txt.gz - * 758185 3454395 78.1% file.txt - */ - - fr_process_set_out_line_func (FR_COMMAND (comm)->process, - list__process_line_gzip, - comm); - - fr_process_begin_command (comm->process, "gzip"); - fr_process_add_arg (comm->process, "-l"); - fr_process_add_arg (comm->process, "-q"); - fr_process_add_arg (comm->process, comm->filename); - fr_process_end_command (comm->process); - fr_process_start (comm->process); + fr_command_cfile_list__gzip(comm); } else { /* ... other compressors do not support this feature so From 0d9af7e9aacd36dddf176671a54805dbc9408ffa Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Fri, 2 Nov 2018 10:01:57 +0200 Subject: [PATCH 3/8] list archive contents: improved readablity --- src/fr-command-cfile.c | 10 ++++++---- src/glib-utils.c | 7 ++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/fr-command-cfile.c b/src/fr-command-cfile.c index 13ae8b772..416c11033 100644 --- a/src/fr-command-cfile.c +++ b/src/fr-command-cfile.c @@ -60,12 +60,13 @@ get_uncompressed_name_from_archive (FrCommand *comm, char buffer[10]; if (g_input_stream_read (stream, buffer, 10, NULL, NULL) >= 0) { + unsigned char flag = buffer[3]; /* Check whether the FLG.FNAME is set */ - if (((unsigned char)(buffer[3]) & 0x08) != 0x08) + if ((flag & 0x08) != 0x08) filename_present = FALSE; /* Check whether the FLG.FEXTRA is set */ - if (((unsigned char)(buffer[3]) & 0x04) == 0x04) + if ((flag & 0x04) == 0x04) filename_present = FALSE; } @@ -110,8 +111,9 @@ list__process_line_gzip(char *line, fdata = file_data_new (); fields = split_line (line, 2); - if (strcmp (fields[1], "-1") != 0) - fdata->size = g_ascii_strtoull (fields[1], NULL, 10); + const char *field_uncompressed = fields[1]; // e.g. "3454395" + if (strcmp (field_uncompressed, "-1") != 0) + fdata->size = g_ascii_strtoull (field_uncompressed, NULL, 10); g_strfreev (fields); if (fdata->size == 0) diff --git a/src/glib-utils.c b/src/glib-utils.c index 27d4548c2..d520cf81a 100644 --- a/src/glib-utils.c +++ b/src/glib-utils.c @@ -456,7 +456,12 @@ eat_void_chars (const char *line) return line; } - +/** + * Parse the line and return array of columns separated by space symbol and size of n_fields + * @param line + * @param n_fields how many colums/fields to parse + * @return NULL terminated array of fields with size of n_fields + */ char ** split_line (const char *line, int n_fields) From 4323a412d91e4f5c17023a1444abbc1b082350be Mon Sep 17 00:00:00 2001 From: sponomarev Date: Fri, 2 Nov 2018 15:03:42 +0200 Subject: [PATCH 4/8] fr-command-cfile.c: fix indention --- src/fr-command-cfile.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/fr-command-cfile.c b/src/fr-command-cfile.c index 416c11033..0b9c4c993 100644 --- a/src/fr-command-cfile.c +++ b/src/fr-command-cfile.c @@ -101,7 +101,7 @@ get_uncompressed_name_from_archive (FrCommand *comm, */ static void list__process_line_gzip(char *line, - gpointer data) + gpointer data) { FrCommand *comm = FR_COMMAND (data); FileData *fdata; @@ -111,7 +111,7 @@ list__process_line_gzip(char *line, fdata = file_data_new (); fields = split_line (line, 2); - const char *field_uncompressed = fields[1]; // e.g. "3454395" + const char *field_uncompressed = fields[1]; // e.g. "3454395" if (strcmp (field_uncompressed, "-1") != 0) fdata->size = g_ascii_strtoull (field_uncompressed, NULL, 10); g_strfreev (fields); @@ -152,16 +152,16 @@ list__process_line_gzip(char *line, static void fr_command_cfile_list__gzip(FrCommand *comm) { - fr_process_set_out_line_func (FR_COMMAND (comm)->process, - list__process_line_gzip, - comm); - - fr_process_begin_command (comm->process, "gzip"); - fr_process_add_arg (comm->process, "-l"); - fr_process_add_arg (comm->process, "-q"); - fr_process_add_arg (comm->process, comm->filename); - fr_process_end_command (comm->process); - fr_process_start (comm->process); + fr_process_set_out_line_func (FR_COMMAND (comm)->process, + list__process_line_gzip, + comm); + + fr_process_begin_command (comm->process, "gzip"); + fr_process_add_arg (comm->process, "-l"); + fr_process_add_arg (comm->process, "-q"); + fr_process_add_arg (comm->process, comm->filename); + fr_process_end_command (comm->process); + fr_process_start (comm->process); } @@ -171,7 +171,7 @@ fr_command_cfile_list (FrCommand *comm) FrCommandCFile *comm_cfile = FR_COMMAND_CFILE (comm); if (is_mime_type (comm->mime_type, "application/x-gzip")) { - fr_command_cfile_list__gzip(comm); + fr_command_cfile_list__gzip(comm); } else { /* ... other compressors do not support this feature so From fcceeaa4a400a7570050bc495e8165df826905d0 Mon Sep 17 00:00:00 2001 From: sponomarev Date: Fri, 2 Nov 2018 18:43:30 +0200 Subject: [PATCH 5/8] Support of lzop --list command --- src/fr-command-cfile.c | 99 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/src/fr-command-cfile.c b/src/fr-command-cfile.c index 0b9c4c993..cc507a898 100644 --- a/src/fr-command-cfile.c +++ b/src/fr-command-cfile.c @@ -164,6 +164,102 @@ fr_command_cfile_list__gzip(FrCommand *comm) fr_process_start (comm->process); } +/** + * Parse `lzop -lvN file.txt.lzo` output: + * Method Length Packed Ratio Date Time Name + * ------ ------ ------ ----- ---- ---- ---- + * LZO1X-1 3454395 758185 78.1% 2018-11-02 12:33 ./original_file.txt + */ +static void +list__process_line_lzop(char *line, + gpointer data) +{ + // Skip first two lines. + // First line have a caption with "Method" in first column + // Second line have "------" in first column + if (strncmp (line, "Method ", 7) == 0 || strncmp (line, "------ ", 7) == 0) + return; + + printf("parsed line: %s\n", line); + FrCommand *comm = FR_COMMAND (data); + FileData *fdata; + char **fields; + char *filename; + + fdata = file_data_new (); + + fields = split_line (line, 6); + const char *field_length = fields[1]; // e.g. "3454395" + const char *field_date = fields[4]; // e.g. "2018-11-02" + const char *field_time = fields[5]; // e.g. "12:33" + + // parse uncompressed size column + if (strcmp (field_length, "-1") != 0) + fdata->size = g_ascii_strtoull (field_length, NULL, 10); + if (fdata->size == 0) { + g_warning("Unable to parse uncompressed size: %s\n", field_length); + fdata->size = get_file_size(comm->filename); + } + + // parse modified date and time columns + char *iso8601_datetime = g_strconcat (field_date, "T", field_time, ":00", NULL); + GTimeZone *local_tz = g_time_zone_new_local (); + GDateTime *modified = g_date_time_new_from_iso8601 (iso8601_datetime, local_tz); + g_time_zone_unref (local_tz); + if (modified != NULL) { + fdata->modified = g_date_time_to_unix(modified); + g_date_time_unref(modified); + } else { + g_warning("Unable to parse modification date: %s %s\n", field_date, field_time); + fdata->modified = get_file_mtime_for_path(comm->filename); + } + g_free (iso8601_datetime); + + g_strfreev (fields); + + // parse original file name + filename = g_strdup (get_last_field (line, 6)); + printf("original_filename: %s\n", filename); + if (filename == NULL) + filename = remove_extension_from_path (comm->filename); + + fdata->full_path = g_strconcat ("/", + file_name_from_path (filename), + NULL); + g_free (filename); + + fdata->original_path = fdata->full_path + 1; + fdata->link = NULL; + + fdata->name = g_strdup (file_name_from_path (fdata->full_path)); + fdata->path = remove_level_from_path (fdata->full_path); + + if (*fdata->name == 0) + file_data_free (fdata); + else + fr_command_add_file (comm, fdata); +} + +/* lzop let us known the uncompressed size, original date and name. To get it run: + * $ lzop -lvN file.txt.lzo + * Method Length Packed Ratio Date Time Name + * ------ ------ ------ ----- ---- ---- ---- + * LZO1X-1 3454395 758185 78.1% 2018-11-02 12:33 ./original_file.txt + */ +static void +fr_command_cfile_list__lzop(FrCommand *comm) +{ + fr_process_set_out_line_func (FR_COMMAND (comm)->process, + list__process_line_lzop, + comm); + + fr_process_begin_command (comm->process, "lzop"); + fr_process_add_arg (comm->process, "-lvN"); + fr_process_add_arg (comm->process, comm->filename); + fr_process_end_command (comm->process); + fr_process_start (comm->process); +} + static void fr_command_cfile_list (FrCommand *comm) @@ -173,6 +269,9 @@ fr_command_cfile_list (FrCommand *comm) if (is_mime_type (comm->mime_type, "application/x-gzip")) { fr_command_cfile_list__gzip(comm); } + else if (is_mime_type (comm->mime_type, "application/x-lzop")) { + fr_command_cfile_list__lzop(comm); + } else { /* ... other compressors do not support this feature so * simply use the archive size, suboptimal but there is no From 73f0e8f9a55d3da8b92d401e1b88504b27ae8ffe Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Sat, 3 Nov 2018 00:33:41 +0200 Subject: [PATCH 6/8] list archive contents: parse xz list output: but it's only give us uncompressed size --- src/fr-command-cfile.c | 99 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 5 deletions(-) diff --git a/src/fr-command-cfile.c b/src/fr-command-cfile.c index cc507a898..66bf51833 100644 --- a/src/fr-command-cfile.c +++ b/src/fr-command-cfile.c @@ -180,7 +180,6 @@ list__process_line_lzop(char *line, if (strncmp (line, "Method ", 7) == 0 || strncmp (line, "------ ", 7) == 0) return; - printf("parsed line: %s\n", line); FrCommand *comm = FR_COMMAND (data); FileData *fdata; char **fields; @@ -219,7 +218,6 @@ list__process_line_lzop(char *line, // parse original file name filename = g_strdup (get_last_field (line, 6)); - printf("original_filename: %s\n", filename); if (filename == NULL) filename = remove_extension_from_path (comm->filename); @@ -254,7 +252,95 @@ fr_command_cfile_list__lzop(FrCommand *comm) comm); fr_process_begin_command (comm->process, "lzop"); - fr_process_add_arg (comm->process, "-lvN"); + fr_process_add_arg (comm->process, "-lvN"); // list verbose and show original name + fr_process_add_arg (comm->process, comm->filename); + fr_process_end_command (comm->process); + fr_process_start (comm->process); +} + + +/** + * Parse file size from xz which was generated by uint64_to_nicestr() function + * @param str_size e.g. "3454395" + * @param str_units "B", "KiB", "MiB", "GiB", "TiB" + * @return + */ +static goffset +parse_file_size (const gchar *str_size, + const gchar *str_units) +{ + gdouble size_in_units = g_ascii_strtod (str_size, NULL); + guint64 size_in_bytes = size_in_units; + static const char eci_suffixes[5][4] = { "B", "KiB", "MiB", "GiB", "TiB" }; + for (int i = 0; i < 5; i++) { + if (strcmp (str_units, eci_suffixes[i]) == 0) { + return size_in_bytes; + } + size_in_bytes *= 1024.0; + } + return size_in_units; +} + +/* Parse xz `xz -l file.txt.xz` output: + * Strms Blocks Compressed Uncompressed Ratio Check Filename + * 1 1 496.3 KiB 3,373.4 KiB 0.147 CRC64 file.txt.xz + */ +static void +list__process_line_xz (gchar *line, + gpointer data) +{ + // Skip first line. + // First line have a caption with "Strms" in first column + if (strncmp (line, "Strms ", 6) == 0) + return; + + FrCommand *comm = FR_COMMAND (data); + FileData *fdata; + gchar **fields; + gchar *filename; + + fdata = file_data_new (); + + fields = split_line (line, 6); + const gchar *field_uncompressed_size = fields[4]; // e.g. "3454395" + const gchar *field_uncompressed_units = fields[5]; // e.g. "KiB" + fdata->size = parse_file_size (field_uncompressed_size, field_uncompressed_units); + g_strfreev (fields); + + filename = remove_extension_from_path (comm->filename); + + fdata->full_path = g_strconcat ("/", + file_name_from_path (filename), + NULL); + g_free (filename); + + fdata->original_path = fdata->full_path + 1; + fdata->link = NULL; + fdata->modified = get_file_mtime_for_path (comm->filename); + + fdata->name = g_strdup (file_name_from_path (fdata->full_path)); + fdata->path = remove_level_from_path (fdata->full_path); + + if (*fdata->name == 0) + file_data_free (fdata); + else + fr_command_add_file (comm, fdata); +} + +/* xz let us known only the uncompressed size. To get it run: + * $ xz -l file.txt.xz + * Strms Blocks Compressed Uncompressed Ratio Check Filename + * 1 1 496.3 KiB 3,373.4 KiB 0.147 CRC64 file.txt.xz + */ +static void +fr_command_cfile_list__xz(FrCommand *comm) +{ + fr_process_set_out_line_func (FR_COMMAND (comm)->process, + list__process_line_xz, + comm); + + fr_process_begin_command (comm->process, "xz"); + fr_process_add_arg (comm->process, "-l"); fr_process_add_arg (comm->process, comm->filename); fr_process_end_command (comm->process); fr_process_start (comm->process); @@ -267,10 +353,13 @@ fr_command_cfile_list (FrCommand *comm) FrCommandCFile *comm_cfile = FR_COMMAND_CFILE (comm); if (is_mime_type (comm->mime_type, "application/x-gzip")) { - fr_command_cfile_list__gzip(comm); + fr_command_cfile_list__gzip (comm); } else if (is_mime_type (comm->mime_type, "application/x-lzop")) { - fr_command_cfile_list__lzop(comm); + fr_command_cfile_list__lzop (comm); + } + else if (is_mime_type (comm->mime_type, "application/x-xz")) { + fr_command_cfile_list__xz (comm); } else { /* ... other compressors do not support this feature so From 9e0d7d6b7e89bf6958f09c407c01fce8111670d4 Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Sun, 4 Nov 2018 20:47:07 +0200 Subject: [PATCH 7/8] #230 fix xz uncompressed size parsing if the size contains thousands separator like 3,373.4 --- src/fr-command-cfile.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/fr-command-cfile.c b/src/fr-command-cfile.c index b6fa9411b..e03730f73 100644 --- a/src/fr-command-cfile.c +++ b/src/fr-command-cfile.c @@ -258,9 +258,9 @@ fr_command_cfile_list__lzop(FrCommand *comm) fr_process_start (comm->process); } - /** - * Parse file size from xz which was generated by uint64_to_nicestr() function + * Parse file size from xz which was generated by uint64_to_nicestr() function and convertes it to bytes. + * NOTE: resulting bytes may be not precise size due to rounding. * @param str_size e.g. "3454395" * @param str_units "B", "KiB", "MiB", "GiB", "TiB" * @return @@ -269,16 +269,18 @@ static goffset parse_file_size (const gchar *str_size, const gchar *str_units) { - gdouble size_in_units = g_ascii_strtod (str_size, NULL); - guint64 size_in_bytes = size_in_units; + gdouble size_in_units = 0.0; + sscanf (str_size, "%'lf", &size_in_units); + gdouble size_in_bytes = size_in_units; static const char eci_suffixes[5][4] = { "B", "KiB", "MiB", "GiB", "TiB" }; for (int i = 0; i < 5; i++) { if (strcmp (str_units, eci_suffixes[i]) == 0) { - return size_in_bytes; + return (goffset) size_in_bytes; } size_in_bytes *= 1024.0; } - return size_in_units; + // if we was unable to determine the unit then let's return the size as is i.e. in bytes + return (goffset) size_in_units; } /* Parse xz `xz -l file.txt.xz` output: From 6200b41db9ddb6ebb38b5f1f3c35578c67cba16c Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Tue, 6 Nov 2018 10:01:30 +0200 Subject: [PATCH 8/8] fr-command-cfile.c: fix indents --- src/fr-command-cfile.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/fr-command-cfile.c b/src/fr-command-cfile.c index e03730f73..6aa638315 100644 --- a/src/fr-command-cfile.c +++ b/src/fr-command-cfile.c @@ -60,7 +60,7 @@ get_uncompressed_name_from_archive (FrCommand *comm, char buffer[10]; if (g_input_stream_read (stream, buffer, 10, NULL, NULL) >= 0) { - unsigned char flag = buffer[3]; + unsigned char flag = buffer[3]; /* Check whether the FLG.FNAME is set */ if ((flag & 0x08) != 0x08) filename_present = FALSE; @@ -100,8 +100,8 @@ get_uncompressed_name_from_archive (FrCommand *comm, * @param data FrCommand* */ static void -list__process_line_gzip(char *line, - gpointer data) +list__process_line_gzip (gchar *line, + gpointer data) { FrCommand *comm = FR_COMMAND (data); FileData *fdata; @@ -150,7 +150,7 @@ list__process_line_gzip(char *line, * 758185 3454395 78.1% file.txt */ static void -fr_command_cfile_list__gzip(FrCommand *comm) +fr_command_cfile_list__gzip (FrCommand *comm) { fr_process_set_out_line_func (FR_COMMAND (comm)->process, list__process_line_gzip, @@ -171,8 +171,8 @@ fr_command_cfile_list__gzip(FrCommand *comm) * LZO1X-1 3454395 758185 78.1% 2018-11-02 12:33 ./original_file.txt */ static void -list__process_line_lzop(char *line, - gpointer data) +list__process_line_lzop (char *line, + gpointer data) { // Skip first two lines. // First line have a caption with "Method" in first column @@ -245,7 +245,7 @@ list__process_line_lzop(char *line, * LZO1X-1 3454395 758185 78.1% 2018-11-02 12:33 ./original_file.txt */ static void -fr_command_cfile_list__lzop(FrCommand *comm) +fr_command_cfile_list__lzop (FrCommand *comm) { fr_process_set_out_line_func (FR_COMMAND (comm)->process, list__process_line_lzop, @@ -288,8 +288,8 @@ parse_file_size (const gchar *str_size, * 1 1 496.3 KiB 3,373.4 KiB 0.147 CRC64 file.txt.xz */ static void -list__process_line_xz (gchar *line, - gpointer data) +list__process_line_xz (gchar *line, + gpointer data) { // Skip first line. // First line have a caption with "Strms" in first column @@ -335,7 +335,7 @@ list__process_line_xz (gchar *line, * 1 1 496.3 KiB 3,373.4 KiB 0.147 CRC64 file.txt.xz */ static void -fr_command_cfile_list__xz(FrCommand *comm) +fr_command_cfile_list__xz (FrCommand *comm) { fr_process_set_out_line_func (FR_COMMAND (comm)->process, list__process_line_xz,