Merge branch 'master' into push-rozmtsmxykmp

.github/workflows/build_mac.yml

@@ -74,22 +74,6 @@ jobs:
       working-directory: build
     - name: Display version information
       run: ./build/ccextractor --version
-  cmake_ocr_hardsubx:
-    runs-on: macos-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: Install dependencies
-      run: brew install pkg-config autoconf automake libtool tesseract leptonica gpac ffmpeg
-    - name: cmake
-      run: |
-        mkdir build && cd build
-        cmake -DWITH_OCR=ON -DWITH_HARDSUBX=ON ../src
-    - name: build
-      run: |
-        make -j$(nproc)
-      working-directory: build
-    - name: Display version information
-      run: ./build/ccextractor --version
   build_rust:
     runs-on: macos-latest
     steps:

.github/workflows/build_windows.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
       - name: Setup MSBuild.exe
-        uses: microsoft/setup-msbuild@v1.3.1
+        uses: microsoft/setup-msbuild@v2.0.0
         with:
           msbuild-architecture: x64
       - name: Install gpac
@@ -38,7 +38,7 @@ jobs:
         run: mkdir C:\vcpkg\.cache
       - name: Cache vcpkg
         id: cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             C:\vcpkg\.cache
@@ -80,7 +80,7 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
       - name: Setup MSBuild.exe
-        uses: microsoft/setup-msbuild@v1.3.1
+        uses: microsoft/setup-msbuild@v2.0.0
         with:
           msbuild-architecture: x64
       - name: Install gpac
@@ -89,7 +89,7 @@ jobs:
         run: mkdir C:\vcpkg\.cache
       - name: Cache vcpkg
         id: cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             C:\vcpkg\.cache

.github/workflows/format.yml

@@ -26,19 +26,22 @@ jobs:
           git diff-index --quiet HEAD -- || (git diff && exit 1)
   format_rust:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        workdir: ['./src/rust', './src/rust/lib_ccxr']
     defaults:
       run:
-        working-directory: ./src/rust
+        working-directory: ${{ matrix.workdir }}
     steps:
       - uses: actions/checkout@v4
       - name: cache 
         uses: actions/cache@v4
         with:
           path: |
-            src/rust/.cargo/registry
-            src/rust/.cargo/git
-            src/rust/target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+            ${{ matrix.workdir }}/.cargo/registry
+            ${{ matrix.workdir }}/.cargo/git
+            ${{ matrix.workdir }}/target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('${{ matrix.workdir }}/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
       - uses: actions-rs/toolchain@v1
         with:
@@ -51,4 +54,4 @@ jobs:
         run: cargo fmt --all -- --check
       - name: clippy
         run: |
-          cargo clippy --all-features -- -D warnings
+          cargo clippy -- -D warnings

.github/workflows/test_rust.yml

@@ -0,0 +1,41 @@
+name: Unit Test Rust
+on:
+  push:
+    paths:
+      - ".github/workflows/test.yml"
+      - "src/rust/**"
+    tags-ignore:
+      - "*.*"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - ".github/workflows/test.yml"
+      - "src/rust/**"
+jobs:
+  test_rust:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./src/rust
+    steps:
+      - uses: actions/checkout@v4
+      - name: cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            src/rust/.cargo/registry
+            src/rust/.cargo/git
+            src/rust/target
+            src/rust/lib_ccxr/target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+      - name: Test main module
+        run: cargo test
+        working-directory: ./src/rust
+      - name: Test lib_ccxr module
+        run: cargo test
+        working-directory: ./src/rust/lib_ccxr

.gitignore

@@ -17,8 +17,10 @@ CVS
 mac/ccextractor
 linux/ccextractor
 linux/depend
+windows/x86_64-pc-windows-msvc/**
 windows/Debug/**
 windows/Debug-OCR/**
+windows/release-with-debug/**
 windows/Release/**
 windows/Release-Full/**
 windows/Release-OCR/**
@@ -154,3 +156,4 @@ windows/ccx_rust.lib
 windows/*/debug/*
 windows/*/CACHEDIR.TAG
 windows/.rustc_info.json
+linux/configure~

docs/CHANGES.TXT

@@ -1,5 +1,7 @@
-0.95 (to be released)
+1.0 (to be released)
 -----------------
+- New: Create unit test for rust code (#1615)
+- Breaking: Major argument flags revamp for CCExtractor (#1564 & #1619)
 - New: Create a Docker image to simplify the CCExtractor usage without any environmental hustle (#1611)
 - New: Add time units module in lib_ccxr (#1623)
 - New: Add bits and levenshtein module in lib_ccxr (#1627)
@@ -32,6 +34,7 @@
 - Cleanup: Reduce the amount of Windows build options in the project file
 - Fix: infinite loop in MP4 file type detector.
 - Improvement: Use Corrosion to build Rust code
+- Improvement: Ignore MXF Caption Essence Container version byte to enhance SRT subtitle extraction compatibility
 
 0.94 (2021-12-14)
 -----------------

docs/FFMPEG.md

@@ -42,7 +42,7 @@ Note:If you installed ffmpeg on non-standard location, please change/update your
 
 ### On Windows:
 #### Set preprocessor flag `ENABLE_FFMPEG=1`
-1. In visual studio 2013 right click <Project> and select property.
+1. In visual studio 2022 right click <Project> and select property.
 2. In the left panel, select Configuration Properties, C/C++, Preprocessor.
 3. In the right panel, in the right-hand column of the Preprocessor Definitions property, open the drop-down menu and choose Edit.
 4. In the Preprocessor Definitions dialog box, add `ENABLE_FFMPEG=1`. Choose OK to save your changes.

docs/OCR.md

@@ -93,15 +93,15 @@ Download prebuild library of leptonica and tesseract from following link
 https://drive.google.com/file/d/0B2ou7ZfB-2nZOTRtc3hJMHBtUFk/view?usp=sharing  
 
 put the path of libs/include of leptonica and tesseract in library paths.  
-1. In visual studio 2013 right click <Project> and select property.
+1. In visual studio 2022 right click <Project> and select property.
 2. Select Configuration properties in left panel(column) of property.
 3. Select VC++ Directory.
 4. In the right pane, in the right-hand column of the VC++ Directory property, open the drop-down menu and choose Edit.
 5. Add path of Directory where you have kept uncompressed library of leptonica and tesseract.
 
 
 Set preprocessor flag ENABLE_OCR=1  
-1. In visual studio 2013 right click <Project> and select property.
+1. In visual studio 2022 right click <Project> and select property.
 2. In the left panel, select Configuration Properties, C/C++, Preprocessor.
 3. In the right panel, in the right-hand column of the Preprocessor Definitions property, open the drop-down menu and choose Edit.
 4. In the Preprocessor Definitions dialog box, add ENABLE_OCR=1. Choose OK to save your changes.

docs/Rust_migration_guide.md

@@ -0,0 +1,71 @@
+# C to Rust Migration Guide
+
+## Porting C Functions to Rust
+
+This guide outlines the process of migrating C functions to Rust while maintaining compatibility with existing C code.
+
+### Step 1: Identify the C Function
+
+First, identify the C function you want to port. For example, let's consider a function named `net_send_cc()` in a file called `networking.c`:
+
+```c
+void net_send_cc() {
+    // Some C code
+}
+```
+
+### Step 2: Create a Pure Rust Equivalent
+
+Write an equivalent function in pure Rust within the `lib_ccxr` module:
+
+```rust
+fn net_send_cc() {
+    // Rust equivalent code to `net_send_cc` function in `networking.c`
+}
+```
+
+### Step 3: Create a C-Compatible Rust Function
+
+In the `libccxr_exports` module, create a new function that will be callable from C:
+
+```rust
+#[no_mangle]
+pub extern "C" fn ccxr_net_send_cc() {
+    net_send_cc() // Call the pure Rust function
+}
+```
+
+### Step 4: Declare the Rust Function in C
+
+In the original C file (`networking.c`), declare the Rust function as an external function:
+
+```rust
+extern void ccxr_net_send_cc();
+```
+
+### Step 5: Modify the Original C Function
+
+Update the original C function to use the Rust implementation when available:
+
+```c
+void net_send_cc() {
+    #ifndef DISABLE_RUST
+        return ccxr_net_send_cc(); // Use the Rust implementation
+    #else
+        // Original C code
+    #endif
+}
+```
+
+## Rust module system
+
+- `lib_ccxr` crate -> **The Idiomatic Rust layer**
+
+  - Path: `src/rust/lib_ccxr`
+  - This layer will contain the migrated idiomatic Rust. It will have complete documentation and tests.
+
+- `libccxr_exports` module -> **The C-like Rust layer**
+
+  - Path: `src/rust/src/libccxr_exports`
+  - This layer will have function names the same as defined in C but with the prefix `ccxr_`. These are the functions defined in the `lib_ccx` crate under appropriate modules. And these functions will be provided to the C library.
+  - Ex: `extern "C" fn ccxr_<function_name>(<args>) {}`

mac/Makefile.am

@@ -249,6 +249,7 @@ GPAC_CPPFLAGS = $(shell pkg-config --cflags gpac)
 
 ccextractor_CPPFLAGS =-I../src/lib_ccx/ -I../src/thirdparty/libpng/ -I../src/thirdparty/zlib/ -I../src/lib_ccx/zvbi/ -I../src/thirdparty/lib_hash/ -I../src/thirdparty/protobuf-c/ -I../src/thirdparty -I../src/ -I../src/thirdparty/freetype/include/
 ccextractor_CPPFLAGS += $(GPAC_CPPFLAGS)
+ccextractor_CPPFLAGS += $(FFMPEG_CPPFLAGS)
 
 ccextractor_LDADD=-lm -lpthread -ldl
 
@@ -271,7 +272,7 @@ if HARDSUBX_IS_ENABLED
 ccextractor_CFLAGS += -DENABLE_HARDSUBX
 ccextractor_CPPFLAGS+= ${libavcodec_CFLAGS}
 ccextractor_CPPFLAGS+= ${libavformat_CFLAGS}
-ccextractor_CPPFLAGS+= ${libavutil_CFALGS}
+ccextractor_CPPFLAGS+= ${libavutil_CFLAGS}
 ccextractor_CPPFLAGS+= ${libswscale_CFLAGS}
 AV_LIB = ${libavcodec_LIBS}
 AV_LIB += ${libavformat_LIBS}

src/ccextractor.c

@@ -446,7 +446,11 @@ int main(int argc, char *argv[])
 	// If "ccextractor.cnf" is present, takes options from it.
 	// See docs/ccextractor.cnf.sample for more info.
 
+#ifndef DISABLE_RUST
+	int compile_ret = ccxr_parse_parameters(api_options, argc, argv);
+#else
 	int compile_ret = parse_parameters(api_options, argc, argv);
+#endif
 
 	if (compile_ret == EXIT_NO_INPUT_FILES)
 	{

src/lib_ccx/ccx_demuxer_mxf.c

@@ -12,6 +12,7 @@
 #define debug(fmt, ...) ccx_common_logging.debug_ftn(CCX_DMT_PARSE, "MXF:%s:%d: " fmt, __FUNCTION__, __LINE__, ##__VA_ARGS__)
 #define log(fmt, ...) ccx_common_logging.log_ftn("MXF:%d: " fmt, __LINE__, ##__VA_ARGS__)
 #define IS_KLV_KEY(x, y) (!memcmp(x, y, sizeof(y)))
+#define IS_KLV_KEY_ANY_VERSION(x, y) (!memcmp(x, y, 7) && !memcmp(x + 8, y + 8, sizeof(y) - 8))
 
 enum MXFCaptionType
 {
@@ -201,7 +202,7 @@ static int mxf_read_header_partition_pack(struct ccx_demuxer *demux, uint64_t si
 		len += 16;
 		for (j = 0; j < sizeof(mxf_caption_essence_container) / sizeof(*mxf_caption_essence_container); j++)
 		{
-			if (IS_KLV_KEY(essence_ul, mxf_caption_essence_container[j].uid))
+			if (IS_KLV_KEY_ANY_VERSION(essence_ul, mxf_caption_essence_container[j].uid))
 			{
 				ctx->type = mxf_caption_essence_container[j].type;
 			}

src/lib_ccx/lib_ccx.h

@@ -161,6 +161,9 @@ extern void ccxr_init_basic_logger(struct ccx_s_options *opts);
 void print_end_msg(void);
 
 //params.c
+#ifndef DISABLE_RUST
+extern int ccxr_parse_parameters(struct ccx_s_options *opt, int argc, char *argv[]);
+#endif
 int parse_parameters (struct ccx_s_options *opt, int argc, char *argv[]);
 void print_usage (void);
 int atoi_hex (char *s);

src/lib_ccx/matroska.c

@@ -1362,8 +1362,8 @@ int matroska_loop(struct lib_ccx_ctx *ctx)
 {
 	if (ccx_options.write_format_rewritten)
 	{
-		mprint(MATROSKA_WARNING "You are using -out=<format>, but Matroska parser extract subtitles in a recorded format\n");
-		mprint("-out=<format> will be ignored\n");
+		mprint(MATROSKA_WARNING "You are using --out=<format>, but Matroska parser extract subtitles in a recorded format\n");
+		mprint("--out=<format> will be ignored\n");
 	}
 
 	// Don't need generated input file

src/lib_ccx/params.c

@@ -127,6 +127,13 @@ int parsedelay(struct ccx_s_options *opt, char *par)
 	return 0;
 }
 
+void set_binary_mode()
+{
+#ifdef WIN32
+	setmode(fileno(stdin), O_BINARY);
+#endif
+}
+
 int append_file_to_queue(struct ccx_s_options *opt, char *filename)
 {
 	if (filename[0] == '\0') // skip files with empty file name (ex : ./ccextractor "")
@@ -978,14 +985,14 @@ void print_usage(void)
 	mprint("  a .d extension. Each .png file will contain an image representing one caption\n");
 	mprint("  and named subNNNN.png, starting with sub0000.png.\n");
 	mprint("  For example, the command:\n");
-	mprint("      ccextractor -out=spupng input.mpg\n");
+	mprint("      ccextractor --out=spupng input.mpg\n");
 	mprint("  will create the files:\n");
 	mprint("      input.xml\n");
 	mprint("      input.d/sub0000.png\n");
 	mprint("      input.d/sub0001.png\n");
 	mprint("      ...\n");
 	mprint("  The command:\n");
-	mprint("      ccextractor -out=spupng -o /tmp/output --12 input.mpg\n");
+	mprint("      ccextractor --out=spupng -o /tmp/output --12 input.mpg\n");
 	mprint("  will create the files:\n");
 	mprint("      /tmp/output_1.xml\n");
 	mprint("      /tmp/output_1.d/sub0000.png\n");
@@ -1245,9 +1252,8 @@ int parse_parameters(struct ccx_s_options *opt, int argc, char *argv[])
 		}
 		if (strcmp(argv[i], "-") == 0 || strcmp(argv[i], "--stdin") == 0)
 		{
-#ifdef WIN32
-			setmode(fileno(stdin), O_BINARY);
-#endif
+			set_binary_mode();
+
 			opt->input_source = CCX_DS_STDIN;
 			if (!opt->live_stream)
 				opt->live_stream = -1;
@@ -2934,7 +2940,7 @@ int parse_parameters(struct ccx_s_options *opt, int argc, char *argv[])
 	}
 	if (opt->write_format == CCX_OF_SPUPNG && opt->cc_to_stdout)
 	{
-		print_error(opt->gui_mode_reports, "You cannot use -out=spupng with -stdout.\n");
+		print_error(opt->gui_mode_reports, "You cannot use --out=spupng with -stdout.\n");
 		return EXIT_INCOMPATIBLE_PARAMETERS;
 	}
 

src/lib_ccx/ts_tables.c

@@ -332,7 +332,7 @@ int parse_PMT(struct ccx_demuxer *ctx, unsigned char *buf, int len, struct progr
 #ifndef ENABLE_OCR
 					if (ccx_options.write_format != CCX_OF_SPUPNG)
 					{
-						mprint("DVB subtitles detected, OCR subsystem not present. Use -out=spupng for graphic output\n");
+						mprint("DVB subtitles detected, OCR subsystem not present. Use --out=spupng for graphic output\n");
 						continue;
 					}
 #endif