Use python3 for all generate scripts (cms-L1TK#200)

* Switch to using python3 for all of the generate scripts. Add some deterministic sorting to the PR and IR scripts so that they return the same module order every time.

* Add a python linter and use the 'needs' GitLab keyword to create a DAG for the pipelines. Run pylint on several of the generate scripts and fix a lot, but not all of the errors.

* Fix some of the linting problems with generate_ME.py.

* Make the python linting non-blocking, more like a recommendation. However, the python3 compatibility checking is still blocking.

* Fix a bug in the generate_TB.py script.

* Exit on failure of any command.

* Split the python linting into two stages, one to check for python3 compatibility and one to do the code formating. The first job must pass, but the second is allowed to fail.

* Make the download CI job depend on the linting jobs. This isn't really a technical requirement, but it short circuits any issues if the python3 linting fails. In the future, the same short circuit could apply if we don't allow the style linting to fail.

.gitlab-ci.yml

@@ -6,6 +6,7 @@ variables:
   CLANG_TIDY_COMP_OPTIONS: '-std=c++11 -I../TrackletAlgorithm -I../TestBenches -I/nfs/data41/software/Xilinx/Vivado/${VIVADO_VERSION}/include'
 
 stages: # ---------------------------------------------------------------------
+  - lint
   - download
   - quality-check
   - hls-build
@@ -20,7 +21,17 @@ stages: # ---------------------------------------------------------------------
   before_script:
     - source $VIVADO_PATH_SH
 
+.job_template: &template_static-check-python
+  tags:
+    - xilinx-tools
+  stage: lint
+  before_script:
+    - which python3
+    - python3 --version
+    - python3 -m pylint --version
+
 .job_template: &template_quality-check
+  needs: ["download"]
   tags:
     - xilinx-tools
   stage: quality-check
@@ -93,11 +104,23 @@ stages: # ---------------------------------------------------------------------
     expire_in: 1 week
 
 # Jobs ------------------------------------------------------------------------
+# Lint ------------
+py3check:
+  <<: *template_static-check-python
+  script:
+    # If changes need to be made you might refer to: https://portingguide.readthedocs.io/en/latest/tools.html#automated-fixer-python-modernize
+    - python3 -m pylint --py3k emData/*.py # Check for python3 compatibility
+lint:
+  <<: *template_static-check-python
+  allow_failure: true
+  script:
+    - python3 -m pylint emData/*.py # Static code checker
 # Download ------------
 download:
   tags:
     - xilinx-tools
   stage: download
+  needs: ["py3check", "lint"]
   script:
     - cd emData
     - ./download.sh
@@ -167,75 +190,87 @@ TB-quality-check:
 # HLS builds ---------------
 IR-vivado-hls-build:
   <<: *template_hls-build
+  needs: ["download", "IR-quality-check"]
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "IR"
 VMR-vivado-hls-build:
   <<: *template_hls-build
+  needs: ["download", "VMR-quality-check"]
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "VMR"
 VMRCM-vivado-hls-build:
   <<: *template_hls-build
+  needs: ["download", "VMRCM-quality-check"]
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "VMRCM"
 TE-vivado-hls-build:
   <<: *template_hls-build
+  needs: ["download", "TE-quality-check"]
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "TE"
 TC-vivado-hls-build:
   <<: *template_hls-build
+  needs: ["download", "TC-quality-check"]
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "TC"
 TP-vivado-hls-build:
   <<: *template_hls-build
+  needs: ["download", "TP-quality-check"]
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "TP"
 PR-vivado-hls-build:
   <<: *template_hls-build
+  needs: ["download", "PR-quality-check"]
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "PR"
 ME-vivado-hls-build:
   <<: *template_hls-build
+  needs: ["download", "ME-quality-check"]
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "ME"
 ME-vitis-hls-build:
   <<: *template_hls-build
-  allow_failure: true # FIXME: remove after avitis updated to 2021.1 and confirmed that pre-synthesis failure is gone.
+  needs: ["download", "ME-quality-check"]
+  allow_failure: true # FIXME: remove after vitis updated to 2021.1 and confirmed that pre-synthesis failure is gone.
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2020.2"
     EXECUTABLE: "vitis_hls"
     PROJ_NAME: "ME"
 MC-vivado-hls-build:
   <<: *template_hls-build
+  needs: ["download", "MC-quality-check"]
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "MC"
 MP-vivado-hls-build:
   <<: *template_hls-build
+  needs: ["download", "MP-quality-check"]
   allow_failure: true # FIXME: remove after all errors are fixed
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "MP"
 TB-hls-build:
   <<: *template_hls-build
+  needs: ["download", "TB-quality-check"]
   variables:
     EXECUTABLE: 'vivado_hls'
     VIVADO_VERSION: "2019.2"
@@ -246,7 +281,7 @@ topPRMEMC-sim:
   variables:
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "PRMEMC"
-  dependencies: # needed to avoid interference from ME-vitis-hls-build
+  needs: # needed to avoid interference from ME-vitis-hls-build
     - download
     - PR-vivado-hls-build
     - ME-vivado-hls-build
@@ -256,7 +291,7 @@ topIRVMR-sim:
   variables:
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "IRVMR"
-  dependencies:
+  needs:
     - download
     - IR-vivado-hls-build
     - VMR-vivado-hls-build
@@ -267,7 +302,7 @@ topPRMEMC-check-results:
   variables:
     VIVADO_VERSION: "2019.2" # Vivado not needed but it is parth of the path that is called
     PROJ_NAME: "PRMEMC"
-  dependencies:
+  needs:
     - download
     - PR-vivado-hls-build
     - ME-vivado-hls-build
@@ -279,7 +314,7 @@ topIRVMR-check-results:
   variables:
     VIVADO_VERSION: "2019.2" # Vivado not needed but it is parth of the path that is called
     PROJ_NAME: "IRVMR"
-  dependencies:
+  needs:
     - download
     - IR-vivado-hls-build
     - VMR-vivado-hls-build
@@ -290,7 +325,7 @@ topPRMEMC-synth:
   variables:
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "PRMEMC"
-  dependencies:
+  needs:
     - PR-vivado-hls-build
     - ME-vivado-hls-build
     - MC-vivado-hls-build
@@ -301,7 +336,7 @@ topIRVMR-synth:
   variables:
     VIVADO_VERSION: "2019.2"
     PROJ_NAME: "IRVMR"
-  dependencies:
+  needs:
     - IR-vivado-hls-build
     - VMR-vivado-hls-build
     - topIRVMR-sim