Merge branch 'develop' into inkz/var-in-href-fix

.codemapignore

@@ -10,7 +10,6 @@
 !libsonnet/
 !scripts/
 !stats/
-# restore also fingerprints/ ? trusted_python/ ? 
 
 # do not skip the rules
 ![a-z]*/**/*.yaml

.github/scripts/validate-metadata.py

@@ -15,7 +15,6 @@
 
 
 class RegistryMetadataValidator(Draft7Validator):
-
     required_property_messages = {
         "references": "Please include at least one URL with more information about this rule in a metadata field called 'references'.",
         "technology": "Please include a metadata field called 'technology' that is a list of relevent tech stacks. For example: [python, flask], or [javascript, jwt].",
@@ -24,12 +23,12 @@ class RegistryMetadataValidator(Draft7Validator):
         "likelihood": "Please include a 'likelihood' metadata field for security rules that is one of that is one of ['LOW', 'MEDIUM', 'HIGH'], See https://semgrep.dev/docs/contributing/contributing-to-semgrep-rules-repository for more info.",
         "impact": "Please include a 'impact' metadata field for security rules that is one of that is one of ['LOW', 'MEDIUM', 'HIGH'], See https://semgrep.dev/docs/contributing/contributing-to-semgrep-rules-repository for more info.",
         "confidence": "Please include a 'confidence' metadata field for security rules that is one of that is one of ['LOW', 'MEDIUM', 'HIGH'], See https://semgrep.dev/docs/contributing/contributing-to-semgrep-rules-repository for more info.",
-        "subcategory": "Please include a 'subcategory' metadata field for security rules that is one of that is one of ['audit', 'vuln', 'guardrail'], See https://semgrep.dev/docs/contributing/contributing-to-semgrep-rules-repository for more info.",
+        "subcategory": "Please include a 'subcategory' metadata field for security rules that is one of that is one of ['audit', 'vuln', 'secure default'], See https://semgrep.dev/docs/contributing/contributing-to-semgrep-rules-repository for more info.",
     }
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        #self.category_enum = self.schema.get('properties', {}).get('category', {}).get('enum', [])
+        # self.category_enum = self.schema.get('properties', {}).get('category', {}).get('enum', [])
         self.category_enum = {}
 
     def _extend_message(self, error: ValidationError) -> None:
@@ -59,7 +58,11 @@ def get_errors(self, instance: dict) -> list[ValidationError]:
         return errors
 
 
-def validate_config_file_metadata(config_path: Path, validator: Draft7Validator, invalid_configs: Optional[list] = None):
+def validate_config_file_metadata(
+    config_path: Path,
+    validator: Draft7Validator,
+    invalid_configs: Optional[list] = None,
+):
     with open(config_path) as fin:
         config = yaml.safe_load(fin)
 
@@ -83,10 +86,12 @@ def validate_config_file_metadata(config_path: Path, validator: Draft7Validator,
         else:
             logger.warning(f"Invalid config {str(config_path)}: {ve.message}")
 
+
 def is_rule(path: Path) -> bool:
     with open(path) as fin:
         return fin.readlines()[0].startswith("rules:")
 
+
 if __name__ == "__main__":
     import argparse
 
@@ -111,7 +116,11 @@ def is_rule(path: Path) -> bool:
     invalid_configs = []
     for config_item in args.config:
         config_path = Path(config_item)
-        if config_path.is_file() and config_path.suffix == ".yaml" and is_rule(config_path):
+        if (
+            config_path.is_file()
+            and config_path.suffix == ".yaml"
+            and is_rule(config_path)
+        ):
             validate_config_file_metadata(config_path, v, invalid_configs)
         elif config_path.is_dir():
             for config_file in config_path.glob("**/*.yaml"):
@@ -122,4 +131,3 @@ def is_rule(path: Path) -> bool:
         for invalid_config in sorted(invalid_configs, key=lambda t: t[0]):
             print(invalid_config)
         sys.exit(1)
-

.github/workflows/semgrep-rule-lints.yaml

@@ -45,5 +45,4 @@ jobs:
             --exclude *.test.yaml \
             --exclude contrib/ \
             --exclude stats/ \
-            --exclude fingerprints/ \
             --exclude yaml/semgrep/

.github/workflows/semgrep-rules-test-develop.yml

@@ -1,3 +1,6 @@
+# Running the tests in the repo using `semgrep test` (osemgrep) and
+# the semgrep/semgrep:pro-develop docker image (the bleeding edge!).
+
 name: semgrep-rules-test-develop
 on:
   pull_request:
@@ -9,26 +12,22 @@ on:
     - develop
     - release
 jobs:
-  # Note: if you change this test there will likely need to be a
-  # corresponding change in returntocorp/semgrep
   test-develop:
     name: rules-test-develop
+    # alt: use directly the semgrep/semgrep:pro-develop container here so we
+    # don't need the calls to 'docker run ...' below
     runs-on: ubuntu-20.04
+    # TODO: remove the with: path: below to simplify
     steps:
     - uses: actions/checkout@v2
       with:
         path: semgrep-rules
-    - name: delete stats directory
-      run: rm -rf semgrep-rules/stats
-    - name: delete fingerprints directory
-      run: rm -rf semgrep-rules/fingerprints
-    - name: delete rules requiring Semgrep Pro
-      run: rm -rf semgrep-rules/apex semgrep-rules/elixir
-    - name: validate rules
-      run: |
-        export SEMGREP="docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src returntocorp/semgrep:develop semgrep"
-        make -C "$GITHUB_WORKSPACE"/semgrep-rules validate
-    - name: test with semgrep develop branch
-      run: |
-        export SEMGREP="docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src returntocorp/semgrep:develop semgrep"
-        make -C "$GITHUB_WORKSPACE"/semgrep-rules test-only
+    # alt: call 'make validate' but would require 'make' in the docker image
+    # alt: export SEMGREP="docker run --rm -w ... semgrep"
+    #    make -C "$GITHUB_WORKSPACE"/semgrep-rules validate
+    #TODO: this actually currently fails because of errors in stats/ but GHA
+    # still continue, weird
+    - name: run osemgrep validate --pro
+      run: docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src semgrep/semgrep:pro-develop semgrep validate --pro .
+    - name: run osemgrep test --pro
+      run: docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src semgrep/semgrep:pro-develop semgrep test --pro .

.github/workflows/semgrep-rules-test-historical.yml

@@ -32,8 +32,6 @@ jobs:
       run: pip3 install semgrep
     - name: delete stats directory
       run: rm -rf semgrep-rules/stats
-    - name: delete fingerprints directory
-      run: rm -rf semgrep-rules/fingerprints
     - name: delete rules requiring Semgrep Pro
       run: rm -rf semgrep-rules/apex semgrep-rules/elixir
     # TODO: remove this in the future, there was a regression in semgrep that

.github/workflows/semgrep-rules-test.yml

@@ -17,15 +17,13 @@ jobs:
     - uses: actions/setup-python@v2
       with:
         python-version: 3.9.2
-    - name: install semgrep
+    - name: install semgrep via pip
       run: pip3 install semgrep
     - name: remove stats directory
       run: rm -rf stats
-    - name: remove fingerprints from testing
-      run: rm -rf fingerprints
     - name: remove rules requiring Semgrep Pro
       run: rm -rf apex elixir
     - name: validate rules
-      run: semgrep --validate --config .
-    - name: run semgrep
-      run: semgrep --test --test-ignore-todo
+      run: semgrep validate .
+    - name: run semgrep test
+      run: semgrep test .

.github/workflows/trigger-semgrep-scanner-initiate-scan.yaml

@@ -21,7 +21,7 @@ jobs:
         env:
           HEAD_REF: ${{ github.head_ref }}
         run: |
-          CHANGED_FILES=$(git diff --name-only origin/develop origin/$HEAD_REF | xargs -0 | sed '/^$/d' | sed -r '/^(.github|bash|contrib|fingerprints|generic|json|problem-based-packs|scripts|stats|trusted_python|yaml)/d' | sed -n '/.*yaml/p' | tr '\n' ' ')
+          CHANGED_FILES=$(git diff --name-only origin/develop origin/$HEAD_REF | xargs -0 | sed '/^$/d' | sed -r '/^(.github|bash|generic|json|problem-based-packs|scripts|stats|trusted_python|yaml)/d' | sed -n '/.*yaml/p' | tr '\n' ' ')
           echo "changed_files=$CHANGED_FILES" >> $GITHUB_ENV
       - id: print-changed-files
         name: debugging step - print changed files
@@ -52,8 +52,6 @@ jobs:
           HEAD_REF: ${{ github.head_ref }}
           REPO_NAME: ${{ github.event.repository.name }}
           PR_HEAD_SHA: ${{github.event.pull_request.head.sha}}
-        if: |
-          github.event_name == 'pull_request' &&
-          env.changed_lang_count > 0
+        if: github.event_name == 'pull_request' && env.changed_lang_count > 0
         run: |
           curl -X POST https://argoworkflows-dev2.corp.r2c.dev/api/v1/events/security-research/initiate-scan-argo -H "Authorization: ${{ secrets.ARGO_WORKFLOWS_TOKEN }}" -d "{\"branch\" : \"$HEAD_REF\", \"repo\" : \"$REPO_NAME\", \"commit\" : \"$PR_HEAD_SHA\", \"changed_files\" : \"$CHANGED_FILES\" , \"langs\" : \"$CHANGED_LANGS\"}"

LICENSE

@@ -6,6 +6,6 @@ Without limiting other conditions in the License, the grant of rights under the
 
 For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you under the License to provide to third parties, for a fee or other consideration (including without limitation fees for hosting or consulting/ support services related to the Software), a product or service whose value derives, entirely or substantially, from the functionality of the Software. Any license notice or attribution required by the License must also include this Commons Clause License Condition notice.
 
-Software: semgrep-rules (https://github.com/returntocorp/semgrep-rules)
+Software: semgrep-rules (https://github.com/semgrep/semgrep-rules)
 License: LGPL 2.1 (GNU Lesser General Public License, Version 2.1)
 Licensor: Semgrep, Inc. (https://semgrep.dev)

Makefile

@@ -1,17 +1,65 @@
 #
 # Check rule validity and check that semgrep finds the expected findings.
+# See https://semgrep.dev/docs/writing-rules/testing-rules for more info.
 #
-# The semgrep repo also runs this as part of its CI for consistency.
+# The semgrep repo (and now semgrep-pro repo) also runs those tests as part
+# of its CI for consistency.
 #
 .PHONY: test
 test:
 	$(MAKE) validate
 	$(MAKE) test-only
 
-.PHONY: validate
-validate:
-	./scripts/run-tests validate
+# Use the SEMGREP env variable to specify a non-standard semgrep command
+SEMGREP ?= semgrep
 
 .PHONY: test-only
+#old: pysemgrep --test was also using flags below but not needed
+# --test-ignore-todo --strict --disable-version-check --metrics=off --verbose
 test-only:
-	./scripts/run-tests test
+	$(SEMGREP) test --pro .
+
+# TODO: semgrep validate use a different targeting than 'semgrep test'
+# so we unfortunately need this whitelist of dirs because it reports
+# errors on stats/ and scripts/ (and .github/workflows/) files otherwise
+# (we also skip libsonnet/ and trusted_python/ which do not contain rules)
+LANG_DIRS=\
+ bash \
+ c \
+ clojure \
+ csharp \
+ dockerfile \
+ generic \
+ go \
+ html \
+ java \
+ javascript \
+ json \
+ kotlin \
+ ocaml \
+ php \
+ python \
+ ruby \
+ rust \
+ scala \
+ solidity \
+ swift \
+ terraform \
+ typescript \
+ yaml
+PRO_DIRS=apex elixir
+OTHER_DIRS=ai problem-based-packs
+DIRS=$(LANG_DIRS) $(PRO_DIRS) $(OTHER_DIRS)
+
+.PHONY: validate
+#old: pysemgrep --validate was also using the flags below but not needed
+# --strict --disable-version-check --metrics=off --verbose
+validate:
+	$(SEMGREP) validate --pro $(DIRS)
+
+.PHONY: test-oss-only
+test-oss-only:
+	@for dir in $(LANG_DIRS) $(OTHER_DIRS); do \
+	   echo "processing $$dir"; \
+           $(SEMGREP) test $$dir; \
+        done