more

.github/workflows/semgrep-rules-test-develop.yml

@@ -14,17 +14,20 @@ on:
 jobs:
   test-develop:
     name: rules-test-develop
+    # alt: use directly the semgrep/semgrep:pro-develop container here so we
+    # don't need the calls to 'docker run ...' below
     runs-on: ubuntu-20.04
     # TODO: remove the with: path: below to simplify
     steps:
     - uses: actions/checkout@v2
       with:
         path: semgrep-rules
     # alt: call 'make validate' but would require 'make' in the docker image
-    # alt: export SEMGREP="docker run semgrep"
+    # alt: export SEMGREP="docker run --rm -w ... semgrep"
     #    make -C "$GITHUB_WORKSPACE"/semgrep-rules validate
+    #TODO: this actually currently fails because of errors in stats/ but GHA
+    # still continue, weird
     - name: run osemgrep validate --pro
       run: docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src semgrep/semgrep:pro-develop semgrep validate --pro .
-    # alt: call 'make test-only' but would require 'make' in the docker image
     - name: run osemgrep test --pro
       run: docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src semgrep/semgrep:pro-develop semgrep test --pro .

.github/workflows/semgrep-rules-test.yml

@@ -17,13 +17,13 @@ jobs:
     - uses: actions/setup-python@v2
       with:
         python-version: 3.9.2
-    - name: install semgrep
+    - name: install semgrep via pip
       run: pip3 install semgrep
     - name: remove stats directory
       run: rm -rf stats
     - name: remove rules requiring Semgrep Pro
       run: rm -rf apex elixir
     - name: validate rules
-      run: semgrep --validate --config .
-    - name: run semgrep
-      run: semgrep --test --test-ignore-todo
+      run: semgrep validate .
+    - name: run semgrep test
+      run: semgrep test .

Makefile

@@ -1,63 +1,65 @@
 #
 # Check rule validity and check that semgrep finds the expected findings.
+# See https://semgrep.dev/docs/writing-rules/testing-rules for more info.
 #
-# The semgrep repo also runs this as part of its CI for consistency.
+# The semgrep repo (and now semgrep-pro repo) also runs those tests as part
+# of its CI for consistency.
 #
 .PHONY: test
 test:
 	$(MAKE) validate
 	$(MAKE) test-only
 
-# Use the SEMGREP environment variable to specify a non-standard semgrep
-# command. This is useful for calling a development version of semgrep
-# e.g.
-#   PIPENV_PIPFILE=~/semgrep/cli/Pipfile SEMGREP='pipenv run semgrep' make test
+# Use the SEMGREP env variable to specify a non-standard semgrep command
 SEMGREP ?= semgrep
 
-# TODO: semgrep validate use a different targeting than semgrep test
+.PHONY: test-only
+#old: pysemgrep --test was also using flags below but not needed
+# --test-ignore-todo --strict --disable-version-check --metrics=off --verbose
+test-only:
+	$(SEMGREP) test --pro .
+
+# TODO: semgrep validate use a different targeting than 'semgrep test'
 # so we unfortunately need this whitelist of dirs because it reports
-# errors on stats/ and scripts/ (and .github yaml) files otherwise
-# NOTE: the apex/ and elixir/ requires --pro (hence the --pro below)
-# alt: we could also skip libsonnet/ and trusted_python/
-DIRS=\
- ai \
- apex \
+# errors on stats/ and scripts/ (and .github/workflows/) files otherwise
+# (we also skip libsonnet/ and trusted_python/ which do not contain rules)
+LANG_DIRS=\
  bash \
  c \
  clojure \
  csharp \
  dockerfile \
- elixir \
  generic \
  go \
  html \
  java \
  javascript \
  json \
  kotlin \
- libsonnet \
  ocaml \
  php \
- problem-based-packs \
  python \
  ruby \
  rust \
  scala \
  solidity \
  swift \
  terraform \
- trusted_python \
  typescript \
  yaml
+PRO_DIRS=apex elixir
+OTHER_DIRS=ai problem-based-packs
+DIRS=$(LANG_DIRS) $(PRO_DIRS) $(OTHER_DIRS)
 
 .PHONY: validate
 #old: pysemgrep --validate was also using the flags below but not needed
 # --strict --disable-version-check --metrics=off --verbose
 validate:
 	$(SEMGREP) validate --pro $(DIRS)
 
-.PHONY: test-only
-#old: pysemgrep --test was also using
-# --test-ignore-todo --strict --disable-version-check --metrics=off --verbose
-test-only:
-	$(SEMGREP) test --pro .
+.PHONY: test-oss-only
+test-oss-only:
+	@for dir in $(LANG_DIRS) $(OTHER_DIRS); do \
+	   echo "processing $$dir"; \
+           $(SEMGREP) test $$dir; \
+        done