From f50e60067149d0fb6929d780f949fc0bcb28b59a Mon Sep 17 00:00:00 2001 From: Shivkant Chauhan <91013793+Shivkant-Chauhan@users.noreply.github.com> Date: Sat, 21 Oct 2023 19:03:51 +0530 Subject: [PATCH] [GSoC'23] - M1.4.2 - migrating local tests under docker (#18775) * migrating some tests to work under docker * migrating local tests * reverting extra changes * reverting changes * fixed mypy checks * added backend associate tests in makefile * added make command for installing node * lighthouse tests under docker * migrated e2e tests * cleaning up + fixing Makefile * fixed karma report coverage * checking prod mode on CI * prod mode action of docker * prod mode action of docker * fixing backend tests * pushing local changes * fixed backend tests + cleanup for the PR * fixed mypy checks + cleanup for the PR * fixed mypy checks + cleanup for the PR * cleanup for the PR * cleanup for the PR + fixing e2e tests node issue * fix node env in docker * backend tests * fixed run-devserver * fixed run-devserver * fixed run-devserver * fixed global node env issue * changes terminal UI changes * fixed lints * fixed backend tests * fixed lints * fixed backend tests * fixed prod mode + lint checks on docker * fixed prod mode + lint checks on docker * fixed prod mode * fixed backend associate tests for Windows users * fixing node issue for Windows in docker: * added missing semicolon * enhance install_node in makefile * added copyright in the new .py script * fixed lints * fixed makefile * pointed chrome version in docker container * changed make command name * resolving PR comments * address review comments * fixed lints * Update run_mypy_checks.py * adding backend tests for the python script * resolving PR comments * Delete mock_app.yaml * Delete mock_app_dev.yaml * reverting wrong change * fixed lints * fixed lints * addressed review comments * fixed build.py backend test * fixed CI tests * fixed webpack prod config * fixed acceptance tests url error --- .lighthouserc-1.js | 2 +- .lighthouserc-accessibility-1.js | 28 ++--- .lighthouserc-accessibility-2.js | 32 +++--- .lighthouserc-base.js | 66 +++++------ Makefile | 107 +++++++++++++++--- core/tests/protractor-browserstack.conf.js | 2 +- .../test-constants.js | 16 +-- core/tests/puppeteer/lighthouse_setup.js | 12 +- core/tests/wdio.conf.js | 4 +- core/tests/webdriverio/accessibility.js | 20 ++-- core/tests/webdriverio/libraryFlow.js | 2 +- core/tests/webdriverio/profileMenuFlow.js | 12 +- core/tests/webdriverio_desktop/embedding.js | 6 +- core/tests/webdriverio_desktop/extensions.js | 2 +- core/tests/webdriverio_desktop/navigation.js | 4 +- core/tests/webdriverio_desktop/preferences.js | 8 +- core/tests/webdriverio_desktop/wipeout.js | 2 +- .../webdriverio_utils/BlogDashboardPage.js | 2 +- .../webdriverio_utils/DeleteAccountPage.js | 2 +- .../TopicsAndSkillsDashboardPage.js | 2 +- core/tests/webdriverio_utils/general.js | 2 +- docker-compose.yml | 10 ++ docker/Dockerfile.backend | 9 ++ docker/install_node.sh | 59 ++++++++++ .../rich_text_components/Image/webdriverio.js | 2 +- .../rich_text_components/Math/webdriverio.js | 2 +- puppeteer-login-script.js | 10 +- scripts/backend_test_shards.json | 1 + scripts/backend_tests_incomplete_coverage.txt | 2 + scripts/build.py | 22 ++-- scripts/build_test.py | 17 ++- scripts/check_backend_associated_test_file.py | 2 +- scripts/common.py | 11 +- scripts/generate_build_directory.py | 35 ++++++ scripts/generate_build_directory_test.py | 41 +++++++ ...install_dependencies_json_packages_test.py | 2 +- scripts/install_third_party_libs.py | 20 +++- scripts/install_third_party_libs_test.py | 11 ++ scripts/run_backend_tests.py | 11 +- scripts/run_backend_tests_test.py | 24 ++++ scripts/run_mypy_checks.py | 20 ++-- webpack.prod.config.ts | 13 ++- 42 files changed, 485 insertions(+), 172 deletions(-) create mode 100644 docker/install_node.sh create mode 100644 scripts/generate_build_directory.py create mode 100644 scripts/generate_build_directory_test.py diff --git a/.lighthouserc-1.js b/.lighthouserc-1.js index 67bea29c486d..c28da266bb7b 100644 --- a/.lighthouserc-1.js +++ b/.lighthouserc-1.js @@ -37,7 +37,7 @@ module.exports = { 'assertions': baseConfig['basePerformanceAssertions'] }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/blog-dashboard$', + 'matchingUrlPattern': '^http://localhost:8181/blog-dashboard$', 'assertions': baseConfig['basePerformanceAssertions'] }, { diff --git a/.lighthouserc-accessibility-1.js b/.lighthouserc-accessibility-1.js index 7e22c9855d05..4e3a9b4cabb2 100644 --- a/.lighthouserc-accessibility-1.js +++ b/.lighthouserc-accessibility-1.js @@ -28,85 +28,85 @@ module.exports = { 'assert': { 'assertMatrix': [ { - 'matchingUrlPattern': '^http://127.0.0.1:8181/$', + 'matchingUrlPattern': '^http://localhost:8181/$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/about$', + 'matchingUrlPattern': '^http://localhost:8181/about$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.98}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/admin$', + 'matchingUrlPattern': '^http://localhost:8181/admin$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/blog-dashboard$', + 'matchingUrlPattern': '^http://localhost:8181/blog-dashboard$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.98}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/community-library$', + 'matchingUrlPattern': '^http://localhost:8181/community-library$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.98}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/contact$', + 'matchingUrlPattern': '^http://localhost:8181/contact$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.98}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/contributor-dashboard$', + 'matchingUrlPattern': '^http://localhost:8181/contributor-dashboard$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/creator-dashboard$', + 'matchingUrlPattern': '^http://localhost:8181/creator-dashboard$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/delete-account$', + 'matchingUrlPattern': '^http://localhost:8181/delete-account$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.98}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/donate$', + 'matchingUrlPattern': '^http://localhost:8181/donate$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.98}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/emailDashboard$', + 'matchingUrlPattern': '^http://localhost:8181/emailDashboard$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/get-started$', + 'matchingUrlPattern': '^http://localhost:8181/get-started$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.98}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/learner-dashboard$', + 'matchingUrlPattern': '^http://localhost:8181/learner-dashboard$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/moderator$', + 'matchingUrlPattern': '^http://localhost:8181/moderator$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.98}] } diff --git a/.lighthouserc-accessibility-2.js b/.lighthouserc-accessibility-2.js index 3742cc07cffa..4b23438768df 100644 --- a/.lighthouserc-accessibility-2.js +++ b/.lighthouserc-accessibility-2.js @@ -28,97 +28,97 @@ module.exports = { 'assert': { 'assertMatrix': [ { - 'matchingUrlPattern': '^http://127.0.0.1:8181/preferences$', + 'matchingUrlPattern': '^http://localhost:8181/preferences$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/privacy-policy$', + 'matchingUrlPattern': '^http://localhost:8181/privacy-policy$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.98}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/profile/username1$', + 'matchingUrlPattern': '^http://localhost:8181/profile/username1$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/signup?return_url=%2F$', + 'matchingUrlPattern': '^http://localhost:8181/signup?return_url=%2F$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/teach$', + 'matchingUrlPattern': '^http://localhost:8181/teach$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/topics-and-skills-dashboard$', + 'matchingUrlPattern': '^http://localhost:8181/topics-and-skills-dashboard$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.9}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/terms$', + 'matchingUrlPattern': '^http://localhost:8181/terms$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.98}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/thanks$', + 'matchingUrlPattern': '^http://localhost:8181/thanks$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/volunteer$', + 'matchingUrlPattern': '^http://localhost:8181/volunteer$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.9}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/learn/staging/dummy-topic-one/story', + 'matchingUrlPattern': '^http://localhost:8181/learn/staging/dummy-topic-one/story', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/learn/staging/dummy-topic-one/story/help-jamie-win-arcade', + 'matchingUrlPattern': '^http://localhost:8181/learn/staging/dummy-topic-one/story/help-jamie-win-arcade', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/learn/math', + 'matchingUrlPattern': '^http://localhost:8181/learn/math', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/create/.*$', + 'matchingUrlPattern': '^http://localhost:8181/create/.*$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.91}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/topic_editor/.*$', + 'matchingUrlPattern': '^http://localhost:8181/topic_editor/.*$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 1}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/skill_editor/.*$', + 'matchingUrlPattern': '^http://localhost:8181/skill_editor/.*$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.91}] } }, { - 'matchingUrlPattern': '^http://127.0.0.1:8181/story_editor/.*$', + 'matchingUrlPattern': '^http://localhost:8181/story_editor/.*$', 'assertions': { 'categories:accessibility': ['error', {'minScore': 0.84}] } diff --git a/.lighthouserc-base.js b/.lighthouserc-base.js index eb75b0419216..afc2a696303d 100644 --- a/.lighthouserc-base.js +++ b/.lighthouserc-base.js @@ -21,41 +21,41 @@ module.exports = { puppeteerScript: 'puppeteer-login-script.js', urlShards: { 1: [ - 'http://127.0.0.1:8181/', - 'http://127.0.0.1:8181/about', - 'http://127.0.0.1:8181/about-foundation', - 'http://127.0.0.1:8181/admin', - 'http://127.0.0.1:8181/blog-dashboard', - 'http://127.0.0.1:8181/community-library', - 'http://127.0.0.1:8181/contact', - 'http://127.0.0.1:8181/contributor-dashboard', - 'http://127.0.0.1:8181/creator-dashboard', - 'http://127.0.0.1:8181/creator-guidelines', - 'http://127.0.0.1:8181/delete-account', - 'http://127.0.0.1:8181/donate', - 'http://127.0.0.1:8181/emaildashboard', - 'http://127.0.0.1:8181/get-started', - 'http://127.0.0.1:8181/learner-dashboard', - 'http://127.0.0.1:8181/license', - 'http://127.0.0.1:8181/moderator', + 'http://localhost:8181/', + 'http://localhost:8181/about', + 'http://localhost:8181/about-foundation', + 'http://localhost:8181/admin', + 'http://localhost:8181/blog-dashboard', + 'http://localhost:8181/community-library', + 'http://localhost:8181/contact', + 'http://localhost:8181/contributor-dashboard', + 'http://localhost:8181/creator-dashboard', + 'http://localhost:8181/creator-guidelines', + 'http://localhost:8181/delete-account', + 'http://localhost:8181/donate', + 'http://localhost:8181/emaildashboard', + 'http://localhost:8181/get-started', + 'http://localhost:8181/learner-dashboard', + 'http://localhost:8181/license', + 'http://localhost:8181/moderator', ], 2: [ - 'http://127.0.0.1:8181/preferences', - 'http://127.0.0.1:8181/privacy-policy', - 'http://127.0.0.1:8181/profile/username1', - 'http://127.0.0.1:8181/signup?return_url=%2F', - 'http://127.0.0.1:8181/teach', - 'http://127.0.0.1:8181/topics-and-skills-dashboard', - 'http://127.0.0.1:8181/terms', - 'http://127.0.0.1:8181/thanks', - 'http://127.0.0.1:8181/volunteer', - 'http://127.0.0.1:8181/learn/staging/dummy-topic-one/story', - 'http://127.0.0.1:8181/learn/staging/dummy-topic-one/story/help-jamie-win-arcade', - 'http://127.0.0.1:8181/learn/math', - `http://127.0.0.1:8181/create/${process.env.exploration_id}`, - `http://127.0.0.1:8181/topic_editor/${process.env.topic_id}`, - `http://127.0.0.1:8181/skill_editor/${process.env.skill_id}`, - `http://127.0.0.1:8181/story_editor/${process.env.story_id}` + 'http://localhost:8181/preferences', + 'http://localhost:8181/privacy-policy', + 'http://localhost:8181/profile/username1', + 'http://localhost:8181/signup?return_url=%2F', + 'http://localhost:8181/teach', + 'http://localhost:8181/topics-and-skills-dashboard', + 'http://localhost:8181/terms', + 'http://localhost:8181/thanks', + 'http://localhost:8181/volunteer', + 'http://localhost:8181/learn/staging/dummy-topic-one/story', + 'http://localhost:8181/learn/staging/dummy-topic-one/story/help-jamie-win-arcade', + 'http://localhost:8181/learn/math', + `http://localhost:8181/create/${process.env.exploration_id}`, + `http://localhost:8181/topic_editor/${process.env.topic_id}`, + `http://localhost:8181/skill_editor/${process.env.skill_id}`, + `http://localhost:8181/story_editor/${process.env.story_id}` ] }, basePerformanceAssertMatrix: { diff --git a/Makefile b/Makefile index 9d01c4058435..a13c92ff7ce3 100644 --- a/Makefile +++ b/Makefile @@ -25,26 +25,37 @@ help: ## Display this help message. build.%: ## Builds the given docker service. Example: make build.datastore docker compose build $* -build: ## Builds the all docker services. +build: ## Builds the all docker setup. + $(MAKE) install_node docker compose build run-devserver: # Runs the dev-server - docker compose up dev-server -d --no-deps - $(MAKE) update.requirements docker compose up angular-build -d $(MAKE) update.package + docker cp oppia-angular-build:/app/oppia/node_modules . + docker compose stop angular-build + docker compose up dev-server -d --no-deps + $(MAKE) update.requirements $(MAKE) run-offline run-offline: # Runs the dev-server in offline mode + $(MAKE) start-devserver + @echo 'Please visit http://localhost:8181 to access the development server.' + @echo 'Check dev-server logs using "make logs.dev-server"' + @echo 'Stop the development server using "make stop"' + +start-devserver: ## Starts the development server for the tests docker compose up dev-server -d @printf 'Please wait while the development server starts...\n\n' @while [[ $$(curl -s -o /tmp/status_code.txt -w '%{http_code}' http://localhost:8181) != "200" ]] || [[ $$(curl -s -o /tmp/status_code.txt -w '%{http_code}' http://localhost:8181/community-library) != "200" ]]; do \ - sleep 5; \ + printf "▓"; \ + if [[ "$(prod_env)" = 'true' ]] && [[ -n $$(docker ps -q -f status=exited -f name=webpack-compiler) ]]; then \ + ${SHELL_PREFIX} dev-server python -m scripts.generate_build_directory; \ + fi; \ + sleep 1; \ done + @printf '\n\n' @echo 'Development server started at port 8181.' - @echo 'Please visit http://localhost:8181 to access the development server.' - @echo 'Check dev-server logs using "make logs.dev-server"' - @echo 'Stop the development server using "make stop"' init: build run-devserver ## Initializes the build and runs dev-server. @@ -80,9 +91,79 @@ logs.%: ## Shows the logs of the given docker service. Example: make logs.datast restart.%: ## Restarts the given docker service. Example: make restart.datastore docker compose restart $* -run_tests.lints: ## Runs the linter tests - docker compose run --no-deps --entrypoint "python -m scripts.linters.pre_commit_linter $(PYTHON_ARGS)" dev-server - -run-backend-tests: ## [Not ready for use] Runs the backend tests - @echo "Run the backend test on the following module: $(RUN_ARGS)" - @echo "Not in use, under construction!" +run_tests.lint: ## Runs the linter tests + docker compose run --no-deps --entrypoint "/bin/sh -c 'git config --global --add safe.directory /app/oppia && python -m scripts.linters.pre_commit_linter $(PYTHON_ARGS)'" dev-server + +run_tests.backend: ## Runs the backend tests + $(MAKE) stop + docker compose up datastore dev-server redis firebase -d --no-deps + @echo '------------------------------------------------------' + @echo ' Backend tests started....' + @echo '------------------------------------------------------' + $(SHELL_PREFIX) dev-server python -m scripts.run_backend_tests $(PYTHON_ARGS) + @echo '------------------------------------------------------' + @echo ' Backend tests has been executed successfully....' + @echo '------------------------------------------------------' + $(MAKE) stop + +run_tests.frontend: ## Runs the frontend unit tests + docker compose run --no-deps --entrypoint "python -m scripts.run_frontend_tests $(PYTHON_ARGS) --skip_install" dev-server + +run_tests.typescript: ## Runs the typescript checks + docker compose run --no-deps --entrypoint "python -m scripts.typescript_checks" dev-server + +run_tests.custom_eslint: ## Runs the custome eslint tests + docker compose run --no-deps --entrypoint "python -m scripts.run_custom_eslint_tests" dev-server + +run_tests.mypy: ## Runs mypy checks + docker compose run --no-deps --entrypoint "python -m scripts.run_mypy_checks" dev-server + +run_tests.check_backend_associated_tests: ## Runs the backend associate tests + docker compose run --no-deps --entrypoint "/bin/sh -c 'git config --global --add safe.directory /app/oppia && python -m scripts.check_backend_associated_test_file'" dev-server + +run_tests.acceptance: ## Runs the acceptance tests for the parsed suite + @echo 'Shutting down any previously started server.' + $(MAKE) stop +# Adding node to the path. + @if [ "$(OS_NAME)" = "Windows" ]; then \ + export PATH=$(cd .. && pwd)/oppia_tools/node-16.13.0:$(PATH); \ + else \ + export PATH=$(shell cd .. && pwd)/oppia_tools/node-16.13.0/bin:$(PATH); \ + fi +# Starting the development server for the acceptance tests. + $(MAKE) start-devserver + @echo '------------------------------------------------------' + @echo ' Starting acceptance test for the suite: $(suite)' + @echo '------------------------------------------------------' + ./node_modules/.bin/jasmine --config="./core/tests/puppeteer-acceptance-tests/jasmine.json" ./core/tests/puppeteer-acceptance-tests/spec/$(suite) + @echo '------------------------------------------------------' + @echo ' Acceptance test has been executed successfully....' + @echo '------------------------------------------------------' + $(MAKE) stop + +CHROME_VERSION := $(shell google-chrome --version | awk '{print $$3}') + +run_tests.e2e: ## Runs the e2e tests for the parsed suite + @echo 'Shutting down any previously started server.' + $(MAKE) stop +# Adding node to the path. + @if [ "$(OS_NAME)" = "Windows" ]; then \ + export PATH=$(cd .. && pwd)/oppia_tools/node-16.13.0:$(PATH); \ + else \ + export PATH=$(shell cd .. && pwd)/oppia_tools/node-16.13.0/bin:$(PATH); \ + fi +# Starting the development server for the e2e tests. + $(MAKE) start-devserver + @echo '------------------------------------------------------' + @echo ' Starting e2e test for the suite: $(suite)' + @echo '------------------------------------------------------' + ../oppia_tools/node-16.13.0/bin/npx wdio ./core/tests/wdio.conf.js --suite $(suite) $(CHROME_VERSION) --params.devMode=True --capabilities[0].maxInstances=3 + @echo '------------------------------------------------------' + @echo ' e2e test has been executed successfully....' + @echo '------------------------------------------------------' + $(MAKE) stop + +OS_NAME := $(shell uname) + +install_node: ## Installs node-16.13.0 in the oppia_tools directory + sh ./docker/install_node.sh diff --git a/core/tests/protractor-browserstack.conf.js b/core/tests/protractor-browserstack.conf.js index d6155519e0ff..5b5334d03da5 100755 --- a/core/tests/protractor-browserstack.conf.js +++ b/core/tests/protractor-browserstack.conf.js @@ -192,7 +192,7 @@ exports.config = { // // A base URL for your application under test. Calls to protractor.get() // with relative paths will be prepended with this. - baseUrl: 'http://localhost:9001', + baseUrl: 'http://localhost:8181', // Selector for the element housing the angular app - this defaults to // body, but is necessary if ng-app is on a descendant of . diff --git a/core/tests/puppeteer-acceptance-tests/puppeteer-testing-utilities/test-constants.js b/core/tests/puppeteer-acceptance-tests/puppeteer-testing-utilities/test-constants.js index 0f186456121a..f74087911085 100644 --- a/core/tests/puppeteer-acceptance-tests/puppeteer-testing-utilities/test-constants.js +++ b/core/tests/puppeteer-acceptance-tests/puppeteer-testing-utilities/test-constants.js @@ -20,14 +20,14 @@ const path = require('path'); let testConstants = { URLs: { - home: 'http://localhost:9001/', - BlogDashboard: 'http://localhost:9001/blog-dashboard', - BlogAdmin: 'http://localhost:9001/blog-admin', - CreatorDashboard: 'http://localhost:9001/creator-dashboard', - ContributorDashboardAdmin: 'http://localhost:9001/contributor-dashboard-admin/', - AdminPage: 'http://localhost:9001/admin', - RolesEditorTab: 'http://localhost:9001/admin#/roles', - logout: 'http://localhost:9001/logout' + home: 'http://localhost:8181/', + BlogDashboard: 'http://localhost:8181/blog-dashboard', + BlogAdmin: 'http://localhost:8181/blog-admin', + CreatorDashboard: 'http://localhost:8181/creator-dashboard', + ContributorDashboardAdmin: 'http://localhost:8181/contributor-dashboard-admin/', + AdminPage: 'http://localhost:8181/admin', + RolesEditorTab: 'http://localhost:8181/admin#/roles', + logout: 'http://localhost:8181/logout' }, Dashboard: { MainDashboard: '.e2e-test-splash-page', diff --git a/core/tests/puppeteer/lighthouse_setup.js b/core/tests/puppeteer/lighthouse_setup.js index 8b2b45e7c460..4bf1e9f61316 100644 --- a/core/tests/puppeteer/lighthouse_setup.js +++ b/core/tests/puppeteer/lighthouse_setup.js @@ -22,9 +22,9 @@ const puppeteer = require('puppeteer'); const { PuppeteerScreenRecorder } = require('puppeteer-screen-recorder'); -const ADMIN_URL = 'http://127.0.0.1:8181/admin'; -const CREATOR_DASHBOARD_URL = 'http://127.0.0.1:8181/creator-dashboard'; -const TOPIC_AND_SKILLS_DASHBOARD_URL = 'http://127.0.0.1:8181/topics-and-skills-dashboard'; +const ADMIN_URL = 'http://localhost:8181/admin'; +const CREATOR_DASHBOARD_URL = 'http://localhost:8181/creator-dashboard'; +const TOPIC_AND_SKILLS_DASHBOARD_URL = 'http://localhost:8181/topics-and-skills-dashboard'; // Read more about networkidle0 // https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagegotourl-options const networkIdle = 'networkidle0'; @@ -129,7 +129,7 @@ const setRole = async function(browser, page, role) { try { // eslint-disable-next-line dot-notation await page.goto( - 'http://127.0.0.1:8181/admin#/roles', { waitUntil: networkIdle }); + 'http://localhost:8181/admin#/roles', { waitUntil: networkIdle }); await page.waitForSelector(usernameInputFieldForRolesEditing); await page.type(usernameInputFieldForRolesEditing, 'username1'); await page.waitForSelector(editUserRoleButton); @@ -280,7 +280,7 @@ const getSkillEditorUrl = async function(browser, page) { const generateDataForTopicAndStoryPlayer = async function(browser, page) { try { // eslint-disable-next-line dot-notation - await page.goto('http://127.0.0.1:8181/admin#/activities', { waitUntil: networkIdle }); + await page.goto('http://localhost:8181/admin#/activities', { waitUntil: networkIdle }); await page.waitForSelector(generateTopicButton); await page.click(generateTopicButton); @@ -306,7 +306,7 @@ const generateDataForTopicAndStoryPlayer = async function(browser, page) { const generateDataForClassroom = async function(browser, page) { try { // eslint-disable-next-line dot-notation - await page.goto('http://127.0.0.1:8181/admin#/activities', { waitUntil: networkIdle }); + await page.goto('http://localhost:8181/admin#/activities', { waitUntil: networkIdle }); await page.waitForSelector(generateClassroomButton); await page.click(generateClassroomButton); diff --git a/core/tests/wdio.conf.js b/core/tests/wdio.conf.js index 27ecc6158e42..8aeeea5f098b 100644 --- a/core/tests/wdio.conf.js +++ b/core/tests/wdio.conf.js @@ -268,7 +268,7 @@ exports.config = { // the path portion of your baseUrl. If your `url` parameter starts // without a scheme or `/` (like `some/path`), the base url gets // prepended directly. - baseUrl: 'http://localhost:9001', + baseUrl: 'http://localhost:8181', // Default timeout in milliseconds for request // if browser driver or grid doesn't send response. @@ -353,7 +353,7 @@ exports.config = { FirebaseAdmin.initializeApp({projectId: 'dev-project-id'}); // Navigate to the splash page so that tests can begin on an Angular page. - browser.url('http://localhost:9001'); + browser.url('http://localhost:8181'); }, /** * Function to be executed before a test (in Mocha/Jasmine only) diff --git a/core/tests/webdriverio/accessibility.js b/core/tests/webdriverio/accessibility.js index c65e7db8b387..3a2308e3464d 100644 --- a/core/tests/webdriverio/accessibility.js +++ b/core/tests/webdriverio/accessibility.js @@ -37,14 +37,14 @@ var EXPLORATION = { describe('screenreader and keyboard user accessibility features', function() { var ERROR_MESSAGE = 'Content container taking too long to load'; var libraryPage = null; - var GET_STARTED_URL = 'http://localhost:9001/get-started'; - var COMMUNITY_LIBRARY_URL = 'http://localhost:9001/community-library'; - var LEARNER_DASHBOARD_URL = 'http://localhost:9001/learner-dashboard'; - var CREATOR_DASHBOARD_URL = 'http://localhost:9001/creator-dashboard'; - var ABOUT_URL = 'http://localhost:9001/about'; - var PREFERENCES_URL = 'http://localhost:9001/preferences'; - var PRIVACY_POLICY_URL = 'http://localhost:9001/privacy-policy'; - var DONATE_URL = 'http://localhost:9001/donate'; + var GET_STARTED_URL = 'http://localhost:8181/get-started'; + var COMMUNITY_LIBRARY_URL = 'http://localhost:8181/community-library'; + var LEARNER_DASHBOARD_URL = 'http://localhost:8181/learner-dashboard'; + var CREATOR_DASHBOARD_URL = 'http://localhost:8181/creator-dashboard'; + var ABOUT_URL = 'http://localhost:8181/about'; + var PREFERENCES_URL = 'http://localhost:8181/preferences'; + var PRIVACY_POLICY_URL = 'http://localhost:8181/privacy-policy'; + var DONATE_URL = 'http://localhost:8181/donate'; var holdCtrlAndPressKey = async function(key) { await browser.keys(['Control', key]); @@ -545,8 +545,8 @@ describe('Cache Slugs', function() { async function() { await browser.url('/console_errors'); var expectedErrors = [ - 'http://localhost:9001/build/fail/logo/288x128_logo_white.png', - 'http://localhost:9001/build/fail/logo/288x128_logo_white.webp' + 'http://localhost:8181/build/fail/logo/288x128_logo_white.png', + 'http://localhost:8181/build/fail/logo/288x128_logo_white.webp' ]; await general.checkForConsoleErrors(expectedErrors); }); diff --git a/core/tests/webdriverio/libraryFlow.js b/core/tests/webdriverio/libraryFlow.js index 55ff1ec8f9a2..75f203af770f 100644 --- a/core/tests/webdriverio/libraryFlow.js +++ b/core/tests/webdriverio/libraryFlow.js @@ -73,7 +73,7 @@ describe('Library pages tour', function() { await libraryPage.get(); await libraryPage.findExploration(SEARCH_TERM); await waitFor.urlToBe( - 'http://localhost:9001/search/find?q=python&language_code=(%22en%22)'); + 'http://localhost:8181/search/find?q=python&language_code=(%22en%22)'); }); it('should visit the library index page', async function() { diff --git a/core/tests/webdriverio/profileMenuFlow.js b/core/tests/webdriverio/profileMenuFlow.js index e4515df784e0..386295c47955 100644 --- a/core/tests/webdriverio/profileMenuFlow.js +++ b/core/tests/webdriverio/profileMenuFlow.js @@ -42,7 +42,7 @@ describe('Profile menu flow', function() { await general.navigateToTopicsAndSkillsDashboardPage(); await waitFor.pageToFullyLoad(); expect(await browser.getUrl()).toEqual( - 'http://localhost:9001/topics-and-skills-dashboard'); + 'http://localhost:8181/topics-and-skills-dashboard'); }); describe('profile dropdown menu', function() { @@ -55,7 +55,7 @@ describe('Profile menu flow', function() { it('should land on the learner dashboard after successful login', async function() { expect(await browser.getUrl()).toEqual( - 'http://localhost:9001/learner-dashboard'); + 'http://localhost:8181/learner-dashboard'); }); it('should visit the profile page from the profile dropdown menu', @@ -64,7 +64,7 @@ describe('Profile menu flow', function() { await action.click('Profile Link', profileLink); await waitFor.pageToFullyLoad(); expect(await browser.getUrl()).toEqual( - 'http://localhost:9001/profile/desktopAndMobileVisitor'); + 'http://localhost:8181/profile/desktopAndMobileVisitor'); }); it('should visit the creator dashboard from the profile dropdown menu', @@ -73,7 +73,7 @@ describe('Profile menu flow', function() { await action.click('Creator Dashboard Link', creatorDashboardLink); await waitFor.pageToFullyLoad(); expect(await browser.getUrl()).toEqual( - 'http://localhost:9001/creator-dashboard'); + 'http://localhost:8181/creator-dashboard'); }); it('should visit the learner dashboard from the profile dropdown menu', @@ -82,7 +82,7 @@ describe('Profile menu flow', function() { await action.click('Learner Dashboard Link', learnerDashboardLink); await waitFor.pageToFullyLoad(); expect(await browser.getUrl()).toEqual( - 'http://localhost:9001/learner-dashboard'); + 'http://localhost:8181/learner-dashboard'); }); it('should not show the topics and skills dashboard link in the profile ' + @@ -97,7 +97,7 @@ describe('Profile menu flow', function() { await action.click('Preferences Link', preferencesLink); await waitFor.pageToFullyLoad(); expect(await browser.getUrl()).toEqual( - 'http://localhost:9001/preferences'); + 'http://localhost:8181/preferences'); }); }); diff --git a/core/tests/webdriverio_desktop/embedding.js b/core/tests/webdriverio_desktop/embedding.js index b8e8c1180880..29a7b2f008ff 100644 --- a/core/tests/webdriverio_desktop/embedding.js +++ b/core/tests/webdriverio_desktop/embedding.js @@ -105,14 +105,14 @@ describe('Embedding', function() { // These errors are to be ignored as 'idToBeReplaced' is not a valid // exploration id. It appears just after the page loads. var EMBEDDING_ERRORS_TO_IGNORE = [ - 'http:\/\/localhost:9001\/assets\/scripts\/' + + 'http:\/\/localhost:8181\/assets\/scripts\/' + 'embedding_tests_dev_i18n_0.0.1.html - Refused to display ' + - '\'http:\/\/localhost:9001\/\' in a frame because it set ' + + '\'http:\/\/localhost:8181\/\' in a frame because it set ' + '\'X-Frame-Options\' to \'deny\'.', 'chrome-error:\/\/chromewebdata\/ - Failed to load resource: the server ' + 'responded with a status of 400 ()', 'chrome-error:\/\/chromewebdata\/ 0 Refused to display ' + - '\'http:\/\/localhost:9001\/\' in a frame because it set ' + + '\'http:\/\/localhost:8181\/\' in a frame because it set ' + '\'X-Frame-Options\' to \'deny\'.', ]; diff --git a/core/tests/webdriverio_desktop/extensions.js b/core/tests/webdriverio_desktop/extensions.js index fb0881b0566c..e7ff2a96e256 100644 --- a/core/tests/webdriverio_desktop/extensions.js +++ b/core/tests/webdriverio_desktop/extensions.js @@ -105,7 +105,7 @@ describe('rich-text components', function() { // in both regexes and strings: https://stackoverflow.com/a/5514380 'The target origin provided \\\(\'https://www\.youtube\.com\'\\\) does ' + 'not match the recipient window\'s ' + - 'origin \\\(\'http://localhost:9001\'\\\).', + 'origin \\\(\'http://localhost:8181\'\\\).', ]); }); }); diff --git a/core/tests/webdriverio_desktop/navigation.js b/core/tests/webdriverio_desktop/navigation.js index 453eb09e59ed..70d5f82f7a57 100644 --- a/core/tests/webdriverio_desktop/navigation.js +++ b/core/tests/webdriverio_desktop/navigation.js @@ -82,7 +82,7 @@ describe('Meta Tags', function() { expect(await getStartedPage.getMetaTagContent('description', 'og')).toEqual( EXPECTED_META_DESCRIPTION); expect(await getStartedPage.getMetaTagContent('url', 'og')).toEqual( - 'http://localhost:9001/get-started'); + 'http://localhost:8181/get-started'); }); it('should set the correct application name', async function() { @@ -149,7 +149,7 @@ describe('Static Pages Tour', function() { (url) => { // Wait for second redirection (splash page to preferred dashboard // page). - return url !== 'http://localhost:9001/'; + return url !== 'http://localhost:8181/'; }, async() => { await waitFor.presenceOf( diff --git a/core/tests/webdriverio_desktop/preferences.js b/core/tests/webdriverio_desktop/preferences.js index 0264bc00a382..155dbe08cf15 100644 --- a/core/tests/webdriverio_desktop/preferences.js +++ b/core/tests/webdriverio_desktop/preferences.js @@ -149,17 +149,17 @@ describe('Preferences', function() { await preferencesPage.get(); await preferencesPage.selectCreatorDashboard(); await general.goToHomePage(); - await waitFor.urlToBe('http://localhost:9001/creator-dashboard'); + await waitFor.urlToBe('http://localhost:8181/creator-dashboard'); await preferencesPage.get(); await preferencesPage.selectContributorDashboard(); await general.goToHomePage(); - await waitFor.urlToBe('http://localhost:9001/contributor-dashboard'); + await waitFor.urlToBe('http://localhost:8181/contributor-dashboard'); await preferencesPage.get(); await preferencesPage.selectLearnerDashboard(); await general.goToHomePage(); - await waitFor.urlToBe('http://localhost:9001/learner-dashboard'); + await waitFor.urlToBe('http://localhost:8181/learner-dashboard'); }); it('should navigate to account deletion page', @@ -168,7 +168,7 @@ describe('Preferences', function() { await users.login('delete@page.com'); await preferencesPage.get(); await preferencesPage.clickDeleteAccountButton(); - await waitFor.urlToBe('http://localhost:9001/delete-account'); + await waitFor.urlToBe('http://localhost:8181/delete-account'); }); it('should export account data', diff --git a/core/tests/webdriverio_desktop/wipeout.js b/core/tests/webdriverio_desktop/wipeout.js index b71f613045b4..8acee12beea5 100644 --- a/core/tests/webdriverio_desktop/wipeout.js +++ b/core/tests/webdriverio_desktop/wipeout.js @@ -58,7 +58,7 @@ describe('When account is deleted it', function() { pendingAccountDeletionHeading, 'Pending Account Deletion Page takes too long to appear'); expect(await browser.getUrl()).toEqual( - 'http://localhost:9001/pending-account-deletion'); + 'http://localhost:8181/pending-account-deletion'); }); it('should delete private exploration', async function() { diff --git a/core/tests/webdriverio_utils/BlogDashboardPage.js b/core/tests/webdriverio_utils/BlogDashboardPage.js index a8c64fd9b3c4..f7ec8623be89 100644 --- a/core/tests/webdriverio_utils/BlogDashboardPage.js +++ b/core/tests/webdriverio_utils/BlogDashboardPage.js @@ -86,7 +86,7 @@ var BlogDashboardPage = function() { await action.click( 'Blog dashboard link from dropdown', blogDashboardLink); await waitFor.pageToFullyLoad(); - await waitFor.urlRedirection('http://localhost:9001/blog-dashboard'); + await waitFor.urlRedirection('http://localhost:8181/blog-dashboard'); }; this.navigateToBlogDashboardPageWithBackButton = async function() { diff --git a/core/tests/webdriverio_utils/DeleteAccountPage.js b/core/tests/webdriverio_utils/DeleteAccountPage.js index e309806ac1f3..05f37975551a 100644 --- a/core/tests/webdriverio_utils/DeleteAccountPage.js +++ b/core/tests/webdriverio_utils/DeleteAccountPage.js @@ -39,7 +39,7 @@ var DeleteAccountPage = function() { await waitFor.clientSideRedirection(async() => { await action.click('Confirm deletion button', confirmDeletionButton); }, (url) => { - return url === 'http://localhost:9001/pending-account-deletion'; + return url === 'http://localhost:8181/pending-account-deletion'; }, async() => { var pendingAccountDeletionHeading = $( '.e2e-test-pending-account-deletion'); diff --git a/core/tests/webdriverio_utils/TopicsAndSkillsDashboardPage.js b/core/tests/webdriverio_utils/TopicsAndSkillsDashboardPage.js index e914317dc158..85911e0982b4 100644 --- a/core/tests/webdriverio_utils/TopicsAndSkillsDashboardPage.js +++ b/core/tests/webdriverio_utils/TopicsAndSkillsDashboardPage.js @@ -111,7 +111,7 @@ var TopicsAndSkillsDashboardPage = function() { }); await general.navigateToTopicsAndSkillsDashboardPage(); expect(await browser.getUrl()).toEqual( - 'http://localhost:9001/topics-and-skills-dashboard'); + 'http://localhost:8181/topics-and-skills-dashboard'); }; // Only use this if the skills count is not zero. This is supposed to be used diff --git a/core/tests/webdriverio_utils/general.js b/core/tests/webdriverio_utils/general.js index 373304442a2a..9e86d2c770cd 100644 --- a/core/tests/webdriverio_utils/general.js +++ b/core/tests/webdriverio_utils/general.js @@ -93,7 +93,7 @@ var isInDevMode = async function() { return await browser.config.params.devMode === 'true'; }; -var SERVER_URL_PREFIX = 'http://localhost:9001'; +var SERVER_URL_PREFIX = 'http://localhost:8181'; var EDITOR_URL_SLICE = '/create/'; var PLAYER_URL_SLICE = '/explore/'; var USER_PREFERENCES_URL = '/preferences'; diff --git a/docker-compose.yml b/docker-compose.yml index aea48dd20029..f2797b0868f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,6 +46,8 @@ services: - frontend_proto_files:/app/oppia/extensions/classifiers/proto - third_party:/app/oppia/third_party - pip_cache:/root/.cache + - node_modules:/app/oppia/node_modules + - ../karma_coverage_reports:/app/karma_coverage_reports environment: - CLOUDSDK_CORE_DISABLE_PROMPTS=1 - PIP_NO_DEPS=True @@ -56,6 +58,14 @@ services: - source_maps=${source_maps:-false} - disable_host_checking=${disable_host_checking:-false} - no_auto_restart=${no_auto_restart:-false} + - DATASTORE_DATASET=dev-project-id + - DATASTORE_EMULATOR_HOST=datastore:8089 + - DATASTORE_EMULATOR_HOST_PATH=datastore:8089/datastore + - DATASTORE_HOST=http://datastore:8089 + - DATASTORE_PROJECT_ID=dev-project-id + - DATASTORE_USE_PROJECT_ID_AS_APP_ID=true + - GOOGLE_CLOUD_PROJECT=dev-project-id + - REDIS_HOST=redis depends_on: - angular-build - webpack-compiler diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend index e18a69b013ff..19d0cc598df1 100644 --- a/docker/Dockerfile.backend +++ b/docker/Dockerfile.backend @@ -15,6 +15,12 @@ RUN apt-get update -y && apt-get upgrade -y \ python3-matplotlib \ unzip \ wget +RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - +RUN apt-get install -y nodejs + +RUN curl -L -o google-chrome.deb https://github.com/webnicer/chrome-downloads/raw/master/x64.deb/google-chrome-stable_117.0.5938.149-1_amd64.deb +RUN apt-get install -y ./google-chrome.deb +RUN rm google-chrome.deb RUN pip install --upgrade pip==21.2.3 RUN pip install pip-tools==6.6.2 setuptools==58.5.3 cmake @@ -54,11 +60,14 @@ RUN chmod -R 744 $BUF_DIR \ # Installing python dependencies from the requirements_dev.txt file COPY requirements.txt . COPY requirements_dev.txt . +COPY mypy_requirements.txt . RUN pip install --require-hashes --no-deps -r requirements.txt -t /app/oppia/third_party/python_libs RUN pip install --require-hashes --no-deps -r requirements_dev.txt +RUN pip install -r mypy_requirements.txt -t /app/oppia/third_party/python3_libs RUN pip download --no-deps -r requirements.txt -d /root/.cache RUN pip download --no-deps -r requirements_dev.txt -d /root/.cache +RUN pip download -r mypy_requirements.txt -d /root/.cache # Installing buf and proto for Linux -- this docker container is based on Linux COPY buf.gen.yaml . diff --git a/docker/install_node.sh b/docker/install_node.sh new file mode 100644 index 000000000000..a12059560cfb --- /dev/null +++ b/docker/install_node.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright 2023 The Oppia Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +OS_NAME=$(uname) +echo "Installing Node.js..." + +if [ "$OS_NAME" = "Windows" ]; then + if [ "$(uname -m)" = "x86_64" ]; then + architecture=x64 + else + architecture=x86 + fi + extension=".zip" + node_file_name="node-v16.13.0-win-$architecture" + url_to_retrieve="https://nodejs.org/dist/v16.13.0/$node_file_name$extension" + curl -o node-download "$url_to_retrieve" + powershell.exe -c "Expand-Archive -Path node-download -DestinationPath ../oppia_tools" +else + extension=".tar.gz" + if [ "$(python -c 'import sys; print(sys.maxsize > 2**32)')" = "True" ] || [ "$(uname -m)" = "x86_64" ]; then + if [ "$OS_NAME" = "Darwin" ]; then + node_file_name="node-v16.13.0-darwin-x64" + elif [ "$OS_NAME" = "Linux" ]; then + node_file_name="node-v16.13.0-linux-x64" + else + echo "System's Operating System is not compatible." + exit 1 + fi + else + node_file_name="node-v16.13.0" + fi + curl -o node-download "https://nodejs.org/dist/v16.13.0/$node_file_name$extension" + mkdir -p ../oppia_tools + tar -xvf node-download -C ../oppia_tools + rm node-download +fi + +if [ "$node_file_name" = "node-v16.13.0" ]; then + cd ../oppia_tools/node-16.13.0 + ./configure + make +fi + +cd ../oppia_tools && find . -maxdepth 1 -type d -name 'node*' -exec mv {} node-16.13.0 \; + +echo "Node.js installation completed." diff --git a/extensions/rich_text_components/Image/webdriverio.js b/extensions/rich_text_components/Image/webdriverio.js index 376a5a0ddac8..7d266c7d5144 100644 --- a/extensions/rich_text_components/Image/webdriverio.js +++ b/extensions/rich_text_components/Image/webdriverio.js @@ -178,7 +178,7 @@ var expectComponentDetailsToMatch = async function( 'SVG Diagram input element takes too long to load.'); var src = await svgDiagramInputElement.getAttribute('src'); var alt = await svgDiagramInputElement.getAttribute('alt'); - var pre = 'http://localhost:9001'; + var pre = 'http://localhost:8181'; var link = pre + src; expect(alt).toEqual(altText); await request(link, function(error, response, body) { diff --git a/extensions/rich_text_components/Math/webdriverio.js b/extensions/rich_text_components/Math/webdriverio.js index e2f180781b9c..33d5dcd01e06 100644 --- a/extensions/rich_text_components/Math/webdriverio.js +++ b/extensions/rich_text_components/Math/webdriverio.js @@ -98,7 +98,7 @@ var expectComponentDetailsToMatch = async function(elem, rawLatex) { mathSvgImage, 'Math SVG takes too long to load.'); var src = await mathSvgImage.getAttribute('src'); - var pre = 'http://localhost:9001'; + var pre = 'http://localhost:8181'; var link = pre + src; await request(link, function(error, response, body) { expect(body).toBe(SVGTAGS[rawLatex]); diff --git a/puppeteer-login-script.js b/puppeteer-login-script.js index 4b8ac7a4f69e..581688f4df1f 100644 --- a/puppeteer-login-script.js +++ b/puppeteer-login-script.js @@ -18,8 +18,8 @@ * @param {puppeteer.Browser} browser * @param {{url: string, options: LHCI.CollectCommand.Options}} context */ -const LOGIN_URL = 'http://127.0.0.1:8181/login'; -const CREATOR_DASHBOARD_URL = 'http://127.0.0.1:8181/creator-dashboard'; +const LOGIN_URL = 'http://localhost:8181/login'; +const CREATOR_DASHBOARD_URL = 'http://localhost:8181/creator-dashboard'; const networkIdle = 'networkidle0'; var emailInput = '.e2e-test-sign-in-email-input'; @@ -88,7 +88,7 @@ const setRole = async function(page, role) { try { // eslint-disable-next-line dot-notation await page.goto( - 'http://127.0.0.1:8181/admin#/roles', { waitUntil: networkIdle }); + 'http://localhost:8181/admin#/roles', { waitUntil: networkIdle }); await page.waitForSelector(usernameInputFieldForRolesEditing); await page.type(usernameInputFieldForRolesEditing, 'username1'); await page.waitForSelector(editUserRoleButton); @@ -117,7 +117,7 @@ const createCollections = async function(context, page) { await setRole(page, 'COLLECTION_EDITOR'); // Load in Collection // eslint-disable-next-line dot-notation - await page.goto('http://127.0.0.1:8181/admin'); + await page.goto('http://localhost:8181/admin'); await page.waitForTimeout(2000); await page.evaluate('window.confirm = () => true'); await page.click('#reload-collection-button-id'); @@ -135,7 +135,7 @@ const createExplorations = async function(context, page) { console.log('Creating Exploration...'); // Load in Exploration // eslint-disable-next-line dot-notation - await page.goto('http://127.0.0.1:8181/admin', { waitUntil: 'networkidle0' }); + await page.goto('http://localhost:8181/admin', { waitUntil: 'networkidle0' }); await page.waitForTimeout(2000); await page.evaluate('window.confirm = () => true'); await page.click( diff --git a/scripts/backend_test_shards.json b/scripts/backend_test_shards.json index 1aeeb1e78028..b65be180353b 100644 --- a/scripts/backend_test_shards.json +++ b/scripts/backend_test_shards.json @@ -93,6 +93,7 @@ "core.domain.learner_goals_services_test", "scripts.scripts_test_utils_test", "scripts.servers_test", + "scripts.generate_build_directory_test", "scripts.concurrent_task_utils_test", "core.storage.storage_models_test", "core.platform.cache.redis_cache_services_test", diff --git a/scripts/backend_tests_incomplete_coverage.txt b/scripts/backend_tests_incomplete_coverage.txt index ae8fedc7d112..54a7066b5f56 100644 --- a/scripts/backend_tests_incomplete_coverage.txt +++ b/scripts/backend_tests_incomplete_coverage.txt @@ -102,3 +102,5 @@ scripts.linters.pre_commit_linter_test scripts.pre_push_hook_test scripts.run_e2e_tests_test scripts.run_frontend_tests_test +scripts.install_third_party_libs_test +scripts.run_mypy_checks_test diff --git a/scripts/build.py b/scripts/build.py index 1936ddd154af..d8297b5dae8a 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -333,21 +333,19 @@ def _minify_and_create_sourcemap( source_map_properties = 'includeSources,url=\'third_party.min.js.map\'' # TODO(#18260): Change this when we permanently move to # the Dockerized Setup. - cmd = '%s %s %s -c -m --source-map %s -o %s ' % ( - common.NODE_BIN_PATH, UGLIFY_FILE, source_path, - source_map_properties, target_file_path) if feconf.OPPIA_IS_DOCKERIZED: - cmd = ' '.join([ - 'bash', '-c', + subprocess.check_call( 'node /app/oppia/node_modules/uglify-js/bin/uglifyjs' ' /app/oppia/third_party/generated/js/third_party.js' - ' -c -m --source-map %s -o /app/oppia/third_party/' - 'generated/js/third_party.min.js' % ( - source_map_properties - ) - ]) - - subprocess.check_call(cmd, shell=True) + ' -c -m --source-map includeSources,url=\'third_party.min.js.map\'' + ' -o /app/oppia/third_party/generated/js/third_party.min.js', + shell=True + ) + else: + cmd = '%s %s %s -c -m --source-map %s -o %s ' % ( + common.NODE_BIN_PATH, UGLIFY_FILE, source_path, + source_map_properties, target_file_path) + subprocess.check_call(cmd, shell=True) def _generate_copy_tasks_for_fonts( diff --git a/scripts/build_test.py b/scripts/build_test.py index a4c97f0974d1..bc2b2d5f1a98 100644 --- a/scripts/build_test.py +++ b/scripts/build_test.py @@ -107,9 +107,22 @@ def test_minify_and_create_sourcemap(self) -> None: def test_minify_and_create_sourcemap_under_docker_environment(self) -> None: """Tests _minify_and_create_sourcemap with an invalid filepath.""" + + def mock_subprocess_check_call(command: str, **kwargs: bool) -> None: # pylint: disable=unused-argument + """Mock method for replacing subprocess.check_call().""" + excepted_cmd = ( + 'node /app/oppia/node_modules/uglify-js/bin/uglifyjs ' + '/app/oppia/third_party/generated/js/third_party.js -c -m' + ' --source-map includeSources,url=\'third_party.min.js.map\' ' + '-o /app/oppia/third_party/generated/js/third_party.min.js' + ) + self.assertEqual(command, excepted_cmd) + with self.swap(feconf, 'OPPIA_IS_DOCKERIZED', True): - build._minify_and_create_sourcemap( # pylint: disable=protected-access - INVALID_INPUT_FILEPATH, INVALID_OUTPUT_FILEPATH) + with self.swap( + subprocess, 'check_call', mock_subprocess_check_call): + build._minify_and_create_sourcemap( # pylint: disable=protected-access + INVALID_INPUT_FILEPATH, INVALID_OUTPUT_FILEPATH) def test_join_files(self) -> None: """Determine third_party.js contains the content of the first 10 JS diff --git a/scripts/check_backend_associated_test_file.py b/scripts/check_backend_associated_test_file.py index 5775c3b06bd7..927e81d6054f 100644 --- a/scripts/check_backend_associated_test_file.py +++ b/scripts/check_backend_associated_test_file.py @@ -51,7 +51,7 @@ 'core/tests/data/image_constants.py', 'core/tests/data/unicode_and_str_handler.py', 'proto_files/text_classifier_pb2.py', - 'proto_files/training_job_response_payload_pb2.py', + 'proto_files/training_job_response_payload_pb2.py' ] diff --git a/scripts/common.py b/scripts/common.py index e284e96094d9..fe166c214c41 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -89,8 +89,10 @@ OPPIA_TOOLS_DIR_ABS_PATH = os.path.abspath(OPPIA_TOOLS_DIR) THIRD_PARTY_DIR = os.path.join(CURR_DIR, 'third_party') THIRD_PARTY_PYTHON_LIBS_DIR = os.path.join(THIRD_PARTY_DIR, 'python_libs') -GOOGLE_CLOUD_SDK_HOME = os.path.join( - OPPIA_TOOLS_DIR_ABS_PATH, 'google-cloud-sdk-364.0.0', 'google-cloud-sdk') +GOOGLE_CLOUD_SDK_HOME = ( + '/google-cloud-sdk' if feconf.OPPIA_IS_DOCKERIZED else os.path.join( + OPPIA_TOOLS_DIR_ABS_PATH, 'google-cloud-sdk-364.0.0', 'google-cloud-sdk' + )) GOOGLE_APP_ENGINE_SDK_HOME = os.path.join( GOOGLE_CLOUD_SDK_HOME, 'platform', 'google_appengine') GOOGLE_CLOUD_SDK_BIN = os.path.join(GOOGLE_CLOUD_SDK_HOME, 'bin') @@ -101,7 +103,8 @@ DEV_APPSERVER_PATH = ( os.path.join(GOOGLE_CLOUD_SDK_BIN, 'dev_appserver.py')) GCLOUD_PATH = os.path.join(GOOGLE_CLOUD_SDK_BIN, 'gcloud') -NODE_PATH = os.path.join(OPPIA_TOOLS_DIR, 'node-%s' % NODE_VERSION) +NODE_PATH = '/usr' if feconf.OPPIA_IS_DOCKERIZED else os.path.join( + OPPIA_TOOLS_DIR, 'node-%s' % NODE_VERSION) NODE_MODULES_PATH = os.path.join(CURR_DIR, 'node_modules') FRONTEND_DIR = os.path.join(CURR_DIR, 'core', 'templates') YARN_PATH = os.path.join(OPPIA_TOOLS_DIR, 'yarn-%s' % YARN_VERSION) @@ -221,7 +224,7 @@ ] -GAE_PORT_FOR_E2E_TESTING: Final = 9001 +GAE_PORT_FOR_E2E_TESTING: Final = 8181 ELASTICSEARCH_SERVER_PORT: Final = 9200 PORTS_USED_BY_OPPIA_PROCESSES_IN_LOCAL_E2E_TESTING: Final = [ GAE_PORT_FOR_E2E_TESTING, diff --git a/scripts/generate_build_directory.py b/scripts/generate_build_directory.py new file mode 100644 index 000000000000..145d21d7a03c --- /dev/null +++ b/scripts/generate_build_directory.py @@ -0,0 +1,35 @@ +# Copyright 2023 The Oppia Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS-IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script for generating the build directory in the prod mode when +the webpack bundling is completed.""" + +from __future__ import annotations + +from scripts import build + + +def main() -> None: + """The main method of this script.""" + + build.safe_delete_directory_tree('build/') + + hashes = build.generate_hashes() + build.generate_build_directory(hashes) + + +# The 'no coverage' pragma is used as this line is un-testable. This is because +# it will only be called when build.py is used as a script. +if __name__ == '__main__': # pragma: no cover + main() diff --git a/scripts/generate_build_directory_test.py b/scripts/generate_build_directory_test.py new file mode 100644 index 000000000000..c978bfaec053 --- /dev/null +++ b/scripts/generate_build_directory_test.py @@ -0,0 +1,41 @@ +# coding: utf-8 +# +# Copyright 2023 The Oppia Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for scripts/generate_build_directory.py.""" + +from __future__ import annotations + +from core import feconf + +from core.tests import test_utils + +from scripts import generate_build_directory + + +class Ret: + """Return object with required attributes.""" + + def __init__(self) -> None: + self.returncode = 0 + + +class GenerateBuildDirectoryTests(test_utils.GenericTestBase): + """Test the methods for generate build directory.""" + + def test_generate_build_dir_under_docker(self) -> None: + with self.assertRaisesRegex(KeyError, 'js/third_party.min.js'): + with self.swap(feconf, 'OPPIA_IS_DOCKERIZED', True): + generate_build_directory.main() diff --git a/scripts/install_dependencies_json_packages_test.py b/scripts/install_dependencies_json_packages_test.py index c8ef04e31582..ee199b7de812 100644 --- a/scripts/install_dependencies_json_packages_test.py +++ b/scripts/install_dependencies_json_packages_test.py @@ -1,6 +1,6 @@ # coding: utf-8 # -# Copyright 2019 The Oppia Authors. All Rights Reserved. +# Copyright 2023 The Oppia Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/install_third_party_libs.py b/scripts/install_third_party_libs.py index bd05e18eef6e..068051359c08 100644 --- a/scripts/install_third_party_libs.py +++ b/scripts/install_third_party_libs.py @@ -22,17 +22,22 @@ import shutil import subprocess import zipfile + +from core import feconf from scripts import install_python_dev_dependencies + from typing import Final, List -install_python_dev_dependencies.main(['--assert_compiled']) +if not feconf.OPPIA_IS_DOCKERIZED: + install_python_dev_dependencies.main(['--assert_compiled']) + + from . import install_third_party # isort:skip pylint: disable=wrong-import-position, wrong-import-order + from . import pre_commit_hook # isort:skip pylint: disable=wrong-import-position, wrong-import-order + from . import pre_push_hook # isort:skip pylint: disable=wrong-import-position, wrong-import-order + from . import setup # isort:skip pylint: disable=wrong-import-position, wrong-import-order + from . import setup_gae # isort:skip pylint: disable=wrong-import-position, wrong-import-order from . import common # isort:skip pylint: disable=wrong-import-position, wrong-import-order -from . import install_third_party # isort:skip pylint: disable=wrong-import-position, wrong-import-order -from . import pre_commit_hook # isort:skip pylint: disable=wrong-import-position, wrong-import-order -from . import pre_push_hook # isort:skip pylint: disable=wrong-import-position, wrong-import-order -from . import setup # isort:skip pylint: disable=wrong-import-position, wrong-import-order -from . import setup_gae # isort:skip pylint: disable=wrong-import-position, wrong-import-order from core import utils # isort:skip pylint: disable=wrong-import-position, wrong-import-order @@ -164,6 +169,9 @@ def compile_protobuf_files(proto_files_paths: List[str]) -> None: def main() -> None: """Install third-party libraries for Oppia.""" + if feconf.OPPIA_IS_DOCKERIZED: + return + setup.main(args=[]) setup_gae.main(args=[]) diff --git a/scripts/install_third_party_libs_test.py b/scripts/install_third_party_libs_test.py index 4da224f75c91..44e8d4e373a4 100644 --- a/scripts/install_third_party_libs_test.py +++ b/scripts/install_third_party_libs_test.py @@ -25,6 +25,7 @@ import tempfile import zipfile +from core import feconf from core import utils from core.tests import test_utils @@ -105,6 +106,16 @@ def mock_ensure_directory_exists(unused_path: str) -> None: self.dir_exists_swap = self.swap( common, 'ensure_directory_exists', mock_ensure_directory_exists) + def test_install_third_party_main_under_docker(self) -> None: + with self.swap(feconf, 'OPPIA_IS_DOCKERIZED', True): + with self.check_call_swap: + install_third_party_libs.main() + + def test_install_third_party_main(self) -> None: + with self.swap(feconf, 'OPPIA_IS_DOCKERIZED', False): + with self.check_call_swap: + install_third_party_libs.main() + def test_tweak_yarn_executable(self) -> None: check_function_calls = { 'mock_rename': False, diff --git a/scripts/run_backend_tests.py b/scripts/run_backend_tests.py index 1b24e399f4a7..6b122dffc136 100644 --- a/scripts/run_backend_tests.py +++ b/scripts/run_backend_tests.py @@ -61,15 +61,17 @@ import sys import threading import time + from typing import Dict, Final, List, Optional, Tuple, cast from . import install_third_party_libs +from core import feconf, utils # isort:skip pylint: disable=wrong-import-position, wrong-import-order + # This installs third party libraries before importing other files or importing # libraries that use the builtins python module (e.g. build, utils). install_third_party_libs.main() -from core import utils # isort:skip pylint: disable=wrong-import-position, wrong-import-order from . import common # isort:skip pylint: disable=wrong-import-position, wrong-import-order from . import concurrent_task_utils # isort:skip pylint: disable=wrong-import-position, wrong-import-order from . import servers # isort:skip pylint: disable=wrong-import-position, wrong-import-order @@ -496,9 +498,10 @@ def main(args: Optional[List[str]] = None) -> None: raise Exception('The delimiter in test_target should be a dot (.)') with contextlib.ExitStack() as stack: - stack.enter_context( - servers.managed_cloud_datastore_emulator(clear_datastore=True)) - stack.enter_context(servers.managed_redis_server()) + if not feconf.OPPIA_IS_DOCKERIZED: + stack.enter_context( + servers.managed_cloud_datastore_emulator(clear_datastore=True)) + stack.enter_context(servers.managed_redis_server()) if parsed_args.test_target: # Check if target either ends with '_test' which means a path to # a test file has been provided or has '_test.' in it which means diff --git a/scripts/run_backend_tests_test.py b/scripts/run_backend_tests_test.py index fbeeda0b6d70..f33c0d4e4d54 100644 --- a/scripts/run_backend_tests_test.py +++ b/scripts/run_backend_tests_test.py @@ -26,6 +26,7 @@ import sys import threading +from core import feconf from core import utils from core.tests import test_utils from scripts import common @@ -630,6 +631,29 @@ def test_invalid_test_target_message_is_displayed_correctly(self) -> None: self.assertIn( 'Redirecting to its corresponding test file...', self.print_arr) + def test_invalid_test_target_message_is_displayed_docker(self) -> None: + with self.swap_install_third_party_libs: + from scripts import run_backend_tests + swap_check_results = self.swap( + run_backend_tests, 'check_test_results', + lambda *unused_args, **unused_kwargs: (100, 0, 0, 0)) + swapcheck_coverage = self.swap( + run_backend_tests, 'check_coverage', + lambda *unused_args, **unused_kwargs: ('', 100.00)) + with self.swap(feconf, 'OPPIA_IS_DOCKERIZED', True): + with self.swap_execute_task, swapcheck_coverage: + with self.swap_cloud_datastore_emulator, swap_check_results: + with self.print_swap, self.swap_redis_server: + run_backend_tests.main( + args=['--test_target', + 'scripts.run_backend_tests.py']) + + self.assertIn( + 'WARNING : test_target flag should point to the test file.', + self.print_arr) + self.assertIn( + 'Redirecting to its corresponding test file...', self.print_arr) + def test_error_in_matching_shards_with_tests_throws_error(self) -> None: with self.swap_install_third_party_libs: from scripts import run_backend_tests diff --git a/scripts/run_mypy_checks.py b/scripts/run_mypy_checks.py index 0e340531f6bd..aa421633f81d 100644 --- a/scripts/run_mypy_checks.py +++ b/scripts/run_mypy_checks.py @@ -24,6 +24,7 @@ import subprocess import sys +from core import feconf from scripts import common from scripts import install_third_party_libs @@ -177,16 +178,19 @@ def main(args: Optional[List[str]] = None) -> int: # https://stackoverflow.com/q/10095037 for more details. sys.path.insert(1, directory) - install_third_party_libraries(parsed_args.skip_install) + if not feconf.OPPIA_IS_DOCKERIZED: + install_third_party_libraries(parsed_args.skip_install) - print('Installing Mypy and stubs for third party libraries.') - return_code, mypy_exec_path = install_mypy_prerequisites( - parsed_args.install_globally) - if return_code != 0: - print('Cannot install Mypy and stubs for third party libraries.') - sys.exit(1) + print('Installing Mypy and stubs for third party libraries.') + return_code, mypy_exec_path = install_mypy_prerequisites( + parsed_args.install_globally) + if return_code != 0: + print('Cannot install Mypy and stubs for third party libraries.') + sys.exit(1) - print('Installed Mypy and stubs for third party libraries.') + print('Installed Mypy and stubs for third party libraries.') + + mypy_exec_path = os.path.join(MYPY_TOOLS_DIR, 'bin', 'mypy') print('Starting Mypy type checks.') cmd = get_mypy_cmd( diff --git a/webpack.prod.config.ts b/webpack.prod.config.ts index 0d2ca05955c2..7f931fcfe03d 100644 --- a/webpack.prod.config.ts +++ b/webpack.prod.config.ts @@ -21,6 +21,7 @@ var common = require('./webpack.common.config.ts'); var path = require('path'); var webpack = require('webpack'); var analyticsConstants = require('./assets/analytics-constants.json'); +const TerserPlugin = require('terser-webpack-plugin'); module.exports = merge(common, { @@ -40,5 +41,15 @@ module.exports = merge(common, { analyticsConstants.SITE_NAME_FOR_ANALYTICS ), }) - ] + ], + optimization: { + ...common.optimization, + minimizer: [ + new TerserPlugin({ + cache: true, + parallel: false, + sourceMap: true, + }), + ], + } });