diff --git a/ansible/artifacts/sunbird/login/login.ftl b/ansible/artifacts/sunbird/login/login.ftl index dd7f0ccf89..fdf5abbbec 100644 --- a/ansible/artifacts/sunbird/login/login.ftl +++ b/ansible/artifacts/sunbird/login/login.ftl @@ -39,10 +39,10 @@ <#if usernameEditDisabled??> - + <#-- TODO: need to find alternative for prepopulating username --> + <#else> - - +
diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index b86ec685dc..4f782b6e41 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -680,3 +680,16 @@ sunbird_portal_azure_storage_account: "{{sunbird_azure_storage_account}}" sunbird_portal_azure_storage_key: "{{sunbird_azure_storage_key}}" sunbird_device_api: "{{proto}}://{{ proxy_server_name }}/api/" sunbird_quartz_shadow_user_migration_timer: "0 0 1 1/1 * ? *" + +# Learning-Service +lp_cassandra_connection: "{{groups['cassandra']|join(':9042,')}}:9042" +dp_cassandra_connection: "{{ groups['dp-cassandra'][0] }}:9042" +environment_id: 10000000 +graph_passport_key: abc123 +learning_neo4j_bolt_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:7687 +language_neo4j_bolt_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:8687 +learning_read_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:7687 +learning_write_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:7687 +language_read_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:8687 +language_write_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:8687 +mw_shard_id: 1 \ No newline at end of file diff --git a/ansible/keycloak.yml b/ansible/keycloak.yml index ab8963bb41..a54b953dc9 100644 --- a/ansible/keycloak.yml +++ b/ansible/keycloak.yml @@ -5,8 +5,29 @@ - ['{{ inventory_dir }}/secrets.yml', 'secrets/{{ env }}.yml'] # --tags provision/deployment will trigger respective roles # --tags bootstrap will trigger user/realm creation + pre_tasks: + - name: Get the list of all the services running in our OS + become: yes + service_facts: + + - name: Stop the monit to ensure that it doesn't start keycloak + service: name=monit state=stopped + become: yes + when: ansible_facts.services.monit is defined + roles: - {role: openjdk, tags: provision} - {role: keycloak-provision, tags: provision} - keycloak-deploy - {role: monit, monit_checks: ['keycloak'], tags: deploy} + + post_tasks: + - name: get the list of all services in the system + become: yes + service_facts: + + - name: After succesful migration and staring of keycloak, start back monit if installed + service: name=monit state=started + become: yes + when: ansible_facts.services.monit is defined + diff --git a/ansible/logging.yml b/ansible/logging.yml index 490398d2bb..ca8beb0e13 100644 --- a/ansible/logging.yml +++ b/ansible/logging.yml @@ -87,11 +87,21 @@ vars_files: - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] roles: - - { role: vm-agents-filebeat, filebeat_log_path: '/mount/data/analytics/logs/services/api-service.log' } + - { role: vm-agents-filebeat, filebeat_log_path: '/mount/data/analytics/logs/services/api-service.log, /var/log/logstash/logstash-plain.log' } tags: - analytics - filebeat +- hosts: dp-kafkaindexer-ps + become: yes + vars_files: + - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] + roles: + - { role: vm-agents-filebeat, filebeat_log_path: '/var/log/logstash/logstash-plain.log' } + tags: + - kafkaindexer + - filebeat + - hosts: zookeeper become: yes vars_files: diff --git a/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/python_keycloak.egg-info/requires.txt b/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/python_keycloak.egg-info/requires.txt index d6eafdf657..f3221c8ab6 100644 --- a/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/python_keycloak.egg-info/requires.txt +++ b/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/python_keycloak.egg-info/requires.txt @@ -1,3 +1,3 @@ -requests==2.18.4 +requests==2.20.0 httmock==1.2.5 python-jose==1.4.0 diff --git a/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/setup.py b/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/setup.py index bf6a505f0a..7f8e077f9a 100644 --- a/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/setup.py +++ b/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/setup.py @@ -12,7 +12,7 @@ keywords='keycloak openid', description=u'python-keycloak is a Python package providing access to the Keycloak API.', packages=['keycloak', 'keycloak.authorization', 'keycloak.tests'], - install_requires=['requests==2.18.4', 'httmock==1.2.5', 'python-jose==1.4.0'], + install_requires=['requests==2.20.0', 'httmock==1.2.5', 'python-jose==1.4.0'], classifiers=[ 'Programming Language :: Python :: 3', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 0634c9a7c7..8395295c9e 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -59,7 +59,7 @@ cert_registry_service_prefix: /certreg desktop_app_prefix: /desktop # Service URLs -content_service_url: "http://content-service:5000" +content_service_url: "http://knowledge-mw-service:5000" learning_service_url: "http://learner-service:9000" vm_learning_service_url: "http://{{learningservice_ip}}:8080/learning-service" telemetry_service_url: "http://telemetry-service:9001" @@ -4661,9 +4661,9 @@ kong_apis: config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: rejectMigrate - uris: "{{ user_service_prefix }}/v1/migrate/reject" - upstream_url: "{{ learning_service_url }}/v1/user/migrate/reject" + - name: userMigrate + uris: "{{ user_service_prefix }}/v1/migrate" + upstream_url: "{{ learning_service_url }}/v1/user/migrate" strip_uri: true plugins: - name: jwt @@ -4709,4 +4709,20 @@ kong_apis: config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: userFeed + uris: "{{ user_service_prefix }}/v1/feed" + upstream_url: "{{ learning_service_url }}/v1/user/feed" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: publicUser + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" diff --git a/ansible/roles/stack-sunbird/defaults/main.yml b/ansible/roles/stack-sunbird/defaults/main.yml index 6b16cc7990..fc8f25b678 100644 --- a/ansible/roles/stack-sunbird/defaults/main.yml +++ b/ansible/roles/stack-sunbird/defaults/main.yml @@ -70,7 +70,6 @@ content_service_blacklisted_channels: sunbird_env_logo_url: desktop_app_storage_url: "https://{{sunbird_offline_azure_storage_account}}.blob.core.windows.net/{{offline_installer_container_name}}" - telemetry_logstash_heap_size: 512m telemetry_logstash_replicas: 1 telemetry_logstash_reservation_memory: 1g diff --git a/ansible/roles/stack-sunbird/tasks/content_service.yml b/ansible/roles/stack-sunbird/tasks/content_service.yml deleted file mode 100644 index d13f312179..0000000000 --- a/ansible/roles/stack-sunbird/tasks/content_service.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Remove content service from stack (previous mode of deploy) - shell: "docker service rm content_service_content_service" - ignore_errors: yes - -- name: Remove content service - shell: "docker service rm content-service" - ignore_errors: yes - -- name: Deploy content service - shell: "docker service create --with-registry-auth --replicas {{ content_replicas }} -p 5000:5000 --name content-service --hostname content-service --reserve-memory {{ content_reservation_memory }} --limit-memory {{ content_limit_memory }} --limit-cpu {{ content_limit_cpu }} --reserve-cpu {{ content_reservation_cpu }} --health-cmd 'wget -qO- content-service:5000/service/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_content-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" - args: - chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/knowledge-mw-service.yml b/ansible/roles/stack-sunbird/tasks/knowledge-mw-service.yml new file mode 100644 index 0000000000..a855ca06db --- /dev/null +++ b/ansible/roles/stack-sunbird/tasks/knowledge-mw-service.yml @@ -0,0 +1,9 @@ +--- +- name: Remove knowledge-mw service + shell: "docker service rm knowledge-mw-service" + ignore_errors: yes + +- name: Deploy knowledge-mw service + shell: "docker service create --with-registry-auth --replicas {{ content_replicas }} -p 5000:5000 --name knowledge-mw-service --hostname knowledge-mw-service --reserve-memory {{ content_reservation_memory }} --limit-memory {{ content_limit_memory }} --limit-cpu {{ content_limit_cpu }} --reserve-cpu {{ content_reservation_cpu }} --health-cmd 'wget -qO- knowledge-mw-service:5000/service/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_knowledge-mw-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" + args: + chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/learning-service.yml b/ansible/roles/stack-sunbird/tasks/learning-service.yml new file mode 100644 index 0000000000..4c2aa1bb43 --- /dev/null +++ b/ansible/roles/stack-sunbird/tasks/learning-service.yml @@ -0,0 +1,9 @@ +--- +- name: Remove learning service + shell: "docker service rm learning-service" + ignore_errors: yes + +- name: Deploy learning service + shell: "docker service create --with-registry-auth --replicas {{ learner_replicas }} -p 9696:9000 --name learning-service --hostname learning-service --reserve-memory {{ learner_reservation_memory }} --limit-memory {{ learner_limit_memory }} --limit-cpu {{ learner_limit_cpu }} --reserve-cpu {{ learner_reservation_cpu }} --health-cmd 'wget -qO- learning-service:9000/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_learning-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" + args: + chdir: /home/deployer/stack \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/tasks/main.yml b/ansible/roles/stack-sunbird/tasks/main.yml index 8dafc5fde9..e691934b89 100644 --- a/ansible/roles/stack-sunbird/tasks/main.yml +++ b/ansible/roles/stack-sunbird/tasks/main.yml @@ -25,8 +25,8 @@ - include: notification_service.yml when: deploy_notification is defined - - include: content_service.yml - when: deploy_content is defined + - include: knowledge-mw-service.yml + when: deploy_knowledge_mw is defined - include: telemetry_service.yml when: deploy_telemetry is defined @@ -39,3 +39,6 @@ - include: telemetry_logstash_datapipeline.yml when: deploy_telemetry_logstash_datapipeline is defined + + - include: learning-service.yml + when: deploy_learning is defined diff --git a/ansible/roles/stack-sunbird/templates/sunbird_content-service.env b/ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env similarity index 100% rename from ansible/roles/stack-sunbird/templates/sunbird_content-service.env rename to ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env diff --git a/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env b/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env index 5e6bace5c3..f24d32f01f 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env @@ -54,14 +54,14 @@ badging_authorization_key={{vault_badging_authorization_key}} sunbird_badger_baseurl=http://badger-service:8004 sunbird_remote_req_router_path=akka.tcp://SunbirdMWSystem@actor-service:8088/user/RequestRouter sunbird_remote_bg_req_router_path=akka.tcp://SunbirdMWSystem@actor-service:8088/user/BackgroundRequestRouter -sunbird_api_base_url=http://content-service:5000 +sunbird_api_base_url=http://knowledge-mw-service:5000 sunbird_authorization={{sunbird_api_auth_token}} telemetry_pdata_id={{sunbird_telemetry_pdata_id}} telemetry_pdata_pid=learner-service sunbird_telemetry_base_url=http://telemetry-service:9001 telemetry_queue_threshold_value=100 sunbird_default_channel={{sunbird_default_channel}} -sunbird_api_mgr_base_url=http://content-service:5000 +sunbird_api_mgr_base_url=http://knowledge-mw-service:5000 sunbird_cs_base_url={{sunbird_cs_base_url}} sunbird_cs_search_path=/v1/content/search diff --git a/ansible/roles/stack-sunbird/templates/sunbird_learning-service.env b/ansible/roles/stack-sunbird/templates/sunbird_learning-service.env new file mode 100644 index 0000000000..8db5e1e1c3 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/sunbird_learning-service.env @@ -0,0 +1,48 @@ +# Cassandra Configuration +cassandra.lp.connection={{ lp_cassandra_connection }} +cassandra.lpa.connection={{ dp_cassandra_connection }} + +# Redis Configuration +redis.host=localhost +redis.port=6379 +redis.maxConnections=128 + +#--Maximum Content Package File Size Limit in Bytes (50 MB) +MAX_CONTENT_PACKAGE_FILE_SIZE_LIMIT=52428800 + +#--Maximum Asset File Size Limit in Bytes (20 MB) +MAX_ASSET_FILE_SIZE_LIMIT=20971520 + +#--No of Retry While File Download Fails +RETRY_ASSET_DOWNLOAD_COUNT=1 + +#Current environment +cloud_storage.env=dev + +# Configuration +graph.dir=/data/graphDB +akka.request_timeout=30 +environment.id={{ environment_id }} +graph.ids=["domain"] +graph.passport.key.base={{ graph_passport_key }} +route.domain={{ learning_neo4j_bolt_url }} +route.bolt.write.domain={{ learning_write_elb_url }} +route.bolt.read.domain={{ learning_read_elb_url }} +route.all={{ language_neo4j_bolt_url }} +route.bolt.write.all={{ language_write_elb_url }} +route.bolt.read.all={{ language_read_elb_url }} + +shard.id={{ mw_shard_id }} +platform.auth.check.enabled=false +platform.cache.ttl=3600000 + +#Top N Config for Search Telemetry +telemetry_env=dev + +installation.id=ekstep + +channel.default=in.ekstep + +content.cache.ttl=86400 +content.cache.read=true +content.cache.hierarchy=true \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env index 65ad37428d..67652c19db 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env @@ -54,14 +54,14 @@ badging_authorization_key={{vault_badging_authorization_key}} sunbird_badger_baseurl=http://badger-service:8004 sunbird_remote_req_router_path=akka.tcp://SunbirdMWSystem@actor-service:8088/user/RequestRouter sunbird_remote_bg_req_router_path=akka.tcp://SunbirdMWSystem@actor-service:8088/user/BackgroundRequestRouter -sunbird_api_base_url=http://content-service:5000 +sunbird_api_base_url=http://knowledge-mw-service:5000 sunbird_authorization={{sunbird_api_auth_token}} telemetry_pdata_id={{sunbird_telemetry_pdata_id}} telemetry_pdata_pid=lms-service sunbird_telemetry_base_url=http://telemetry-service:9001 telemetry_queue_threshold_value=100 sunbird_default_channel={{sunbird_default_channel}} -sunbird_api_mgr_base_url=http://content-service:5000 +sunbird_api_mgr_base_url=http://knowledge-mw-service:5000 sunbird_cs_base_url={{sunbird_cs_base_url}} sunbird_cs_search_path=/v1/content/search diff --git a/ansible/roles/stack-sunbird/templates/sunbird_player.env b/ansible/roles/stack-sunbird/templates/sunbird_player.env index 092ad8d358..571a02583c 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_player.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_player.env @@ -1,7 +1,7 @@ sunbird_port=3000 sunbird_content_player_url={{sunbird_content_player_url}} sunbird_learner_player_url={{sunbird_learner_player_url}} -sunbird_content_proxy_url=http://content-service:5000 +sunbird_content_proxy_url=http://knowledge-mw-service:5000 sunbird_echo_api_url={{sunbird_echo_api_url}} sunbird_autocreate_trampoline_user=false sunbird_portal_realm={{keycloak_realm}} diff --git a/pipelines/deploy/knowledge-mw-service/Jenkinsfile b/pipelines/deploy/knowledge-mw-service/Jenkinsfile new file mode 100644 index 0000000000..629ca40e07 --- /dev/null +++ b/pipelines/deploy/knowledge-mw-service/Jenkinsfile @@ -0,0 +1,30 @@ +@Library('deploy-conf') _ +node() { + try { + stage('checkout public repo') { + cleanWs() + checkout scm + } + + stage('deploy') { + values = docker_params() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "$currentWs/ansible/deploy.yml" + ansibleExtraArgs = "--tags \"stack-sunbird\" --extra-vars \"hub_org=$hub_org image_name=$values.image_name image_tag=$values.image_tag service_name=knowledge-mw-service deploy_knowledge_mw=True\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + ansible_playbook_run(values) + archiveArtifacts 'metadata.json' + currentBuild.description = "Image: ${values.image_tag}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/pipelines/deploy/content/Jenkinsfile b/pipelines/deploy/learning/Jenkinsfile similarity index 88% rename from pipelines/deploy/content/Jenkinsfile rename to pipelines/deploy/learning/Jenkinsfile index 0be3b9cc35..e440154d5b 100644 --- a/pipelines/deploy/content/Jenkinsfile +++ b/pipelines/deploy/learning/Jenkinsfile @@ -10,7 +10,7 @@ node() { values = docker_params() currentWs = sh(returnStdout: true, script: 'pwd').trim() ansiblePlaybook = "$currentWs/ansible/deploy.yml" - ansibleExtraArgs = "--tags \"stack-sunbird\" --extra-vars \"hub_org=$hub_org image_name=$values.image_name image_tag=$values.image_tag service_name=content-service deploy_content=True\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + ansibleExtraArgs = "--tags \"stack-sunbird\" --extra-vars \"hub_org=$hub_org image_name=$values.image_name image_tag=$values.image_tag service_name=learning-service deploy_learning=True\" --vault-password-file /var/lib/jenkins/secrets/vault-pass" values.put('currentWs', currentWs) values.put('ansiblePlaybook', ansiblePlaybook) values.put('ansibleExtraArgs', ansibleExtraArgs) @@ -27,4 +27,4 @@ node() { slack_notify(currentBuild.result) email_notify() } -} +} \ No newline at end of file