Merge branch 'release-2.6.0' into release-2.5.0

ansible/artifacts/sunbird/login/login.ftl

@@ -39,10 +39,10 @@
                             </#if>
                         </label>
                         <#if usernameEditDisabled??>
-                        <input class="mt-8" id="username" name="username" value="${(login.username!'')?html}" type="text" disabled />
+                          <#-- TODO: need to find alternative for prepopulating username -->
+                        <input class="mt-8" id="username" name="username" type="text" disabled />
                         <#else>
-                        <input class="mt-8" id="username" name="username" onfocusin="inputBoxFocusIn(this)" onfocusout="inputBoxFocusOut(this)" value="${(login.username!'')?html}" type="text" autofocus autocomplete="off" />
-                        </#if>
+                        <input class="mt-8" id="username" name="username" onfocusin="inputBoxFocusIn(this)" onfocusout="inputBoxFocusOut(this)" type="text" autofocus autocomplete="off" />                        </#if>
                     </div>
                     <div class="field">
                         <div>

ansible/inventory/env/group_vars/all.yml

@@ -680,3 +680,16 @@ sunbird_portal_azure_storage_account: "{{sunbird_azure_storage_account}}"
 sunbird_portal_azure_storage_key: "{{sunbird_azure_storage_key}}"
 sunbird_device_api: "{{proto}}://{{ proxy_server_name }}/api/"
 sunbird_quartz_shadow_user_migration_timer: "0 0 1 1/1 * ? *"
+
+# Learning-Service 
+lp_cassandra_connection: "{{groups['cassandra']|join(':9042,')}}:9042"
+dp_cassandra_connection: "{{ groups['dp-cassandra'][0] }}:9042"
+environment_id: 10000000
+graph_passport_key: abc123
+learning_neo4j_bolt_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:7687
+language_neo4j_bolt_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:8687
+learning_read_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:7687
+learning_write_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:7687
+language_read_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:8687
+language_write_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:8687
+mw_shard_id: 1

ansible/keycloak.yml

@@ -5,8 +5,29 @@
     - ['{{ inventory_dir }}/secrets.yml', 'secrets/{{ env }}.yml']
   # --tags provision/deployment will trigger respective roles
   # --tags bootstrap will trigger user/realm creation
+  pre_tasks:
+    - name: Get the list of all the services running in our OS
+      become: yes
+      service_facts:
+
+    - name: Stop the monit to ensure that it doesn't start keycloak
+      service: name=monit state=stopped
+      become: yes
+      when: ansible_facts.services.monit is defined
+
   roles:
     - {role: openjdk, tags: provision}
     - {role: keycloak-provision, tags: provision}
     - keycloak-deploy
     - {role: monit, monit_checks: ['keycloak'], tags: deploy}
+
+  post_tasks:
+    - name: get the list of all services in the system
+      become: yes
+      service_facts:
+
+    - name: After succesful migration and staring of keycloak, start back monit if installed
+      service: name=monit state=started
+      become: yes
+      when: ansible_facts.services.monit is defined
+

ansible/logging.yml

@@ -87,11 +87,21 @@
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   roles:
-    - { role: vm-agents-filebeat, filebeat_log_path: '/mount/data/analytics/logs/services/api-service.log' }
+    - { role: vm-agents-filebeat, filebeat_log_path: '/mount/data/analytics/logs/services/api-service.log, /var/log/logstash/logstash-plain.log' }
   tags:            
     - analytics
     - filebeat
 
+- hosts: dp-kafkaindexer-ps
+  become: yes
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+  roles:
+    - { role: vm-agents-filebeat, filebeat_log_path: '/var/log/logstash/logstash-plain.log' }
+  tags:            
+    - kafkaindexer
+    - filebeat
+
 - hosts: zookeeper
   become: yes
   vars_files:

ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/python_keycloak.egg-info/requires.txt

@@ -1,3 +1,3 @@
-requests==2.18.4
+requests==2.20.0
 httmock==1.2.5
 python-jose==1.4.0

ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/setup.py

@@ -12,7 +12,7 @@
     keywords='keycloak openid',
     description=u'python-keycloak is a Python package providing access to the Keycloak API.',
     packages=['keycloak', 'keycloak.authorization', 'keycloak.tests'],
-    install_requires=['requests==2.18.4', 'httmock==1.2.5', 'python-jose==1.4.0'],
+    install_requires=['requests==2.20.0', 'httmock==1.2.5', 'python-jose==1.4.0'],
     classifiers=[
         'Programming Language :: Python :: 3',
         'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',

ansible/roles/kong-api/defaults/main.yml

@@ -59,7 +59,7 @@ cert_registry_service_prefix: /certreg
 desktop_app_prefix: /desktop
 
 # Service URLs
-content_service_url: "http://content-service:5000"
+content_service_url: "http://knowledge-mw-service:5000"
 learning_service_url: "http://learner-service:9000"
 vm_learning_service_url: "http://{{learningservice_ip}}:8080/learning-service"
 telemetry_service_url: "http://telemetry-service:9001"
@@ -4661,9 +4661,9 @@ kong_apis:
       config.limit_by: credential
     - name: request-size-limiting
       config.allowed_payload_size: "{{ small_request_size_limit }}"
-  - name: rejectMigrate
-    uris: "{{ user_service_prefix }}/v1/migrate/reject"
-    upstream_url: "{{ learning_service_url }}/v1/user/migrate/reject"
+  - name: userMigrate
+    uris: "{{ user_service_prefix }}/v1/migrate"
+    upstream_url: "{{ learning_service_url }}/v1/user/migrate"
     strip_uri: true
     plugins:
     - name: jwt
@@ -4709,4 +4709,20 @@ kong_apis:
       config.limit_by: credential
     - name: request-size-limiting
       config.allowed_payload_size: "{{ small_request_size_limit }}"
+  - name: userFeed
+    uris: "{{ user_service_prefix }}/v1/feed"
+    upstream_url: "{{ learning_service_url }}/v1/user/feed"
+    strip_uri: true
+    plugins:
+    - name: jwt
+    - name: cors
+    - "{{ statsd_pulgin }}"
+    - name: acl
+      config.whitelist: publicUser
+    - name: rate-limiting
+      config.policy: local
+      config.hour: "{{ medium_rate_limit_per_hour }}"
+      config.limit_by: credential
+    - name: request-size-limiting
+      config.allowed_payload_size: "{{ small_request_size_limit }}"
 

ansible/roles/stack-sunbird/defaults/main.yml

@@ -70,7 +70,6 @@ content_service_blacklisted_channels:
 sunbird_env_logo_url:
 desktop_app_storage_url: "https://{{sunbird_offline_azure_storage_account}}.blob.core.windows.net/{{offline_installer_container_name}}"
 
-
 telemetry_logstash_heap_size: 512m
 telemetry_logstash_replicas: 1
 telemetry_logstash_reservation_memory: 1g

ansible/roles/stack-sunbird/tasks/knowledge-mw-service.yml

@@ -0,0 +1,9 @@
+---
+- name: Remove knowledge-mw service
+  shell: "docker service rm knowledge-mw-service"
+  ignore_errors: yes
+
+- name: Deploy knowledge-mw service
+  shell: "docker service create --with-registry-auth --replicas {{ content_replicas }} -p 5000:5000  --name knowledge-mw-service --hostname knowledge-mw-service --reserve-memory {{ content_reservation_memory }} --limit-memory {{ content_limit_memory }} --limit-cpu {{ content_limit_cpu }} --reserve-cpu {{ content_reservation_cpu }} --health-cmd 'wget -qO- knowledge-mw-service:5000/service/health || exit 1' --health-timeout 3s --health-retries 3  --network application_default --env-file /home/deployer/env/sunbird_knowledge-mw-service.env  {{hub_org}}/{{image_name}}:{{image_tag}}"
+  args:
+    chdir: /home/deployer/stack

ansible/roles/stack-sunbird/tasks/learning-service.yml

@@ -0,0 +1,9 @@
+---
+- name: Remove learning service
+  shell: "docker service rm learning-service"
+  ignore_errors: yes
+
+- name: Deploy learning service
+  shell: "docker service create --with-registry-auth --replicas {{ learner_replicas }} -p 9696:9000  --name learning-service --hostname learning-service --reserve-memory {{ learner_reservation_memory }} --limit-memory {{ learner_limit_memory }} --limit-cpu {{ learner_limit_cpu }} --reserve-cpu {{ learner_reservation_cpu }} --health-cmd 'wget -qO- learning-service:9000/health || exit 1' --health-timeout 3s --health-retries 3  --network application_default --env-file /home/deployer/env/sunbird_learning-service.env  {{hub_org}}/{{image_name}}:{{image_tag}}"
+  args:
+    chdir: /home/deployer/stack

ansible/roles/stack-sunbird/tasks/main.yml

@@ -25,8 +25,8 @@
   - include: notification_service.yml
     when: deploy_notification is defined
 
-  - include: content_service.yml
-    when: deploy_content is defined
+  - include: knowledge-mw-service.yml
+    when: deploy_knowledge_mw is defined
 
   - include: telemetry_service.yml
     when: deploy_telemetry is defined
@@ -39,3 +39,6 @@
 
   - include: telemetry_logstash_datapipeline.yml
     when: deploy_telemetry_logstash_datapipeline is defined
+
+  - include: learning-service.yml
+    when: deploy_learning is defined

ansible/roles/stack-sunbird/templates/sunbird_learner-service.env

@@ -54,14 +54,14 @@ badging_authorization_key={{vault_badging_authorization_key}}
 sunbird_badger_baseurl=http://badger-service:8004
 sunbird_remote_req_router_path=akka.tcp://SunbirdMWSystem@actor-service:8088/user/RequestRouter
 sunbird_remote_bg_req_router_path=akka.tcp://SunbirdMWSystem@actor-service:8088/user/BackgroundRequestRouter
-sunbird_api_base_url=http://content-service:5000
+sunbird_api_base_url=http://knowledge-mw-service:5000
 sunbird_authorization={{sunbird_api_auth_token}}
 telemetry_pdata_id={{sunbird_telemetry_pdata_id}}
 telemetry_pdata_pid=learner-service
 sunbird_telemetry_base_url=http://telemetry-service:9001
 telemetry_queue_threshold_value=100
 sunbird_default_channel={{sunbird_default_channel}}
-sunbird_api_mgr_base_url=http://content-service:5000
+sunbird_api_mgr_base_url=http://knowledge-mw-service:5000
 sunbird_cs_base_url={{sunbird_cs_base_url}}
 sunbird_cs_search_path=/v1/content/search
 

ansible/roles/stack-sunbird/templates/sunbird_learning-service.env

@@ -0,0 +1,48 @@
+# Cassandra Configuration
+cassandra.lp.connection={{ lp_cassandra_connection }}
+cassandra.lpa.connection={{ dp_cassandra_connection }}
+
+# Redis Configuration
+redis.host=localhost
+redis.port=6379
+redis.maxConnections=128
+
+#--Maximum Content Package File Size Limit in Bytes (50 MB)
+MAX_CONTENT_PACKAGE_FILE_SIZE_LIMIT=52428800
+
+#--Maximum Asset File Size Limit in Bytes (20 MB)
+MAX_ASSET_FILE_SIZE_LIMIT=20971520
+
+#--No of Retry While File Download Fails
+RETRY_ASSET_DOWNLOAD_COUNT=1
+
+#Current environment
+cloud_storage.env=dev
+
+# Configuration
+graph.dir=/data/graphDB
+akka.request_timeout=30
+environment.id={{ environment_id }}
+graph.ids=["domain"]
+graph.passport.key.base={{ graph_passport_key }}
+route.domain={{ learning_neo4j_bolt_url }}
+route.bolt.write.domain={{ learning_write_elb_url }}
+route.bolt.read.domain={{ learning_read_elb_url }}
+route.all={{ language_neo4j_bolt_url }}
+route.bolt.write.all={{ language_write_elb_url }}
+route.bolt.read.all={{ language_read_elb_url }}
+
+shard.id={{ mw_shard_id }}
+platform.auth.check.enabled=false
+platform.cache.ttl=3600000
+
+#Top N Config for Search Telemetry
+telemetry_env=dev
+
+installation.id=ekstep
+
+channel.default=in.ekstep
+
+content.cache.ttl=86400
+content.cache.read=true
+content.cache.hierarchy=true

ansible/roles/stack-sunbird/templates/sunbird_lms-service.env

@@ -54,14 +54,14 @@ badging_authorization_key={{vault_badging_authorization_key}}
 sunbird_badger_baseurl=http://badger-service:8004
 sunbird_remote_req_router_path=akka.tcp://SunbirdMWSystem@actor-service:8088/user/RequestRouter
 sunbird_remote_bg_req_router_path=akka.tcp://SunbirdMWSystem@actor-service:8088/user/BackgroundRequestRouter
-sunbird_api_base_url=http://content-service:5000
+sunbird_api_base_url=http://knowledge-mw-service:5000
 sunbird_authorization={{sunbird_api_auth_token}}
 telemetry_pdata_id={{sunbird_telemetry_pdata_id}}
 telemetry_pdata_pid=lms-service
 sunbird_telemetry_base_url=http://telemetry-service:9001
 telemetry_queue_threshold_value=100
 sunbird_default_channel={{sunbird_default_channel}}
-sunbird_api_mgr_base_url=http://content-service:5000
+sunbird_api_mgr_base_url=http://knowledge-mw-service:5000
 sunbird_cs_base_url={{sunbird_cs_base_url}}
 sunbird_cs_search_path=/v1/content/search
 

ansible/roles/stack-sunbird/templates/sunbird_player.env

@@ -1,7 +1,7 @@
 sunbird_port=3000
 sunbird_content_player_url={{sunbird_content_player_url}}
 sunbird_learner_player_url={{sunbird_learner_player_url}}
-sunbird_content_proxy_url=http://content-service:5000
+sunbird_content_proxy_url=http://knowledge-mw-service:5000
 sunbird_echo_api_url={{sunbird_echo_api_url}}
 sunbird_autocreate_trampoline_user=false
 sunbird_portal_realm={{keycloak_realm}}

pipelines/deploy/knowledge-mw-service/Jenkinsfile

@@ -0,0 +1,30 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        stage('checkout public repo') {
+            cleanWs()
+            checkout scm
+        }
+
+        stage('deploy') {
+            values = docker_params()
+            currentWs = sh(returnStdout: true, script: 'pwd').trim()
+            ansiblePlaybook = "$currentWs/ansible/deploy.yml"
+            ansibleExtraArgs = "--tags \"stack-sunbird\" --extra-vars \"hub_org=$hub_org image_name=$values.image_name image_tag=$values.image_tag service_name=knowledge-mw-service deploy_knowledge_mw=True\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+            values.put('currentWs', currentWs)
+            values.put('ansiblePlaybook', ansiblePlaybook)
+            values.put('ansibleExtraArgs', ansibleExtraArgs)
+            ansible_playbook_run(values)
+            archiveArtifacts 'metadata.json'
+            currentBuild.description = "Image: ${values.image_tag}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+        }
+    }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}

pipelines/deploy/content/Jenkinsfile → pipelines/deploy/learning/Jenkinsfile

@@ -10,7 +10,7 @@ node() {
             values = docker_params()
             currentWs = sh(returnStdout: true, script: 'pwd').trim()
             ansiblePlaybook = "$currentWs/ansible/deploy.yml"
-            ansibleExtraArgs = "--tags \"stack-sunbird\" --extra-vars \"hub_org=$hub_org image_name=$values.image_name image_tag=$values.image_tag service_name=content-service deploy_content=True\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+            ansibleExtraArgs = "--tags \"stack-sunbird\" --extra-vars \"hub_org=$hub_org image_name=$values.image_name image_tag=$values.image_tag service_name=learning-service deploy_learning=True\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
             values.put('currentWs', currentWs)
             values.put('ansiblePlaybook', ansiblePlaybook)
             values.put('ansibleExtraArgs', ansibleExtraArgs)
@@ -27,4 +27,4 @@ node() {
         slack_notify(currentBuild.result)
         email_notify()
     }
-}
+}