diff --git a/contrib/docker/superset_config.py b/contrib/docker/superset_config.py index ace7503c7edd9..dd2c6968d96d0 100644 --- a/contrib/docker/superset_config.py +++ b/contrib/docker/superset_config.py @@ -1,21 +1,17 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. import os +import jwt +import requests +from flask_appbuilder.security.manager import AUTH_REMOTE_USER +from superset.security import SupersetSecurityManager +from flask import redirect, g, flash, request +from flask_appbuilder._compat import as_unicode +from flask_appbuilder.security.views import AuthRemoteUserView +from flask_appbuilder.security.views import expose +from flask_login import login_user + + +# Create CM kubectl create configmap -n dev superset-configmap-data --from-file=../../contrib/docker/superset_config.py def get_env_variable(var_name, default=None): """Get the environment variable or raise exception.""" @@ -42,13 +38,13 @@ def get_env_variable(var_name, default=None): POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB) - REDIS_HOST = get_env_variable('REDIS_HOST') REDIS_PORT = get_env_variable('REDIS_PORT') +REDIS_PASSWORD = get_env_variable('REDIS_PASSWORD') class CeleryConfig(object): - BROKER_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) + BROKER_URL = 'redis://%s@%s:%s/0' % (REDIS_PASSWORD, REDIS_HOST, REDIS_PORT) CELERY_IMPORTS = ('superset.sql_lab', ) CELERY_RESULT_BACKEND = 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT) CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} @@ -56,3 +52,95 @@ class CeleryConfig(object): CELERY_CONFIG = CeleryConfig + + +class RemoteUserMiddleware(object): + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + return self.app(environ, start_response) + + +ADDITIONAL_MIDDLEWARE = [RemoteUserMiddleware] + + +class OpenAMValidator: + def __init__(self, token): + self.token = token + + def is_valid(self): + url = "https://217.182.160.199:8443/openam/identity/isTokenValid" # noqa + + resp = requests.post(url, verify=False, data=f"tokenid={self.token}", headers={ + "Host": "openam.beedata.beedataanalytics.com", + "Content-Type": "application/x-www-form-urlencoded", + }) + if resp.text.rstrip("\n") != "boolean=true": + return None + return True + + +class MiCustomRemoteUserView(AuthRemoteUserView): + @expose('/login/') + def login(self): + jwt_options = {'verify_signature': False} + token = request.args.get('token', '') + if not token: + redirect("https://dashboard.bee.iskra.cat/") + try: + # Get the token from the url + token_decoded = jwt.decode(token, options=jwt_options) + # Decode it and get the OpenAM token + open_am_token = token_decoded["access_token"] + roles = token_decoded["roles"] + username = token_decoded["id"] + open_am_validator = OpenAMValidator(open_am_token) + is_valid = open_am_validator.is_valid() + user_mapping = { + "username": username, + "first_name": username, + "last_name": username, + "email": f"{username}@beedata.cat" + } + except (jwt.exceptions.ExpiredSignatureError, jwt.exceptions.DecodeError): + return redirect("https://dashboard.bee.iskra.cat/login") + sm = self.appbuilder.sm + role = sm.find_role(sm.auth_role_public) + # If user has role beedata.SuperUser assign admin role + if 'beedata.SuperUser' in roles: + role = sm.find_role(sm.auth_role_admin) + user_mapping["role"] = role + if g.user is not None and g.user.is_authenticated: + return redirect(self.appbuilder.get_url_for_index) + session = sm.get_session + user = session.query(sm.user_model).filter_by(username=username).first() + if user: + if not user.is_active: + return ( + "Your account is not activated, " + "ask an admin to check the 'Is Active?' box in your " + "user profile") + if username and is_valid: + user = self.appbuilder.sm.auth_user_remote_user(username) + if user is None: + # Create User if it does not exists and log in + user = sm.add_user(**user_mapping) + msg = ("Welcome to Superset, {}".format(username)) + flash(as_unicode(msg), 'info') + user = sm.auth_user_remote_user(username) + login_user(user) + return redirect(self.appbuilder.get_url_for_index) + else: + # Log in if user exists + login_user(user) + return redirect(self.appbuilder.get_url_for_index) + return redirect("https://dashboard.bee.iskra.cat/") + + +class MiCustomSecurityManager(SupersetSecurityManager): + authremoteuserview = MiCustomRemoteUserView + + +AUTH_TYPE = AUTH_REMOTE_USER +CUSTOM_SECURITY_MANAGER = MiCustomSecurityManager diff --git a/install/helm/Makefile b/install/helm/Makefile new file mode 100644 index 0000000000000..1d63ece237fd8 --- /dev/null +++ b/install/helm/Makefile @@ -0,0 +1,22 @@ +context = beedata +namespace = dev +FILE_VERSION = ./VERSION +version = $(shell cat $(FILE_VERSION)) + +purge-superset: + helm del --kube-context $(context) --purge superset-bee + +deploy-superset: + sed -i.bak 's#99999.99999.99999#$(version)#' ./superset/Chart.yaml + helm package ./superset + helm install -n superset-bee ./superset-$(version).tgz --namespace $(namespace) + sed -i.bak 's#$(version)#99999.99999.99999#' ./superset/Chart.yaml + +rollback-superset: + sed -i.bak 's#99999.99999.99999#$(version)#' ./superset/Chart.yaml + helm package ./superset + helm upgrade --install --kube-context $(context) superset-bee ./superset-$(version).tgz + sed -i.bak 's#$(version)#99999.99999.99999#' ./superset/Chart.yaml + +deploy-redis: + helm install --name redis-bee stable/redis --values ./redis/values.yaml diff --git a/install/helm/VERSION b/install/helm/VERSION new file mode 100644 index 0000000000000..7f207341d5d9b --- /dev/null +++ b/install/helm/VERSION @@ -0,0 +1 @@ +1.0.1 \ No newline at end of file diff --git a/install/helm/redis/values.yaml b/install/helm/redis/values.yaml new file mode 100644 index 0000000000000..b5411a7cc137e --- /dev/null +++ b/install/helm/redis/values.yaml @@ -0,0 +1,544 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami Redis image version +## ref: https://hub.docker.com/r/bitnami/redis/tags/ +## +image: + registry: docker.io + repository: bitnami/redis + ## Bitnami Redis image tag + ## ref: https://github.com/bitnami/bitnami-docker-redis#supported-tags-and-respective-dockerfile-links + ## + tag: 5.0.5-debian-9-r118 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + +## String to partially override redis.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override redis.fullname template +## +# fullnameOverride: + +## Cluster settings +cluster: + enabled: true + slaveCount: 2 + +## Use redis sentinel in the redis pod. This will disable the master and slave services and +## create one redis service with ports to the sentinel and the redis instances +sentinel: + enabled: true + ## Bitnami Redis Sentintel image version + ## ref: https://hub.docker.com/r/bitnami/redis-sentinel/tags/ + ## + image: + registry: docker.io + repository: bitnami/redis-sentinel + ## Bitnami Redis image tag + ## ref: https://github.com/bitnami/bitnami-docker-redis-sentinel#supported-tags-and-respective-dockerfile-links + ## + tag: 5.0.5-debian-9-r37 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + masterSet: mymaster + initialCheckTimeout: 5 + quorum: 2 + downAfterMilliseconds: 60000 + failoverTimeout: 18000 + parallelSyncs: 1 + port: 26379 + ## Additional Redis configuration for the sentinel nodes + ## ref: https://redis.io/topics/config + ## + configmap: + ## Configure extra options for Redis Sentinel liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + ## Redis Sentinel resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + # resources: + # requests: + # memory: 256Mi + # cpu: 100m + ## Redis Sentinel Service properties + service: + ## Redis Sentinel Service type + type: ClusterIP + sentinelPort: 26379 + redisPort: 6379 + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # sentinelNodePort: + # redisNodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + loadBalancerIP: + +## Specifies the Kubernetes Cluster's Domain Name. +## +clusterDomain: cluster.local + +networkPolicy: + ## Specifies whether a NetworkPolicy should be created + ## + enabled: false + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port Redis is listening + ## on. When true, Redis will accept connections from any source + ## (with the correct destination port). + ## + # allowExternal: true + +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: false + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the fullname template + name: + +rbac: + ## Specifies whether RBAC resources should be created + ## + create: false + + role: + ## Rules to create. It follows the role specification + # rules: + # - apiGroups: + # - extensions + # resources: + # - podsecuritypolicies + # verbs: + # - use + # resourceNames: + # - gce.unprivileged + rules: [] + +## Redis pod Security Context +securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + +## Use password authentication +usePassword: true +## Redis password (both master and slave) +## Defaults to a random 10-character alphanumeric string if not set and usePassword is true +## ref: https://github.com/bitnami/bitnami-docker-redis#setting-the-server-password-on-first-run +## +password: +## Use existing secret (ignores previous password) +existingSecret: redis-password-file + +## Mount secrets as files instead of environment variables +usePasswordFile: true + +## Persist data to a persistent volume (Redis Master) +persistence: {} + ## A manually managed Persistent Volume and Claim + ## Requires persistence.enabled: true + ## If defined, PVC must be created manually before volume will be bound + # existingClaim: + +# Redis port +redisPort: 6379 + +## +## Redis Master parameters +## +master: + ## Redis command arguments + ## + ## Can be used to specify command line arguments, for example: + ## + command: "/run.sh" + ## Additional Redis configuration for the master nodes + ## ref: https://redis.io/topics/config + ## + configmap: + ## Redis additional command line flags + ## + ## Can be used to specify command line flags, for example: + ## + ## extraFlags: + ## - "--maxmemory-policy volatile-ttl" + ## - "--repl-backlog-size 1024mb" + extraFlags: [] + ## Comma-separated list of Redis commands to disable + ## + ## Can be used to disable Redis commands for security reasons. + ## Commands will be completely disabled by renaming each to an empty string. + ## ref: https://redis.io/topics/security#disabling-of-specific-commands + ## + disableCommands: + - FLUSHDB + - FLUSHALL + + ## Redis Master additional pod labels and annotations + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + podLabels: {} + podAnnotations: {} + + ## Redis Master resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + # resources: + # requests: + # memory: 256Mi + # cpu: 100m + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: + + ## Configure extra options for Redis Master liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + + ## Redis Master Node selectors and tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature + ## + # nodeSelector: {"beta.kubernetes.io/arch": "amd64"} + # tolerations: [] + ## Redis Master pod/node affinity/anti-affinity + ## + affinity: {} + + ## Redis Master Service properties + service: + ## Redis Master Service type + type: ClusterIP + port: 6379 + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + loadBalancerIP: + + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + enabled: true + ## The path the volume will be mounted at, useful when using different + ## Redis images. + path: /data + ## The subdirectory of the volume to mount to, useful in dev environments + ## and one PV for multiple services. + subPath: "" + ## redis data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + + ## Update strategy, can be set to RollingUpdate or onDelete by default. + ## https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets + statefulset: + updateStrategy: RollingUpdate + ## Partition update strategy + ## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions + # rollingUpdatePartition: + + ## Redis Master pod priorityClassName + # priorityClassName: {} + +## +## Redis Slave properties +## Note: service.type is a mandatory parameter +## The rest of the parameters are either optional or, if undefined, will inherit those declared in Redis Master +## +slave: + ## Slave Service properties + service: + ## Redis Slave Service type + type: ClusterIP + ## Redis port + port: 6379 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + loadBalancerIP: + + ## Redis slave port + port: 6379 + ## Can be used to specify command line arguments, for example: + ## + command: "/run.sh" + ## Additional Redis configuration for the slave nodes + ## ref: https://redis.io/topics/config + ## + configmap: + ## Redis extra flags + extraFlags: [] + ## List of Redis commands to disable + disableCommands: + - FLUSHDB + - FLUSHALL + + ## Redis Slave pod/node affinity/anti-affinity + ## + affinity: {} + + ## Configure extra options for Redis Slave liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## + livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 5 + + ## Redis slave Resource + # resources: + # requests: + # memory: 256Mi + # cpu: 100m + + ## Redis slave selectors and tolerations for pod assignment + # nodeSelector: {"beta.kubernetes.io/arch": "amd64"} + # tolerations: [] + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: + + ## Redis slave pod Annotation and Labels + podLabels: {} + podAnnotations: {} + + ## Redis slave pod priorityClassName + # priorityClassName: {} + + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + enabled: true + ## The path the volume will be mounted at, useful when using different + ## Redis images. + path: /data + ## The subdirectory of the volume to mount to, useful in dev environments + ## and one PV for multiple services. + subPath: "" + ## redis data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + + ## Update strategy, can be set to RollingUpdate or onDelete by default. + ## https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets + statefulset: + updateStrategy: RollingUpdate + ## Partition update strategy + ## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions + # rollingUpdatePartition: + +## Prometheus Exporter / Metrics +## +metrics: + enabled: true + + image: + registry: docker.io + repository: bitnami/redis-exporter + tag: 1.1.0-debian-9-r12 + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Metrics exporter resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + # resources: {} + + ## Extra arguments for Metrics exporter, for example: + ## extraArgs: + ## check-keys: myKey,myOtherKey + # extraArgs: {} + + ## Metrics exporter pod Annotation and Labels + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9121" + # podLabels: {} + + # Enable this if you're using https://github.com/coreos/prometheus-operator + serviceMonitor: + enabled: false + ## Specify a namespace if needed + # namespace: monitoring + # fallback to the prometheus default unless specified + # interval: 10s + ## Defaults to what's used if you follow CoreOS [Prometheus Install Instructions](https://github.com/helm/charts/tree/master/stable/prometheus-operator#tldr) + ## [Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#prometheus-operator-1) + ## [Kube Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#exporters) + selector: + prometheus: kube-prometheus + + ## Metrics exporter pod priorityClassName + # priorityClassName: {} + service: + type: ClusterIP + ## Use serviceLoadBalancerIP to request a specific static IP, + ## otherwise leave blank + # loadBalancerIP: + annotations: {} + +## +## Init containers parameters: +## volumePermissions: Change the owner of the persist volume mountpoint to RunAsUser:fsGroup +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/minideb + tag: stretch + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + resources: {} + # resources: + # requests: + # memory: 128Mi + # cpu: 100m + +## Redis config file +## ref: https://redis.io/topics/config +## +configmap: |- + # Enable AOF https://redis.io/topics/persistence#append-only-file + appendonly yes + # Disable RDB persistence, AOF persistence already enabled. + save "" + +## Sysctl InitContainer +## used to perform sysctl operation to modify Kernel settings (needed sometimes to avoid warnings) +sysctlImage: + enabled: false + command: [] + registry: docker.io + repository: bitnami/minideb + tag: stretch + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + mountHostSys: false + resources: {} + # resources: + # requests: + # memory: 128Mi + # cpu: 100m diff --git a/install/helm/superset/Chart.yaml b/install/helm/superset/Chart.yaml index 83bbd394d1766..bfd0af5d16fe9 100644 --- a/install/helm/superset/Chart.yaml +++ b/install/helm/superset/Chart.yaml @@ -16,10 +16,9 @@ # apiVersion: v1 appVersion: "1.0" -description: A Helm chart for Kubernetes +description: A superset Helm chart for Kubernetes name: superset +version: 99999.99999.99999 maintainers: - - name: Chuan-Yen Chiang - email: cychiang0823@gmail.com - url: https://github.com/cychiang -version: 0.1.0 + - name: iskra + email: n.bacardit@iskra.cat diff --git a/install/helm/superset/templates/configmap-env.yaml b/install/helm/superset/templates/configmap-env.yaml new file mode 100644 index 0000000000000..57fc34579e690 --- /dev/null +++ b/install/helm/superset/templates/configmap-env.yaml @@ -0,0 +1,150 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: superset-configmap-data +data: + superset_config.py: | + import os + import jwt + import requests + + from flask_appbuilder.security.manager import AUTH_REMOTE_USER + from superset.security import SupersetSecurityManager + from flask import redirect, g, flash, request + from flask_appbuilder._compat import as_unicode + from flask_appbuilder.security.views import AuthRemoteUserView + from flask_appbuilder.security.views import expose + from flask_login import login_user + + + def get_env_variable(var_name, default=None): + """Get the environment variable or raise exception.""" + try: + return os.environ[var_name] + except KeyError: + if default is not None: + return default + else: + error_msg = 'The environment variable {} was missing, abort...'\ + .format(var_name) + raise EnvironmentError(error_msg) + + + POSTGRES_USER = get_env_variable('POSTGRES_USER') + POSTGRES_PASSWORD = get_env_variable('POSTGRES_PASSWORD') + POSTGRES_HOST = get_env_variable('POSTGRES_HOST') + POSTGRES_PORT = get_env_variable('POSTGRES_PORT') + POSTGRES_DB = get_env_variable('POSTGRES_DB') + + # The SQLAlchemy connection string. + SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%s/%s' % (POSTGRES_USER, + POSTGRES_PASSWORD, + POSTGRES_HOST, + POSTGRES_PORT, + POSTGRES_DB) + REDIS_HOST = get_env_variable('REDIS_HOST') + REDIS_PORT = get_env_variable('REDIS_PORT') + REDIS_PASSWORD = get_env_variable('REDIS_PASSWORD') + + + class CeleryConfig(object): + BROKER_URL = 'redis://%s@%s:%s/0' % (REDIS_PASSWORD, REDIS_HOST, REDIS_PORT) + CELERY_IMPORTS = ('superset.sql_lab', ) + CELERY_RESULT_BACKEND = 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT) + CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}} + CELERY_TASK_PROTOCOL = 1 + + + CELERY_CONFIG = CeleryConfig + + + class RemoteUserMiddleware(object): + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + return self.app(environ, start_response) + + + ADDITIONAL_MIDDLEWARE = [RemoteUserMiddleware] + + + class OpenAMValidator: + def __init__(self, token): + self.token = token + + def is_valid(self): + url = "https://217.182.160.199:8443/openam/identity/isTokenValid" # noqa + + resp = requests.post(url, verify=False, data=f"tokenid={self.token}", headers={ + "Host": "openam.beedata.beedataanalytics.com", + "Content-Type": "application/x-www-form-urlencoded", + }) + if resp.text.rstrip("\n") != "boolean=true": + return None + return True + + + class MiCustomRemoteUserView(AuthRemoteUserView): + @expose('/login/') + def login(self): + jwt_options = {'verify_signature': False} + token = request.args.get('token', '') + if not token: + return redirect("https://dashboard.bee.iskra.cat/") + try: + # Get the token from the url + token_decoded = jwt.decode(token, options=jwt_options) + # Decode it and get the OpenAM token + open_am_token = token_decoded["access_token"] + roles = token_decoded["roles"] + username = token_decoded["id"] + open_am_validator = OpenAMValidator(open_am_token) + is_valid = open_am_validator.is_valid() + user_mapping = { + "username": username, + "first_name": username, + "last_name": username, + "email": f"{username}@beedata.cat" + } + except (jwt.exceptions.ExpiredSignatureError, jwt.exceptions.DecodeError): + return redirect("https://dashboard.bee.iskra.cat/login") + sm = self.appbuilder.sm + role = sm.find_role(sm.auth_role_public) + # If user has role beedata.SuperUser assign admin role + if 'beedata.SuperUser' in roles: + role = sm.find_role(sm.auth_role_admin) + user_mapping["role"] = role + if g.user is not None and g.user.is_authenticated: + return redirect(self.appbuilder.get_url_for_index) + session = sm.get_session + user = session.query(sm.user_model).filter_by(username=username).first() + if user: + if not user.is_active: + return ( + "Your account is not activated, " + "ask an admin to check the 'Is Active?' box in your " + "user profile") + if username and is_valid: + user = self.appbuilder.sm.auth_user_remote_user(username) + if user is None: + # Create User if it does not exists and log in + user = sm.add_user(**user_mapping) + msg = ("Welcome to Superset, {}".format(username)) + flash(as_unicode(msg), 'info') + user = sm.auth_user_remote_user(username) + login_user(user) + return redirect(self.appbuilder.get_url_for_index) + else: + # Log in if user exists + login_user(user) + return redirect(self.appbuilder.get_url_for_index) + return redirect("https://dashboard.bee.iskra.cat/") + + + class MiCustomSecurityManager(SupersetSecurityManager): + authremoteuserview = MiCustomRemoteUserView + + + AUTH_TYPE = AUTH_REMOTE_USER + CUSTOM_SECURITY_MANAGER = MiCustomSecurityManager diff --git a/install/helm/superset/templates/configmap.yaml b/install/helm/superset/templates/configmap.yaml index 7a4e5151d1095..e2bf51bec92c5 100644 --- a/install/helm/superset/templates/configmap.yaml +++ b/install/helm/superset/templates/configmap.yaml @@ -23,5 +23,3 @@ metadata: chart: {{ template "superset.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} -data: -{{ (.Files.Glob "config/*").AsConfig | indent 2 }} diff --git a/install/helm/superset/templates/deployment.yaml b/install/helm/superset/templates/deployment.yaml index d56c59e282597..3f0711bd1c76c 100644 --- a/install/helm/superset/templates/deployment.yaml +++ b/install/helm/superset/templates/deployment.yaml @@ -35,13 +35,41 @@ spec: app: {{ template "superset.name" . }} release: {{ .Release.Name }} spec: + volumes: + - name: superset-config + configMap: + name: superset-configmap-data containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + env: + - name: POSTGRES_USER + value: "{{ .Values.postgresql.user }}" + - name: POSTGRES_DB + value: "{{ .Values.postgresql.database }}" + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-bi-dev-postgresql + key: postgresql-password + - name: POSTGRES_HOST + value: "{{ .Values.postgresql.host }}" + - name: POSTGRES_PORT + value: "{{ .Values.postgresql.port }}" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-password-file + key: redis-password + - name: REDIS_HOST + value: "{{ .Values.redis.host }}" + - name: REDIS_PORT + value: "{{ .Values.redis.port }}" imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: - name: superset-config - mountPath: /etc/superset/ + mountPath: /etc/superset/superset_config.py + subPath: superset_config.py ports: - name: http containerPort: 8088 @@ -60,7 +88,3 @@ spec: tolerations: {{ toYaml . | indent 8 }} {{- end }} - volumes: - - name: "superset-config" - configMap: - name: superset-configmap diff --git a/install/helm/superset/values.yaml b/install/helm/superset/values.yaml index 0fc3c76588db2..4004d3877fa6a 100644 --- a/install/helm/superset/values.yaml +++ b/install/helm/superset/values.yaml @@ -27,7 +27,7 @@ image: pullPolicy: IfNotPresent service: - type: NodePort + type: ClusterIP port: 8088 ingress: @@ -60,3 +60,13 @@ nodeSelector: {} tolerations: [] affinity: {} + +postgresql: + user: bi + database: storage + host: postgres-bi-dev-postgresql.dev.svc + port: 5432 + +redis: + host: redis-superset-master.dev.svc + port: 6379