Skip to content

Commit

Permalink
chore: model transfers
Browse files Browse the repository at this point in the history
  • Loading branch information
@katebygrace
katebygrace committed Mar 25, 2024
1 parent b325997 commit 0da7e1f
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 2 deletions.
1 change: 1 addition & 0 deletions dataeng/jobs/analytics/ModelTransfers.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class ModelTransfers{
}
publishers common_publishers(allVars)
steps {
shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager-setup.sh'))
shell(dslFactory.readFileFromWorkspace('dataeng/resources/model-transfers.sh'))
}
}
Expand Down
11 changes: 9 additions & 2 deletions dataeng/resources/model-transfers.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,14 @@ fi

ARGS="{mart: ${MART_NAME} }"

dbt deps --profiles-dir $WORKSPACE/analytics-secure/warehouse-transforms/ --profile $DBT_PROFILE --target $DBT_TARGET
source $WORKSPACE/secrets-manager.sh
# Fetch the secrets from AWS
set +x
get_secret_value analytics-secure/warehouse-transforms/profiles DBT_PASSWORD
set -x
export DBT_PASSWORD

dbt deps --profiles-dir $WORKSPACE/warehouse-transforms/profiles --profile $DBT_PROFILE --target $DBT_TARGET

# Call DBT to perform all transfers for this mart.
dbt run-operation perform_s3_transfers --args "${ARGS}" --profile $DBT_PROFILE --target $DBT_TARGET --profiles-dir $WORKSPACE/analytics-secure/warehouse-transforms/
dbt run-operation perform_s3_transfers --args "${ARGS}" --profile $DBT_PROFILE --target $DBT_TARGET --profiles-dir $WORKSPACE/warehouse-transforms/profiles/

0 comments on commit 0da7e1f

Please sign in to comment.