feat(mlop-2517): create new stable butterfree releases

.checklist.yaml

@@ -0,0 +1,30 @@
+apiVersion: quintoandar.com.br/checklist/v2
+kind: ServiceChecklist
+metadata:
+  name: butterfree
+spec:
+  description: >-
+    A solution for Feature Stores.
+
+  costCenter: C055
+  department: engineering
+  lifecycle: production
+  docs: true
+
+  ownership:
+    team: data_products_mlops
+    line: tech_platform
+    owner: [email protected]
+
+  libraries:
+    - name: butterfree
+      type: common-usage
+      path: https://quintoandar.github.io/python-package-server/
+      description: A lib to build Feature Stores.
+      registries:
+      - github-packages
+      tier: T0
+
+  channels:
+    squad: 'mlops'
+    alerts: 'data-products-reports'

.github/workflows/publish.yml

@@ -4,16 +4,26 @@ on:
     paths:
       - 'setup.py'
 
-
 jobs:
   Pipeline:
-    if: github.ref == 'refs/heads/master' || contains(github.ref, 'hotfix/')
-
-    runs-on: ubuntu-16.04
-    container: quintoandar/python-3-7-java
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+
+    - uses: actions/setup-java@v4
+      with:
+        java-version: '17'
+        distribution: microsoft
+
+    - uses: vemonet/setup-spark@v1
+      with:
+        spark-version: '3.5.1'
+        hadoop-version: '3'
 
     - name: Install dependencies
       run: make ci-install
@@ -24,13 +34,18 @@ jobs:
     - name: Get version
       run: echo "version=$(grep __version__ setup.py | head -1 | cut -d \" -f2 | cut -d \' -f2)" >> $GITHUB_ENV
 
+    - name: Get release notes
+      id: get_release_notes
+      uses: ffurrer2/extract-release-notes@v1
+
     - name: Create release
       uses: actions/create-release@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         tag_name: ${{ env.version }}
         release_name: Release ${{ env.version }}
+        body: ${{ steps.get_release_notes.outputs.release_notes }}
 
     - name: Release already exist
       if: ${{ failure() }}

.github/workflows/skip_lint.yml

@@ -0,0 +1,17 @@
+# This step is used only because we want to mark the runner-linter check as required
+# for PRs to develop, but not for the merge queue to merge into develop,
+# github does not have this functionality yet
+
+name: 'Skip github-actions/runner-linter check at merge queue'
+
+on:
+  merge_group:
+
+jobs:
+  empty_job:
+    name: 'github-actions/runner-linter'
+    runs-on: github-actions-developers-runner
+    steps:
+      - name: Skip github-actions/runner-linter check at merge queue
+        run: |
+          echo "Done"

.github/workflows/staging.yml

@@ -0,0 +1,55 @@
+name: "Publish Dev Package"
+on:
+  push:
+    paths:
+      - "setup.py"
+
+jobs:
+  Pipeline:
+    if: github.ref == 'refs/heads/staging'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: microsoft
+
+      - uses: vemonet/setup-spark@v1
+        with:
+          spark-version: '3.5.1'
+          hadoop-version: '3'
+
+      - name: Install dependencies
+        run: make ci-install
+
+      - name: Get version
+        run: echo "version=$(grep __version__ setup.py | head -1 | cut -d \" -f2 | cut -d \' -f2 )" >> $GITHUB_ENV
+
+      - name: Build package
+        run: make package
+
+      - name: Create release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.version }}
+          release_name: Release ${{ env.version }}
+          prerelease: true
+
+      - name: Release already exist
+        if: ${{ failure() }}
+        run: echo Release already exist
+
+      - name: Publish release to pypi.org
+        if: ${{ success() }}
+        env:
+          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: PYTHONPATH=./pip/deps python -m twine upload -u $PYPI_USERNAME -p $PYPI_PASSWORD --verbose dist/*

.github/workflows/test.yml

@@ -3,16 +3,29 @@ on:
   push:
     branches:
       - master
+      - staging
       - hotfix/**
   pull_request:
 
 jobs:
   Pipeline:
-    runs-on: ubuntu-16.04
-    container: quintoandar/python-3-7-java
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+
+    - uses: actions/setup-java@v4
+      with:
+        java-version: '17'
+        distribution: microsoft
+
+    - uses: vemonet/setup-spark@v1
+      with:
+        spark-version: '3.5.1'
+        hadoop-version: '3'
 
     - name: Install dependencies
       run: make ci-install

.gitignore

@@ -47,6 +47,7 @@ coverage.xml
 *.cover
 .hypothesis/
 *cov.xml
+test_folder/
 
 # Translations
 *.mo
@@ -65,6 +66,7 @@ instance/
 
 # PyBuilder
 target/
+pip/
 
 # Jupyter Notebook
 .ipynb_checkpoints

CHANGELOG.md

@@ -3,6 +3,108 @@ All notable changes to this project will be documented in this file.
 
 Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each release or unreleased log for a better organization.
 
+## [Unreleased]
+
+## [1.4.0](https://github.com/quintoandar/butterfree/releases/tag/1.4.0)
+* Add Delta support ([#370](https://github.com/quintoandar/butterfree/pull/370))
+
+## [1.3.5](https://github.com/quintoandar/butterfree/releases/tag/1.3.5)
+* Auto create feature sets ([#368](https://github.com/quintoandar/butterfree/pull/368))
+
+## [1.3.4](https://github.com/quintoandar/butterfree/releases/tag/1.3.4)
+* Fix Cassandra Config and tests ([#366](https://github.com/quintoandar/butterfree/pull/366))
+
+## [1.3.3](https://github.com/quintoandar/butterfree/releases/tag/1.3.3)
+* Fix Cassandra Config and Numpy version ([#364](https://github.com/quintoandar/butterfree/pull/364))
+
+## [1.3.2](https://github.com/quintoandar/butterfree/releases/tag/1.3.2)
+* Fix publish script ([#362](https://github.com/quintoandar/butterfree/pull/362))
+
+## [1.3.2](https://github.com/quintoandar/butterfree/releases/tag/1.3.2)
+* Fix publish script ([#360](https://github.com/quintoandar/butterfree/pull/362))
+
+## [1.3.1](https://github.com/quintoandar/butterfree/releases/tag/1.3.1)
+* Timestamp NTZ available ([#360](https://github.com/quintoandar/butterfree/pull/360))
+
+## [1.3.0](https://github.com/quintoandar/butterfree/releases/tag/1.3.0)
+* Bump versions  ([#355](https://github.com/quintoandar/butterfree/pull/355))
+* Sphinx version ([#356](https://github.com/quintoandar/butterfree/pull/356))
+
+## [1.2.4](https://github.com/quintoandar/butterfree/releases/tag/1.2.4)
+* Auto create feature sets ([#351](https://github.com/quintoandar/butterfree/pull/351))
+
+## [1.2.3](https://github.com/quintoandar/butterfree/releases/tag/1.2.3)
+* Optional params ([#347](https://github.com/quintoandar/butterfree/pull/347))
+
+### Changed
+* Optional row count validation ([#340](https://github.com/quintoandar/butterfree/pull/340))
+
+## [1.2.2](https://github.com/quintoandar/butterfree/releases/tag/1.2.2)
+
+### Changed
+* Optional row count validation ([#284](https://github.com/quintoandar/butterfree/pull/284))
+* Bump several libs versions ([#333](https://github.com/quintoandar/butterfree/pull/333))
+
+## [1.2.1](https://github.com/quintoandar/butterfree/releases/tag/1.2.1)
+
+### Changed
+* Update README.md ([#331](https://github.com/quintoandar/butterfree/pull/331))
+* Update Github Actions Workflow runner ([#332](https://github.com/quintoandar/butterfree/pull/332))
+* Delete sphinx version. ([#334](https://github.com/quintoandar/butterfree/pull/334))
+
+### Fixed
+* Add the missing link for H3 geohash ([#330](https://github.com/quintoandar/butterfree/pull/330))
+
+## [1.2.0](https://github.com/quintoandar/butterfree/releases/tag/1.2.0)
+
+### Added
+* [MLOP-636] Create migration classes ([#282](https://github.com/quintoandar/butterfree/pull/282))
+* [MLOP-635] Rebase Incremental Job/Interval Run branch for test on selected feature sets ([#278](https://github.com/quintoandar/butterfree/pull/278))
+* Allow slide selection ([#293](https://github.com/quintoandar/butterfree/pull/293))
+* [MLOP-637] Implement diff method ([#292](https://github.com/quintoandar/butterfree/pull/292))
+* [MLOP-640] Create CLI with migrate command ([#298](https://github.com/quintoandar/butterfree/pull/298))
+* [MLOP-645] Implement query method, cassandra ([#291](https://github.com/quintoandar/butterfree/pull/291))
+* [MLOP-671] Implement get_schema on Spark client ([#301](https://github.com/quintoandar/butterfree/pull/301))
+* [MLOP-648] Implement query method, metastore ([#294](https://github.com/quintoandar/butterfree/pull/294))
+* [MLOP-647] / [MLOP-646] Apply migrations ([#300](https://github.com/quintoandar/butterfree/pull/300))
+* [MLOP-639] Track logs in S3 ([#306](https://github.com/quintoandar/butterfree/pull/306))
+* [MLOP-702] Debug mode for Automate Migration ([#322](https://github.com/quintoandar/butterfree/pull/322))
+
+### Changed
+* Keep milliseconds when using 'from_ms' argument in timestamp feature ([#284](https://github.com/quintoandar/butterfree/pull/284))
+* Read and write consistency level options ([#309](https://github.com/quintoandar/butterfree/pull/309))
+* [MLOP-691] Include step to add partition to SparkMetastore during writing of Butterfree ([#327](https://github.com/quintoandar/butterfree/pull/327))
+
+### Fixed
+* [BUG] Apply create_partitions to historical validate ([#303](https://github.com/quintoandar/butterfree/pull/303))
+* [BUG] Fix key path for validate read ([#304](https://github.com/quintoandar/butterfree/pull/304))
+* [FIX] Add Partition types for Metastore ([#305](https://github.com/quintoandar/butterfree/pull/305))
+* Change solution for tracking logs ([#308](https://github.com/quintoandar/butterfree/pull/308))
+* [BUG] Fix Cassandra Connect Session ([#316](https://github.com/quintoandar/butterfree/pull/316))
+* Fix method to generate agg feature name. ([#326](https://github.com/quintoandar/butterfree/pull/326))
+
+## [1.1.3](https://github.com/quintoandar/butterfree/releases/tag/1.1.3)
+### Added
+* [MLOP-599] Apply mypy to ButterFree ([#273](https://github.com/quintoandar/butterfree/pull/273))
+
+### Changed
+* [MLOP-634] Butterfree dev workflow, set triggers for branches staging and master ([#280](https://github.com/quintoandar/butterfree/pull/280))
+* Keep milliseconds when using 'from_ms' argument in timestamp feature ([#284](https://github.com/quintoandar/butterfree/pull/284))
+* [MLOP-633] Butterfree dev workflow, update documentation ([#281](https://github.com/quintoandar/butterfree/commit/74278986a49f1825beee0fd8df65a585764e5524))
+* [MLOP-632] Butterfree dev workflow, automate release description ([#279](https://github.com/quintoandar/butterfree/commit/245eaa594846166972241b03fddc61ee5117b1f7))
+
+### Fixed
+* Change trigger for pipeline staging ([#287](https://github.com/quintoandar/butterfree/pull/287))
+
+## [1.1.2](https://github.com/quintoandar/butterfree/releases/tag/1.1.2)
+### Fixed
+* [HOTFIX] Add both cache and count back to Butterfree ([#274](https://github.com/quintoandar/butterfree/pull/274))
+* [MLOP-606] Change docker image in Github Actions Pipeline ([#275](https://github.com/quintoandar/butterfree/pull/275))
+* FIX Read the Docs build ([#272](https://github.com/quintoandar/butterfree/pull/272))
+* [BUG] Fix style ([#271](https://github.com/quintoandar/butterfree/pull/271))
+* [MLOP-594] Remove from_column in some transforms ([#270](https://github.com/quintoandar/butterfree/pull/270))
+* [MLOP-536] Rename S3 config to Metastore config ([#269](https://github.com/quintoandar/butterfree/pull/269))
+
 ## [1.1.1](https://github.com/quintoandar/butterfree/releases/tag/1.2.0)
 ### Added
 * [MLOP-590] Adapt KafkaConfig to receive a custom topic name ([#266](https://github.com/quintoandar/butterfree/pull/266))
@@ -15,11 +117,11 @@ Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each
 * Update README ([#257](https://github.com/quintoandar/butterfree/pull/257))
 
 ### Fixed
-* Fix Butterfree's workflow ([#262](https://github.com/quintoandar/butterfree/pull/262)) 
+* Fix Butterfree's workflow ([#262](https://github.com/quintoandar/butterfree/pull/262))
 * [FIX] Downgrade Python Version in Pyenv ([#227](https://github.com/quintoandar/butterfree/pull/227))
-* [FIX] Fix docs ([#229](https://github.com/quintoandar/butterfree/pull/229)) 
+* [FIX] Fix docs ([#229](https://github.com/quintoandar/butterfree/pull/229))
 * [FIX] Fix Docs - Add more dependencies ([#230](https://github.com/quintoandar/butterfree/pull/230))
-* Fix broken notebook URL ([#236](https://github.com/quintoandar/butterfree/pull/236)) 
+* Fix broken notebook URL ([#236](https://github.com/quintoandar/butterfree/pull/236))
 * Issue #77 Fix ([#245](https://github.com/quintoandar/butterfree/pull/245))
 
 ## [1.0.2](https://github.com/quintoandar/butterfree/releases/tag/1.0.2)
@@ -30,7 +132,7 @@ Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each
 * [MLOP-426] Change branching strategy on butterfree to use only master branch ([#216](https://github.com/quintoandar/butterfree/pull/216))
 
 ### Fixed
-* [MLOP-440] Python 3.7 bump and Fixing Dependencies ([#220](https://github.com/quintoandar/butterfree/pull/220)) 
+* [MLOP-440] Python 3.7 bump and Fixing Dependencies ([#220](https://github.com/quintoandar/butterfree/pull/220))
 
 ## [1.0.1](https://github.com/quintoandar/butterfree/releases/tag/1.0.1)
 ### Added
@@ -229,4 +331,4 @@ Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each
 * [MLOP-143] Fix Bugs for HouseMain FeatureSet ([#62](https://github.com/quintoandar/butterfree/pull/62))
 
 ## [0.1.0](https://github.com/quintoandar/butterfree/releases/tag/0.1.0)
-* First modules and entities of butterfree package.
+* First modules and entities of butterfree package.