From c65abfd34713a612d22280285222e5ab28903303 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 24 Dec 2024 16:57:57 +0100 Subject: [PATCH 01/25] refactor: update file upload logic to use UUID for storage --- .../dashboard/file/uploadFile/usecase.go | 10 +++--- .../dashboard/file/uploadFile/usecase_test.go | 31 +++++++++++++------ .../http/api/dashboard/file/upload_test.go | 6 ++-- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/backend/application/dashboard/file/uploadFile/usecase.go b/backend/application/dashboard/file/uploadFile/usecase.go index e029ad1c..086c3a28 100644 --- a/backend/application/dashboard/file/uploadFile/usecase.go +++ b/backend/application/dashboard/file/uploadFile/usecase.go @@ -32,10 +32,6 @@ func (uc *UseCase) Execute(request *Request) (*Response, error) { }, nil } - if err := uc.storage.Store(context.Background(), request.Name, request.FileReader, request.Size); err != nil { - return nil, err - } - uuid, err := uc.filesRepository.Save(&file.File{ Name: request.Name, Size: request.Size, @@ -46,5 +42,11 @@ func (uc *UseCase) Execute(request *Request) (*Response, error) { return nil, err } + if err := uc.storage.Store(context.Background(), uuid, request.FileReader, request.Size); err != nil { + _ = uc.filesRepository.Delete(uuid) + + return nil, err + } + return &Response{UUID: uuid}, nil } diff --git a/backend/application/dashboard/file/uploadFile/usecase_test.go b/backend/application/dashboard/file/uploadFile/usecase_test.go index e574c456..9e129735 100644 --- a/backend/application/dashboard/file/uploadFile/usecase_test.go +++ b/backend/application/dashboard/file/uploadFile/usecase_test.go @@ -51,12 +51,12 @@ func TestUseCase_Execute(t *testing.T) { validator.On("Validate", &r).Once().Return(nil) defer validator.AssertExpectations(t) - storage.On("Store", context.Background(), r.Name, r.FileReader, r.Size).Once().Return(nil) - defer storage.AssertExpectations(t) - filesRepository.On("Save", &f).Once().Return(fileUUID, nil) defer filesRepository.AssertExpectations(t) + storage.On("Store", context.Background(), fileUUID, r.FileReader, r.Size).Once().Return(nil) + defer storage.AssertExpectations(t) + response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) assert.NoError(t, err) @@ -87,8 +87,8 @@ func TestUseCase_Execute(t *testing.T) { response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) - storage.AssertNotCalled(t, "Store") filesRepository.AssertNotCalled(t, "Save") + storage.AssertNotCalled(t, "Store") assert.NoError(t, err) assert.Equal(t, &expectedResponse, response) @@ -111,18 +111,30 @@ func TestUseCase_Execute(t *testing.T) { Size: int64(len(fileContent)), } + f = file.File{ + Name: r.Name, + Size: r.Size, + OwnerUUID: r.OwnerUUID, + } + + fileUUID = "test-file-uuid" + expectedErr = errors.New("storage error") ) validator.On("Validate", &r).Once().Return(nil) defer validator.AssertExpectations(t) - storage.On("Store", context.Background(), r.Name, r.FileReader, r.Size).Once().Return(expectedErr) + filesRepository.On("Save", &f).Once().Return(fileUUID, nil) + defer filesRepository.AssertExpectations(t) + + storage.On("Store", context.Background(), fileUUID, r.FileReader, r.Size).Once().Return(expectedErr) defer storage.AssertExpectations(t) - response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) + filesRepository.On("Delete", fileUUID).Once().Return(nil) + defer filesRepository.AssertExpectations(t) - filesRepository.AssertNotCalled(t, "Save") + response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) assert.ErrorIs(t, err, expectedErr) assert.Nil(t, response) @@ -157,14 +169,13 @@ func TestUseCase_Execute(t *testing.T) { validator.On("Validate", &r).Once().Return(nil) defer validator.AssertExpectations(t) - storage.On("Store", context.Background(), r.Name, r.FileReader, r.Size).Once().Return(nil) - defer storage.AssertExpectations(t) - filesRepository.On("Save", &f).Once().Return("", expectedErr) defer filesRepository.AssertExpectations(t) response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) + storage.AssertNotCalled(t, "Store") + assert.ErrorIs(t, err, expectedErr) assert.Nil(t, response) }) diff --git a/backend/presentation/http/api/dashboard/file/upload_test.go b/backend/presentation/http/api/dashboard/file/upload_test.go index 3b99c54c..2553bb2a 100644 --- a/backend/presentation/http/api/dashboard/file/upload_test.go +++ b/backend/presentation/http/api/dashboard/file/upload_test.go @@ -70,12 +70,12 @@ func TestUploadHandler(t *testing.T) { requestValidator.On("Validate", mock.Anything).Once().Return(nil) defer requestValidator.AssertExpectations(t) - storage.On("Store", context.Background(), r.Name, mock.Anything, r.Size).Once().Return(nil) - defer storage.AssertExpectations(t) - filesRepository.On("Save", &f).Once().Return(fileUUID, nil) defer filesRepository.AssertExpectations(t) + storage.On("Store", context.Background(), fileUUID, mock.Anything, r.Size).Once().Return(nil) + defer storage.AssertExpectations(t) + handler := NewUploadHandler(createfile.NewUseCase(&filesRepository, &storage, &requestValidator), &authorizer) request := httptest.NewRequest(http.MethodPost, "/", &payload) From b96f813a2230c19fcce3f91cd5fab074ad9b13c7 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 24 Dec 2024 17:25:47 +0100 Subject: [PATCH 02/25] test: ensure Delete method is not called in file upload tests --- backend/application/dashboard/file/uploadFile/usecase_test.go | 4 ++++ backend/presentation/http/api/dashboard/file/upload_test.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/backend/application/dashboard/file/uploadFile/usecase_test.go b/backend/application/dashboard/file/uploadFile/usecase_test.go index 9e129735..d49a8068 100644 --- a/backend/application/dashboard/file/uploadFile/usecase_test.go +++ b/backend/application/dashboard/file/uploadFile/usecase_test.go @@ -59,6 +59,8 @@ func TestUseCase_Execute(t *testing.T) { response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) + filesRepository.AssertNotCalled(t, "Delete") + assert.NoError(t, err) assert.Equal(t, &expectedResponse, response) }) @@ -89,6 +91,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.AssertNotCalled(t, "Save") storage.AssertNotCalled(t, "Store") + filesRepository.AssertNotCalled(t, "Delete") assert.NoError(t, err) assert.Equal(t, &expectedResponse, response) @@ -175,6 +178,7 @@ func TestUseCase_Execute(t *testing.T) { response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) storage.AssertNotCalled(t, "Store") + filesRepository.AssertNotCalled(t, "Delete") assert.ErrorIs(t, err, expectedErr) assert.Nil(t, response) diff --git a/backend/presentation/http/api/dashboard/file/upload_test.go b/backend/presentation/http/api/dashboard/file/upload_test.go index 2553bb2a..e558e977 100644 --- a/backend/presentation/http/api/dashboard/file/upload_test.go +++ b/backend/presentation/http/api/dashboard/file/upload_test.go @@ -85,6 +85,8 @@ func TestUploadHandler(t *testing.T) { handler.ServeHTTP(response, request) + filesRepository.AssertNotCalled(t, "Delete") + expectedBody, err := os.ReadFile("testdata/upload-files-response.json") assert.NoError(t, err) From c9f0eb61cc68a957938482ed457244095ccc0523 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 24 Dec 2024 19:49:45 +0100 Subject: [PATCH 03/25] refactor: update file retrieval to use UUID instead of name --- backend/application/dashboard/file/getFile/useCase.go | 2 +- backend/application/dashboard/file/getFile/useCase_test.go | 4 ++-- backend/application/file/getFile/useCase.go | 2 +- backend/application/file/getFile/useCase_test.go | 4 ++-- backend/presentation/http/api/dashboard/file/show_test.go | 4 ++-- backend/presentation/http/api/file/show_test.go | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/application/dashboard/file/getFile/useCase.go b/backend/application/dashboard/file/getFile/useCase.go index 70c24533..48d8e20f 100644 --- a/backend/application/dashboard/file/getFile/useCase.go +++ b/backend/application/dashboard/file/getFile/useCase.go @@ -24,7 +24,7 @@ func (uc *UseCase) Execute(UUID string) (*Response, error) { return nil, err } - reader, err := uc.storage.Read(context.Background(), f.Name) + reader, err := uc.storage.Read(context.Background(), f.UUID) if err != nil { return nil, err } diff --git a/backend/application/dashboard/file/getFile/useCase_test.go b/backend/application/dashboard/file/getFile/useCase_test.go index 53ff3dce..9b6be3b5 100644 --- a/backend/application/dashboard/file/getFile/useCase_test.go +++ b/backend/application/dashboard/file/getFile/useCase_test.go @@ -51,7 +51,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.On("GetOne", uuid).Once().Return(f, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), f.Name).Return(reader, nil) + storage.On("Read", context.Background(), f.UUID).Return(reader, nil) defer storage.AssertExpectations(t) response, err := NewUseCase(&filesRepository, &storage).Execute(uuid) @@ -101,7 +101,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.On("GetOne", uuid).Once().Return(f, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), f.Name).Return(nil, expectedErr) + storage.On("Read", context.Background(), f.UUID).Return(nil, expectedErr) defer storage.AssertExpectations(t) response, err := NewUseCase(&filesRepository, &storage).Execute(uuid) diff --git a/backend/application/file/getFile/useCase.go b/backend/application/file/getFile/useCase.go index 70c24533..48d8e20f 100644 --- a/backend/application/file/getFile/useCase.go +++ b/backend/application/file/getFile/useCase.go @@ -24,7 +24,7 @@ func (uc *UseCase) Execute(UUID string) (*Response, error) { return nil, err } - reader, err := uc.storage.Read(context.Background(), f.Name) + reader, err := uc.storage.Read(context.Background(), f.UUID) if err != nil { return nil, err } diff --git a/backend/application/file/getFile/useCase_test.go b/backend/application/file/getFile/useCase_test.go index 53ff3dce..9b6be3b5 100644 --- a/backend/application/file/getFile/useCase_test.go +++ b/backend/application/file/getFile/useCase_test.go @@ -51,7 +51,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.On("GetOne", uuid).Once().Return(f, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), f.Name).Return(reader, nil) + storage.On("Read", context.Background(), f.UUID).Return(reader, nil) defer storage.AssertExpectations(t) response, err := NewUseCase(&filesRepository, &storage).Execute(uuid) @@ -101,7 +101,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.On("GetOne", uuid).Once().Return(f, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), f.Name).Return(nil, expectedErr) + storage.On("Read", context.Background(), f.UUID).Return(nil, expectedErr) defer storage.AssertExpectations(t) response, err := NewUseCase(&filesRepository, &storage).Execute(uuid) diff --git a/backend/presentation/http/api/dashboard/file/show_test.go b/backend/presentation/http/api/dashboard/file/show_test.go index 27f2a2ae..cd3ca4b7 100644 --- a/backend/presentation/http/api/dashboard/file/show_test.go +++ b/backend/presentation/http/api/dashboard/file/show_test.go @@ -50,7 +50,7 @@ func TestShowHandler(t *testing.T) { filesRepository.On("GetOne", f.UUID).Once().Return(f, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), f.Name).Once().Return(reader, nil) + storage.On("Read", context.Background(), f.UUID).Once().Return(reader, nil) defer storage.AssertExpectations(t) useCase := getfile.NewUseCase(&filesRepository, &storage) @@ -167,7 +167,7 @@ func TestShowHandler(t *testing.T) { filesRepository.On("GetOne", file.UUID).Once().Return(file, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), file.Name).Once().Return(reader, errors.New("some error")) + storage.On("Read", context.Background(), file.UUID).Once().Return(reader, errors.New("some error")) defer storage.AssertExpectations(t) useCase := getfile.NewUseCase(&filesRepository, &storage) diff --git a/backend/presentation/http/api/file/show_test.go b/backend/presentation/http/api/file/show_test.go index b5440e15..23f64843 100644 --- a/backend/presentation/http/api/file/show_test.go +++ b/backend/presentation/http/api/file/show_test.go @@ -41,7 +41,7 @@ func TestShowHandler(t *testing.T) { filesRepository.On("GetOne", f.UUID).Once().Return(f, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), f.Name).Once().Return(reader, nil) + storage.On("Read", context.Background(), f.UUID).Once().Return(reader, nil) defer storage.AssertExpectations(t) useCase := getfile.NewUseCase(&filesRepository, &storage) @@ -108,7 +108,7 @@ func TestShowHandler(t *testing.T) { filesRepository.On("GetOne", file.UUID).Once().Return(file, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), file.Name).Once().Return(reader, errors.New("some error")) + storage.On("Read", context.Background(), file.UUID).Once().Return(reader, errors.New("some error")) defer storage.AssertExpectations(t) useCase := getfile.NewUseCase(&filesRepository, &storage) From a8550f406e07931a3c6fd76ddd6abf54252b8d5f Mon Sep 17 00:00:00 2001 From: Amir Date: Wed, 25 Dec 2024 22:09:00 +0100 Subject: [PATCH 04/25] refactor: update environment variables for file URLs to use protocol, host, and port (#44) --- .env | 11 +++++++---- compose.yaml | 5 +++-- frontend/.env.local.example | 5 +++-- frontend/next.config.mjs | 5 +++-- frontend/src/constants/envs.ts | 4 +--- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.env b/.env index 6e509032..f2b86087 100644 --- a/.env +++ b/.env @@ -28,9 +28,12 @@ NATS_URL=nats:4222 ## frontend INTERNAL_BACKEND_BASE_URL=https://backend.tarhche.com -NEXT_PUBLIC_FILES_BASE_URL=https://backend.tarhche.com -NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL=https://tarhche.com +NEXT_PUBLIC_FILES_PROTOCOL=https +NEXT_PUBLIC_FILES_HOST=backend.tarhche.com +NEXT_PUBLIC_FILES_PORT=80 +## frontend local # INTERNAL_BACKEND_BASE_URL=http://app -# NEXT_PUBLIC_FILES_BASE_URL=http://127.0.0.1:8000 -# NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL=http://127.0.0.1:3000 +# NEXT_PUBLIC_FILES_PROTOCOL=http +# NEXT_PUBLIC_FILES_HOST=127.0.0.1 +# NEXT_PUBLIC_FILES_PORT=8000 diff --git a/compose.yaml b/compose.yaml index be320dd4..62d60e81 100644 --- a/compose.yaml +++ b/compose.yaml @@ -43,9 +43,10 @@ services: volumes: - ./frontend:/opt/app environment: - NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL: ${NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL} INTERNAL_BACKEND_BASE_URL: ${INTERNAL_BACKEND_BASE_URL} - NEXT_PUBLIC_FILES_BASE_URL: ${NEXT_PUBLIC_FILES_BASE_URL} + NEXT_PUBLIC_FILES_PROTOCOL: ${NEXT_PUBLIC_FILES_PROTOCOL} + NEXT_PUBLIC_FILES_HOST: ${NEXT_PUBLIC_FILES_HOST} + NEXT_PUBLIC_FILES_PORT: ${NEXT_PUBLIC_FILES_PORT} command: > sh -c "npm install; npm run dev -- --hostname \"0.0.0.0\" --port 3000" diff --git a/frontend/.env.local.example b/frontend/.env.local.example index 2ccb6b69..9769d98d 100644 --- a/frontend/.env.local.example +++ b/frontend/.env.local.example @@ -1,3 +1,4 @@ INTERNAL_BACKEND_BASE_URL=https://backend.tarhche.com -NEXT_PUBLIC_FILES_BASE_URL=https://backend.tarhche.com -NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL=https://tarhche.com +NEXT_PUBLIC_FILES_PROTOCOL=https +NEXT_PUBLIC_FILES_HOST=backend.tarhche.com +NEXT_PUBLIC_FILES_PORT=80 diff --git a/frontend/next.config.mjs b/frontend/next.config.mjs index 00aa818b..98c889e3 100644 --- a/frontend/next.config.mjs +++ b/frontend/next.config.mjs @@ -6,8 +6,9 @@ const nextConfig = { images: { remotePatterns: [ { - protocol: "https", - hostname: "backend.tarhche.com", + protocol: process.env.NEXT_PUBLIC_FILES_PROTOCOL, + hostname: process.env.NEXT_PUBLIC_FILES_HOST, + port: process.env.NEXT_PUBLIC_FILES_PORT, }, ], }, diff --git a/frontend/src/constants/envs.ts b/frontend/src/constants/envs.ts index 4e9a130d..8def7fe8 100644 --- a/frontend/src/constants/envs.ts +++ b/frontend/src/constants/envs.ts @@ -1,4 +1,2 @@ export const INTERNAL_BACKEND_URL = process.env.INTERNAL_BACKEND_BASE_URL; -export const EXTERNAL_BACKEND_URL = - process.env.NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL; -export const FILES_PUBLIC_URL = `https://backend.tarhche.com/files`; +export const FILES_PUBLIC_URL = `${process.env.NEXT_PUBLIC_FILES_PROTOCOL}://${process.env.NEXT_PUBLIC_FILES_HOST}:${process.env.NEXT_PUBLIC_FILES_PORT}/files`; From 39e3e463efd3ec00fd022437c5d44a9def8edcdd Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Sat, 28 Dec 2024 14:47:13 +0100 Subject: [PATCH 05/25] provision aws resources (#41) * provision aws resources * fix --- .github/workflows/backend.yaml | 104 +++++------ .github/workflows/frontend.yaml | 102 +++++------ .github/workflows/infrastructure.yaml | 147 ++++++++++++++++ .gitignore | 3 +- backend/.gitignore | 1 + compose.yaml | 42 +++-- infrastructure/.gitignore | 13 ++ infrastructure/.terraform.lock.hcl | 24 +++ infrastructure/Makefile | 110 ++++++++++++ infrastructure/compose.backend.yaml | 49 ++++++ infrastructure/compose.docker.yaml | 24 +++ infrastructure/compose.frontend.yaml | 21 +++ infrastructure/compose.mongodb.yaml | 27 +++ infrastructure/compose.nats.yaml | 43 +++++ infrastructure/compose.proxy.yaml | 27 +++ infrastructure/proxy/Dockerfile | 5 + infrastructure/proxy/nginx.conf | 56 ++++++ infrastructure/resources.tf | 239 ++++++++++++++++++++++++++ 18 files changed, 921 insertions(+), 116 deletions(-) create mode 100644 .github/workflows/infrastructure.yaml create mode 100644 infrastructure/.gitignore create mode 100644 infrastructure/.terraform.lock.hcl create mode 100644 infrastructure/Makefile create mode 100644 infrastructure/compose.backend.yaml create mode 100644 infrastructure/compose.docker.yaml create mode 100644 infrastructure/compose.frontend.yaml create mode 100644 infrastructure/compose.mongodb.yaml create mode 100644 infrastructure/compose.nats.yaml create mode 100644 infrastructure/compose.proxy.yaml create mode 100644 infrastructure/proxy/Dockerfile create mode 100644 infrastructure/proxy/nginx.conf create mode 100644 infrastructure/resources.tf diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index ea63340a..1c343a05 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -11,6 +11,10 @@ on: - .github/** - backend/** +defaults: + run: + working-directory: ./backend + env: REGISTRY: ghcr.io IMAGE_NAME: backend @@ -30,7 +34,7 @@ jobs: - name: Run unit tests run: | - cd ./backend && go test ./... -v -race -cover + go test ./... -v -race -cover - name: Provide image name and version run: | @@ -41,52 +45,52 @@ jobs: - name: Build image run: | - cd ./backend && docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest - - cd: - runs-on: ubuntu-latest - - # This job will be invoked only on default branch - if: ${{ always() && format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} - - permissions: - packages: write - contents: read - - needs: - - ci - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Provide image name and version - run: | - IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') - IMAGE_VERSION=${{ github.sha }} - echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" - echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" - - - name: Build image - run: | - cd ./backend && docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest - - - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Push image - run: | - docker push $IMAGE_ID:$IMAGE_VERSION - docker push $IMAGE_ID:latest - - - uses: actions/setup-node@v3 - with: - node-version: "18" - - - name: deploy to production - env: - LIARA_TOKEN: ${{ secrets.LIARA_API_TOKEN }} - LIARA_BACKEND_APP_NAME: ${{secrets.LIARA_BACKEND_APP_NAME}} - run: | - npm i -g @liara/cli@7 - liara deploy --image $IMAGE_ID:$IMAGE_VERSION --platform=docker --port="80" --app="$LIARA_BACKEND_APP_NAME" --api-token="$LIARA_TOKEN" --detach + docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest + + # cd: + # runs-on: ubuntu-latest + + # # This job will be invoked only on default branch + # if: ${{ always() && format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} + + # permissions: + # packages: write + # contents: read + + # needs: + # - ci + + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + + # - name: Provide image name and version + # run: | + # IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') + # IMAGE_VERSION=${{ github.sha }} + # echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" + # echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" + + # - name: Build image + # run: | + # docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest + + # - name: Log in to registry + # run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + # - name: Push image + # run: | + # docker push $IMAGE_ID:$IMAGE_VERSION + # docker push $IMAGE_ID:latest + + # - uses: actions/setup-node@v3 + # with: + # node-version: "18" + + # - name: deploy to production + # env: + # LIARA_TOKEN: ${{ secrets.LIARA_API_TOKEN }} + # LIARA_BACKEND_APP_NAME: ${{secrets.LIARA_BACKEND_APP_NAME}} + # run: | + # npm i -g @liara/cli@7 + # liara deploy --image $IMAGE_ID:$IMAGE_VERSION --platform=docker --port="80" --app="$LIARA_BACKEND_APP_NAME" --api-token="$LIARA_TOKEN" --detach diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index dfff995a..16a1bb1d 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -11,6 +11,10 @@ on: - .github/** - frontend/** +defaults: + run: + working-directory: ./frontend + env: REGISTRY: ghcr.io IMAGE_NAME: frontend @@ -36,52 +40,52 @@ jobs: - name: Build image run: | - cd ./frontend && docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest - - cd: - runs-on: ubuntu-latest - - # This job will be invoked only on default branch - if: ${{ always() && format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} - - permissions: - packages: write - contents: read - - needs: - - ci - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Provide image name and version - run: | - IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') - IMAGE_VERSION=${{ github.sha }} - echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" - echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" - - - name: Build image - run: | - cd ./frontend && docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest - - - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Push image - run: | - docker push $IMAGE_ID:$IMAGE_VERSION - docker push $IMAGE_ID:latest - - - uses: actions/setup-node@v3 - with: - node-version: "18" - - - name: deploy to production - env: - LIARA_TOKEN: ${{ secrets.LIARA_API_TOKEN }} - LIARA_FRONTEND_APP_NAME: ${{secrets.LIARA_FRONTEND_APP_NAME}} - run: | - npm i -g @liara/cli@7 - liara deploy --image $IMAGE_ID:$IMAGE_VERSION --platform=docker --port="3000" --app="$LIARA_FRONTEND_APP_NAME" --api-token="$LIARA_TOKEN" --detach + docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest + + # cd: + # runs-on: ubuntu-latest + + # # This job will be invoked only on default branch + # if: ${{ always() && format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} + + # permissions: + # packages: write + # contents: read + + # needs: + # - ci + + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + + # - name: Provide image name and version + # run: | + # IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') + # IMAGE_VERSION=${{ github.sha }} + # echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" + # echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" + + # - name: Build image + # run: | + # docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest + + # - name: Log in to registry + # run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + # - name: Push image + # run: | + # docker push $IMAGE_ID:$IMAGE_VERSION + # docker push $IMAGE_ID:latest + + # - uses: actions/setup-node@v3 + # with: + # node-version: "18" + + # - name: deploy to production + # env: + # LIARA_TOKEN: ${{ secrets.LIARA_API_TOKEN }} + # LIARA_FRONTEND_APP_NAME: ${{secrets.LIARA_FRONTEND_APP_NAME}} + # run: | + # npm i -g @liara/cli@7 + # liara deploy --image $IMAGE_ID:$IMAGE_VERSION --platform=docker --port="3000" --app="$LIARA_FRONTEND_APP_NAME" --api-token="$LIARA_TOKEN" --detach diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml new file mode 100644 index 00000000..1ce824a6 --- /dev/null +++ b/.github/workflows/infrastructure.yaml @@ -0,0 +1,147 @@ +name: Infrastructure CI and CD +on: + push: + branches: + - main + paths: + - .github/** + - infrastructure/** + pull_request: + paths: + - .github/** + - infrastructure/** + +defaults: + run: + working-directory: ./infrastructure + +env: + TF_VAR_project_name: tarhche + TF_VAR_instance_name: backend + + EC2_SSH_ADDRESS: ${{ secrets.EC2_SSH_ADDRESS }} + +jobs: + ci: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Format + id: fmt + run: terraform fmt -check + + - name: Terraform Init + id: init + run: terraform init + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + + - name: Terraform Plan + run: terraform plan -no-color -input=false + continue-on-error: false + + cd: + runs-on: ubuntu-latest + + # This job will be invoked only on default branch + if: ${{ always() && format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} + + needs: + - ci + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Apply + run: terraform apply -auto-approve -input=false + continue-on-error: false + + - name: Deploy services + run: | + # setup ssh key + echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" | base64 --decode > ~/ec2-key.pem + chmod 400 ~/ec2-key.pem + + # copy files + scp -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ./* ubuntu@${{ secrets.EC2_PUBLIC_IP }}:/opt/deployment/ + + # connect and deploy services + ssh -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${{ secrets.EC2_PUBLIC_IP }} << 'EOF' + VOLUME_PATH="${{ secrets.VOLUME_PATH }}" + + MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" + MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" + + DASHBOARD_MONGO_USERNAME="${{ secrets.DASHBOARD_MONGO_USERNAME }}" + DASHBOARD_MONGO_PASSWORD="${{ secrets.DASHBOARD_MONGO_PASSWORD }}" + DASHBOARD_MONGO_MONGODB_URL="mongodb://${{ secrets.MONGO_USERNAME }}:${{ secrets.MONGO_PASSWORD }}@mongodb:27017" + + BACKEND_NATS_URL="${{ secrets.BACKEND_NATS_URL }}" + BACKEND_PRIVATE_KEY="${{ secrets.BACKEND_PRIVATE_KEY }}" + + BACKEND_MONGO_HOST="mongodb" + BACKEND_MONGO_PORT="27017" + BACKEND_MONGO_SCHEME="mongodb" + BACKEND_MONGO_DATABASE_NAME="${{ secrets.BACKEND_MONGO_DATABASE_NAME }}" + BACKEND_MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" + BACKEND_MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" + + BACKEND_MAIL_SMTP_PASSWORD="${{ secrets.BACKEND_MAIL_SMTP_PASSWORD }}" + BACKEND_MAIL_SMTP_HOST="${{ secrets.BACKEND_MAIL_SMTP_HOST }}" + BACKEND_MAIL_SMTP_FROM="${{ secrets.BACKEND_MAIL_SMTP_FROM }}" + BACKEND_MAIL_SMTP_USERNAME="${{ secrets.BACKEND_MAIL_SMTP_USERNAME }}" + BACKEND_MAIL_SMTP_PORT="${{ secrets.BACKEND_MAIL_SMTP_PORT }}" + + BACKEND_S3_ENDPOINT="${{ secrets.BACKEND_S3_ENDPOINT }}" + BACKEND_S3_SECRET_KEY="${{ secrets.BACKEND_S3_SECRET_KEY }}" + BACKEND_S3_ACCESS_KEY="${{ secrets.BACKEND_S3_ACCESS_KEY }}" + BACKEND_S3_USE_SSL="${{ secrets.BACKEND_S3_USE_SSL }}" + BACKEND_S3_BUCKET_NAME="${{ secrets.BACKEND_S3_BUCKET_NAME }}" + + APP_IMAGE="${{ secrets.APP_IMAGE }}" + + PORTAINER_ADMIN_PASSWORD="${{ secrets.PORTAINER_ADMIN_PASSWORD }}" + + FRONTEND_IMAGE="${{ secrets.FRONTEND_IMAGE }}" + NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL="${{ secrets.NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL }}" + INTERNAL_BACKEND_BASE_URL="${{ secrets.INTERNAL_BACKEND_BASE_URL }}" + NEXT_PUBLIC_FILES_BASE_URL="${{ secrets.NEXT_PUBLIC_FILES_BASE_URL }}" + + # Run Docker Compose + cd /opt/deployment/ + + docker compose \ + -f compose.mongodb.yaml \ + -f compose.nats.yaml \ + -f compose.docker.yaml \ + -f compose.backend.yaml \ + -f compose.frontend.yaml \ + -f compose.proxy.yaml \ + up -d + EOF diff --git a/.gitignore b/.gitignore index c899b176..2fc028ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ -.idea .DS_Store +.vscode +.idea /private /private.pub diff --git a/backend/.gitignore b/backend/.gitignore index 070fa0bd..1a637777 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,3 +1,4 @@ +.DS_Store .vscode .idea /tmp \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 62d60e81..70b34230 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,6 +4,7 @@ x-app: &app context: ./backend dockerfile: Dockerfile target: develop + restart: unless-stopped depends_on: - mongodb - minio @@ -36,6 +37,7 @@ services: context: ./frontend dockerfile: Dockerfile target: develop + restart: unless-stopped depends_on: - app ports: @@ -57,8 +59,6 @@ services: runner: <<: *app - volumes: - - podman-data:/tmp ports: - "8001:80" command: > @@ -66,7 +66,7 @@ services: mongodb: image: mongo - restart: always + restart: unless-stopped ports: - "27017:27017" environment: @@ -75,6 +75,7 @@ services: mongodashboard: image: mongo-express + restart: unless-stopped depends_on: - mongodb ports: @@ -86,6 +87,7 @@ services: minio: image: minio/minio + restart: unless-stopped ports: - "9000:9000" - "9001:9001" @@ -116,17 +118,25 @@ services: ports: - "8082:80" - podman: - image: quay.io/podman/stable - volumes: - - podman-data:/tmp - security_opt: - - label=disable - devices: - - /dev/fuse - user: podman - command: > - podman system service --time=0 unix:///tmp/podman.sock + docker: + image: docker:27-dind + restart: unless-stopped + privileged: true + ports: + - "2375:2375" + environment: + DOCKER_TLS_CERTDIR: "" # disable certs -volumes: - podman-data: + portainer: + image: portainer/portainer-ce + restart: unless-stopped + depends_on: + - docker + ports: + - "9100:9000" + - "9443:9443" + command: > + --admin-password="$$2a$$12$$4xcOa82Ni5rjgQF.v.JWi.i71OyUm3fwmfWiumgJHIAPGU.uOw3qu" + # username: admin + # password: admin-password + # standalone docker API url: docker:2375 diff --git a/infrastructure/.gitignore b/infrastructure/.gitignore new file mode 100644 index 00000000..15044452 --- /dev/null +++ b/infrastructure/.gitignore @@ -0,0 +1,13 @@ +.DS_Store +.vscode +.idea +/tmp + +# SSH keys +/*.pem +/*.pub + +# Terraform files +*.tfstate +*.tfstate.backup +.terraform/ diff --git a/infrastructure/.terraform.lock.hcl b/infrastructure/.terraform.lock.hcl new file mode 100644 index 00000000..d3e4f5ce --- /dev/null +++ b/infrastructure/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.82.1" + hashes = [ + "h1:QTOtDMehUfiD3wDbbDuXYuTqGgLDkKK9Agkd5NCUEic=", + "zh:0fde8533282973f1f5d33b2c4f82d962a2c78860d39b42ac20a9ce399f06f62c", + "zh:1fd1a252bffe91668f35be8eac4e0a980f022120254eae1674c3c05049aff88a", + "zh:31bbd380cd7d74bf9a8c961fc64da4222bed40ffbdb27b011e637fa8b2d33641", + "zh:333ee400cf6f62fa199dc1270bf8efac6ffe56659f86918070b8351b8636e03b", + "zh:42ea9fee0a152d344d548eab43583299a13bcd73fae9e53e7e1a708720ac1315", + "zh:4b78f25a8cda3316eb56aa01909a403ec2f325a2eb0512c9a73966068c26cf29", + "zh:5e9cf9a275eda8f7940a41e32abe0b92ba76b5744def4af5124b343b5f33eb94", + "zh:6a46c8630c16b9e1338c2daed6006118db951420108b58b8b886403c69317439", + "zh:6efe11cf1a01f98a8d8043cdcd8c0ee5fe93a0e582c2b69ebb73ea073f5068c3", + "zh:88ab5c768c7d8133dab94eff48071e764424ad2b7cfeee5abe6d5bb16e4b85c6", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a614beb312574342b27dbc34d65b450997f63fa3e948d0d30f441e4f69337380", + "zh:c1f486e27130610a9b64cacb0bd928009c433d62b3be515488185e6467b4aa1f", + "zh:dccd166e89e1a02e7ce658df3c42d040edec4b09c6f7906aa5743938518148b1", + "zh:e75a3ae0fb42b7ea5a0bb5dffd8f8468004c9700fcc934eb04c264fda2ba9984", + ] +} diff --git a/infrastructure/Makefile b/infrastructure/Makefile new file mode 100644 index 00000000..7d6a734f --- /dev/null +++ b/infrastructure/Makefile @@ -0,0 +1,110 @@ +export TF_VAR_project_name = tarhche +export TF_VAR_instance_name = backend + +export EC2_SSH_ADDRESS = +export VOLUME_PATH = ./tmp/volume_01 + +export MONGO_USERNAME = test +export MONGO_PASSWORD = test‍ + +export DASHBOARD_MONGO_USERNAME = username +export DASHBOARD_MONGO_PASSWORD = password +export DASHBOARD_MONGO_MONGODB_URL = mongodb://${MONGO_USERNAME}:${MONGO_PASSWORD}@mongodb:27017 + +export BACKEND_NATS_URL = +export BACKEND_PRIVATE_KEY = + +export BACKEND_MONGO_HOST = mongodb +export BACKEND_MONGO_PORT = 27017 +export BACKEND_MONGO_SCHEME = mongodb +export BACKEND_MONGO_DATABASE_NAME = test +export BACKEND_MONGO_USERNAME = ${MONGO_USERNAME} +export BACKEND_MONGO_PASSWORD = ${MONGO_PASSWORD} + +export BACKEND_MAIL_SMTP_PASSWORD = +export BACKEND_MAIL_SMTP_HOST = +export BACKEND_MAIL_SMTP_FROM = +export BACKEND_MAIL_SMTP_USERNAME = +export BACKEND_MAIL_SMTP_PORT = + +export BACKEND_S3_ENDPOINT = +export BACKEND_S3_SECRET_KEY = +export BACKEND_S3_ACCESS_KEY = +export BACKEND_S3_USE_SSL = false +export BACKEND_S3_BUCKET_NAME = + +export APP_IMAGE = ghcr.io/tarhche/backend:latest + +# username: admin +# password: admin-password (in bcrypt, a dollar-sign should be escaped by an arbitrary dollar-sign ($ --> $$)) +export PORTAINER_ADMIN_PASSWORD = $$2a$$12$$4xcOa82Ni5rjgQF.v.JWi.i71OyUm3fwmfWiumgJHIAPGU.uOw3qu + +export FRONTEND_IMAGE = ghcr.io/tarhche/frontend:latest +export NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL = +export INTERNAL_BACKEND_BASE_URL = +export NEXT_PUBLIC_FILES_BASE_URL = + +validate: + terraform validate + +fmt: + terraform fmt + +init: + terraform init + +state: + terraform state list + +plan: + terraform plan + +apply: + terraform apply + rm -f terraform.tfstate *.tfstate.* + +public_key: + ssh-keygen -y -f ssh-private-key.pem > ssh-public-key.pub + +ssh: + ssh -i "ssh-private-key.pem" ${EC2_SSH_ADDRESS} + +up: + docker compose \ + -f compose.mongodb.yaml \ + -f compose.nats.yaml \ + -f compose.docker.yaml \ + -f compose.backend.yaml \ + -f compose.frontend.yaml \ + -f compose.proxy.yaml \ + up -d + +down: + docker compose \ + -f compose.mongodb.yaml \ + -f compose.nats.yaml \ + -f compose.docker.yaml \ + -f compose.backend.yaml \ + -f compose.frontend.yaml \ + -f compose.proxy.yaml \ + down --volumes --remove-orphans + +ps: + docker compose \ + -f compose.mongodb.yaml \ + -f compose.nats.yaml \ + -f compose.docker.yaml \ + -f compose.backend.yaml \ + -f compose.frontend.yaml \ + -f compose.proxy.yaml \ + ps -a + +logs%: + docker compose \ + -f compose.mongodb.yaml \ + -f compose.nats.yaml \ + -f compose.docker.yaml \ + -f compose.backend.yaml \ + -f compose.frontend.yaml \ + -f compose.proxy.yaml \ + logs $* diff --git a/infrastructure/compose.backend.yaml b/infrastructure/compose.backend.yaml new file mode 100644 index 00000000..ca5bab0b --- /dev/null +++ b/infrastructure/compose.backend.yaml @@ -0,0 +1,49 @@ +# app placement +x-app: &app + restart: unless-stopped + environment: + PRIVATE_KEY: ${BACKEND_PRIVATE_KEY} + S3_ENDPOINT: ${BACKEND_S3_ENDPOINT} + S3_ACCESS_KEY: ${BACKEND_S3_ACCESS_KEY} + S3_SECRET_KEY: ${BACKEND_S3_SECRET_KEY} + S3_BUCKET_NAME: ${BACKEND_S3_BUCKET_NAME} + S3_USE_SSL: ${BACKEND_S3_USE_SSL} + MONGO_SCHEME: ${BACKEND_MONGO_SCHEME} + MONGO_USERNAME: ${BACKEND_MONGO_USERNAME} + MONGO_PASSWORD: ${BACKEND_MONGO_PASSWORD} + MONGO_HOST: ${BACKEND_MONGO_HOST} + MONGO_PORT: ${BACKEND_MONGO_PORT} + MONGO_DATABASE_NAME: ${BACKEND_MONGO_DATABASE_NAME} + MAIL_SMTP_FROM: ${BACKEND_MAIL_SMTP_FROM} + MAIL_SMTP_USERNAME: ${BACKEND_MAIL_SMTP_USERNAME} + MAIL_SMTP_PASSWORD: ${BACKEND_MAIL_SMTP_PASSWORD} + MAIL_SMTP_HOST: ${BACKEND_MAIL_SMTP_HOST} + MAIL_SMTP_PORT: ${BACKEND_MAIL_SMTP_PORT} + NATS_URL: ${BACKEND_NATS_URL} + +services: + app: + <<: *app + image: ${APP_IMAGE} + networks: + - backend + - frontend + - mongodb + - nats + - docker + deploy: + mode: replicated + replicas: 2 + endpoint_mode: vip + +networks: + backend: + name: backend + frontend: + name: frontend + mongodb: + name: mongodb + nats: + name: nats + docker: + name: docker diff --git a/infrastructure/compose.docker.yaml b/infrastructure/compose.docker.yaml new file mode 100644 index 00000000..b896e0c2 --- /dev/null +++ b/infrastructure/compose.docker.yaml @@ -0,0 +1,24 @@ +services: + docker: + image: docker:27-dind + restart: unless-stopped + networks: + - docker + runtime: sysbox-runc + # privileged: true # uncomment this to test locally and comment runtime. + environment: + DOCKER_TLS_CERTDIR: "" # disable certs + + portainer: + image: portainer/portainer-ce + restart: unless-stopped + networks: + - docker + depends_on: + - docker + command: > + --admin-password="${PORTAINER_ADMIN_PASSWORD}" + +networks: + docker: + name: docker diff --git a/infrastructure/compose.frontend.yaml b/infrastructure/compose.frontend.yaml new file mode 100644 index 00000000..15e92213 --- /dev/null +++ b/infrastructure/compose.frontend.yaml @@ -0,0 +1,21 @@ +services: + frontend: + image: ${FRONTEND_IMAGE} + restart: unless-stopped + networks: + - frontend + - backend + deploy: + mode: replicated + replicas: 2 + endpoint_mode: vip + environment: + NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL: ${NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL} + INTERNAL_BACKEND_BASE_URL: ${INTERNAL_BACKEND_BASE_URL} + NEXT_PUBLIC_FILES_BASE_URL: ${NEXT_PUBLIC_FILES_BASE_URL} + +networks: + frontend: + name: frontend + backend: + name: backend diff --git a/infrastructure/compose.mongodb.yaml b/infrastructure/compose.mongodb.yaml new file mode 100644 index 00000000..24a7727c --- /dev/null +++ b/infrastructure/compose.mongodb.yaml @@ -0,0 +1,27 @@ +services: + mongodb: + image: mongo:8.0 + restart: unless-stopped + networks: + - mongodb + environment: + MONGO_INITDB_ROOT_USERNAME: ${MONGO_USERNAME} + MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD} + volumes: + - ./${VOLUME_PATH}/mongodb:/data + + mongodashboard: + image: mongo-express + restart: unless-stopped + networks: + - mongodb + depends_on: + - mongodb + environment: + ME_CONFIG_BASICAUTH_USERNAME: ${DASHBOARD_MONGO_USERNAME} + ME_CONFIG_BASICAUTH_PASSWORD: ${DASHBOARD_MONGO_PASSWORD} + ME_CONFIG_MONGODB_URL: ${DASHBOARD_MONGO_MONGODB_URL} + +networks: + mongodb: + name: mongodb diff --git a/infrastructure/compose.nats.yaml b/infrastructure/compose.nats.yaml new file mode 100644 index 00000000..81cc7097 --- /dev/null +++ b/infrastructure/compose.nats.yaml @@ -0,0 +1,43 @@ +services: + nats: + image: nats:2.10 + restart: unless-stopped + networks: + - nats + volumes: + - ./${VOLUME_PATH}/nats:/data + command: "--jetstream --store_dir /data --cluster_name NATS --cluster nats://0.0.0.0:6222 --http_port 8222" + nats-1: + image: nats:2.10 + restart: unless-stopped + networks: + - nats + depends_on: + - nats + volumes: + - ./${VOLUME_PATH}/nats-1:/data + command: "--jetstream --store_dir /data --cluster_name NATS --cluster nats://0.0.0.0:6222 --routes=nats://ruser:T0pS3cr3t@nats:6222" + nats-2: + image: nats:2.10 + restart: unless-stopped + networks: + - nats + depends_on: + - nats + volumes: + - ./${VOLUME_PATH}/nats-2:/data + command: "--jetstream --store_dir /data --cluster_name NATS --cluster nats://0.0.0.0:6222 --routes=nats://ruser:T0pS3cr3t@nats:6222" + nats-3: + image: nats:2.10 + restart: unless-stopped + networks: + - nats + depends_on: + - nats + volumes: + - ./${VOLUME_PATH}/nats-3:/data + command: "--jetstream --store_dir /data --cluster_name NATS --cluster nats://0.0.0.0:6222 --routes=nats://ruser:T0pS3cr3t@nats:6222" + +networks: + nats: + name: nats diff --git a/infrastructure/compose.proxy.yaml b/infrastructure/compose.proxy.yaml new file mode 100644 index 00000000..ff2907ce --- /dev/null +++ b/infrastructure/compose.proxy.yaml @@ -0,0 +1,27 @@ +services: + proxy: + build: + dockerfile: Dockerfile + context: ./proxy + restart: unless-stopped + networks: + - backend + - frontend + - mongodb + - docker + deploy: + mode: replicated + replicas: 1 + endpoint_mode: vip + ports: + - 80:80 + +networks: + backend: + name: backend + frontend: + name: frontend + mongodb: + name: mongodb + docker: + name: docker diff --git a/infrastructure/proxy/Dockerfile b/infrastructure/proxy/Dockerfile new file mode 100644 index 00000000..070ba3e8 --- /dev/null +++ b/infrastructure/proxy/Dockerfile @@ -0,0 +1,5 @@ +FROM nginx:1.26-alpine + +COPY ./nginx.conf /etc/nginx/conf.d/default.conf + +EXPOSE 80 diff --git a/infrastructure/proxy/nginx.conf b/infrastructure/proxy/nginx.conf new file mode 100644 index 00000000..919886cd --- /dev/null +++ b/infrastructure/proxy/nginx.conf @@ -0,0 +1,56 @@ +# Server block to catch-all unmatched subdomains +server { + listen 80 default_server; + + server_name "_"; + + location / { + proxy_pass http://frontend:3000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +# Server block for backend subdomain +server { + listen 80; + server_name "backend.*"; + + location / { + proxy_pass http://app:80; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +# Server block for dockerdashboard subdomain +server { + listen 80; + server_name "dockerdashboard.*"; + + location / { + proxy_pass http://portainer:9000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + +# Server block for mongodashboard subdomain +server { + listen 80; + server_name "mongodashboard.*"; + + location / { + proxy_pass http://mongodashboard:8081; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/infrastructure/resources.tf b/infrastructure/resources.tf new file mode 100644 index 00000000..f9911832 --- /dev/null +++ b/infrastructure/resources.tf @@ -0,0 +1,239 @@ +provider "aws" { + region = "eu-central-1" +} + +variable "project_name" { + description = "Project tag given to each deployed Instance" + type = string +} + +variable "instance_name" { + description = "instance_name" + type = string +} + +import { + to = aws_security_group.backend + id = "sg-0c4446cdf14777251" +} + +resource "aws_security_group" "backend" { + name = var.instance_name + description = "Allow HTTP, HTTPS, and SSH inbound traffic" + + tags = { + project_name = var.project_name + } + + # Allow SSH (port 22) from any IP address + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + # Allow HTTP (port 80) from any IP address + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # Allow HTTP from anywhere + } + + # Allow HTTPS (port 443) from any IP address + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + # Allow all outbound traffic + egress { + from_port = 0 + to_port = 0 + protocol = "-1" # all protocols + cidr_blocks = ["0.0.0.0/0"] + } +} + +import { + to = aws_ebs_volume.backend + id = "vol-0d2bab5e75ac580e9" +} + +resource "aws_ebs_volume" "backend" { + availability_zone = aws_instance.backend.availability_zone + encrypted = false + size = 10 + + tags = { + project_name = var.project_name + } +} + +import { + to = aws_volume_attachment.backend + id = "/dev/xvdf:vol-0d2bab5e75ac580e9:${aws_instance.backend.id}" +} + +resource "aws_volume_attachment" "backend" { + device_name = "/dev/xvdf" + instance_id = aws_instance.backend.id + volume_id = aws_ebs_volume.backend.id +} + +import { + to = aws_instance.backend + id = "i-026c60a5a3cdec06e" +} + +resource "aws_instance" "backend" { + ami = "ami-0a628e1e89aaedf80" # Canonical, Ubuntu, 24.04, amd64 noble image + instance_type = "t2.micro" + key_name = "backend" + availability_zone = "eu-central-1b" + + user_data = <<-EOT + #!/bin/bash + + # volumes + sudo mkfs.ext4 /dev/xvdf + sudo mkdir /volume_01 + sudo mount /dev/xvdf /volume_01 + sudo echo "/dev/xvdf /volume_01 ext4 defaults,nofail 0 0" | sudo tee -a /etc/fstab + + # tools + sudo apt install -y wget python3 ca-certificates curl htop jq vim + + # Add Docker's official GPG key: + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + + # install docker and sysbox + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + wget https://downloads.nestybox.com/sysbox/releases/v0.6.5/sysbox-ce_0.6.5-0.linux_amd64.deb + sudo apt install -y ./sysbox-ce_0.6.5-0.linux_amd64.deb + rm ./sysbox-ce_0.6.5-0.linux_amd64.deb + + # setup + sudo systemctl enable docker.service + sudo systemctl start docker.service + sudo usermod -a -G docker ubuntu + id ubuntu + newgrp docker + docker swarm init --advertise-addr 192.168.99.100 + EOT + + root_block_device { + delete_on_termination = true + encrypted = false + volume_size = 20 + volume_type = "gp3" + + tags = { + project_name = var.project_name + } + } + + security_groups = [ + aws_security_group.backend.name + ] + + tags = { + project_name = var.project_name + } +} + +import { + to = aws_eip.backend + id = "eipalloc-0adaac6f91269c716" +} + +resource "aws_eip" "backend" { + instance = aws_instance.backend.id + domain = "vpc" + + tags = { + project_name = var.project_name + } +} + +import { + to = aws_route53_zone.tarhche_com + id = "Z0951095A7CDVGITDCUP" +} + +resource "aws_route53_zone" "tarhche_com" { + name = "tarhche.com" + force_destroy = false + comment = "" + + tags = { + project_name = var.project_name + } +} + +import { + to = aws_route53_record.a_record_tarhche_com + id = "${aws_route53_zone.tarhche_com.id}_tarhche.com_A" +} + +resource "aws_route53_record" "a_record_tarhche_com" { + zone_id = aws_route53_zone.tarhche_com.id + name = "tarhche.com" + type = "A" + ttl = 300 + records = ["3.125.118.7"] +} + +import { + to = aws_route53_zone.tarhche_ir + id = "Z07817351L3HY3TPTD5IU" +} + +resource "aws_route53_zone" "tarhche_ir" { + name = "tarhche.ir" + force_destroy = false + comment = "" + + tags = { + project_name = var.project_name + } +} + +import { + to = aws_route53_record.a_record_tarhche_ir + id = "${aws_route53_zone.tarhche_ir.id}_tarhche.ir_A" +} + +resource "aws_route53_record" "a_record_tarhche_ir" { + zone_id = aws_route53_zone.tarhche_ir.id + name = "tarhche.ir" + type = "A" + ttl = 300 + records = ["3.125.118.7"] +} + +import { + to = aws_s3_bucket.tarhche-backend + id = "tarhche-backend" +} + +resource "aws_s3_bucket" "tarhche-backend" { + bucket = "tarhche-backend" + force_destroy = false + + tags = { + project_name = var.project_name + } +} From 1d1ec806dd22cac75f6abeb447c3b16786178d3e Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Sat, 28 Dec 2024 15:00:53 +0100 Subject: [PATCH 06/25] Provision infrastructure (#45) * provision aws resources * fix * fix --- .github/workflows/infrastructure.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 1ce824a6..2644d4a4 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -78,6 +78,14 @@ jobs: - name: Setup Terraform uses: hashicorp/setup-terraform@v3 + - name: Terraform Init + id: init + run: terraform init + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + - name: Terraform Apply run: terraform apply -auto-approve -input=false continue-on-error: false From 82220dbfc290202e4a46b4340f80fe81674a896d Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Sat, 28 Dec 2024 15:52:36 +0100 Subject: [PATCH 07/25] Provision infrastructure (#46) * provision aws resources * fix * fix * fix --- .github/workflows/infrastructure.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 2644d4a4..9a146c28 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -93,7 +93,7 @@ jobs: - name: Deploy services run: | # setup ssh key - echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" | base64 --decode > ~/ec2-key.pem + echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" > ~/ec2-key.pem chmod 400 ~/ec2-key.pem # copy files From 05fc440fa90b0d837b6708f6966ce5f66cc47bd9 Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Sat, 28 Dec 2024 16:06:32 +0100 Subject: [PATCH 08/25] fix ssh key (#48) --- .github/workflows/infrastructure.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 9a146c28..7bfaa562 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -97,10 +97,10 @@ jobs: chmod 400 ~/ec2-key.pem # copy files - scp -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ./* ubuntu@${{ secrets.EC2_PUBLIC_IP }}:/opt/deployment/ + scp -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ./* ${{ secrets.EC2_SSH_ADDRESS }}:/opt/deployment/ # connect and deploy services - ssh -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${{ secrets.EC2_PUBLIC_IP }} << 'EOF' + ssh -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${{ secrets.EC2_SSH_ADDRESS }} << 'EOF' VOLUME_PATH="${{ secrets.VOLUME_PATH }}" MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" From 27463d9024bbe15f552c57f6969b90580cad619a Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Wed, 1 Jan 2025 17:40:42 +0100 Subject: [PATCH 09/25] Create deployment path (#49) * ensure remote directory exists * ensure remote directory exists --- .github/workflows/backend.yaml | 84 ++++------ .github/workflows/frontend.yaml | 84 ++++------ .github/workflows/infrastructure.yaml | 102 ++++++------ infrastructure/Makefile | 6 +- infrastructure/compose.mongodb.yaml | 2 +- infrastructure/compose.nats.yaml | 34 +--- infrastructure/resources.tf | 229 +++++++++++++++++++++++++- 7 files changed, 353 insertions(+), 188 deletions(-) diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index 1c343a05..d2822696 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -1,8 +1,7 @@ name: Backend CI and CD + on: push: - branches: - - main paths: - .github/** - backend/** @@ -47,50 +46,37 @@ jobs: run: | docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest - # cd: - # runs-on: ubuntu-latest - - # # This job will be invoked only on default branch - # if: ${{ always() && format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} - - # permissions: - # packages: write - # contents: read - - # needs: - # - ci - - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - - # - name: Provide image name and version - # run: | - # IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') - # IMAGE_VERSION=${{ github.sha }} - # echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" - # echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" - - # - name: Build image - # run: | - # docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest - - # - name: Log in to registry - # run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - # - name: Push image - # run: | - # docker push $IMAGE_ID:$IMAGE_VERSION - # docker push $IMAGE_ID:latest - - # - uses: actions/setup-node@v3 - # with: - # node-version: "18" - - # - name: deploy to production - # env: - # LIARA_TOKEN: ${{ secrets.LIARA_API_TOKEN }} - # LIARA_BACKEND_APP_NAME: ${{secrets.LIARA_BACKEND_APP_NAME}} - # run: | - # npm i -g @liara/cli@7 - # liara deploy --image $IMAGE_ID:$IMAGE_VERSION --platform=docker --port="80" --app="$LIARA_BACKEND_APP_NAME" --api-token="$LIARA_TOKEN" --detach + cd: + runs-on: ubuntu-latest + + if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} + + permissions: + packages: write + contents: read + + needs: + - ci + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Provide image name and version + run: | + IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') + IMAGE_VERSION=${{ github.sha }} + echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" + echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" + + - name: Build image + run: | + docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest + + - name: Log in to registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push image + run: | + docker push $IMAGE_ID:$IMAGE_VERSION + docker push $IMAGE_ID:latest diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 16a1bb1d..00b4e20d 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -1,8 +1,7 @@ name: Frontend CI and CD + on: push: - branches: - - main paths: - .github/** - frontend/** @@ -42,50 +41,37 @@ jobs: run: | docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest - # cd: - # runs-on: ubuntu-latest - - # # This job will be invoked only on default branch - # if: ${{ always() && format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} - - # permissions: - # packages: write - # contents: read - - # needs: - # - ci - - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - - # - name: Provide image name and version - # run: | - # IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') - # IMAGE_VERSION=${{ github.sha }} - # echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" - # echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" - - # - name: Build image - # run: | - # docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest - - # - name: Log in to registry - # run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - # - name: Push image - # run: | - # docker push $IMAGE_ID:$IMAGE_VERSION - # docker push $IMAGE_ID:latest - - # - uses: actions/setup-node@v3 - # with: - # node-version: "18" - - # - name: deploy to production - # env: - # LIARA_TOKEN: ${{ secrets.LIARA_API_TOKEN }} - # LIARA_FRONTEND_APP_NAME: ${{secrets.LIARA_FRONTEND_APP_NAME}} - # run: | - # npm i -g @liara/cli@7 - # liara deploy --image $IMAGE_ID:$IMAGE_VERSION --platform=docker --port="3000" --app="$LIARA_FRONTEND_APP_NAME" --api-token="$LIARA_TOKEN" --detach + cd: + runs-on: ubuntu-latest + + if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} + + permissions: + packages: write + contents: read + + needs: + - ci + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Provide image name and version + run: | + IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') + IMAGE_VERSION=${{ github.sha }} + echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" + echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" + + - name: Build image + run: | + docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest + + - name: Log in to registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push image + run: | + docker push $IMAGE_ID:$IMAGE_VERSION + docker push $IMAGE_ID:latest diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 7bfaa562..92594ad4 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -1,15 +1,12 @@ name: Infrastructure CI and CD + on: - push: - branches: - - main - paths: - - .github/** - - infrastructure/** - pull_request: - paths: - - .github/** - - infrastructure/** + workflow_run: + workflows: + - Frontend CI and CD + - Backend CI and CD + types: + - completed defaults: run: @@ -20,6 +17,7 @@ env: TF_VAR_instance_name: backend EC2_SSH_ADDRESS: ${{ secrets.EC2_SSH_ADDRESS }} + EC2_SSH_ENDPOINT: ${{ secrets.EC2_SSH_USER }}@${{ secrets.EC2_SSH_ADDRESS }} jobs: ci: @@ -58,9 +56,6 @@ jobs: cd: runs-on: ubuntu-latest - # This job will be invoked only on default branch - if: ${{ always() && format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} - needs: - ci @@ -92,54 +87,63 @@ jobs: - name: Deploy services run: | - # setup ssh key + # Setup ssh key echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" > ~/ec2-key.pem chmod 400 ~/ec2-key.pem - # copy files - scp -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ./* ${{ secrets.EC2_SSH_ADDRESS }}:/opt/deployment/ + mkdir -p ~/.ssh + ssh-keyscan -H $EC2_SSH_ADDRESS >> ~/.ssh/known_hosts + + # Ensure remote directory exists + ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' + sudo mkdir -p /opt/deployment + sudo chown ${{ secrets.EC2_SSH_USER }}:${{ secrets.EC2_SSH_USER }} /opt/deployment + EOF + + # Copy files + scp -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r ./* $EC2_SSH_ENDPOINT:/opt/deployment/ > /dev/null 2>&1 - # connect and deploy services - ssh -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${{ secrets.EC2_SSH_ADDRESS }} << 'EOF' - VOLUME_PATH="${{ secrets.VOLUME_PATH }}" + # Connect and deploy services + ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' + export VOLUME_PATH="${{ secrets.VOLUME_PATH }}" - MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" - MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" + export MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" + export MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" - DASHBOARD_MONGO_USERNAME="${{ secrets.DASHBOARD_MONGO_USERNAME }}" - DASHBOARD_MONGO_PASSWORD="${{ secrets.DASHBOARD_MONGO_PASSWORD }}" - DASHBOARD_MONGO_MONGODB_URL="mongodb://${{ secrets.MONGO_USERNAME }}:${{ secrets.MONGO_PASSWORD }}@mongodb:27017" + export DASHBOARD_MONGO_USERNAME="${{ secrets.DASHBOARD_MONGO_USERNAME }}" + export DASHBOARD_MONGO_PASSWORD="${{ secrets.DASHBOARD_MONGO_PASSWORD }}" + export DASHBOARD_MONGO_MONGODB_URL="mongodb://${{ secrets.MONGO_USERNAME }}:${{ secrets.MONGO_PASSWORD }}@mongodb:27017" - BACKEND_NATS_URL="${{ secrets.BACKEND_NATS_URL }}" - BACKEND_PRIVATE_KEY="${{ secrets.BACKEND_PRIVATE_KEY }}" + export BACKEND_NATS_URL="${{ secrets.BACKEND_NATS_URL }}" + export BACKEND_PRIVATE_KEY="${{ secrets.BACKEND_PRIVATE_KEY }}" - BACKEND_MONGO_HOST="mongodb" - BACKEND_MONGO_PORT="27017" - BACKEND_MONGO_SCHEME="mongodb" - BACKEND_MONGO_DATABASE_NAME="${{ secrets.BACKEND_MONGO_DATABASE_NAME }}" - BACKEND_MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" - BACKEND_MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" + export BACKEND_MONGO_HOST="mongodb" + export BACKEND_MONGO_PORT="27017" + export BACKEND_MONGO_SCHEME="mongodb" + export BACKEND_MONGO_DATABASE_NAME="${{ secrets.BACKEND_MONGO_DATABASE_NAME }}" + export BACKEND_MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" + export BACKEND_MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" - BACKEND_MAIL_SMTP_PASSWORD="${{ secrets.BACKEND_MAIL_SMTP_PASSWORD }}" - BACKEND_MAIL_SMTP_HOST="${{ secrets.BACKEND_MAIL_SMTP_HOST }}" - BACKEND_MAIL_SMTP_FROM="${{ secrets.BACKEND_MAIL_SMTP_FROM }}" - BACKEND_MAIL_SMTP_USERNAME="${{ secrets.BACKEND_MAIL_SMTP_USERNAME }}" - BACKEND_MAIL_SMTP_PORT="${{ secrets.BACKEND_MAIL_SMTP_PORT }}" + export BACKEND_MAIL_SMTP_PASSWORD="${{ secrets.BACKEND_MAIL_SMTP_PASSWORD }}" + export BACKEND_MAIL_SMTP_HOST="${{ secrets.BACKEND_MAIL_SMTP_HOST }}" + export BACKEND_MAIL_SMTP_FROM="${{ secrets.BACKEND_MAIL_SMTP_FROM }}" + export BACKEND_MAIL_SMTP_USERNAME="${{ secrets.BACKEND_MAIL_SMTP_USERNAME }}" + export BACKEND_MAIL_SMTP_PORT="${{ secrets.BACKEND_MAIL_SMTP_PORT }}" - BACKEND_S3_ENDPOINT="${{ secrets.BACKEND_S3_ENDPOINT }}" - BACKEND_S3_SECRET_KEY="${{ secrets.BACKEND_S3_SECRET_KEY }}" - BACKEND_S3_ACCESS_KEY="${{ secrets.BACKEND_S3_ACCESS_KEY }}" - BACKEND_S3_USE_SSL="${{ secrets.BACKEND_S3_USE_SSL }}" - BACKEND_S3_BUCKET_NAME="${{ secrets.BACKEND_S3_BUCKET_NAME }}" + export BACKEND_S3_ENDPOINT="${{ secrets.BACKEND_S3_ENDPOINT }}" + export BACKEND_S3_SECRET_KEY="${{ secrets.BACKEND_S3_SECRET_KEY }}" + export BACKEND_S3_ACCESS_KEY="${{ secrets.BACKEND_S3_ACCESS_KEY }}" + export BACKEND_S3_USE_SSL="${{ secrets.BACKEND_S3_USE_SSL }}" + export BACKEND_S3_BUCKET_NAME="${{ secrets.BACKEND_S3_BUCKET_NAME }}" - APP_IMAGE="${{ secrets.APP_IMAGE }}" + export APP_IMAGE="${{ secrets.APP_IMAGE }}" - PORTAINER_ADMIN_PASSWORD="${{ secrets.PORTAINER_ADMIN_PASSWORD }}" + export PORTAINER_ADMIN_PASSWORD="${{ secrets.PORTAINER_ADMIN_PASSWORD }}" - FRONTEND_IMAGE="${{ secrets.FRONTEND_IMAGE }}" - NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL="${{ secrets.NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL }}" - INTERNAL_BACKEND_BASE_URL="${{ secrets.INTERNAL_BACKEND_BASE_URL }}" - NEXT_PUBLIC_FILES_BASE_URL="${{ secrets.NEXT_PUBLIC_FILES_BASE_URL }}" + export FRONTEND_IMAGE="${{ secrets.FRONTEND_IMAGE }}" + export NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL="${{ secrets.NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL }}" + export INTERNAL_BACKEND_BASE_URL="${{ secrets.INTERNAL_BACKEND_BASE_URL }}" + export NEXT_PUBLIC_FILES_BASE_URL="${{ secrets.NEXT_PUBLIC_FILES_BASE_URL }}" # Run Docker Compose cd /opt/deployment/ @@ -151,5 +155,5 @@ jobs: -f compose.backend.yaml \ -f compose.frontend.yaml \ -f compose.proxy.yaml \ - up -d + up --detach --pull always EOF diff --git a/infrastructure/Makefile b/infrastructure/Makefile index 7d6a734f..2aaff00c 100644 --- a/infrastructure/Makefile +++ b/infrastructure/Makefile @@ -2,6 +2,8 @@ export TF_VAR_project_name = tarhche export TF_VAR_instance_name = backend export EC2_SSH_ADDRESS = +export EC2_SSH_USER = +export EC2_SSH_ENDPOINT = ${EC2_SSH_USER}@${EC2_SSH_ADDRESS} export VOLUME_PATH = ./tmp/volume_01 export MONGO_USERNAME = test @@ -67,7 +69,7 @@ public_key: ssh-keygen -y -f ssh-private-key.pem > ssh-public-key.pub ssh: - ssh -i "ssh-private-key.pem" ${EC2_SSH_ADDRESS} + ssh -i "ssh-private-key.pem" ${EC2_SSH_ENDPOINT} up: docker compose \ @@ -77,7 +79,7 @@ up: -f compose.backend.yaml \ -f compose.frontend.yaml \ -f compose.proxy.yaml \ - up -d + up --detach --pull always down: docker compose \ diff --git a/infrastructure/compose.mongodb.yaml b/infrastructure/compose.mongodb.yaml index 24a7727c..1c1ac3e1 100644 --- a/infrastructure/compose.mongodb.yaml +++ b/infrastructure/compose.mongodb.yaml @@ -8,7 +8,7 @@ services: MONGO_INITDB_ROOT_USERNAME: ${MONGO_USERNAME} MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD} volumes: - - ./${VOLUME_PATH}/mongodb:/data + - ${VOLUME_PATH}/mongodb:/data mongodashboard: image: mongo-express diff --git a/infrastructure/compose.nats.yaml b/infrastructure/compose.nats.yaml index 81cc7097..d3e621db 100644 --- a/infrastructure/compose.nats.yaml +++ b/infrastructure/compose.nats.yaml @@ -5,38 +5,8 @@ services: networks: - nats volumes: - - ./${VOLUME_PATH}/nats:/data - command: "--jetstream --store_dir /data --cluster_name NATS --cluster nats://0.0.0.0:6222 --http_port 8222" - nats-1: - image: nats:2.10 - restart: unless-stopped - networks: - - nats - depends_on: - - nats - volumes: - - ./${VOLUME_PATH}/nats-1:/data - command: "--jetstream --store_dir /data --cluster_name NATS --cluster nats://0.0.0.0:6222 --routes=nats://ruser:T0pS3cr3t@nats:6222" - nats-2: - image: nats:2.10 - restart: unless-stopped - networks: - - nats - depends_on: - - nats - volumes: - - ./${VOLUME_PATH}/nats-2:/data - command: "--jetstream --store_dir /data --cluster_name NATS --cluster nats://0.0.0.0:6222 --routes=nats://ruser:T0pS3cr3t@nats:6222" - nats-3: - image: nats:2.10 - restart: unless-stopped - networks: - - nats - depends_on: - - nats - volumes: - - ./${VOLUME_PATH}/nats-3:/data - command: "--jetstream --store_dir /data --cluster_name NATS --cluster nats://0.0.0.0:6222 --routes=nats://ruser:T0pS3cr3t@nats:6222" + - ${VOLUME_PATH}/nats:/data + command: ["--jetstream", "-m", "8222", "-p", "4222"] networks: nats: diff --git a/infrastructure/resources.tf b/infrastructure/resources.tf index f9911832..992c0b3d 100644 --- a/infrastructure/resources.tf +++ b/infrastructure/resources.tf @@ -105,7 +105,7 @@ resource "aws_instance" "backend" { sudo echo "/dev/xvdf /volume_01 ext4 defaults,nofail 0 0" | sudo tee -a /etc/fstab # tools - sudo apt install -y wget python3 ca-certificates curl htop jq vim + sudo apt install -y wget python3 ca-certificates curl htop jq vim make # Add Docker's official GPG key: sudo install -m 0755 -d /etc/apt/keyrings @@ -156,7 +156,7 @@ resource "aws_instance" "backend" { import { to = aws_eip.backend - id = "eipalloc-0adaac6f91269c716" + id = "eipalloc-02bceef376bc05f89" } resource "aws_eip" "backend" { @@ -168,6 +168,162 @@ resource "aws_eip" "backend" { } } +import { + to = aws_lb.tarhche + id = "arn:aws:elasticloadbalancing:eu-central-1:381491955644:loadbalancer/app/tarhche/6953bf38e49158d7" +} + +resource "aws_lb" "tarhche" { + name = "tarhche" + internal = false + load_balancer_type = "application" + idle_timeout = 60 + ip_address_type = "ipv4" + enable_deletion_protection = true + + security_groups = [ + aws_security_group.backend.id, + ] + + subnets = [ + "subnet-0d68a01f5a4861c65", + "subnet-0fca4d198b88d68d6", + "subnet-0c8f8df628e715018", + ] + + tags = { + project_name = var.project_name + } +} + +import { + to = aws_lb_target_group.http + id = "arn:aws:elasticloadbalancing:eu-central-1:381491955644:targetgroup/HTTP/374d0a16b08c8d4a" +} + +resource "aws_lb_target_group" "http" { + name = "HTTP" + port = 80 + protocol = "HTTP" + vpc_id = "vpc-04db3e4490d90be8e" + ip_address_type = "ipv4" + proxy_protocol_v2 = false + + lambda_multi_value_headers_enabled = false + + health_check { + path = "/" + interval = 30 + timeout = 5 + healthy_threshold = 5 + unhealthy_threshold = 2 + } + + tags = { + project_name = var.project_name + } +} + +# resource "aws_lb_target_group_attachment" "backend_http" { +# target_group_arn = aws_lb_target_group.http.arn +# target_id = aws_instance.backend.id +# port = 80 +# } + +import { + to = aws_lb_listener.http + id = "arn:aws:elasticloadbalancing:eu-central-1:381491955644:listener/app/tarhche/6953bf38e49158d7/637c8770b5e4d6ed" +} + +resource "aws_lb_listener" "http" { + load_balancer_arn = aws_lb.tarhche.arn + port = 80 + protocol = "HTTP" + + default_action { + order = 1 + type = "redirect" + target_group_arn = aws_lb_target_group.http.arn + + redirect { + host = "#{host}" + path = "/#{path}" + port = "443" + protocol = "HTTPS" + query = "#{query}" + status_code = "HTTP_301" + } + } + + tags = { + project_name = var.project_name + } +} + +import { + to = aws_lb_listener.https + id = "arn:aws:elasticloadbalancing:eu-central-1:381491955644:listener/app/tarhche/6953bf38e49158d7/ab1c7847cbb6f739" +} + +resource "aws_lb_listener" "https" { + load_balancer_arn = aws_lb.tarhche.arn + port = 443 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" + certificate_arn = aws_acm_certificate.tarhche_com.arn + + default_action { + order = 1 + type = "forward" + target_group_arn = aws_lb_target_group.http.arn + + forward { + stickiness { + duration = 3600 + enabled = false + } + + target_group { + arn = aws_lb_target_group.http.arn + weight = 1 + } + } + } + + tags = { + project_name = var.project_name + } +} + +import { + to = aws_route53domains_registered_domain.tarhche-com + id = "tarhche.com" +} + +resource "aws_route53domains_registered_domain" "tarhche-com" { + domain_name = "tarhche.com" + + name_server { + name = "ns-1611.awsdns-09.co.uk" + } + + name_server { + name = "ns-1254.awsdns-28.org" + } + + name_server { + name = "ns-143.awsdns-17.com" + } + + name_server { + name = "ns-769.awsdns-32.net" + } + + tags = { + project_name = var.project_name + } +} + import { to = aws_route53_zone.tarhche_com id = "Z0951095A7CDVGITDCUP" @@ -192,8 +348,29 @@ resource "aws_route53_record" "a_record_tarhche_com" { zone_id = aws_route53_zone.tarhche_com.id name = "tarhche.com" type = "A" - ttl = 300 - records = ["3.125.118.7"] + + alias { + name = aws_lb.tarhche.dns_name + zone_id = aws_lb.tarhche.zone_id + evaluate_target_health = true + } +} + +import { + to = aws_route53_record.a_record_all_tarhche_com + id = "${aws_route53_zone.tarhche_com.id}_*.tarhche.com_A" +} + +resource "aws_route53_record" "a_record_all_tarhche_com" { + zone_id = aws_route53_zone.tarhche_com.id + name = "*.tarhche.com" + type = "A" + + alias { + name = aws_lb.tarhche.dns_name + zone_id = aws_lb.tarhche.zone_id + evaluate_target_health = true + } } import { @@ -220,8 +397,12 @@ resource "aws_route53_record" "a_record_tarhche_ir" { zone_id = aws_route53_zone.tarhche_ir.id name = "tarhche.ir" type = "A" - ttl = 300 - records = ["3.125.118.7"] + + alias { + evaluate_target_health = true + name = aws_lb.tarhche.dns_name + zone_id = aws_lb.tarhche.zone_id + } } import { @@ -237,3 +418,39 @@ resource "aws_s3_bucket" "tarhche-backend" { project_name = var.project_name } } + +import { + to = aws_acm_certificate.tarhche_com + id = "arn:aws:acm:eu-central-1:381491955644:certificate/a446a0ad-9cac-479f-a1d6-59b983d633d6" +} + +resource "aws_acm_certificate" "tarhche_com" { + domain_name = "tarhche.com" + validation_method = "DNS" + + subject_alternative_names = [ + "tarhche.com", + "*.tarhche.com", + ] + + lifecycle { + create_before_destroy = true + } + + tags = { + project_name = var.project_name + } +} + +import { + to = aws_route53_record.tarhche_com_ssl_validation + id = "${aws_route53_zone.tarhche_com.id}__e7a6f01cbe22cb6d1db5c70fb80299a8.tarhche.com_CNAME" +} + +resource "aws_route53_record" "tarhche_com_ssl_validation" { + zone_id = aws_route53_zone.tarhche_com.id + name = "_e7a6f01cbe22cb6d1db5c70fb80299a8.tarhche.com" + type = "CNAME" + records = ["_0fdeb4d57a8f62c9a90a8f77b0146a14.zfyfvmchrl.acm-validations.aws."] + ttl = 60 +} From 0b3e79d21d29cc1a242bae29485a3872caf04c0c Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Wed, 1 Jan 2025 18:15:06 +0100 Subject: [PATCH 10/25] fix (#50) --- .github/workflows/infrastructure.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 92594ad4..dcb6e622 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -56,6 +56,10 @@ jobs: cd: runs-on: ubuntu-latest + if: | + (format('refs/heads/{0}', github.event.repository.default_branch) == github.ref) + && (github.event.workflow_run.conclusion == 'success') + needs: - ci From 4e1fb96c716b9b41752e7b34585f0c92b57213e6 Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Wed, 1 Jan 2025 23:22:01 +0100 Subject: [PATCH 11/25] avoid unintended interpolation (#51) --- .github/workflows/infrastructure.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index dcb6e622..7d2b47bb 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -90,6 +90,8 @@ jobs: continue-on-error: false - name: Deploy services + env: + PORTAINER_ADMIN_PASSWORD: ${{ secrets.PORTAINER_ADMIN_PASSWORD }} # Set the secret as an environment variable run: | # Setup ssh key echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" > ~/ec2-key.pem @@ -142,7 +144,7 @@ jobs: export APP_IMAGE="${{ secrets.APP_IMAGE }}" - export PORTAINER_ADMIN_PASSWORD="${{ secrets.PORTAINER_ADMIN_PASSWORD }}" + export PORTAINER_ADMIN_PASSWORD=$PORTAINER_ADMIN_PASSWORD export FRONTEND_IMAGE="${{ secrets.FRONTEND_IMAGE }}" export NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL="${{ secrets.NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL }}" From 44abbf505c30358034a189504282f5f316e28451 Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Thu, 2 Jan 2025 09:08:33 +0100 Subject: [PATCH 12/25] prevent portainer secret interpolation (#52) --- .github/workflows/infrastructure.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 7d2b47bb..1ad0394a 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -90,8 +90,6 @@ jobs: continue-on-error: false - name: Deploy services - env: - PORTAINER_ADMIN_PASSWORD: ${{ secrets.PORTAINER_ADMIN_PASSWORD }} # Set the secret as an environment variable run: | # Setup ssh key echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" > ~/ec2-key.pem @@ -144,7 +142,7 @@ jobs: export APP_IMAGE="${{ secrets.APP_IMAGE }}" - export PORTAINER_ADMIN_PASSWORD=$PORTAINER_ADMIN_PASSWORD + export PORTAINER_ADMIN_PASSWORD='${{ secrets.PORTAINER_ADMIN_PASSWORD }}' export FRONTEND_IMAGE="${{ secrets.FRONTEND_IMAGE }}" export NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL="${{ secrets.NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL }}" From 360e5bf835adf03d12d9aa83c6069e531a8917ce Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:00:10 +0100 Subject: [PATCH 13/25] switch to docker stack (#53) --- .github/workflows/infrastructure.yaml | 41 +++++++++---- .gitignore | 1 + compose.yaml | 11 +++- infrastructure/Makefile | 60 +++++++------------ infrastructure/compose.app.yaml | 49 +++++++++++++++ infrastructure/compose.backend.yaml | 49 --------------- infrastructure/compose.docker.yaml | 21 +++---- infrastructure/compose.docker_dashboard.yaml | 24 ++++++++ infrastructure/compose.frontend.yaml | 13 ++-- infrastructure/compose.mongodb.yaml | 25 ++++---- infrastructure/compose.mongodb_dashboard.yaml | 28 +++++++++ infrastructure/compose.nats.yaml | 14 ++++- infrastructure/compose.proxy.yaml | 35 +++++++---- infrastructure/proxy/nginx.conf | 4 +- 14 files changed, 225 insertions(+), 150 deletions(-) create mode 100644 infrastructure/compose.app.yaml delete mode 100644 infrastructure/compose.backend.yaml create mode 100644 infrastructure/compose.docker_dashboard.yaml create mode 100644 infrastructure/compose.mongodb_dashboard.yaml diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 1ad0394a..2ca1fc65 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -16,6 +16,9 @@ env: TF_VAR_project_name: tarhche TF_VAR_instance_name: backend + DOCKER_REGISTRY: ghcr.io + PROXY_IMAGE_NAME: proxy + EC2_SSH_ADDRESS: ${{ secrets.EC2_SSH_ADDRESS }} EC2_SSH_ENDPOINT: ${{ secrets.EC2_SSH_USER }}@${{ secrets.EC2_SSH_ADDRESS }} @@ -56,9 +59,7 @@ jobs: cd: runs-on: ubuntu-latest - if: | - (format('refs/heads/{0}', github.event.repository.default_branch) == github.ref) - && (github.event.workflow_run.conclusion == 'success') + # if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} needs: - ci @@ -89,6 +90,22 @@ jobs: run: terraform apply -auto-approve -input=false continue-on-error: false + - name: Build images + run: | + PROXY_IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$PROXY_IMAGE_NAME | tr '[A-Z]' '[a-z]') + PROXY_IMAGE_VERSION=${{ github.sha }} + echo "PROXY_IMAGE_ID=$PROXY_IMAGE_ID" >> "$GITHUB_ENV" + echo "PROXY_IMAGE_VERSION=$PROXY_IMAGE_VERSION" >> "$GITHUB_ENV" + docker build . --file Dockerfile --target production --tag $PROXY_IMAGE_ID:$PROXY_IMAGE_VERSION --tag $PROXY_IMAGE_ID:latest + + - name: Log in to registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push images + run: | + docker push $PROXY_IMAGE_ID:$PROXY_IMAGE_VERSION + docker push $PROXY_IMAGE_ID:latest + - name: Deploy services run: | # Setup ssh key @@ -140,6 +157,8 @@ jobs: export BACKEND_S3_USE_SSL="${{ secrets.BACKEND_S3_USE_SSL }}" export BACKEND_S3_BUCKET_NAME="${{ secrets.BACKEND_S3_BUCKET_NAME }}" + export PROXY_IMAGE=${{ secrets.PROXY_IMAGE }} + export APP_IMAGE="${{ secrets.APP_IMAGE }}" export PORTAINER_ADMIN_PASSWORD='${{ secrets.PORTAINER_ADMIN_PASSWORD }}' @@ -152,12 +171,12 @@ jobs: # Run Docker Compose cd /opt/deployment/ - docker compose \ - -f compose.mongodb.yaml \ - -f compose.nats.yaml \ - -f compose.docker.yaml \ - -f compose.backend.yaml \ - -f compose.frontend.yaml \ - -f compose.proxy.yaml \ - up --detach --pull always + docker stack deploy -c compose.mongodb.yaml mongodb --detach=false + docker stack deploy -c compose.mongodb_dashboard.yaml mongodb_dashboard --detach=false + docker stack deploy -c compose.nats.yaml nats --detach=false + docker stack deploy -c compose.docker.yaml docker --detach=false + docker stack deploy -c compose.docker_dashboard.yaml docker_dashboard --detach=false + docker stack deploy -c compose.backend.yaml backend --detach=false + docker stack deploy -c compose.frontend.yaml frontend --detach=false + docker stack deploy -c compose.proxy.yaml proxy --detach=false EOF diff --git a/.gitignore b/.gitignore index 2fc028ce..6e512438 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /private /private.pub +/tmp \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 70b34230..9fb6ca46 100644 --- a/compose.yaml +++ b/compose.yaml @@ -69,6 +69,9 @@ services: restart: unless-stopped ports: - "27017:27017" + volumes: + - ./tmp/mongodb/db:/data/db + - ./tmp/mongodb/configdb:/data/configdb environment: MONGO_INITDB_ROOT_USERNAME: ${MONGO_USERNAME} MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD} @@ -89,7 +92,7 @@ services: image: minio/minio restart: unless-stopped ports: - - "9000:9000" + - "9100:9000" - "9001:9001" environment: MINIO_ROOT_USER: ${S3_ACCESS_KEY} @@ -107,11 +110,13 @@ services: nats: image: nats:2.10 - command: ["--jetstream", "-m", "8222", "-p", "4222"] # --config /etc/nats/nats-server.conf + command: ["--jetstream", "--http_port", "8222", "--port", "4222", "--store_dir", "/data"] ports: - "4222:4222" # client port - "6222:6222" # cluster post - "8222:8222" # HTTP monitoring port + volumes: + - ./tmp/nats:/data natsdashboard: image: mdawar/nats-dashboard @@ -133,7 +138,7 @@ services: depends_on: - docker ports: - - "9100:9000" + - "9200:9000" - "9443:9443" command: > --admin-password="$$2a$$12$$4xcOa82Ni5rjgQF.v.JWi.i71OyUm3fwmfWiumgJHIAPGU.uOw3qu" diff --git a/infrastructure/Makefile b/infrastructure/Makefile index 2aaff00c..6221ed95 100644 --- a/infrastructure/Makefile +++ b/infrastructure/Makefile @@ -1,8 +1,8 @@ export TF_VAR_project_name = tarhche export TF_VAR_instance_name = backend -export EC2_SSH_ADDRESS = -export EC2_SSH_USER = +export EC2_SSH_ADDRESS = ec2-3-124-72-48.eu-central-1.compute.amazonaws.com +export EC2_SSH_USER = ubuntu export EC2_SSH_ENDPOINT = ${EC2_SSH_USER}@${EC2_SSH_ADDRESS} export VOLUME_PATH = ./tmp/volume_01 @@ -35,6 +35,8 @@ export BACKEND_S3_ACCESS_KEY = export BACKEND_S3_USE_SSL = false export BACKEND_S3_BUCKET_NAME = +export PROXY_IMAGE = ghcr.io/tarhche/proxy:latest + export APP_IMAGE = ghcr.io/tarhche/backend:latest # username: admin @@ -43,7 +45,7 @@ export PORTAINER_ADMIN_PASSWORD = $$2a$$12$$4xcOa82Ni5rjgQF.v.JWi.i71OyUm3fwmfWi export FRONTEND_IMAGE = ghcr.io/tarhche/frontend:latest export NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL = -export INTERNAL_BACKEND_BASE_URL = +export INTERNAL_BACKEND_BASE_URL = http://app export NEXT_PUBLIC_FILES_BASE_URL = validate: @@ -72,41 +74,21 @@ ssh: ssh -i "ssh-private-key.pem" ${EC2_SSH_ENDPOINT} up: - docker compose \ - -f compose.mongodb.yaml \ - -f compose.nats.yaml \ - -f compose.docker.yaml \ - -f compose.backend.yaml \ - -f compose.frontend.yaml \ - -f compose.proxy.yaml \ - up --detach --pull always + docker stack deploy -c compose.mongodb.yaml mongodb --detach=false + docker stack deploy -c compose.mongodb_dashboard.yaml mongodb_dashboard --detach=false + docker stack deploy -c compose.nats.yaml nats --detach=false + docker stack deploy -c compose.docker.yaml docker --detach=false + docker stack deploy -c compose.docker_dashboard.yaml docker_dashboard --detach=false + docker stack deploy -c compose.backend.yaml backend --detach=false + docker stack deploy -c compose.frontend.yaml frontend --detach=false + docker stack deploy -c compose.proxy.yaml proxy --detach=false down: - docker compose \ - -f compose.mongodb.yaml \ - -f compose.nats.yaml \ - -f compose.docker.yaml \ - -f compose.backend.yaml \ - -f compose.frontend.yaml \ - -f compose.proxy.yaml \ - down --volumes --remove-orphans - -ps: - docker compose \ - -f compose.mongodb.yaml \ - -f compose.nats.yaml \ - -f compose.docker.yaml \ - -f compose.backend.yaml \ - -f compose.frontend.yaml \ - -f compose.proxy.yaml \ - ps -a - -logs%: - docker compose \ - -f compose.mongodb.yaml \ - -f compose.nats.yaml \ - -f compose.docker.yaml \ - -f compose.backend.yaml \ - -f compose.frontend.yaml \ - -f compose.proxy.yaml \ - logs $* + docker stack rm proxy + docker stack rm frontend + docker stack rm backend + docker stack rm docker_dashboard + docker stack rm docker + docker stack rm nats + docker stack rm mongodb_dashboard + docker stack rm mongodb diff --git a/infrastructure/compose.app.yaml b/infrastructure/compose.app.yaml new file mode 100644 index 00000000..c950acf5 --- /dev/null +++ b/infrastructure/compose.app.yaml @@ -0,0 +1,49 @@ +services: + app: + image: ${APP_IMAGE} + networks: + - app + - mongodb + - nats + - docker + deploy: + mode: replicated + replicas: 1 + endpoint_mode: vip + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + environment: + PRIVATE_KEY: ${BACKEND_PRIVATE_KEY} + S3_ENDPOINT: ${BACKEND_S3_ENDPOINT} + S3_ACCESS_KEY: ${BACKEND_S3_ACCESS_KEY} + S3_SECRET_KEY: ${BACKEND_S3_SECRET_KEY} + S3_BUCKET_NAME: ${BACKEND_S3_BUCKET_NAME} + S3_USE_SSL: ${BACKEND_S3_USE_SSL} + MONGO_SCHEME: ${BACKEND_MONGO_SCHEME} + MONGO_USERNAME: ${BACKEND_MONGO_USERNAME} + MONGO_PASSWORD: ${BACKEND_MONGO_PASSWORD} + MONGO_HOST: ${BACKEND_MONGO_HOST} + MONGO_PORT: ${BACKEND_MONGO_PORT} + MONGO_DATABASE_NAME: ${BACKEND_MONGO_DATABASE_NAME} + MAIL_SMTP_FROM: ${BACKEND_MAIL_SMTP_FROM} + MAIL_SMTP_USERNAME: ${BACKEND_MAIL_SMTP_USERNAME} + MAIL_SMTP_PASSWORD: ${BACKEND_MAIL_SMTP_PASSWORD} + MAIL_SMTP_HOST: ${BACKEND_MAIL_SMTP_HOST} + MAIL_SMTP_PORT: ${BACKEND_MAIL_SMTP_PORT} + NATS_URL: ${BACKEND_NATS_URL} + +networks: + app: + name: app + driver: overlay + mongodb: + name: mongodb + external: true + nats: + name: nats + external: true + docker: + name: docker + external: true diff --git a/infrastructure/compose.backend.yaml b/infrastructure/compose.backend.yaml deleted file mode 100644 index ca5bab0b..00000000 --- a/infrastructure/compose.backend.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# app placement -x-app: &app - restart: unless-stopped - environment: - PRIVATE_KEY: ${BACKEND_PRIVATE_KEY} - S3_ENDPOINT: ${BACKEND_S3_ENDPOINT} - S3_ACCESS_KEY: ${BACKEND_S3_ACCESS_KEY} - S3_SECRET_KEY: ${BACKEND_S3_SECRET_KEY} - S3_BUCKET_NAME: ${BACKEND_S3_BUCKET_NAME} - S3_USE_SSL: ${BACKEND_S3_USE_SSL} - MONGO_SCHEME: ${BACKEND_MONGO_SCHEME} - MONGO_USERNAME: ${BACKEND_MONGO_USERNAME} - MONGO_PASSWORD: ${BACKEND_MONGO_PASSWORD} - MONGO_HOST: ${BACKEND_MONGO_HOST} - MONGO_PORT: ${BACKEND_MONGO_PORT} - MONGO_DATABASE_NAME: ${BACKEND_MONGO_DATABASE_NAME} - MAIL_SMTP_FROM: ${BACKEND_MAIL_SMTP_FROM} - MAIL_SMTP_USERNAME: ${BACKEND_MAIL_SMTP_USERNAME} - MAIL_SMTP_PASSWORD: ${BACKEND_MAIL_SMTP_PASSWORD} - MAIL_SMTP_HOST: ${BACKEND_MAIL_SMTP_HOST} - MAIL_SMTP_PORT: ${BACKEND_MAIL_SMTP_PORT} - NATS_URL: ${BACKEND_NATS_URL} - -services: - app: - <<: *app - image: ${APP_IMAGE} - networks: - - backend - - frontend - - mongodb - - nats - - docker - deploy: - mode: replicated - replicas: 2 - endpoint_mode: vip - -networks: - backend: - name: backend - frontend: - name: frontend - mongodb: - name: mongodb - nats: - name: nats - docker: - name: docker diff --git a/infrastructure/compose.docker.yaml b/infrastructure/compose.docker.yaml index b896e0c2..f66b8b1b 100644 --- a/infrastructure/compose.docker.yaml +++ b/infrastructure/compose.docker.yaml @@ -1,24 +1,21 @@ services: docker: image: docker:27-dind - restart: unless-stopped networks: - docker + deploy: + mode: replicated + replicas: 1 + endpoint_mode: vip + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 runtime: sysbox-runc - # privileged: true # uncomment this to test locally and comment runtime. environment: DOCKER_TLS_CERTDIR: "" # disable certs - portainer: - image: portainer/portainer-ce - restart: unless-stopped - networks: - - docker - depends_on: - - docker - command: > - --admin-password="${PORTAINER_ADMIN_PASSWORD}" - networks: docker: name: docker + driver: overlay diff --git a/infrastructure/compose.docker_dashboard.yaml b/infrastructure/compose.docker_dashboard.yaml new file mode 100644 index 00000000..7a4b84fa --- /dev/null +++ b/infrastructure/compose.docker_dashboard.yaml @@ -0,0 +1,24 @@ +services: + docker_dashboard: + image: portainer/portainer-ce + networks: + - docker + - docker_dashboard + deploy: + mode: replicated + replicas: 1 + endpoint_mode: vip + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + command: > + --admin-password="${PORTAINER_ADMIN_PASSWORD}" + +networks: + docker_dashboard: + name: docker_dashboard + driver: overlay + docker: + name: docker + external: true diff --git a/infrastructure/compose.frontend.yaml b/infrastructure/compose.frontend.yaml index 15e92213..d06c2081 100644 --- a/infrastructure/compose.frontend.yaml +++ b/infrastructure/compose.frontend.yaml @@ -1,14 +1,17 @@ services: frontend: image: ${FRONTEND_IMAGE} - restart: unless-stopped networks: - frontend - - backend + - app deploy: mode: replicated replicas: 2 endpoint_mode: vip + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 environment: NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL: ${NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL} INTERNAL_BACKEND_BASE_URL: ${INTERNAL_BACKEND_BASE_URL} @@ -17,5 +20,7 @@ services: networks: frontend: name: frontend - backend: - name: backend + driver: overlay + app: + name: app + external: true diff --git a/infrastructure/compose.mongodb.yaml b/infrastructure/compose.mongodb.yaml index 1c1ac3e1..704d61be 100644 --- a/infrastructure/compose.mongodb.yaml +++ b/infrastructure/compose.mongodb.yaml @@ -1,27 +1,24 @@ services: mongodb: image: mongo:8.0 - restart: unless-stopped networks: - mongodb + deploy: + mode: replicated + replicas: 1 + endpoint_mode: vip + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 environment: MONGO_INITDB_ROOT_USERNAME: ${MONGO_USERNAME} MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD} volumes: - - ${VOLUME_PATH}/mongodb:/data - - mongodashboard: - image: mongo-express - restart: unless-stopped - networks: - - mongodb - depends_on: - - mongodb - environment: - ME_CONFIG_BASICAUTH_USERNAME: ${DASHBOARD_MONGO_USERNAME} - ME_CONFIG_BASICAUTH_PASSWORD: ${DASHBOARD_MONGO_PASSWORD} - ME_CONFIG_MONGODB_URL: ${DASHBOARD_MONGO_MONGODB_URL} + - ${VOLUME_PATH}/mongodb/db:/data/db + - ${VOLUME_PATH}/mongodb/configdb:/data/configdb networks: mongodb: name: mongodb + driver: overlay diff --git a/infrastructure/compose.mongodb_dashboard.yaml b/infrastructure/compose.mongodb_dashboard.yaml new file mode 100644 index 00000000..bbc9a2d0 --- /dev/null +++ b/infrastructure/compose.mongodb_dashboard.yaml @@ -0,0 +1,28 @@ +services: + mongodb_dashboard: + image: mongo-express + networks: + - mongodb + - mongodb_dashboard + deploy: + mode: replicated + replicas: 1 + endpoint_mode: vip + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + depends_on: + - mongodb + environment: + ME_CONFIG_BASICAUTH_USERNAME: ${DASHBOARD_MONGO_USERNAME} + ME_CONFIG_BASICAUTH_PASSWORD: ${DASHBOARD_MONGO_PASSWORD} + ME_CONFIG_MONGODB_URL: ${DASHBOARD_MONGO_MONGODB_URL} + +networks: + mongodb_dashboard: + name: mongodb_dashboard + driver: overlay + mongodb: + name: mongodb + external: true diff --git a/infrastructure/compose.nats.yaml b/infrastructure/compose.nats.yaml index d3e621db..da55a2dc 100644 --- a/infrastructure/compose.nats.yaml +++ b/infrastructure/compose.nats.yaml @@ -1,13 +1,21 @@ services: nats: image: nats:2.10 - restart: unless-stopped networks: - nats + deploy: + mode: replicated + replicas: 1 + endpoint_mode: vip + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 volumes: - - ${VOLUME_PATH}/nats:/data - command: ["--jetstream", "-m", "8222", "-p", "4222"] + - ${VOLUME_PATH}/nats:/jetstream + command: ["--jetstream", "--http_port", "8222", "--port", "4222", "--store_dir", "/data"] networks: nats: name: nats + driver: overlay diff --git a/infrastructure/compose.proxy.yaml b/infrastructure/compose.proxy.yaml index ff2907ce..b7791a27 100644 --- a/infrastructure/compose.proxy.yaml +++ b/infrastructure/compose.proxy.yaml @@ -1,27 +1,36 @@ services: proxy: - build: - dockerfile: Dockerfile - context: ./proxy - restart: unless-stopped + image: ${PROXY_IMAGE} networks: - - backend + - proxy - frontend - - mongodb - - docker + - app + - mongodb_dashboard + - docker_dashboard deploy: mode: replicated replicas: 1 endpoint_mode: vip + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 ports: - 80:80 networks: - backend: - name: backend + proxy: + name: proxy + driver: overlay frontend: name: frontend - mongodb: - name: mongodb - docker: - name: docker + external: true + app: + name: app + external: true + mongodb_dashboard: + name: mongodb_dashboard + external: true + docker_dashboard: + name: docker_dashboard + external: true diff --git a/infrastructure/proxy/nginx.conf b/infrastructure/proxy/nginx.conf index 919886cd..749055f2 100644 --- a/infrastructure/proxy/nginx.conf +++ b/infrastructure/proxy/nginx.conf @@ -33,7 +33,7 @@ server { server_name "dockerdashboard.*"; location / { - proxy_pass http://portainer:9000; + proxy_pass http://docker_dashboard:9000; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -47,7 +47,7 @@ server { server_name "mongodashboard.*"; location / { - proxy_pass http://mongodashboard:8081; + proxy_pass http://mongodb_dashboard:8081; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; From d511e60e75e602da8d1fe77d61911a9fe5047784 Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:09:50 +0100 Subject: [PATCH 14/25] fix proxy image id (#54) --- .github/workflows/infrastructure.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 2ca1fc65..f3035337 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -92,7 +92,7 @@ jobs: - name: Build images run: | - PROXY_IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$PROXY_IMAGE_NAME | tr '[A-Z]' '[a-z]') + PROXY_IMAGE_ID=$(echo $DOCKER_REGISTRY/${{ github.repository_owner }}/$PROXY_IMAGE_NAME | tr '[A-Z]' '[a-z]') PROXY_IMAGE_VERSION=${{ github.sha }} echo "PROXY_IMAGE_ID=$PROXY_IMAGE_ID" >> "$GITHUB_ENV" echo "PROXY_IMAGE_VERSION=$PROXY_IMAGE_VERSION" >> "$GITHUB_ENV" From 5981946fd9002777a96779db98ddc7615e15d5f7 Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:43:50 +0100 Subject: [PATCH 15/25] fix proxy path (#55) --- .github/workflows/infrastructure.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index f3035337..1fa55c18 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -96,7 +96,7 @@ jobs: PROXY_IMAGE_VERSION=${{ github.sha }} echo "PROXY_IMAGE_ID=$PROXY_IMAGE_ID" >> "$GITHUB_ENV" echo "PROXY_IMAGE_VERSION=$PROXY_IMAGE_VERSION" >> "$GITHUB_ENV" - docker build . --file Dockerfile --target production --tag $PROXY_IMAGE_ID:$PROXY_IMAGE_VERSION --tag $PROXY_IMAGE_ID:latest + docker build ./proxy --file ./proxy/Dockerfile --target production --tag $PROXY_IMAGE_ID:$PROXY_IMAGE_VERSION --tag $PROXY_IMAGE_ID:latest - name: Log in to registry run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin From 04df042cc66a83e926e98114238017590693c900 Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:58:25 +0100 Subject: [PATCH 16/25] Fix context path (#56) * fix proxy path * remove target --- .github/workflows/infrastructure.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 1fa55c18..4350f412 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -96,7 +96,7 @@ jobs: PROXY_IMAGE_VERSION=${{ github.sha }} echo "PROXY_IMAGE_ID=$PROXY_IMAGE_ID" >> "$GITHUB_ENV" echo "PROXY_IMAGE_VERSION=$PROXY_IMAGE_VERSION" >> "$GITHUB_ENV" - docker build ./proxy --file ./proxy/Dockerfile --target production --tag $PROXY_IMAGE_ID:$PROXY_IMAGE_VERSION --tag $PROXY_IMAGE_ID:latest + docker build ./proxy --file ./proxy/Dockerfile --tag $PROXY_IMAGE_ID:$PROXY_IMAGE_VERSION --tag $PROXY_IMAGE_ID:latest - name: Log in to registry run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin From 43944ab5e27dad80f7f1f53cc9aff5ea92820d81 Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Thu, 2 Jan 2025 18:07:36 +0100 Subject: [PATCH 17/25] github action write access (#57) --- .github/workflows/infrastructure.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 4350f412..62086ee1 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -59,7 +59,11 @@ jobs: cd: runs-on: ubuntu-latest - # if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} + if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} + + permissions: + packages: write + contents: read needs: - ci From 326221dc087bd7f80b7d62e95e4d846ab4dd319a Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Thu, 2 Jan 2025 21:09:10 +0100 Subject: [PATCH 18/25] fix docker stack deploy (#58) --- .github/workflows/infrastructure.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 62086ee1..7878197b 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -175,12 +175,12 @@ jobs: # Run Docker Compose cd /opt/deployment/ - docker stack deploy -c compose.mongodb.yaml mongodb --detach=false - docker stack deploy -c compose.mongodb_dashboard.yaml mongodb_dashboard --detach=false - docker stack deploy -c compose.nats.yaml nats --detach=false - docker stack deploy -c compose.docker.yaml docker --detach=false - docker stack deploy -c compose.docker_dashboard.yaml docker_dashboard --detach=false - docker stack deploy -c compose.backend.yaml backend --detach=false - docker stack deploy -c compose.frontend.yaml frontend --detach=false - docker stack deploy -c compose.proxy.yaml proxy --detach=false + docker stack deploy -c compose.mongodb.yaml mongodb --detach=true + docker stack deploy -c compose.mongodb_dashboard.yaml mongodb_dashboard --detach=true + docker stack deploy -c compose.nats.yaml nats --detach=true + docker stack deploy -c compose.docker.yaml docker --detach=true + docker stack deploy -c compose.docker_dashboard.yaml docker_dashboard --detach=true + docker stack deploy -c compose.backend.yaml backend --detach=true + docker stack deploy -c compose.frontend.yaml frontend --detach=true + docker stack deploy -c compose.proxy.yaml proxy --detach=true EOF From d3bc4ca31000be847d8a4eb8c1f07b3910b1a0b7 Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Thu, 2 Jan 2025 22:06:42 +0100 Subject: [PATCH 19/25] Fix docker stack deploy (#59) * fix docker stack deploy * fix --- .github/workflows/infrastructure.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index 7878197b..ac6fd2e5 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -121,8 +121,16 @@ jobs: # Ensure remote directory exists ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' + export VOLUME_PATH="${{ secrets.VOLUME_PATH }}" + sudo mkdir -p /opt/deployment sudo chown ${{ secrets.EC2_SSH_USER }}:${{ secrets.EC2_SSH_USER }} /opt/deployment + + # create volumes directories + sudo mkdir -p $VOLUME_PATH/mongodb/db + sudo mkdir -p $VOLUME_PATH/mongodb/configdb + sudo mkdir -p $VOLUME_PATH/nats + sudo chown -R ${{ secrets.EC2_SSH_USER }}:${{ secrets.EC2_SSH_USER }} $VOLUME_PATH/* EOF # Copy files From 8282e8412cc565477199055fa11bf6bad86a3158 Mon Sep 17 00:00:00 2001 From: mahdikhanzadi <6291970+khanzadimahdi@users.noreply.github.com> Date: Fri, 3 Jan 2025 05:39:07 +0100 Subject: [PATCH 20/25] fix chown (#60) --- .github/workflows/infrastructure.yaml | 17 +++++----- infrastructure/Makefile | 32 +++++++++---------- infrastructure/compose.app.yaml | 1 - infrastructure/compose.docker.yaml | 2 +- infrastructure/compose.docker_dashboard.yaml | 1 - infrastructure/compose.frontend.yaml | 1 - infrastructure/compose.mongodb.yaml | 1 - infrastructure/compose.mongodb_dashboard.yaml | 3 -- infrastructure/compose.nats.yaml | 3 +- infrastructure/compose.proxy.yaml | 1 - 10 files changed, 26 insertions(+), 36 deletions(-) diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml index ac6fd2e5..e27cce41 100644 --- a/.github/workflows/infrastructure.yaml +++ b/.github/workflows/infrastructure.yaml @@ -130,7 +130,6 @@ jobs: sudo mkdir -p $VOLUME_PATH/mongodb/db sudo mkdir -p $VOLUME_PATH/mongodb/configdb sudo mkdir -p $VOLUME_PATH/nats - sudo chown -R ${{ secrets.EC2_SSH_USER }}:${{ secrets.EC2_SSH_USER }} $VOLUME_PATH/* EOF # Copy files @@ -183,12 +182,12 @@ jobs: # Run Docker Compose cd /opt/deployment/ - docker stack deploy -c compose.mongodb.yaml mongodb --detach=true - docker stack deploy -c compose.mongodb_dashboard.yaml mongodb_dashboard --detach=true - docker stack deploy -c compose.nats.yaml nats --detach=true - docker stack deploy -c compose.docker.yaml docker --detach=true - docker stack deploy -c compose.docker_dashboard.yaml docker_dashboard --detach=true - docker stack deploy -c compose.backend.yaml backend --detach=true - docker stack deploy -c compose.frontend.yaml frontend --detach=true - docker stack deploy -c compose.proxy.yaml proxy --detach=true + docker compose -f compose.mongodb.yaml --project-name mongodb up --pull always --detach + docker compose -f compose.mongodb_dashboard.yaml --project-name mongodb_dashboard up --pull always --detach + docker compose -f compose.nats.yaml --project-name nats up --pull always --detach + docker compose -f compose.docker.yaml --project-name docker up --pull always --detach + docker compose -f compose.docker_dashboard.yaml --project-name docker_dashboard up --pull always --detach + docker compose -f compose.app.yaml --project-name app up --pull always --detach + docker compose -f compose.frontend.yaml --project-name frontend up --pull always --detach + docker compose -f compose.proxy.yaml --project-name proxy up --pull always --detach EOF diff --git a/infrastructure/Makefile b/infrastructure/Makefile index 6221ed95..40269854 100644 --- a/infrastructure/Makefile +++ b/infrastructure/Makefile @@ -74,21 +74,21 @@ ssh: ssh -i "ssh-private-key.pem" ${EC2_SSH_ENDPOINT} up: - docker stack deploy -c compose.mongodb.yaml mongodb --detach=false - docker stack deploy -c compose.mongodb_dashboard.yaml mongodb_dashboard --detach=false - docker stack deploy -c compose.nats.yaml nats --detach=false - docker stack deploy -c compose.docker.yaml docker --detach=false - docker stack deploy -c compose.docker_dashboard.yaml docker_dashboard --detach=false - docker stack deploy -c compose.backend.yaml backend --detach=false - docker stack deploy -c compose.frontend.yaml frontend --detach=false - docker stack deploy -c compose.proxy.yaml proxy --detach=false + docker compose -f compose.mongodb.yaml --project-name mongodb up --pull always --detach + docker compose -f compose.mongodb_dashboard.yaml --project-name mongodb_dashboard up --pull always --detach + docker compose -f compose.nats.yaml --project-name nats up --pull always --detach + docker compose -f compose.docker.yaml --project-name docker up --pull always --detach + docker compose -f compose.docker_dashboard.yaml --project-name docker_dashboard up --pull always --detach + docker compose -f compose.app.yaml --project-name app up --pull always --detach + docker compose -f compose.frontend.yaml --project-name frontend up --pull always --detach + docker compose -f compose.proxy.yaml --project-name proxy up --pull always --detach down: - docker stack rm proxy - docker stack rm frontend - docker stack rm backend - docker stack rm docker_dashboard - docker stack rm docker - docker stack rm nats - docker stack rm mongodb_dashboard - docker stack rm mongodb + docker compose -f compose.frontend.yaml --project-name frontend down --volumes --remove-orphans + docker compose -f compose.app.yaml --project-name app down --volumes --remove-orphans + docker compose -f compose.proxy.yaml --project-name proxy down --volumes --remove-orphans + docker compose -f compose.nats.yaml --project-name nats down --volumes --remove-orphans + docker compose -f compose.docker_dashboard.yaml --project-name docker_dashboard down --volumes --remove-orphans + docker compose -f compose.docker.yaml --project-name docker down --volumes --remove-orphans + docker compose -f compose.mongodb_dashboard.yaml --project-name mongodb_dashboard down --volumes --remove-orphans + docker compose -f compose.mongodb.yaml --project-name mongodb down --volumes --remove-orphans diff --git a/infrastructure/compose.app.yaml b/infrastructure/compose.app.yaml index c950acf5..36c4cba0 100644 --- a/infrastructure/compose.app.yaml +++ b/infrastructure/compose.app.yaml @@ -37,7 +37,6 @@ services: networks: app: name: app - driver: overlay mongodb: name: mongodb external: true diff --git a/infrastructure/compose.docker.yaml b/infrastructure/compose.docker.yaml index f66b8b1b..0b09267a 100644 --- a/infrastructure/compose.docker.yaml +++ b/infrastructure/compose.docker.yaml @@ -12,10 +12,10 @@ services: delay: 5s max_attempts: 3 runtime: sysbox-runc + # privileged: true # To test localy: enable this line and disable runtime environment: DOCKER_TLS_CERTDIR: "" # disable certs networks: docker: name: docker - driver: overlay diff --git a/infrastructure/compose.docker_dashboard.yaml b/infrastructure/compose.docker_dashboard.yaml index 7a4b84fa..ec875938 100644 --- a/infrastructure/compose.docker_dashboard.yaml +++ b/infrastructure/compose.docker_dashboard.yaml @@ -18,7 +18,6 @@ services: networks: docker_dashboard: name: docker_dashboard - driver: overlay docker: name: docker external: true diff --git a/infrastructure/compose.frontend.yaml b/infrastructure/compose.frontend.yaml index d06c2081..35f2c02a 100644 --- a/infrastructure/compose.frontend.yaml +++ b/infrastructure/compose.frontend.yaml @@ -20,7 +20,6 @@ services: networks: frontend: name: frontend - driver: overlay app: name: app external: true diff --git a/infrastructure/compose.mongodb.yaml b/infrastructure/compose.mongodb.yaml index 704d61be..55389b26 100644 --- a/infrastructure/compose.mongodb.yaml +++ b/infrastructure/compose.mongodb.yaml @@ -21,4 +21,3 @@ services: networks: mongodb: name: mongodb - driver: overlay diff --git a/infrastructure/compose.mongodb_dashboard.yaml b/infrastructure/compose.mongodb_dashboard.yaml index bbc9a2d0..3076e8dc 100644 --- a/infrastructure/compose.mongodb_dashboard.yaml +++ b/infrastructure/compose.mongodb_dashboard.yaml @@ -12,8 +12,6 @@ services: condition: on-failure delay: 5s max_attempts: 3 - depends_on: - - mongodb environment: ME_CONFIG_BASICAUTH_USERNAME: ${DASHBOARD_MONGO_USERNAME} ME_CONFIG_BASICAUTH_PASSWORD: ${DASHBOARD_MONGO_PASSWORD} @@ -22,7 +20,6 @@ services: networks: mongodb_dashboard: name: mongodb_dashboard - driver: overlay mongodb: name: mongodb external: true diff --git a/infrastructure/compose.nats.yaml b/infrastructure/compose.nats.yaml index da55a2dc..a9bb63e9 100644 --- a/infrastructure/compose.nats.yaml +++ b/infrastructure/compose.nats.yaml @@ -12,10 +12,9 @@ services: delay: 5s max_attempts: 3 volumes: - - ${VOLUME_PATH}/nats:/jetstream + - ${VOLUME_PATH}/nats:/data command: ["--jetstream", "--http_port", "8222", "--port", "4222", "--store_dir", "/data"] networks: nats: name: nats - driver: overlay diff --git a/infrastructure/compose.proxy.yaml b/infrastructure/compose.proxy.yaml index b7791a27..94957c5f 100644 --- a/infrastructure/compose.proxy.yaml +++ b/infrastructure/compose.proxy.yaml @@ -21,7 +21,6 @@ services: networks: proxy: name: proxy - driver: overlay frontend: name: frontend external: true From 7c02a3dd835b7f342743251b71025a55cb5b69a2 Mon Sep 17 00:00:00 2001 From: Mahdi Khanzadi Date: Sun, 5 Jan 2025 19:22:14 +0100 Subject: [PATCH 21/25] deploy on aws --- .github/actions/docker-build/action.yaml | 74 +++ .github/workflows/backend.yaml | 97 +++- .github/workflows/frontend.yaml | 78 +-- .github/workflows/infrastructure.yaml | 193 -------- {infrastructure => backend}/compose.app.yaml | 1 + .../compose.frontend.yaml | 1 + infrastructure/.gitignore | 13 - infrastructure/.terraform.lock.hcl | 24 - infrastructure/Makefile | 94 ---- infrastructure/compose.docker.yaml | 21 - infrastructure/compose.docker_dashboard.yaml | 23 - infrastructure/compose.mongodb.yaml | 23 - infrastructure/compose.mongodb_dashboard.yaml | 25 - infrastructure/compose.nats.yaml | 20 - infrastructure/compose.proxy.yaml | 35 -- infrastructure/proxy/Dockerfile | 5 - infrastructure/proxy/nginx.conf | 56 --- infrastructure/resources.tf | 456 ------------------ 18 files changed, 201 insertions(+), 1038 deletions(-) create mode 100644 .github/actions/docker-build/action.yaml delete mode 100644 .github/workflows/infrastructure.yaml rename {infrastructure => backend}/compose.app.yaml (98%) rename {infrastructure => frontend}/compose.frontend.yaml (96%) delete mode 100644 infrastructure/.gitignore delete mode 100644 infrastructure/.terraform.lock.hcl delete mode 100644 infrastructure/Makefile delete mode 100644 infrastructure/compose.docker.yaml delete mode 100644 infrastructure/compose.docker_dashboard.yaml delete mode 100644 infrastructure/compose.mongodb.yaml delete mode 100644 infrastructure/compose.mongodb_dashboard.yaml delete mode 100644 infrastructure/compose.nats.yaml delete mode 100644 infrastructure/compose.proxy.yaml delete mode 100644 infrastructure/proxy/Dockerfile delete mode 100644 infrastructure/proxy/nginx.conf delete mode 100644 infrastructure/resources.tf diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml new file mode 100644 index 00000000..da1b6843 --- /dev/null +++ b/.github/actions/docker-build/action.yaml @@ -0,0 +1,74 @@ +name: 'Docker Build Action' +description: 'Builds a Docker image using a specified Dockerfile and context' + +inputs: + context: + description: 'The Docker build context (path to the directory containing the Dockerfile)' + required: true + default: '.' + dockerfile: + description: 'The path to the Dockerfile (relative to the context)' + required: true + default: 'Dockerfile' + target: + description: 'The build target' + required: false + image-name: + description: 'The name of the image to build' + required: true + push: + description: 'Determines if the built image should be pushed' + required: true + default: 'no' + container-registry: + description: 'container registry address (example: ghcr.io)' + required: false + container-registry-username: + description: 'container registry username' + required: false + container-registry-password: + description: 'container registry password' + required: false + +runs: + using: 'composite' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + if: ${{ inputs.push == 'true' }} + with: + logout: false + registry: ${{ inputs.container-registry }} + username: ${{ inputs.container-registry-username }} + password: ${{ inputs.container-registry-password }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract metadata (tags, labels) for Docker image + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ inputs.container-registry }}/${{ github.repository_owner }}/${{ inputs.image-name }} + tags: | + type=raw,value=latest,priority=200,enable={{is_default_branch}} + type=sha,enable=true,priority=100,prefix=,suffix=,format=short + + - name: Build image and push (optional) + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64,linux/arm64 + push: ${{ inputs.push == 'true' }} + context: ${{ inputs.context }} + file: ${{ inputs.dockerfile }} + target: ${{ inputs.target }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index d2822696..e264bd57 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -18,6 +18,9 @@ env: REGISTRY: ghcr.io IMAGE_NAME: backend + EC2_SSH_ADDRESS: ${{ secrets.EC2_SSH_ADDRESS }} + EC2_SSH_ENDPOINT: ${{ secrets.EC2_SSH_USER }}@${{ secrets.EC2_SSH_ADDRESS }} + jobs: ci: runs-on: ubuntu-latest @@ -35,16 +38,15 @@ jobs: run: | go test ./... -v -race -cover - - name: Provide image name and version - run: | - IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') - IMAGE_VERSION=${{ github.sha }} - echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" - echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" - - name: Build image - run: | - docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest + uses: ./.github/actions/docker-build + with: + context: ./backend + dockerfile: ./backend/Dockerfile + image-name: ${{ env.IMAGE_NAME }} + target: production + push: false + container-registry: ${{ env.REGISTRY }} cd: runs-on: ubuntu-latest @@ -62,21 +64,70 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Provide image name and version + - name: Build and push image + uses: ./.github/actions/docker-build + with: + context: ./backend + dockerfile: ./backend/Dockerfile + image-name: ${{ env.IMAGE_NAME }} + target: production + push: true + container-registry: ${{ env.REGISTRY }} + container-registry-username: ${{ github.actor }} + container-registry-password: ${{ secrets.GITHUB_TOKEN }} + + - name: Deploy services run: | - IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') - IMAGE_VERSION=${{ github.sha }} - echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" - echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" + # Setup ssh key + echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" > ~/ec2-key.pem + chmod 400 ~/ec2-key.pem - - name: Build image - run: | - docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest + mkdir -p ~/.ssh + ssh-keyscan -H $EC2_SSH_ADDRESS >> ~/.ssh/known_hosts - - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + # Ensure remote directory exists + ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' + sudo mkdir -p /tmp/deployment_backend + sudo chown ${{ secrets.EC2_SSH_USER }}:${{ secrets.EC2_SSH_USER }} /tmp/deployment_backend - - name: Push image - run: | - docker push $IMAGE_ID:$IMAGE_VERSION - docker push $IMAGE_ID:latest + EOF + + # Copy files + scp -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r ./compose.app.yaml $EC2_SSH_ENDPOINT:/tmp/deployment_backend/ > /dev/null 2>&1 + + # Connect and deploy services + ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' + export MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" + export MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" + + export BACKEND_NATS_URL="${{ secrets.BACKEND_NATS_URL }}" + export BACKEND_PRIVATE_KEY="${{ secrets.BACKEND_PRIVATE_KEY }}" + + export BACKEND_MONGO_HOST="mongodb" + export BACKEND_MONGO_PORT="27017" + export BACKEND_MONGO_SCHEME="mongodb" + export BACKEND_MONGO_DATABASE_NAME="${{ secrets.BACKEND_MONGO_DATABASE_NAME }}" + export BACKEND_MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" + export BACKEND_MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" + + export BACKEND_MAIL_SMTP_PASSWORD="${{ secrets.BACKEND_MAIL_SMTP_PASSWORD }}" + export BACKEND_MAIL_SMTP_HOST="${{ secrets.BACKEND_MAIL_SMTP_HOST }}" + export BACKEND_MAIL_SMTP_FROM="${{ secrets.BACKEND_MAIL_SMTP_FROM }}" + export BACKEND_MAIL_SMTP_USERNAME="${{ secrets.BACKEND_MAIL_SMTP_USERNAME }}" + export BACKEND_MAIL_SMTP_PORT="${{ secrets.BACKEND_MAIL_SMTP_PORT }}" + + export BACKEND_S3_ENDPOINT="${{ secrets.BACKEND_S3_ENDPOINT }}" + export BACKEND_S3_SECRET_KEY="${{ secrets.BACKEND_S3_SECRET_KEY }}" + export BACKEND_S3_ACCESS_KEY="${{ secrets.BACKEND_S3_ACCESS_KEY }}" + export BACKEND_S3_USE_SSL="${{ secrets.BACKEND_S3_USE_SSL }}" + export BACKEND_S3_BUCKET_NAME="${{ secrets.BACKEND_S3_BUCKET_NAME }}" + + export APP_IMAGE="${{ secrets.APP_IMAGE }}" + + # Run Docker Compose + cd /tmp/deployment_backend/ + + docker compose -f compose.app.yaml --project-name app up --pull always --detach + + sudo rm -rf /tmp/deployment_backend + EOF diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 00b4e20d..2c29ffbe 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -18,6 +18,9 @@ env: REGISTRY: ghcr.io IMAGE_NAME: frontend + EC2_SSH_ADDRESS: ${{ secrets.EC2_SSH_ADDRESS }} + EC2_SSH_ENDPOINT: ${{ secrets.EC2_SSH_USER }}@${{ secrets.EC2_SSH_ADDRESS }} + jobs: ci: runs-on: ubuntu-latest @@ -26,20 +29,15 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - uses: actions/setup-node@v3 - with: - node-version: "18" - - - name: Provide image name and version - run: | - IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') - IMAGE_VERSION=${{ github.sha }} - echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" - echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" - - name: Build image - run: | - docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest + uses: ./.github/actions/docker-build + with: + context: ./frontend + dockerfile: ./frontend/Dockerfile + image-name: ${{ env.IMAGE_NAME }} + target: production + push: false + container-registry: ${{ env.REGISTRY }} cd: runs-on: ubuntu-latest @@ -57,21 +55,47 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Provide image name and version + - name: Build and push image + uses: ./.github/actions/docker-build + with: + context: ./frontend + dockerfile: ./frontend/Dockerfile + image-name: ${{ env.IMAGE_NAME }} + target: production + push: true + container-registry: ${{ env.REGISTRY }} + container-registry-username: ${{ github.actor }} + container-registry-password: ${{ secrets.GITHUB_TOKEN }} + + - name: Deploy services run: | - IMAGE_ID=$(echo $REGISTRY/${{ github.repository_owner }}/$IMAGE_NAME | tr '[A-Z]' '[a-z]') - IMAGE_VERSION=${{ github.sha }} - echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" - echo "IMAGE_VERSION=$IMAGE_VERSION" >> "$GITHUB_ENV" + # Setup ssh key + echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" > ~/ec2-key.pem + chmod 400 ~/ec2-key.pem - - name: Build image - run: | - docker build . --file Dockerfile --target production --tag $IMAGE_ID:$IMAGE_VERSION --tag $IMAGE_ID:latest + mkdir -p ~/.ssh + ssh-keyscan -H $EC2_SSH_ADDRESS >> ~/.ssh/known_hosts - - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + # Ensure remote directory exists + ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' + sudo mkdir -p /tmp/deployment_frontend + sudo chown ${{ secrets.EC2_SSH_USER }}:${{ secrets.EC2_SSH_USER }} /tmp/deployment_frontend + EOF - - name: Push image - run: | - docker push $IMAGE_ID:$IMAGE_VERSION - docker push $IMAGE_ID:latest + # Copy files + scp -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r ./compose.frontend.yaml $EC2_SSH_ENDPOINT:/tmp/deployment_frontend/ > /dev/null 2>&1 + + # Connect and deploy services + ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' + export FRONTEND_IMAGE="${{ secrets.FRONTEND_IMAGE }}" + export NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL="${{ secrets.NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL }}" + export INTERNAL_BACKEND_BASE_URL="${{ secrets.INTERNAL_BACKEND_BASE_URL }}" + export NEXT_PUBLIC_FILES_BASE_URL="${{ secrets.NEXT_PUBLIC_FILES_BASE_URL }}" + + # Run Docker Compose + cd /tmp/deployment_frontend/ + + docker compose -f compose.frontend.yaml --project-name frontend up --pull always --detach + + sudo rm -rf /tmp/deployment_frontend + EOF diff --git a/.github/workflows/infrastructure.yaml b/.github/workflows/infrastructure.yaml deleted file mode 100644 index e27cce41..00000000 --- a/.github/workflows/infrastructure.yaml +++ /dev/null @@ -1,193 +0,0 @@ -name: Infrastructure CI and CD - -on: - workflow_run: - workflows: - - Frontend CI and CD - - Backend CI and CD - types: - - completed - -defaults: - run: - working-directory: ./infrastructure - -env: - TF_VAR_project_name: tarhche - TF_VAR_instance_name: backend - - DOCKER_REGISTRY: ghcr.io - PROXY_IMAGE_NAME: proxy - - EC2_SSH_ADDRESS: ${{ secrets.EC2_SSH_ADDRESS }} - EC2_SSH_ENDPOINT: ${{ secrets.EC2_SSH_USER }}@${{ secrets.EC2_SSH_ADDRESS }} - -jobs: - ci: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - - - name: Terraform Format - id: fmt - run: terraform fmt -check - - - name: Terraform Init - id: init - run: terraform init - - - name: Terraform Validate - id: validate - run: terraform validate -no-color - - - name: Terraform Plan - run: terraform plan -no-color -input=false - continue-on-error: false - - cd: - runs-on: ubuntu-latest - - if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} - - permissions: - packages: write - contents: read - - needs: - - ci - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - - - name: Terraform Init - id: init - run: terraform init - - - name: Terraform Validate - id: validate - run: terraform validate -no-color - - - name: Terraform Apply - run: terraform apply -auto-approve -input=false - continue-on-error: false - - - name: Build images - run: | - PROXY_IMAGE_ID=$(echo $DOCKER_REGISTRY/${{ github.repository_owner }}/$PROXY_IMAGE_NAME | tr '[A-Z]' '[a-z]') - PROXY_IMAGE_VERSION=${{ github.sha }} - echo "PROXY_IMAGE_ID=$PROXY_IMAGE_ID" >> "$GITHUB_ENV" - echo "PROXY_IMAGE_VERSION=$PROXY_IMAGE_VERSION" >> "$GITHUB_ENV" - docker build ./proxy --file ./proxy/Dockerfile --tag $PROXY_IMAGE_ID:$PROXY_IMAGE_VERSION --tag $PROXY_IMAGE_ID:latest - - - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Push images - run: | - docker push $PROXY_IMAGE_ID:$PROXY_IMAGE_VERSION - docker push $PROXY_IMAGE_ID:latest - - - name: Deploy services - run: | - # Setup ssh key - echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" > ~/ec2-key.pem - chmod 400 ~/ec2-key.pem - - mkdir -p ~/.ssh - ssh-keyscan -H $EC2_SSH_ADDRESS >> ~/.ssh/known_hosts - - # Ensure remote directory exists - ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' - export VOLUME_PATH="${{ secrets.VOLUME_PATH }}" - - sudo mkdir -p /opt/deployment - sudo chown ${{ secrets.EC2_SSH_USER }}:${{ secrets.EC2_SSH_USER }} /opt/deployment - - # create volumes directories - sudo mkdir -p $VOLUME_PATH/mongodb/db - sudo mkdir -p $VOLUME_PATH/mongodb/configdb - sudo mkdir -p $VOLUME_PATH/nats - EOF - - # Copy files - scp -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r ./* $EC2_SSH_ENDPOINT:/opt/deployment/ > /dev/null 2>&1 - - # Connect and deploy services - ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' - export VOLUME_PATH="${{ secrets.VOLUME_PATH }}" - - export MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" - export MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" - - export DASHBOARD_MONGO_USERNAME="${{ secrets.DASHBOARD_MONGO_USERNAME }}" - export DASHBOARD_MONGO_PASSWORD="${{ secrets.DASHBOARD_MONGO_PASSWORD }}" - export DASHBOARD_MONGO_MONGODB_URL="mongodb://${{ secrets.MONGO_USERNAME }}:${{ secrets.MONGO_PASSWORD }}@mongodb:27017" - - export BACKEND_NATS_URL="${{ secrets.BACKEND_NATS_URL }}" - export BACKEND_PRIVATE_KEY="${{ secrets.BACKEND_PRIVATE_KEY }}" - - export BACKEND_MONGO_HOST="mongodb" - export BACKEND_MONGO_PORT="27017" - export BACKEND_MONGO_SCHEME="mongodb" - export BACKEND_MONGO_DATABASE_NAME="${{ secrets.BACKEND_MONGO_DATABASE_NAME }}" - export BACKEND_MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" - export BACKEND_MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" - - export BACKEND_MAIL_SMTP_PASSWORD="${{ secrets.BACKEND_MAIL_SMTP_PASSWORD }}" - export BACKEND_MAIL_SMTP_HOST="${{ secrets.BACKEND_MAIL_SMTP_HOST }}" - export BACKEND_MAIL_SMTP_FROM="${{ secrets.BACKEND_MAIL_SMTP_FROM }}" - export BACKEND_MAIL_SMTP_USERNAME="${{ secrets.BACKEND_MAIL_SMTP_USERNAME }}" - export BACKEND_MAIL_SMTP_PORT="${{ secrets.BACKEND_MAIL_SMTP_PORT }}" - - export BACKEND_S3_ENDPOINT="${{ secrets.BACKEND_S3_ENDPOINT }}" - export BACKEND_S3_SECRET_KEY="${{ secrets.BACKEND_S3_SECRET_KEY }}" - export BACKEND_S3_ACCESS_KEY="${{ secrets.BACKEND_S3_ACCESS_KEY }}" - export BACKEND_S3_USE_SSL="${{ secrets.BACKEND_S3_USE_SSL }}" - export BACKEND_S3_BUCKET_NAME="${{ secrets.BACKEND_S3_BUCKET_NAME }}" - - export PROXY_IMAGE=${{ secrets.PROXY_IMAGE }} - - export APP_IMAGE="${{ secrets.APP_IMAGE }}" - - export PORTAINER_ADMIN_PASSWORD='${{ secrets.PORTAINER_ADMIN_PASSWORD }}' - - export FRONTEND_IMAGE="${{ secrets.FRONTEND_IMAGE }}" - export NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL="${{ secrets.NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL }}" - export INTERNAL_BACKEND_BASE_URL="${{ secrets.INTERNAL_BACKEND_BASE_URL }}" - export NEXT_PUBLIC_FILES_BASE_URL="${{ secrets.NEXT_PUBLIC_FILES_BASE_URL }}" - - # Run Docker Compose - cd /opt/deployment/ - - docker compose -f compose.mongodb.yaml --project-name mongodb up --pull always --detach - docker compose -f compose.mongodb_dashboard.yaml --project-name mongodb_dashboard up --pull always --detach - docker compose -f compose.nats.yaml --project-name nats up --pull always --detach - docker compose -f compose.docker.yaml --project-name docker up --pull always --detach - docker compose -f compose.docker_dashboard.yaml --project-name docker_dashboard up --pull always --detach - docker compose -f compose.app.yaml --project-name app up --pull always --detach - docker compose -f compose.frontend.yaml --project-name frontend up --pull always --detach - docker compose -f compose.proxy.yaml --project-name proxy up --pull always --detach - EOF diff --git a/infrastructure/compose.app.yaml b/backend/compose.app.yaml similarity index 98% rename from infrastructure/compose.app.yaml rename to backend/compose.app.yaml index 36c4cba0..c4a07ec5 100644 --- a/infrastructure/compose.app.yaml +++ b/backend/compose.app.yaml @@ -37,6 +37,7 @@ services: networks: app: name: app + external: true mongodb: name: mongodb external: true diff --git a/infrastructure/compose.frontend.yaml b/frontend/compose.frontend.yaml similarity index 96% rename from infrastructure/compose.frontend.yaml rename to frontend/compose.frontend.yaml index 35f2c02a..eb396d2d 100644 --- a/infrastructure/compose.frontend.yaml +++ b/frontend/compose.frontend.yaml @@ -20,6 +20,7 @@ services: networks: frontend: name: frontend + external: true app: name: app external: true diff --git a/infrastructure/.gitignore b/infrastructure/.gitignore deleted file mode 100644 index 15044452..00000000 --- a/infrastructure/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ -.DS_Store -.vscode -.idea -/tmp - -# SSH keys -/*.pem -/*.pub - -# Terraform files -*.tfstate -*.tfstate.backup -.terraform/ diff --git a/infrastructure/.terraform.lock.hcl b/infrastructure/.terraform.lock.hcl deleted file mode 100644 index d3e4f5ce..00000000 --- a/infrastructure/.terraform.lock.hcl +++ /dev/null @@ -1,24 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "5.82.1" - hashes = [ - "h1:QTOtDMehUfiD3wDbbDuXYuTqGgLDkKK9Agkd5NCUEic=", - "zh:0fde8533282973f1f5d33b2c4f82d962a2c78860d39b42ac20a9ce399f06f62c", - "zh:1fd1a252bffe91668f35be8eac4e0a980f022120254eae1674c3c05049aff88a", - "zh:31bbd380cd7d74bf9a8c961fc64da4222bed40ffbdb27b011e637fa8b2d33641", - "zh:333ee400cf6f62fa199dc1270bf8efac6ffe56659f86918070b8351b8636e03b", - "zh:42ea9fee0a152d344d548eab43583299a13bcd73fae9e53e7e1a708720ac1315", - "zh:4b78f25a8cda3316eb56aa01909a403ec2f325a2eb0512c9a73966068c26cf29", - "zh:5e9cf9a275eda8f7940a41e32abe0b92ba76b5744def4af5124b343b5f33eb94", - "zh:6a46c8630c16b9e1338c2daed6006118db951420108b58b8b886403c69317439", - "zh:6efe11cf1a01f98a8d8043cdcd8c0ee5fe93a0e582c2b69ebb73ea073f5068c3", - "zh:88ab5c768c7d8133dab94eff48071e764424ad2b7cfeee5abe6d5bb16e4b85c6", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:a614beb312574342b27dbc34d65b450997f63fa3e948d0d30f441e4f69337380", - "zh:c1f486e27130610a9b64cacb0bd928009c433d62b3be515488185e6467b4aa1f", - "zh:dccd166e89e1a02e7ce658df3c42d040edec4b09c6f7906aa5743938518148b1", - "zh:e75a3ae0fb42b7ea5a0bb5dffd8f8468004c9700fcc934eb04c264fda2ba9984", - ] -} diff --git a/infrastructure/Makefile b/infrastructure/Makefile deleted file mode 100644 index 40269854..00000000 --- a/infrastructure/Makefile +++ /dev/null @@ -1,94 +0,0 @@ -export TF_VAR_project_name = tarhche -export TF_VAR_instance_name = backend - -export EC2_SSH_ADDRESS = ec2-3-124-72-48.eu-central-1.compute.amazonaws.com -export EC2_SSH_USER = ubuntu -export EC2_SSH_ENDPOINT = ${EC2_SSH_USER}@${EC2_SSH_ADDRESS} -export VOLUME_PATH = ./tmp/volume_01 - -export MONGO_USERNAME = test -export MONGO_PASSWORD = test‍ - -export DASHBOARD_MONGO_USERNAME = username -export DASHBOARD_MONGO_PASSWORD = password -export DASHBOARD_MONGO_MONGODB_URL = mongodb://${MONGO_USERNAME}:${MONGO_PASSWORD}@mongodb:27017 - -export BACKEND_NATS_URL = -export BACKEND_PRIVATE_KEY = - -export BACKEND_MONGO_HOST = mongodb -export BACKEND_MONGO_PORT = 27017 -export BACKEND_MONGO_SCHEME = mongodb -export BACKEND_MONGO_DATABASE_NAME = test -export BACKEND_MONGO_USERNAME = ${MONGO_USERNAME} -export BACKEND_MONGO_PASSWORD = ${MONGO_PASSWORD} - -export BACKEND_MAIL_SMTP_PASSWORD = -export BACKEND_MAIL_SMTP_HOST = -export BACKEND_MAIL_SMTP_FROM = -export BACKEND_MAIL_SMTP_USERNAME = -export BACKEND_MAIL_SMTP_PORT = - -export BACKEND_S3_ENDPOINT = -export BACKEND_S3_SECRET_KEY = -export BACKEND_S3_ACCESS_KEY = -export BACKEND_S3_USE_SSL = false -export BACKEND_S3_BUCKET_NAME = - -export PROXY_IMAGE = ghcr.io/tarhche/proxy:latest - -export APP_IMAGE = ghcr.io/tarhche/backend:latest - -# username: admin -# password: admin-password (in bcrypt, a dollar-sign should be escaped by an arbitrary dollar-sign ($ --> $$)) -export PORTAINER_ADMIN_PASSWORD = $$2a$$12$$4xcOa82Ni5rjgQF.v.JWi.i71OyUm3fwmfWiumgJHIAPGU.uOw3qu - -export FRONTEND_IMAGE = ghcr.io/tarhche/frontend:latest -export NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL = -export INTERNAL_BACKEND_BASE_URL = http://app -export NEXT_PUBLIC_FILES_BASE_URL = - -validate: - terraform validate - -fmt: - terraform fmt - -init: - terraform init - -state: - terraform state list - -plan: - terraform plan - -apply: - terraform apply - rm -f terraform.tfstate *.tfstate.* - -public_key: - ssh-keygen -y -f ssh-private-key.pem > ssh-public-key.pub - -ssh: - ssh -i "ssh-private-key.pem" ${EC2_SSH_ENDPOINT} - -up: - docker compose -f compose.mongodb.yaml --project-name mongodb up --pull always --detach - docker compose -f compose.mongodb_dashboard.yaml --project-name mongodb_dashboard up --pull always --detach - docker compose -f compose.nats.yaml --project-name nats up --pull always --detach - docker compose -f compose.docker.yaml --project-name docker up --pull always --detach - docker compose -f compose.docker_dashboard.yaml --project-name docker_dashboard up --pull always --detach - docker compose -f compose.app.yaml --project-name app up --pull always --detach - docker compose -f compose.frontend.yaml --project-name frontend up --pull always --detach - docker compose -f compose.proxy.yaml --project-name proxy up --pull always --detach - -down: - docker compose -f compose.frontend.yaml --project-name frontend down --volumes --remove-orphans - docker compose -f compose.app.yaml --project-name app down --volumes --remove-orphans - docker compose -f compose.proxy.yaml --project-name proxy down --volumes --remove-orphans - docker compose -f compose.nats.yaml --project-name nats down --volumes --remove-orphans - docker compose -f compose.docker_dashboard.yaml --project-name docker_dashboard down --volumes --remove-orphans - docker compose -f compose.docker.yaml --project-name docker down --volumes --remove-orphans - docker compose -f compose.mongodb_dashboard.yaml --project-name mongodb_dashboard down --volumes --remove-orphans - docker compose -f compose.mongodb.yaml --project-name mongodb down --volumes --remove-orphans diff --git a/infrastructure/compose.docker.yaml b/infrastructure/compose.docker.yaml deleted file mode 100644 index 0b09267a..00000000 --- a/infrastructure/compose.docker.yaml +++ /dev/null @@ -1,21 +0,0 @@ -services: - docker: - image: docker:27-dind - networks: - - docker - deploy: - mode: replicated - replicas: 1 - endpoint_mode: vip - restart_policy: - condition: on-failure - delay: 5s - max_attempts: 3 - runtime: sysbox-runc - # privileged: true # To test localy: enable this line and disable runtime - environment: - DOCKER_TLS_CERTDIR: "" # disable certs - -networks: - docker: - name: docker diff --git a/infrastructure/compose.docker_dashboard.yaml b/infrastructure/compose.docker_dashboard.yaml deleted file mode 100644 index ec875938..00000000 --- a/infrastructure/compose.docker_dashboard.yaml +++ /dev/null @@ -1,23 +0,0 @@ -services: - docker_dashboard: - image: portainer/portainer-ce - networks: - - docker - - docker_dashboard - deploy: - mode: replicated - replicas: 1 - endpoint_mode: vip - restart_policy: - condition: on-failure - delay: 5s - max_attempts: 3 - command: > - --admin-password="${PORTAINER_ADMIN_PASSWORD}" - -networks: - docker_dashboard: - name: docker_dashboard - docker: - name: docker - external: true diff --git a/infrastructure/compose.mongodb.yaml b/infrastructure/compose.mongodb.yaml deleted file mode 100644 index 55389b26..00000000 --- a/infrastructure/compose.mongodb.yaml +++ /dev/null @@ -1,23 +0,0 @@ -services: - mongodb: - image: mongo:8.0 - networks: - - mongodb - deploy: - mode: replicated - replicas: 1 - endpoint_mode: vip - restart_policy: - condition: on-failure - delay: 5s - max_attempts: 3 - environment: - MONGO_INITDB_ROOT_USERNAME: ${MONGO_USERNAME} - MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD} - volumes: - - ${VOLUME_PATH}/mongodb/db:/data/db - - ${VOLUME_PATH}/mongodb/configdb:/data/configdb - -networks: - mongodb: - name: mongodb diff --git a/infrastructure/compose.mongodb_dashboard.yaml b/infrastructure/compose.mongodb_dashboard.yaml deleted file mode 100644 index 3076e8dc..00000000 --- a/infrastructure/compose.mongodb_dashboard.yaml +++ /dev/null @@ -1,25 +0,0 @@ -services: - mongodb_dashboard: - image: mongo-express - networks: - - mongodb - - mongodb_dashboard - deploy: - mode: replicated - replicas: 1 - endpoint_mode: vip - restart_policy: - condition: on-failure - delay: 5s - max_attempts: 3 - environment: - ME_CONFIG_BASICAUTH_USERNAME: ${DASHBOARD_MONGO_USERNAME} - ME_CONFIG_BASICAUTH_PASSWORD: ${DASHBOARD_MONGO_PASSWORD} - ME_CONFIG_MONGODB_URL: ${DASHBOARD_MONGO_MONGODB_URL} - -networks: - mongodb_dashboard: - name: mongodb_dashboard - mongodb: - name: mongodb - external: true diff --git a/infrastructure/compose.nats.yaml b/infrastructure/compose.nats.yaml deleted file mode 100644 index a9bb63e9..00000000 --- a/infrastructure/compose.nats.yaml +++ /dev/null @@ -1,20 +0,0 @@ -services: - nats: - image: nats:2.10 - networks: - - nats - deploy: - mode: replicated - replicas: 1 - endpoint_mode: vip - restart_policy: - condition: on-failure - delay: 5s - max_attempts: 3 - volumes: - - ${VOLUME_PATH}/nats:/data - command: ["--jetstream", "--http_port", "8222", "--port", "4222", "--store_dir", "/data"] - -networks: - nats: - name: nats diff --git a/infrastructure/compose.proxy.yaml b/infrastructure/compose.proxy.yaml deleted file mode 100644 index 94957c5f..00000000 --- a/infrastructure/compose.proxy.yaml +++ /dev/null @@ -1,35 +0,0 @@ -services: - proxy: - image: ${PROXY_IMAGE} - networks: - - proxy - - frontend - - app - - mongodb_dashboard - - docker_dashboard - deploy: - mode: replicated - replicas: 1 - endpoint_mode: vip - restart_policy: - condition: on-failure - delay: 5s - max_attempts: 3 - ports: - - 80:80 - -networks: - proxy: - name: proxy - frontend: - name: frontend - external: true - app: - name: app - external: true - mongodb_dashboard: - name: mongodb_dashboard - external: true - docker_dashboard: - name: docker_dashboard - external: true diff --git a/infrastructure/proxy/Dockerfile b/infrastructure/proxy/Dockerfile deleted file mode 100644 index 070ba3e8..00000000 --- a/infrastructure/proxy/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM nginx:1.26-alpine - -COPY ./nginx.conf /etc/nginx/conf.d/default.conf - -EXPOSE 80 diff --git a/infrastructure/proxy/nginx.conf b/infrastructure/proxy/nginx.conf deleted file mode 100644 index 749055f2..00000000 --- a/infrastructure/proxy/nginx.conf +++ /dev/null @@ -1,56 +0,0 @@ -# Server block to catch-all unmatched subdomains -server { - listen 80 default_server; - - server_name "_"; - - location / { - proxy_pass http://frontend:3000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } -} - -# Server block for backend subdomain -server { - listen 80; - server_name "backend.*"; - - location / { - proxy_pass http://app:80; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } -} - -# Server block for dockerdashboard subdomain -server { - listen 80; - server_name "dockerdashboard.*"; - - location / { - proxy_pass http://docker_dashboard:9000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } -} - -# Server block for mongodashboard subdomain -server { - listen 80; - server_name "mongodashboard.*"; - - location / { - proxy_pass http://mongodb_dashboard:8081; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } -} diff --git a/infrastructure/resources.tf b/infrastructure/resources.tf deleted file mode 100644 index 992c0b3d..00000000 --- a/infrastructure/resources.tf +++ /dev/null @@ -1,456 +0,0 @@ -provider "aws" { - region = "eu-central-1" -} - -variable "project_name" { - description = "Project tag given to each deployed Instance" - type = string -} - -variable "instance_name" { - description = "instance_name" - type = string -} - -import { - to = aws_security_group.backend - id = "sg-0c4446cdf14777251" -} - -resource "aws_security_group" "backend" { - name = var.instance_name - description = "Allow HTTP, HTTPS, and SSH inbound traffic" - - tags = { - project_name = var.project_name - } - - # Allow SSH (port 22) from any IP address - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - # Allow HTTP (port 80) from any IP address - ingress { - from_port = 80 - to_port = 80 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] # Allow HTTP from anywhere - } - - # Allow HTTPS (port 443) from any IP address - ingress { - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - # Allow all outbound traffic - egress { - from_port = 0 - to_port = 0 - protocol = "-1" # all protocols - cidr_blocks = ["0.0.0.0/0"] - } -} - -import { - to = aws_ebs_volume.backend - id = "vol-0d2bab5e75ac580e9" -} - -resource "aws_ebs_volume" "backend" { - availability_zone = aws_instance.backend.availability_zone - encrypted = false - size = 10 - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_volume_attachment.backend - id = "/dev/xvdf:vol-0d2bab5e75ac580e9:${aws_instance.backend.id}" -} - -resource "aws_volume_attachment" "backend" { - device_name = "/dev/xvdf" - instance_id = aws_instance.backend.id - volume_id = aws_ebs_volume.backend.id -} - -import { - to = aws_instance.backend - id = "i-026c60a5a3cdec06e" -} - -resource "aws_instance" "backend" { - ami = "ami-0a628e1e89aaedf80" # Canonical, Ubuntu, 24.04, amd64 noble image - instance_type = "t2.micro" - key_name = "backend" - availability_zone = "eu-central-1b" - - user_data = <<-EOT - #!/bin/bash - - # volumes - sudo mkfs.ext4 /dev/xvdf - sudo mkdir /volume_01 - sudo mount /dev/xvdf /volume_01 - sudo echo "/dev/xvdf /volume_01 ext4 defaults,nofail 0 0" | sudo tee -a /etc/fstab - - # tools - sudo apt install -y wget python3 ca-certificates curl htop jq vim make - - # Add Docker's official GPG key: - sudo install -m 0755 -d /etc/apt/keyrings - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc - - # Add the repository to Apt sources: - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update - - # install docker and sysbox - sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - wget https://downloads.nestybox.com/sysbox/releases/v0.6.5/sysbox-ce_0.6.5-0.linux_amd64.deb - sudo apt install -y ./sysbox-ce_0.6.5-0.linux_amd64.deb - rm ./sysbox-ce_0.6.5-0.linux_amd64.deb - - # setup - sudo systemctl enable docker.service - sudo systemctl start docker.service - sudo usermod -a -G docker ubuntu - id ubuntu - newgrp docker - docker swarm init --advertise-addr 192.168.99.100 - EOT - - root_block_device { - delete_on_termination = true - encrypted = false - volume_size = 20 - volume_type = "gp3" - - tags = { - project_name = var.project_name - } - } - - security_groups = [ - aws_security_group.backend.name - ] - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_eip.backend - id = "eipalloc-02bceef376bc05f89" -} - -resource "aws_eip" "backend" { - instance = aws_instance.backend.id - domain = "vpc" - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_lb.tarhche - id = "arn:aws:elasticloadbalancing:eu-central-1:381491955644:loadbalancer/app/tarhche/6953bf38e49158d7" -} - -resource "aws_lb" "tarhche" { - name = "tarhche" - internal = false - load_balancer_type = "application" - idle_timeout = 60 - ip_address_type = "ipv4" - enable_deletion_protection = true - - security_groups = [ - aws_security_group.backend.id, - ] - - subnets = [ - "subnet-0d68a01f5a4861c65", - "subnet-0fca4d198b88d68d6", - "subnet-0c8f8df628e715018", - ] - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_lb_target_group.http - id = "arn:aws:elasticloadbalancing:eu-central-1:381491955644:targetgroup/HTTP/374d0a16b08c8d4a" -} - -resource "aws_lb_target_group" "http" { - name = "HTTP" - port = 80 - protocol = "HTTP" - vpc_id = "vpc-04db3e4490d90be8e" - ip_address_type = "ipv4" - proxy_protocol_v2 = false - - lambda_multi_value_headers_enabled = false - - health_check { - path = "/" - interval = 30 - timeout = 5 - healthy_threshold = 5 - unhealthy_threshold = 2 - } - - tags = { - project_name = var.project_name - } -} - -# resource "aws_lb_target_group_attachment" "backend_http" { -# target_group_arn = aws_lb_target_group.http.arn -# target_id = aws_instance.backend.id -# port = 80 -# } - -import { - to = aws_lb_listener.http - id = "arn:aws:elasticloadbalancing:eu-central-1:381491955644:listener/app/tarhche/6953bf38e49158d7/637c8770b5e4d6ed" -} - -resource "aws_lb_listener" "http" { - load_balancer_arn = aws_lb.tarhche.arn - port = 80 - protocol = "HTTP" - - default_action { - order = 1 - type = "redirect" - target_group_arn = aws_lb_target_group.http.arn - - redirect { - host = "#{host}" - path = "/#{path}" - port = "443" - protocol = "HTTPS" - query = "#{query}" - status_code = "HTTP_301" - } - } - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_lb_listener.https - id = "arn:aws:elasticloadbalancing:eu-central-1:381491955644:listener/app/tarhche/6953bf38e49158d7/ab1c7847cbb6f739" -} - -resource "aws_lb_listener" "https" { - load_balancer_arn = aws_lb.tarhche.arn - port = 443 - protocol = "HTTPS" - ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" - certificate_arn = aws_acm_certificate.tarhche_com.arn - - default_action { - order = 1 - type = "forward" - target_group_arn = aws_lb_target_group.http.arn - - forward { - stickiness { - duration = 3600 - enabled = false - } - - target_group { - arn = aws_lb_target_group.http.arn - weight = 1 - } - } - } - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_route53domains_registered_domain.tarhche-com - id = "tarhche.com" -} - -resource "aws_route53domains_registered_domain" "tarhche-com" { - domain_name = "tarhche.com" - - name_server { - name = "ns-1611.awsdns-09.co.uk" - } - - name_server { - name = "ns-1254.awsdns-28.org" - } - - name_server { - name = "ns-143.awsdns-17.com" - } - - name_server { - name = "ns-769.awsdns-32.net" - } - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_route53_zone.tarhche_com - id = "Z0951095A7CDVGITDCUP" -} - -resource "aws_route53_zone" "tarhche_com" { - name = "tarhche.com" - force_destroy = false - comment = "" - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_route53_record.a_record_tarhche_com - id = "${aws_route53_zone.tarhche_com.id}_tarhche.com_A" -} - -resource "aws_route53_record" "a_record_tarhche_com" { - zone_id = aws_route53_zone.tarhche_com.id - name = "tarhche.com" - type = "A" - - alias { - name = aws_lb.tarhche.dns_name - zone_id = aws_lb.tarhche.zone_id - evaluate_target_health = true - } -} - -import { - to = aws_route53_record.a_record_all_tarhche_com - id = "${aws_route53_zone.tarhche_com.id}_*.tarhche.com_A" -} - -resource "aws_route53_record" "a_record_all_tarhche_com" { - zone_id = aws_route53_zone.tarhche_com.id - name = "*.tarhche.com" - type = "A" - - alias { - name = aws_lb.tarhche.dns_name - zone_id = aws_lb.tarhche.zone_id - evaluate_target_health = true - } -} - -import { - to = aws_route53_zone.tarhche_ir - id = "Z07817351L3HY3TPTD5IU" -} - -resource "aws_route53_zone" "tarhche_ir" { - name = "tarhche.ir" - force_destroy = false - comment = "" - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_route53_record.a_record_tarhche_ir - id = "${aws_route53_zone.tarhche_ir.id}_tarhche.ir_A" -} - -resource "aws_route53_record" "a_record_tarhche_ir" { - zone_id = aws_route53_zone.tarhche_ir.id - name = "tarhche.ir" - type = "A" - - alias { - evaluate_target_health = true - name = aws_lb.tarhche.dns_name - zone_id = aws_lb.tarhche.zone_id - } -} - -import { - to = aws_s3_bucket.tarhche-backend - id = "tarhche-backend" -} - -resource "aws_s3_bucket" "tarhche-backend" { - bucket = "tarhche-backend" - force_destroy = false - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_acm_certificate.tarhche_com - id = "arn:aws:acm:eu-central-1:381491955644:certificate/a446a0ad-9cac-479f-a1d6-59b983d633d6" -} - -resource "aws_acm_certificate" "tarhche_com" { - domain_name = "tarhche.com" - validation_method = "DNS" - - subject_alternative_names = [ - "tarhche.com", - "*.tarhche.com", - ] - - lifecycle { - create_before_destroy = true - } - - tags = { - project_name = var.project_name - } -} - -import { - to = aws_route53_record.tarhche_com_ssl_validation - id = "${aws_route53_zone.tarhche_com.id}__e7a6f01cbe22cb6d1db5c70fb80299a8.tarhche.com_CNAME" -} - -resource "aws_route53_record" "tarhche_com_ssl_validation" { - zone_id = aws_route53_zone.tarhche_com.id - name = "_e7a6f01cbe22cb6d1db5c70fb80299a8.tarhche.com" - type = "CNAME" - records = ["_0fdeb4d57a8f62c9a90a8f77b0146a14.zfyfvmchrl.acm-validations.aws."] - ttl = 60 -} From 14be04acb104fa7723b58c201a654f601bec446b Mon Sep 17 00:00:00 2001 From: Mahdi Khanzadi Date: Mon, 6 Jan 2025 22:21:25 +0100 Subject: [PATCH 22/25] seperate build and deploy steps --- .github/workflows/backend.yaml | 14 +++++++++++++- .github/workflows/frontend.yaml | 14 +++++++++++++- backend/presentation/http/api/home/home.go | 2 ++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index e264bd57..d7c4f438 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -48,7 +48,7 @@ jobs: push: false container-registry: ${{ env.REGISTRY }} - cd: + build-and-push-images: runs-on: ubuntu-latest if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} @@ -76,6 +76,18 @@ jobs: container-registry-username: ${{ github.actor }} container-registry-password: ${{ secrets.GITHUB_TOKEN }} + deploy: + runs-on: ubuntu-latest + + if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} + + needs: + - build-and-push-images + + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Deploy services run: | # Setup ssh key diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 2c29ffbe..5eee3cab 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -39,7 +39,7 @@ jobs: push: false container-registry: ${{ env.REGISTRY }} - cd: + build-and-push-images: runs-on: ubuntu-latest if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} @@ -67,6 +67,18 @@ jobs: container-registry-username: ${{ github.actor }} container-registry-password: ${{ secrets.GITHUB_TOKEN }} + deploy: + runs-on: ubuntu-latest + + if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }} + + needs: + - build-and-push-images + + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Deploy services run: | # Setup ssh key diff --git a/backend/presentation/http/api/home/home.go b/backend/presentation/http/api/home/home.go index 4ffa8e89..c73047ad 100644 --- a/backend/presentation/http/api/home/home.go +++ b/backend/presentation/http/api/home/home.go @@ -2,6 +2,7 @@ package home import ( "encoding/json" + "log" "net/http" "github.com/khanzadimahdi/testproject/application/home" @@ -22,6 +23,7 @@ func (h *homeHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { switch { case err != nil: + log.Println(err) rw.WriteHeader(http.StatusInternalServerError) default: rw.Header().Add("Content-Type", "application/json") From 29f8945ad0c2c67d50f048a94402ee0ac97d2b91 Mon Sep 17 00:00:00 2001 From: Mahdi Khanzadi Date: Sat, 11 Jan 2025 12:07:46 +0100 Subject: [PATCH 23/25] introduce storedName for files, so files can be stored with a different name from their real uploaded name --- .../dashboard/file/deleteFile/usecase.go | 2 +- .../dashboard/file/deleteFile/usecase_test.go | 21 +++--- .../dashboard/file/deleteUserFile/usecase.go | 2 +- .../file/deleteUserFile/usecase_test.go | 27 ++++---- .../dashboard/file/getFile/useCase.go | 2 +- .../dashboard/file/uploadFile/request.go | 16 +++++ .../dashboard/file/uploadFile/usecase.go | 20 +++--- .../dashboard/file/uploadFile/usecase_test.go | 66 ++++++++----------- backend/application/file/getFile/useCase.go | 2 +- backend/domain/file/file.go | 13 ++-- .../repository/mongodb/files/model.go | 13 ++-- .../repository/mongodb/files/repository.go | 65 +++++++++--------- .../api/dashboard/file/deleteUser_test.go | 7 +- .../http/api/dashboard/file/delete_test.go | 7 +- .../http/api/dashboard/file/show_test.go | 14 ++-- .../http/api/dashboard/file/upload_test.go | 20 +++--- .../presentation/http/api/file/show_test.go | 14 ++-- 17 files changed, 169 insertions(+), 142 deletions(-) diff --git a/backend/application/dashboard/file/deleteFile/usecase.go b/backend/application/dashboard/file/deleteFile/usecase.go index fe0bc054..976c1ad9 100644 --- a/backend/application/dashboard/file/deleteFile/usecase.go +++ b/backend/application/dashboard/file/deleteFile/usecase.go @@ -24,7 +24,7 @@ func (uc *UseCase) Execute(request Request) error { return err } - if err := uc.storage.Delete(context.Background(), file.Name); err != nil { + if err := uc.storage.Delete(context.Background(), file.StoredName); err != nil { return err } diff --git a/backend/application/dashboard/file/deleteFile/usecase_test.go b/backend/application/dashboard/file/deleteFile/usecase_test.go index c024e36f..63d8a88e 100644 --- a/backend/application/dashboard/file/deleteFile/usecase_test.go +++ b/backend/application/dashboard/file/deleteFile/usecase_test.go @@ -23,8 +23,9 @@ func TestUseCase_Execute(t *testing.T) { r = Request{FileUUID: "file-uuid"} f = file.File{ - UUID: r.FileUUID, - Name: "file-name", + UUID: r.FileUUID, + Name: "file-name", + StoredName: "store-name", } ) @@ -32,7 +33,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.On("Delete", r.FileUUID).Return(nil) defer filesRepository.AssertExpectations(t) - storage.On("Delete", context.Background(), f.Name).Once().Return(nil) + storage.On("Delete", context.Background(), f.StoredName).Once().Return(nil) defer storage.AssertExpectations(t) err := NewUseCase(&filesRepository, &storage).Execute(r) @@ -73,8 +74,9 @@ func TestUseCase_Execute(t *testing.T) { r = Request{FileUUID: "file-uuid"} f = file.File{ - UUID: r.FileUUID, - Name: "file-name", + UUID: r.FileUUID, + Name: "file-name", + StoredName: "store-name", } expectedErr = errors.New("error") @@ -83,7 +85,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.On("GetOne", r.FileUUID).Once().Return(f, nil) defer filesRepository.AssertExpectations(t) - storage.On("Delete", context.Background(), f.Name).Once().Return(expectedErr) + storage.On("Delete", context.Background(), f.StoredName).Once().Return(expectedErr) defer storage.AssertExpectations(t) err := NewUseCase(&filesRepository, &storage).Execute(r) @@ -103,8 +105,9 @@ func TestUseCase_Execute(t *testing.T) { r = Request{FileUUID: "file-uuid"} f = file.File{ - UUID: r.FileUUID, - Name: "file-name", + UUID: r.FileUUID, + Name: "file-name", + StoredName: "store-name", } expectedErr = errors.New("error") @@ -114,7 +117,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.On("Delete", r.FileUUID).Return(expectedErr) defer filesRepository.AssertExpectations(t) - storage.On("Delete", context.Background(), f.Name).Once().Return(nil) + storage.On("Delete", context.Background(), f.StoredName).Once().Return(nil) defer storage.AssertExpectations(t) err := NewUseCase(&filesRepository, &storage).Execute(r) diff --git a/backend/application/dashboard/file/deleteUserFile/usecase.go b/backend/application/dashboard/file/deleteUserFile/usecase.go index 9e401e56..c52f1461 100644 --- a/backend/application/dashboard/file/deleteUserFile/usecase.go +++ b/backend/application/dashboard/file/deleteUserFile/usecase.go @@ -24,7 +24,7 @@ func (uc *UseCase) Execute(request Request) error { return err } - if err := uc.storage.Delete(context.Background(), file.Name); err != nil { + if err := uc.storage.Delete(context.Background(), file.StoredName); err != nil { return err } diff --git a/backend/application/dashboard/file/deleteUserFile/usecase_test.go b/backend/application/dashboard/file/deleteUserFile/usecase_test.go index 07f222ae..a802dce2 100644 --- a/backend/application/dashboard/file/deleteUserFile/usecase_test.go +++ b/backend/application/dashboard/file/deleteUserFile/usecase_test.go @@ -26,9 +26,10 @@ func TestUseCase_Execute(t *testing.T) { } f = file.File{ - UUID: r.FileUUID, - Name: "file-name", - OwnerUUID: r.OwnerUUID, + UUID: r.FileUUID, + Name: "file-name", + StoredName: "store-name", + OwnerUUID: r.OwnerUUID, } ) @@ -36,7 +37,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.On("DeleteByOwnerUUID", r.OwnerUUID, r.FileUUID).Return(nil) defer filesRepository.AssertExpectations(t) - storage.On("Delete", context.Background(), f.Name).Once().Return(nil) + storage.On("Delete", context.Background(), f.StoredName).Once().Return(nil) defer storage.AssertExpectations(t) err := NewUseCase(&filesRepository, &storage).Execute(r) @@ -83,9 +84,10 @@ func TestUseCase_Execute(t *testing.T) { } f = file.File{ - UUID: r.FileUUID, - Name: "file-name", - OwnerUUID: r.OwnerUUID, + UUID: r.FileUUID, + Name: "file-name", + StoredName: "store-name", + OwnerUUID: r.OwnerUUID, } expectedErr = errors.New("error") @@ -94,7 +96,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.On("GetOneByOwnerUUID", r.OwnerUUID, r.FileUUID).Once().Return(f, nil) defer filesRepository.AssertExpectations(t) - storage.On("Delete", context.Background(), f.Name).Once().Return(expectedErr) + storage.On("Delete", context.Background(), f.StoredName).Once().Return(expectedErr) defer storage.AssertExpectations(t) err := NewUseCase(&filesRepository, &storage).Execute(r) @@ -117,9 +119,10 @@ func TestUseCase_Execute(t *testing.T) { } f = file.File{ - UUID: r.FileUUID, - Name: "file-name", - OwnerUUID: r.OwnerUUID, + UUID: r.FileUUID, + Name: "file-name", + StoredName: "store-name", + OwnerUUID: r.OwnerUUID, } expectedErr = errors.New("error") @@ -129,7 +132,7 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.On("DeleteByOwnerUUID", r.OwnerUUID, r.FileUUID).Return(expectedErr) defer filesRepository.AssertExpectations(t) - storage.On("Delete", context.Background(), f.Name).Once().Return(nil) + storage.On("Delete", context.Background(), f.StoredName).Once().Return(nil) defer storage.AssertExpectations(t) err := NewUseCase(&filesRepository, &storage).Execute(r) diff --git a/backend/application/dashboard/file/getFile/useCase.go b/backend/application/dashboard/file/getFile/useCase.go index 48d8e20f..bfb7177a 100644 --- a/backend/application/dashboard/file/getFile/useCase.go +++ b/backend/application/dashboard/file/getFile/useCase.go @@ -24,7 +24,7 @@ func (uc *UseCase) Execute(UUID string) (*Response, error) { return nil, err } - reader, err := uc.storage.Read(context.Background(), f.UUID) + reader, err := uc.storage.Read(context.Background(), f.StoredName) if err != nil { return nil, err } diff --git a/backend/application/dashboard/file/uploadFile/request.go b/backend/application/dashboard/file/uploadFile/request.go index 4697c049..e7d31fb4 100644 --- a/backend/application/dashboard/file/uploadFile/request.go +++ b/backend/application/dashboard/file/uploadFile/request.go @@ -2,7 +2,9 @@ package createfile import ( "io" + "path/filepath" + "github.com/gofrs/uuid/v5" "github.com/khanzadimahdi/testproject/domain" ) @@ -37,3 +39,17 @@ func (r *Request) Validate() domain.ValidationErrors { return validationErrors } + +func (r *Request) StoredName() (string, error) { + var filename string + + extension := filepath.Ext(r.Name) + + uuid, err := uuid.NewV7() + if err != nil { + return filename, err + } + filename = uuid.String() + extension + + return filename, nil +} diff --git a/backend/application/dashboard/file/uploadFile/usecase.go b/backend/application/dashboard/file/uploadFile/usecase.go index 086c3a28..3b8b86b7 100644 --- a/backend/application/dashboard/file/uploadFile/usecase.go +++ b/backend/application/dashboard/file/uploadFile/usecase.go @@ -32,19 +32,23 @@ func (uc *UseCase) Execute(request *Request) (*Response, error) { }, nil } - uuid, err := uc.filesRepository.Save(&file.File{ - Name: request.Name, - Size: request.Size, - OwnerUUID: request.OwnerUUID, - MimeType: request.MimeType, - }) + storedName, err := request.StoredName() if err != nil { return nil, err } - if err := uc.storage.Store(context.Background(), uuid, request.FileReader, request.Size); err != nil { - _ = uc.filesRepository.Delete(uuid) + if err := uc.storage.Store(context.Background(), storedName, request.FileReader, request.Size); err != nil { + return nil, err + } + uuid, err := uc.filesRepository.Save(&file.File{ + Name: request.Name, + StoredName: storedName, + Size: request.Size, + OwnerUUID: request.OwnerUUID, + MimeType: request.MimeType, + }) + if err != nil { return nil, err } diff --git a/backend/application/dashboard/file/uploadFile/usecase_test.go b/backend/application/dashboard/file/uploadFile/usecase_test.go index d49a8068..60640bf1 100644 --- a/backend/application/dashboard/file/uploadFile/usecase_test.go +++ b/backend/application/dashboard/file/uploadFile/usecase_test.go @@ -3,10 +3,12 @@ package createfile import ( "context" "errors" + "path/filepath" "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/khanzadimahdi/testproject/domain" "github.com/khanzadimahdi/testproject/domain/file" @@ -33,12 +35,7 @@ func TestUseCase_Execute(t *testing.T) { OwnerUUID: "owner-uuid", FileReader: strings.NewReader(fileContent), Size: int64(len(fileContent)), - } - - f = file.File{ - Name: r.Name, - Size: r.Size, - OwnerUUID: r.OwnerUUID, + MimeType: "application/octet-stream", } fileUUID = "test-file-uuid" @@ -51,15 +48,17 @@ func TestUseCase_Execute(t *testing.T) { validator.On("Validate", &r).Once().Return(nil) defer validator.AssertExpectations(t) - filesRepository.On("Save", &f).Once().Return(fileUUID, nil) - defer filesRepository.AssertExpectations(t) - - storage.On("Store", context.Background(), fileUUID, r.FileReader, r.Size).Once().Return(nil) + storage.On("Store", context.Background(), mock.Anything, r.FileReader, r.Size).Once().Return(nil) defer storage.AssertExpectations(t) - response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) + matchingFile := mock.MatchedBy(func(f *file.File) bool { + return f.Name == r.Name && f.Size == r.Size && f.OwnerUUID == r.OwnerUUID && f.MimeType == r.MimeType && filepath.Ext(f.Name) == filepath.Ext(f.StoredName) + }) - filesRepository.AssertNotCalled(t, "Delete") + filesRepository.On("Save", matchingFile).Once().Return(fileUUID, nil) + defer filesRepository.AssertExpectations(t) + + response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) assert.NoError(t, err) assert.Equal(t, &expectedResponse, response) @@ -91,7 +90,6 @@ func TestUseCase_Execute(t *testing.T) { filesRepository.AssertNotCalled(t, "Save") storage.AssertNotCalled(t, "Store") - filesRepository.AssertNotCalled(t, "Delete") assert.NoError(t, err) assert.Equal(t, &expectedResponse, response) @@ -112,33 +110,24 @@ func TestUseCase_Execute(t *testing.T) { OwnerUUID: "owner-uuid", FileReader: strings.NewReader(fileContent), Size: int64(len(fileContent)), + MimeType: "application/octet-stream", } - f = file.File{ - Name: r.Name, - Size: r.Size, - OwnerUUID: r.OwnerUUID, - } - - fileUUID = "test-file-uuid" - expectedErr = errors.New("storage error") ) validator.On("Validate", &r).Once().Return(nil) defer validator.AssertExpectations(t) - filesRepository.On("Save", &f).Once().Return(fileUUID, nil) - defer filesRepository.AssertExpectations(t) - - storage.On("Store", context.Background(), fileUUID, r.FileReader, r.Size).Once().Return(expectedErr) + storage.On("Store", context.Background(), mock.MatchedBy(func(storedName string) bool { + return filepath.Ext(r.Name) == filepath.Ext(storedName) && storedName != r.Name + }), r.FileReader, r.Size).Once().Return(expectedErr) defer storage.AssertExpectations(t) - filesRepository.On("Delete", fileUUID).Once().Return(nil) - defer filesRepository.AssertExpectations(t) - response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) + filesRepository.AssertNotCalled(t, "Save") + assert.ErrorIs(t, err, expectedErr) assert.Nil(t, response) }) @@ -158,12 +147,7 @@ func TestUseCase_Execute(t *testing.T) { OwnerUUID: "owner-uuid", FileReader: strings.NewReader(fileContent), Size: int64(len(fileContent)), - } - - f = file.File{ - Name: r.Name, - Size: r.Size, - OwnerUUID: r.OwnerUUID, + MimeType: "application/octet-stream", } expectedErr = errors.New("error") @@ -172,14 +156,20 @@ func TestUseCase_Execute(t *testing.T) { validator.On("Validate", &r).Once().Return(nil) defer validator.AssertExpectations(t) - filesRepository.On("Save", &f).Once().Return("", expectedErr) + storage.On("Store", context.Background(), mock.MatchedBy(func(storedName string) bool { + return filepath.Ext(r.Name) == filepath.Ext(storedName) && storedName != r.Name + }), r.FileReader, r.Size).Once().Return(nil) + defer storage.AssertExpectations(t) + + matchingFile := mock.MatchedBy(func(f *file.File) bool { + return f.Name == r.Name && f.Size == r.Size && f.OwnerUUID == r.OwnerUUID && f.MimeType == r.MimeType && filepath.Ext(f.Name) == filepath.Ext(f.StoredName) + }) + + filesRepository.On("Save", matchingFile).Once().Return("", expectedErr) defer filesRepository.AssertExpectations(t) response, err := NewUseCase(&filesRepository, &storage, &validator).Execute(&r) - storage.AssertNotCalled(t, "Store") - filesRepository.AssertNotCalled(t, "Delete") - assert.ErrorIs(t, err, expectedErr) assert.Nil(t, response) }) diff --git a/backend/application/file/getFile/useCase.go b/backend/application/file/getFile/useCase.go index 48d8e20f..bfb7177a 100644 --- a/backend/application/file/getFile/useCase.go +++ b/backend/application/file/getFile/useCase.go @@ -24,7 +24,7 @@ func (uc *UseCase) Execute(UUID string) (*Response, error) { return nil, err } - reader, err := uc.storage.Read(context.Background(), f.UUID) + reader, err := uc.storage.Read(context.Background(), f.StoredName) if err != nil { return nil, err } diff --git a/backend/domain/file/file.go b/backend/domain/file/file.go index 178830a2..9f243922 100644 --- a/backend/domain/file/file.go +++ b/backend/domain/file/file.go @@ -7,12 +7,13 @@ import ( ) type File struct { - UUID string - Name string - Size int64 - OwnerUUID string - MimeType string - CreatedAt time.Time + UUID string + Name string + StoredName string + Size int64 + OwnerUUID string + MimeType string + CreatedAt time.Time } type Repository interface { diff --git a/backend/infrastructure/repository/mongodb/files/model.go b/backend/infrastructure/repository/mongodb/files/model.go index 6fb90e40..8fe91622 100644 --- a/backend/infrastructure/repository/mongodb/files/model.go +++ b/backend/infrastructure/repository/mongodb/files/model.go @@ -5,12 +5,13 @@ import ( ) type FileBson struct { - UUID string `bson:"_id,omitempty"` - Name string `bson:"name"` - Size int64 `bson:"size"` - OwnerUUID string `bson:"owner_uuid"` - MimeType string `bson:"mimetype"` - CreatedAt time.Time `bson:"created_at,omitempty"` + UUID string `bson:"_id,omitempty"` + Name string `bson:"name"` + StoredName string `bson:"stored_name"` + Size int64 `bson:"size"` + OwnerUUID string `bson:"owner_uuid"` + MimeType string `bson:"mimetype"` + CreatedAt time.Time `bson:"created_at,omitempty"` } type SetWrapper struct { diff --git a/backend/infrastructure/repository/mongodb/files/repository.go b/backend/infrastructure/repository/mongodb/files/repository.go index ff2893cc..8fcc49d3 100644 --- a/backend/infrastructure/repository/mongodb/files/repository.go +++ b/backend/infrastructure/repository/mongodb/files/repository.go @@ -62,12 +62,13 @@ func (r *FilesRepository) GetAll(offset uint, limit uint) ([]file.File, error) { return nil, err } items = append(items, file.File{ - UUID: a.UUID, - Name: a.Name, - Size: a.Size, - OwnerUUID: a.OwnerUUID, - MimeType: a.MimeType, - CreatedAt: a.CreatedAt, + UUID: a.UUID, + Name: a.Name, + StoredName: a.StoredName, + Size: a.Size, + OwnerUUID: a.OwnerUUID, + MimeType: a.MimeType, + CreatedAt: a.CreatedAt, }) } @@ -91,12 +92,13 @@ func (r *FilesRepository) GetOne(UUID string) (file.File, error) { } return file.File{ - UUID: a.UUID, - Name: a.Name, - Size: a.Size, - OwnerUUID: a.OwnerUUID, - MimeType: a.MimeType, - CreatedAt: a.CreatedAt, + UUID: a.UUID, + Name: a.Name, + StoredName: a.StoredName, + Size: a.Size, + OwnerUUID: a.OwnerUUID, + MimeType: a.MimeType, + CreatedAt: a.CreatedAt, }, nil } @@ -113,12 +115,13 @@ func (r *FilesRepository) Save(a *file.File) (string, error) { } update := FileBson{ - UUID: a.UUID, - Name: a.Name, - Size: a.Size, - OwnerUUID: a.OwnerUUID, - MimeType: a.MimeType, - CreatedAt: time.Now(), + UUID: a.UUID, + Name: a.Name, + StoredName: a.StoredName, + Size: a.Size, + OwnerUUID: a.OwnerUUID, + MimeType: a.MimeType, + CreatedAt: time.Now(), } upsert := true @@ -187,12 +190,13 @@ func (r *FilesRepository) GetAllByOwnerUUID(ownerUUID string, offset uint, limit return nil, err } items = append(items, file.File{ - UUID: a.UUID, - Name: a.Name, - Size: a.Size, - OwnerUUID: a.OwnerUUID, - MimeType: a.MimeType, - CreatedAt: a.CreatedAt, + UUID: a.UUID, + Name: a.Name, + StoredName: a.StoredName, + Size: a.Size, + OwnerUUID: a.OwnerUUID, + MimeType: a.MimeType, + CreatedAt: a.CreatedAt, }) } @@ -221,12 +225,13 @@ func (r *FilesRepository) GetOneByOwnerUUID(ownerUUID string, UUID string) (file } return file.File{ - UUID: a.UUID, - Name: a.Name, - Size: a.Size, - OwnerUUID: a.OwnerUUID, - MimeType: a.MimeType, - CreatedAt: a.CreatedAt, + UUID: a.UUID, + Name: a.Name, + StoredName: a.StoredName, + Size: a.Size, + OwnerUUID: a.OwnerUUID, + MimeType: a.MimeType, + CreatedAt: a.CreatedAt, }, nil } diff --git a/backend/presentation/http/api/dashboard/file/deleteUser_test.go b/backend/presentation/http/api/dashboard/file/deleteUser_test.go index 43226c43..c626716e 100644 --- a/backend/presentation/http/api/dashboard/file/deleteUser_test.go +++ b/backend/presentation/http/api/dashboard/file/deleteUser_test.go @@ -38,8 +38,9 @@ func TestDeleteUserHandler(t *testing.T) { } f = file.File{ - UUID: r.FileUUID, - Name: "file-name", + UUID: r.FileUUID, + Name: "file-name", + StoredName: "stored-name", } ) @@ -50,7 +51,7 @@ func TestDeleteUserHandler(t *testing.T) { filesRepository.On("DeleteByOwnerUUID", r.OwnerUUID, r.FileUUID).Return(nil) defer filesRepository.AssertExpectations(t) - storage.On("Delete", context.Background(), f.Name).Once().Return(nil) + storage.On("Delete", context.Background(), f.StoredName).Once().Return(nil) defer storage.AssertExpectations(t) handler := NewDeleteUserHandler(deleteuserfile.NewUseCase(&filesRepository, &storage), &authorizer) diff --git a/backend/presentation/http/api/dashboard/file/delete_test.go b/backend/presentation/http/api/dashboard/file/delete_test.go index 10ba96f9..548bbeae 100644 --- a/backend/presentation/http/api/dashboard/file/delete_test.go +++ b/backend/presentation/http/api/dashboard/file/delete_test.go @@ -35,8 +35,9 @@ func TestDeleteHandler(t *testing.T) { r = deletefile.Request{FileUUID: "file-uuid"} f = file.File{ - UUID: r.FileUUID, - Name: "file-name", + UUID: r.FileUUID, + Name: "file-name", + StoredName: "stored-name", } ) @@ -47,7 +48,7 @@ func TestDeleteHandler(t *testing.T) { filesRepository.On("Delete", r.FileUUID).Return(nil) defer filesRepository.AssertExpectations(t) - storage.On("Delete", context.Background(), f.Name).Once().Return(nil) + storage.On("Delete", context.Background(), f.StoredName).Once().Return(nil) defer storage.AssertExpectations(t) handler := NewDeleteHandler(deletefile.NewUseCase(&filesRepository, &storage), &authorizer) diff --git a/backend/presentation/http/api/dashboard/file/show_test.go b/backend/presentation/http/api/dashboard/file/show_test.go index cd3ca4b7..77b1f3cc 100644 --- a/backend/presentation/http/api/dashboard/file/show_test.go +++ b/backend/presentation/http/api/dashboard/file/show_test.go @@ -36,8 +36,9 @@ func TestShowHandler(t *testing.T) { u = user.User{UUID: "auth-user-uuid"} f = file.File{ - UUID: "file-test-uuid", - Name: "file-test-name", + UUID: "file-test-uuid", + Name: "file-test-name", + StoredName: "stored-file-name", } ) @@ -50,7 +51,7 @@ func TestShowHandler(t *testing.T) { filesRepository.On("GetOne", f.UUID).Once().Return(f, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), f.UUID).Once().Return(reader, nil) + storage.On("Read", context.Background(), f.StoredName).Once().Return(reader, nil) defer storage.AssertExpectations(t) useCase := getfile.NewUseCase(&filesRepository, &storage) @@ -153,8 +154,9 @@ func TestShowHandler(t *testing.T) { u = user.User{UUID: "auth-user-uuid"} file = file.File{ - UUID: "file-test-uuid", - Name: "file-test-name", + UUID: "file-test-uuid", + Name: "file-test-name", + StoredName: "stored-file-name", } ) @@ -167,7 +169,7 @@ func TestShowHandler(t *testing.T) { filesRepository.On("GetOne", file.UUID).Once().Return(file, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), file.UUID).Once().Return(reader, errors.New("some error")) + storage.On("Read", context.Background(), file.StoredName).Once().Return(reader, errors.New("some error")) defer storage.AssertExpectations(t) useCase := getfile.NewUseCase(&filesRepository, &storage) diff --git a/backend/presentation/http/api/dashboard/file/upload_test.go b/backend/presentation/http/api/dashboard/file/upload_test.go index e558e977..6cc334fb 100644 --- a/backend/presentation/http/api/dashboard/file/upload_test.go +++ b/backend/presentation/http/api/dashboard/file/upload_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -46,13 +47,6 @@ func TestUploadHandler(t *testing.T) { } fileUUID = "test-file-uuid" - - f = file.File{ - Name: r.Name, - Size: r.Size, - OwnerUUID: u.UUID, - MimeType: "application/octet-stream", - } ) var payload bytes.Buffer @@ -70,12 +64,16 @@ func TestUploadHandler(t *testing.T) { requestValidator.On("Validate", mock.Anything).Once().Return(nil) defer requestValidator.AssertExpectations(t) - filesRepository.On("Save", &f).Once().Return(fileUUID, nil) - defer filesRepository.AssertExpectations(t) - - storage.On("Store", context.Background(), fileUUID, mock.Anything, r.Size).Once().Return(nil) + storage.On("Store", context.Background(), mock.Anything, mock.Anything, r.Size).Once().Return(nil) defer storage.AssertExpectations(t) + matchingFile := mock.MatchedBy(func(f *file.File) bool { + return f.Name == r.Name && f.Size == r.Size && f.OwnerUUID == u.UUID && f.MimeType == "application/octet-stream" && filepath.Ext(f.Name) == filepath.Ext(f.StoredName) + }) + + filesRepository.On("Save", matchingFile).Once().Return(fileUUID, nil) + defer filesRepository.AssertExpectations(t) + handler := NewUploadHandler(createfile.NewUseCase(&filesRepository, &storage, &requestValidator), &authorizer) request := httptest.NewRequest(http.MethodPost, "/", &payload) diff --git a/backend/presentation/http/api/file/show_test.go b/backend/presentation/http/api/file/show_test.go index 23f64843..e2ef6225 100644 --- a/backend/presentation/http/api/file/show_test.go +++ b/backend/presentation/http/api/file/show_test.go @@ -31,8 +31,9 @@ func TestShowHandler(t *testing.T) { ) f := file.File{ - UUID: "file-test-uuid", - Name: "file-test-name", + UUID: "file-test-uuid", + Name: "file-test-name", + StoredName: "stored-file-name", } fileData := []byte("this is the file payload") @@ -41,7 +42,7 @@ func TestShowHandler(t *testing.T) { filesRepository.On("GetOne", f.UUID).Once().Return(f, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), f.UUID).Once().Return(reader, nil) + storage.On("Read", context.Background(), f.StoredName).Once().Return(reader, nil) defer storage.AssertExpectations(t) useCase := getfile.NewUseCase(&filesRepository, &storage) @@ -98,8 +99,9 @@ func TestShowHandler(t *testing.T) { ) file := file.File{ - UUID: "file-test-uuid", - Name: "file-test-name", + UUID: "file-test-uuid", + Name: "file-test-name", + StoredName: "stored-file-name", } fileData := "this is the file payload" @@ -108,7 +110,7 @@ func TestShowHandler(t *testing.T) { filesRepository.On("GetOne", file.UUID).Once().Return(file, nil) defer filesRepository.AssertExpectations(t) - storage.On("Read", context.Background(), file.UUID).Once().Return(reader, errors.New("some error")) + storage.On("Read", context.Background(), file.StoredName).Once().Return(reader, errors.New("some error")) defer storage.AssertExpectations(t) useCase := getfile.NewUseCase(&filesRepository, &storage) From af8fdf4bbd98cc62bcacb6fddf79e307f7f6aba2 Mon Sep 17 00:00:00 2001 From: Mahdi Khanzadi Date: Sat, 11 Jan 2025 14:01:08 +0100 Subject: [PATCH 24/25] remove stale env variables --- .env | 2 -- .github/workflows/backend.yaml | 54 ++++++++++++++++----------------- .github/workflows/frontend.yaml | 14 ++++++--- compose.yaml | 1 - frontend/.env.local.example | 1 - frontend/compose.frontend.yaml | 4 +-- frontend/next.config.mjs | 1 - frontend/src/constants/envs.ts | 2 +- 8 files changed, 39 insertions(+), 40 deletions(-) diff --git a/.env b/.env index f2b86087..c6b2cdba 100644 --- a/.env +++ b/.env @@ -30,10 +30,8 @@ NATS_URL=nats:4222 INTERNAL_BACKEND_BASE_URL=https://backend.tarhche.com NEXT_PUBLIC_FILES_PROTOCOL=https NEXT_PUBLIC_FILES_HOST=backend.tarhche.com -NEXT_PUBLIC_FILES_PORT=80 ## frontend local # INTERNAL_BACKEND_BASE_URL=http://app # NEXT_PUBLIC_FILES_PROTOCOL=http # NEXT_PUBLIC_FILES_HOST=127.0.0.1 -# NEXT_PUBLIC_FILES_PORT=8000 diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index d7c4f438..e452fbfa 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -91,7 +91,7 @@ jobs: - name: Deploy services run: | # Setup ssh key - echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" > ~/ec2-key.pem + echo '${{ secrets.EC2_SSH_PRIVATE_KEY }}' > ~/ec2-key.pem chmod 400 ~/ec2-key.pem mkdir -p ~/.ssh @@ -109,32 +109,32 @@ jobs: # Connect and deploy services ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' - export MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" - export MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" - - export BACKEND_NATS_URL="${{ secrets.BACKEND_NATS_URL }}" - export BACKEND_PRIVATE_KEY="${{ secrets.BACKEND_PRIVATE_KEY }}" - - export BACKEND_MONGO_HOST="mongodb" - export BACKEND_MONGO_PORT="27017" - export BACKEND_MONGO_SCHEME="mongodb" - export BACKEND_MONGO_DATABASE_NAME="${{ secrets.BACKEND_MONGO_DATABASE_NAME }}" - export BACKEND_MONGO_USERNAME="${{ secrets.MONGO_USERNAME }}" - export BACKEND_MONGO_PASSWORD="${{ secrets.MONGO_PASSWORD }}" - - export BACKEND_MAIL_SMTP_PASSWORD="${{ secrets.BACKEND_MAIL_SMTP_PASSWORD }}" - export BACKEND_MAIL_SMTP_HOST="${{ secrets.BACKEND_MAIL_SMTP_HOST }}" - export BACKEND_MAIL_SMTP_FROM="${{ secrets.BACKEND_MAIL_SMTP_FROM }}" - export BACKEND_MAIL_SMTP_USERNAME="${{ secrets.BACKEND_MAIL_SMTP_USERNAME }}" - export BACKEND_MAIL_SMTP_PORT="${{ secrets.BACKEND_MAIL_SMTP_PORT }}" - - export BACKEND_S3_ENDPOINT="${{ secrets.BACKEND_S3_ENDPOINT }}" - export BACKEND_S3_SECRET_KEY="${{ secrets.BACKEND_S3_SECRET_KEY }}" - export BACKEND_S3_ACCESS_KEY="${{ secrets.BACKEND_S3_ACCESS_KEY }}" - export BACKEND_S3_USE_SSL="${{ secrets.BACKEND_S3_USE_SSL }}" - export BACKEND_S3_BUCKET_NAME="${{ secrets.BACKEND_S3_BUCKET_NAME }}" - - export APP_IMAGE="${{ secrets.APP_IMAGE }}" + export MONGO_USERNAME='${{ secrets.MONGO_USERNAME }}' + export MONGO_PASSWORD='${{ secrets.MONGO_PASSWORD }}' + + export BACKEND_NATS_URL='${{ secrets.BACKEND_NATS_URL }}' + export BACKEND_PRIVATE_KEY='${{ secrets.BACKEND_PRIVATE_KEY }}' + + export BACKEND_MONGO_HOST='mongodb' + export BACKEND_MONGO_PORT='27017' + export BACKEND_MONGO_SCHEME='mongodb' + export BACKEND_MONGO_DATABASE_NAME='${{ secrets.BACKEND_MONGO_DATABASE_NAME }}' + export BACKEND_MONGO_USERNAME='${{ secrets.MONGO_USERNAME }}' + export BACKEND_MONGO_PASSWORD='${{ secrets.MONGO_PASSWORD }}' + + export BACKEND_MAIL_SMTP_PASSWORD='${{ secrets.BACKEND_MAIL_SMTP_PASSWORD }}' + export BACKEND_MAIL_SMTP_HOST='${{ secrets.BACKEND_MAIL_SMTP_HOST }}' + export BACKEND_MAIL_SMTP_FROM='${{ secrets.BACKEND_MAIL_SMTP_FROM }}' + export BACKEND_MAIL_SMTP_USERNAME='${{ secrets.BACKEND_MAIL_SMTP_USERNAME }}' + export BACKEND_MAIL_SMTP_PORT='${{ secrets.BACKEND_MAIL_SMTP_PORT }}' + + export BACKEND_S3_ENDPOINT='${{ secrets.BACKEND_S3_ENDPOINT }}' + export BACKEND_S3_SECRET_KEY='${{ secrets.BACKEND_S3_SECRET_KEY }}' + export BACKEND_S3_ACCESS_KEY='${{ secrets.BACKEND_S3_ACCESS_KEY }}' + export BACKEND_S3_USE_SSL='${{ secrets.BACKEND_S3_USE_SSL }}' + export BACKEND_S3_BUCKET_NAME='${{ secrets.BACKEND_S3_BUCKET_NAME }}' + + export APP_IMAGE='${{ secrets.APP_IMAGE }}' # Run Docker Compose cd /tmp/deployment_backend/ diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 5eee3cab..3899c86f 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -21,6 +21,10 @@ env: EC2_SSH_ADDRESS: ${{ secrets.EC2_SSH_ADDRESS }} EC2_SSH_ENDPOINT: ${{ secrets.EC2_SSH_USER }}@${{ secrets.EC2_SSH_ADDRESS }} + INTERNAL_BACKEND_BASE_URL: ${{ secrets.INTERNAL_BACKEND_BASE_URL }} + NEXT_PUBLIC_FILES_PROTOCOL: https + NEXT_PUBLIC_FILES_HOST: ${{ secrets.NEXT_PUBLIC_FILES_HOST }} + jobs: ci: runs-on: ubuntu-latest @@ -82,7 +86,7 @@ jobs: - name: Deploy services run: | # Setup ssh key - echo "${{ secrets.EC2_SSH_PRIVATE_KEY }}" > ~/ec2-key.pem + echo '${{ secrets.EC2_SSH_PRIVATE_KEY }}' > ~/ec2-key.pem chmod 400 ~/ec2-key.pem mkdir -p ~/.ssh @@ -99,10 +103,10 @@ jobs: # Connect and deploy services ssh -q -i ~/ec2-key.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $EC2_SSH_ENDPOINT > /dev/null 2>&1 << 'EOF' - export FRONTEND_IMAGE="${{ secrets.FRONTEND_IMAGE }}" - export NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL="${{ secrets.NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL }}" - export INTERNAL_BACKEND_BASE_URL="${{ secrets.INTERNAL_BACKEND_BASE_URL }}" - export NEXT_PUBLIC_FILES_BASE_URL="${{ secrets.NEXT_PUBLIC_FILES_BASE_URL }}" + export FRONTEND_IMAGE='${{ secrets.FRONTEND_IMAGE }}' + export INTERNAL_BACKEND_BASE_URL='${{ env.INTERNAL_BACKEND_BASE_URL }}' + export NEXT_PUBLIC_FILES_PROTOCOL='${{ env.NEXT_PUBLIC_FILES_PROTOCOL }}' + export NEXT_PUBLIC_FILES_HOST='${{ env.NEXT_PUBLIC_FILES_HOST }}' # Run Docker Compose cd /tmp/deployment_frontend/ diff --git a/compose.yaml b/compose.yaml index 9fb6ca46..0a8114be 100644 --- a/compose.yaml +++ b/compose.yaml @@ -48,7 +48,6 @@ services: INTERNAL_BACKEND_BASE_URL: ${INTERNAL_BACKEND_BASE_URL} NEXT_PUBLIC_FILES_PROTOCOL: ${NEXT_PUBLIC_FILES_PROTOCOL} NEXT_PUBLIC_FILES_HOST: ${NEXT_PUBLIC_FILES_HOST} - NEXT_PUBLIC_FILES_PORT: ${NEXT_PUBLIC_FILES_PORT} command: > sh -c "npm install; npm run dev -- --hostname \"0.0.0.0\" --port 3000" diff --git a/frontend/.env.local.example b/frontend/.env.local.example index 9769d98d..84a29ce9 100644 --- a/frontend/.env.local.example +++ b/frontend/.env.local.example @@ -1,4 +1,3 @@ INTERNAL_BACKEND_BASE_URL=https://backend.tarhche.com NEXT_PUBLIC_FILES_PROTOCOL=https NEXT_PUBLIC_FILES_HOST=backend.tarhche.com -NEXT_PUBLIC_FILES_PORT=80 diff --git a/frontend/compose.frontend.yaml b/frontend/compose.frontend.yaml index eb396d2d..b2155cf7 100644 --- a/frontend/compose.frontend.yaml +++ b/frontend/compose.frontend.yaml @@ -13,9 +13,9 @@ services: delay: 5s max_attempts: 3 environment: - NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL: ${NEXT_PUBLIC_EXTERNAL_BACKEND_BASE_URL} INTERNAL_BACKEND_BASE_URL: ${INTERNAL_BACKEND_BASE_URL} - NEXT_PUBLIC_FILES_BASE_URL: ${NEXT_PUBLIC_FILES_BASE_URL} + NEXT_PUBLIC_FILES_PROTOCOL: ${NEXT_PUBLIC_FILES_PROTOCOL} + NEXT_PUBLIC_FILES_HOST: ${NEXT_PUBLIC_FILES_HOST} networks: frontend: diff --git a/frontend/next.config.mjs b/frontend/next.config.mjs index 98c889e3..354c3602 100644 --- a/frontend/next.config.mjs +++ b/frontend/next.config.mjs @@ -8,7 +8,6 @@ const nextConfig = { { protocol: process.env.NEXT_PUBLIC_FILES_PROTOCOL, hostname: process.env.NEXT_PUBLIC_FILES_HOST, - port: process.env.NEXT_PUBLIC_FILES_PORT, }, ], }, diff --git a/frontend/src/constants/envs.ts b/frontend/src/constants/envs.ts index 8def7fe8..e408dd91 100644 --- a/frontend/src/constants/envs.ts +++ b/frontend/src/constants/envs.ts @@ -1,2 +1,2 @@ export const INTERNAL_BACKEND_URL = process.env.INTERNAL_BACKEND_BASE_URL; -export const FILES_PUBLIC_URL = `${process.env.NEXT_PUBLIC_FILES_PROTOCOL}://${process.env.NEXT_PUBLIC_FILES_HOST}:${process.env.NEXT_PUBLIC_FILES_PORT}/files`; +export const FILES_PUBLIC_URL = `${process.env.NEXT_PUBLIC_FILES_PROTOCOL}://${process.env.NEXT_PUBLIC_FILES_HOST}/files`; From 8bd946606fb202c7d3f300039b39bc0bce4d6222 Mon Sep 17 00:00:00 2001 From: Mahdi Khanzadi Date: Sat, 11 Jan 2025 20:36:39 +0100 Subject: [PATCH 25/25] introduce resource limitation --- backend/compose.app.yaml | 6 +++++- frontend/compose.frontend.yaml | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/compose.app.yaml b/backend/compose.app.yaml index c4a07ec5..d871f12b 100644 --- a/backend/compose.app.yaml +++ b/backend/compose.app.yaml @@ -8,12 +8,16 @@ services: - docker deploy: mode: replicated - replicas: 1 + replicas: 2 endpoint_mode: vip restart_policy: condition: on-failure delay: 5s max_attempts: 3 + resources: + limits: + cpus: '0.20' + memory: 150M environment: PRIVATE_KEY: ${BACKEND_PRIVATE_KEY} S3_ENDPOINT: ${BACKEND_S3_ENDPOINT} diff --git a/frontend/compose.frontend.yaml b/frontend/compose.frontend.yaml index b2155cf7..396ab31f 100644 --- a/frontend/compose.frontend.yaml +++ b/frontend/compose.frontend.yaml @@ -12,6 +12,10 @@ services: condition: on-failure delay: 5s max_attempts: 3 + resources: + limits: + cpus: '0.20' + memory: 150M environment: INTERNAL_BACKEND_BASE_URL: ${INTERNAL_BACKEND_BASE_URL} NEXT_PUBLIC_FILES_PROTOCOL: ${NEXT_PUBLIC_FILES_PROTOCOL}