Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: added git resource map consumption #38470

Merged
merged 6 commits into from
Jan 5, 2025
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -674,6 +674,13 @@ public Mono<ApplicationGitReference> reconstructApplicationReferenceFromGitRepo(
.subscribeOn(scheduler);
}

@Override
public Mono<GitResourceMap> constructGitResourceMapFromGitRepo(Path repositorySuffix, String refName) {
// TODO: check that we need to checkout to the ref
Path repositoryPath = Paths.get(gitServiceConfig.getGitRootPath()).resolve(repositorySuffix);
return Mono.fromCallable(() -> fetchGitResourceMap(repositoryPath)).subscribeOn(scheduler);
}

/**
* This is used to initialize repo with Readme file when the application is connected to remote repo
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ Mono<Path> saveArtifactToGitRepo(Path baseRepoSuffix, GitResourceMap gitResource
Mono<ApplicationGitReference> reconstructApplicationReferenceFromGitRepo(
String organisationId, String baseApplicationId, String repoName, String branchName);

Mono<GitResourceMap> constructGitResourceMapFromGitRepo(Path repositorySuffix, String refName);

/**
* This method just reconstructs the metdata of the json from git repo.
*
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.appsmith.server.applications.git;

import com.appsmith.external.git.FileInterface;
import com.appsmith.external.git.constants.ce.RefType;
import com.appsmith.external.git.models.GitResourceIdentity;
import com.appsmith.external.git.models.GitResourceMap;
import com.appsmith.external.git.models.GitResourceType;
Expand Down Expand Up @@ -29,6 +30,7 @@
import com.appsmith.server.dtos.PageDTO;
import com.appsmith.server.exceptions.AppsmithError;
import com.appsmith.server.exceptions.AppsmithException;
import com.appsmith.server.git.dtos.ArtifactJsonTransformationDTO;
import com.appsmith.server.helpers.CollectionUtils;
import com.appsmith.server.helpers.ce.ArtifactGitFileUtilsCE;
import com.appsmith.server.migrations.JsonSchemaMigration;
Expand Down Expand Up @@ -467,10 +469,19 @@ public Mono<ArtifactExchangeJson> reconstructArtifactExchangeJsonFromFilesInRepo
ApplicationJson applicationJson = getApplicationJsonFromGitReference(applicationReference);
copyNestedNonNullProperties(metadata, applicationJson);
return jsonSchemaMigration.migrateApplicationJsonToLatestSchema(
applicationJson, baseArtifactId, branchName);
applicationJson, baseArtifactId, branchName, RefType.branch);
});
}

@Override
public Mono<? extends ArtifactExchangeJson> performJsonMigration(
ArtifactJsonTransformationDTO jsonTransformationDTO, ArtifactExchangeJson artifactExchangeJson) {
String baseArtifactId = jsonTransformationDTO.getBaseArtifactId();
String refName = jsonTransformationDTO.getRefName();
return jsonSchemaMigration.migrateArtifactExchangeJsonToLatestSchema(
nidhi-nair marked this conversation as resolved.
Show resolved Hide resolved
artifactExchangeJson, baseArtifactId, refName, null);
}

protected <T> List<T> getApplicationResource(Map<String, Object> resources, Type type) {

List<T> deserializedResources = new ArrayList<>();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.appsmith.server.applications.imports;

import com.appsmith.external.git.constants.ce.RefType;
import com.appsmith.external.models.Datasource;
import com.appsmith.external.models.DatasourceStorageDTO;
import com.appsmith.server.applications.base.ApplicationService;
Expand Down Expand Up @@ -643,18 +644,21 @@ public Mono<ApplicationJson> migrateArtifactExchangeJson(
ApplicationJson applicationJson = (ApplicationJson) artifactExchangeJson;

if (!hasText(branchedArtifactId)) {
return jsonSchemaMigration.migrateApplicationJsonToLatestSchema(applicationJson, null, null);
return jsonSchemaMigration.migrateApplicationJsonToLatestSchema(applicationJson, null, null, null);
nidhi-nair marked this conversation as resolved.
Show resolved Hide resolved
}

return applicationService.findById(branchedArtifactId).flatMap(application -> {
String baseArtifactId = application.getBaseId();
String refName = null;
RefType refType = null;

if (application.getGitArtifactMetadata() != null) {
refName = application.getGitArtifactMetadata().getRefName();
refType = application.getGitArtifactMetadata().getRefType();
}

return jsonSchemaMigration.migrateApplicationJsonToLatestSchema(applicationJson, baseArtifactId, refName);
return jsonSchemaMigration.migrateApplicationJsonToLatestSchema(
applicationJson, baseArtifactId, refName, refType);
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ Mono<? extends ArtifactImportDTO> importArtifactFromGit(

Mono<? extends Artifact> connectArtifactToGit(
String baseArtifactId,
ArtifactType artifactType,
GitConnectDTO gitConnectDTO,
String originHeader,
ArtifactType artifactType,
GitType gitType);

Mono<String> commitArtifact(
Expand All @@ -36,44 +36,44 @@ Mono<String> commitArtifact(
Mono<? extends Artifact> detachRemote(String branchedArtifactId, ArtifactType artifactType, GitType gitType);

Mono<List<GitBranchDTO>> listBranchForArtifact(
String branchedArtifactId, Boolean pruneBranches, ArtifactType artifactType, GitType gitType);
String branchedArtifactId, ArtifactType artifactType, Boolean pruneBranches, GitType gitType);

Mono<String> fetchRemoteChanges(
String referenceArtifactId,
boolean isFileLock,
ArtifactType artifactType,
boolean isFileLock,
GitType gitType,
RefType refType);

Mono<? extends Artifact> discardChanges(String branchedArtifactId, ArtifactType artifactType, GitType gitType);

Mono<GitStatusDTO> getStatus(
String branchedArtifactId, boolean compareRemote, ArtifactType artifactType, GitType gitType);
String branchedArtifactId, ArtifactType artifactType, boolean compareRemote, GitType gitType);

Mono<GitPullDTO> pullArtifact(String branchedArtifactId, ArtifactType artifactType, GitType gitType);

Mono<? extends Artifact> checkoutReference(
String referenceArtifactId,
ArtifactType artifactType,
GitRefDTO gitRefDTO,
boolean addFileLock,
ArtifactType artifactType,
GitType gitType);

Mono<? extends Artifact> createReference(
String referencedArtifactId, GitRefDTO refDTO, ArtifactType artifactType, GitType gitType);
String referencedArtifactId, ArtifactType artifactType, GitRefDTO refDTO, GitType gitType);

Mono<? extends Artifact> deleteGitReference(
String baseArtifactId, GitRefDTO gitRefDTO, ArtifactType artifactType, GitType gitType);
String baseArtifactId, ArtifactType artifactType, GitRefDTO gitRefDTO, GitType gitType);

Mono<List<String>> updateProtectedBranches(
String baseArtifactId, List<String> branchNames, ArtifactType artifactType);
String baseArtifactId, ArtifactType artifactType, List<String> branchNames);

Mono<List<String>> getProtectedBranches(String baseArtifactId, ArtifactType artifactType);

Mono<Boolean> toggleAutoCommitEnabled(String baseArtifactId, ArtifactType artifactType);

Mono<AutoCommitResponseDTO> getAutoCommitProgress(
String baseArtifactId, String branchName, ArtifactType artifactType);
String baseArtifactId, ArtifactType artifactType, String branchName);

Mono<GitAuth> generateSSHKey(String keyType);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -364,9 +364,9 @@ private boolean checkIsDatasourceNameConflict(
@Override
public Mono<? extends Artifact> checkoutReference(
String referenceArtifactId,
ArtifactType artifactType,
GitRefDTO gitRefDTO,
boolean addFileLock,
ArtifactType artifactType,
GitType gitType) {

if (gitRefDTO == null || !hasText(gitRefDTO.getRefName())) {
Expand Down Expand Up @@ -423,6 +423,7 @@ protected Mono<? extends Artifact> checkoutReference(
ArtifactJsonTransformationDTO jsonTransformationDTO = new ArtifactJsonTransformationDTO();
jsonTransformationDTO.setWorkspaceId(baseArtifact.getWorkspaceId());
jsonTransformationDTO.setBaseArtifactId(baseGitMetadata.getDefaultArtifactId());
jsonTransformationDTO.setRefName(finalRefName);
jsonTransformationDTO.setRefType(refType);
jsonTransformationDTO.setArtifactType(baseArtifact.getArtifactType());
jsonTransformationDTO.setRepoName(baseGitMetadata.getRepoName());
Expand Down Expand Up @@ -474,7 +475,7 @@ protected Mono<? extends Artifact> checkoutReference(
}

protected Mono<? extends Artifact> checkoutRemoteReference(
String baseArtifactId, GitRefDTO gitRefDTO, ArtifactType artifactType, GitType gitType) {
String baseArtifactId, ArtifactType artifactType, GitRefDTO gitRefDTO, GitType gitType) {

GitArtifactHelper<?> gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
AclPermission artifactEditPermission = gitArtifactHelper.getArtifactEditPermission();
Expand Down Expand Up @@ -559,7 +560,7 @@ private Mono<? extends Artifact> checkoutRemoteReference(

@Override
public Mono<? extends Artifact> createReference(
String referencedArtifactId, GitRefDTO refDTO, ArtifactType artifactType, GitType gitType) {
String referencedArtifactId, ArtifactType artifactType, GitRefDTO refDTO, GitType gitType) {

/*
1. Check if the src artifact is available and user have sufficient permissions
Expand Down Expand Up @@ -689,7 +690,7 @@ protected Mono<? extends Artifact> createReference(
})
// after the branch is created, we need to reset the older branch to the
// clean status, i.e. last commit
.doOnSuccess(newImportedArtifact -> discardChanges(sourceArtifact, gitType));
.flatMap(newImportedArtifact -> discardChanges(sourceArtifact, gitType));
})
.flatMap(newImportedArtifact -> gitRedisUtils
.releaseFileLock(
Expand Down Expand Up @@ -729,7 +730,7 @@ protected Mono<Boolean> isValidationForRefCreationComplete(

@Override
public Mono<? extends Artifact> deleteGitReference(
String baseArtifactId, GitRefDTO gitRefDTO, ArtifactType artifactType, GitType gitType) {
String baseArtifactId, ArtifactType artifactType, GitRefDTO gitRefDTO, GitType gitType) {

String refName = gitRefDTO.getRefName();
RefType refType = gitRefDTO.getRefType();
Expand Down Expand Up @@ -867,19 +868,19 @@ protected Mono<? extends Artifact> deleteGitReference(
* Each artifact is equal to a repo in the git(and each branch creates a new artifact with default artifact as parent)
*
* @param baseArtifactId : artifactId of the artifact which is getting connected to git
* @param gitConnectDTO artifactId - this is used to link the local git repo to an artifact
* remoteUrl - used for connecting to remote repo etc
* @param originHeader
* @param artifactType
* @param gitConnectDTO artifactId - this is used to link the local git repo to an artifact
* remoteUrl - used for connecting to remote repo etc
* @param originHeader
* @param gitType
* @return an artifact with git metadata
*/
@Override
public Mono<? extends Artifact> connectArtifactToGit(
String baseArtifactId,
ArtifactType artifactType,
GitConnectDTO gitConnectDTO,
String originHeader,
ArtifactType artifactType,
GitType gitType) {
/*
* Connecting the artifact for the first time
Expand Down Expand Up @@ -1473,31 +1474,31 @@ private boolean isBaseGitMetadataInvalid(GitArtifactMetadata gitArtifactMetadata
}

private Mono<GitStatusDTO> getStatusAfterComparingWithRemote(
String baseArtifactId, boolean isFileLock, ArtifactType artifactType, GitType gitType) {
return getStatus(baseArtifactId, isFileLock, true, artifactType, gitType);
String baseArtifactId, ArtifactType artifactType, boolean isFileLock, GitType gitType) {
return getStatus(baseArtifactId, artifactType, isFileLock, true, gitType);
}

@Override
public Mono<GitStatusDTO> getStatus(
String branchedArtifactId, boolean compareRemote, ArtifactType artifactType, GitType gitType) {
return getStatus(branchedArtifactId, true, compareRemote, artifactType, gitType);
String branchedArtifactId, ArtifactType artifactType, boolean compareRemote, GitType gitType) {
return getStatus(branchedArtifactId, artifactType, true, compareRemote, gitType);
}

/**
* Get the status of the artifact for given branched id
*
* @param branchedArtifactId branched id of the artifact
* @param artifactType Type of artifact in context
* @param isFileLock if the locking is required, since the status API is used in the other flows of git
* Only for the direct hits from the client the locking will be added
* @param artifactType Type of artifact in context
* @param gitType Type of the service
* @return Map of json file names which are added, modified, conflicting, removed and the working tree if this is clean
*/
private Mono<GitStatusDTO> getStatus(
String branchedArtifactId,
ArtifactType artifactType,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Look out for potential concurrency pitfalls.

Multiple calls to getBaseAndBranchedArtifacts in a short time frame might need additional synchronization to prevent race conditions in code branches that depend on artifact states.

boolean isFileLock,
boolean compareRemote,
ArtifactType artifactType,
GitType gitType) {

Mono<Tuple2<? extends Artifact, ? extends Artifact>> baseAndBranchedArtifacts =
Expand Down Expand Up @@ -1867,14 +1868,14 @@ public Mono<String> fetchRemoteChanges(
* 1. Checkout (if required) to the branch to make sure we are comparing the right branch
* 2. Run a git fetch command to fetch the latest changes from the remote
*
* @param refArtifactId id of the reference
* @param isFileLock whether to add file lock or not
* @param refArtifactId id of the reference
* @param artifactType
* @param isFileLock whether to add file lock or not
* @return Mono of {@link BranchTrackingStatus}
*/
@Override
public Mono<String> fetchRemoteChanges(
String refArtifactId, boolean isFileLock, ArtifactType artifactType, GitType gitType, RefType refType) {
String refArtifactId, ArtifactType artifactType, boolean isFileLock, GitType gitType, RefType refType) {
GitArtifactHelper<?> artifactGitHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
AclPermission artifactEditPermission = artifactGitHelper.getArtifactEditPermission();

Expand Down Expand Up @@ -2066,15 +2067,15 @@ protected Mono<? extends Artifact> discardChanges(Artifact branchedArtifact, Git
}

public Mono<List<GitBranchDTO>> listBranchForArtifact(
String branchedArtifactId, Boolean pruneBranches, ArtifactType artifactType, GitType gitType) {
return getBranchList(branchedArtifactId, pruneBranches, true, artifactType, gitType);
String branchedArtifactId, ArtifactType artifactType, Boolean pruneBranches, GitType gitType) {
return getBranchList(branchedArtifactId, artifactType, pruneBranches, true, gitType);
}

protected Mono<List<GitBranchDTO>> getBranchList(
String branchedArtifactId,
ArtifactType artifactType,
Boolean pruneBranches,
boolean syncDefaultBranchWithRemote,
ArtifactType artifactType,
GitType gitType) {

GitArtifactHelper<?> gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
Expand Down Expand Up @@ -2176,9 +2177,9 @@ private Mono<String> syncDefaultBranchNameFromRemote(
// default branch has been changed in remote
return updateDefaultBranchName(
metadata.getDefaultArtifactId(),
artifactType,
defaultBranchNameInRemote,
jsonTransformationDTO,
artifactType,
gitType)
.then()
.thenReturn(defaultBranchNameInRemote);
Expand All @@ -2191,9 +2192,9 @@ private Mono<String> syncDefaultBranchNameFromRemote(

private Flux<? extends Artifact> updateDefaultBranchName(
String baseArtifactId,
ArtifactType artifactType,
String newDefaultBranchName,
ArtifactJsonTransformationDTO jsonTransformationDTO,
ArtifactType artifactType,
GitType gitType) {
// Get the artifact from DB by new defaultBranchName
GitArtifactHelper<?> gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
Expand Down Expand Up @@ -2391,7 +2392,7 @@ private Mono<List<GitBranchDTO>> getBranchListWithDefaultBranchName(

@Override
public Mono<List<String>> updateProtectedBranches(
String baseArtifactId, List<String> branchNames, ArtifactType artifactType) {
String baseArtifactId, ArtifactType artifactType, List<String> branchNames) {

GitArtifactHelper<?> gitArtifactHelper = gitArtifactHelperResolver.getArtifactHelper(artifactType);
AclPermission artifactManageProtectedBranchPermission =
Expand Down Expand Up @@ -2508,7 +2509,7 @@ public Mono<Boolean> toggleAutoCommitEnabled(String baseArtifactId, ArtifactType

@Override
public Mono<AutoCommitResponseDTO> getAutoCommitProgress(
String artifactId, String branchName, ArtifactType artifactType) {
String artifactId, ArtifactType artifactType, String branchName) {
return gitAutoCommitHelper.getAutoCommitProgress(artifactId, branchName);
}

Expand Down
Loading
Loading