diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java index e41928d..0234f13 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java @@ -11,14 +11,12 @@ import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; public interface NewDatasetService { String newCreateDataset(NewDatasetVo datasetVo) throws Exception; - CompletableFuture newCreateVersion(NewDatasetVo datasetVo)throws Exception; - + String newCreateVersion(NewDatasetVo datasetVo) throws Exception; List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception; ResponseEntity downloadDatasetlocal(String filePath) throws Exception; ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java index 040ca4c..5c04914 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentInsServiceImpl.java @@ -53,10 +53,15 @@ public class ExperimentInsServiceImpl implements ExperimentInsService { private String argoWorkflowPodLog; @Value("${argo.ins.logsLines}") private int logsLines; + @Value("${minio.endpoint}") + private String minioEndpoint; @Resource private K8sClientUtil k8sClientUtil; - @Resource - private MinioUtil minioUtil; + private final MinioUtil minioUtil; + public ExperimentInsServiceImpl(MinioUtil minioUtil) { + this.minioUtil = minioUtil; + } + public enum ExperimentInsStatus { Running, diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java index 56f8ec5..1ecccaa 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/GitServiceImpl.java @@ -114,6 +114,7 @@ public class GitServiceImpl implements GitService { resMap.put("new_branch_name", branchName); resMap.put("old_branch_name", oldBranchName); String req = HttpUtils.sendPostWithToken(createBranchUrl, JsonUtils.objectToJson(resMap), token); + System.out.println(req); } @Override diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java index dff6c6b..eb0cca0 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -104,12 +103,12 @@ public class NewDatasetServiceImpl implements NewDatasetService { datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); datasetVo.setUpdateTime(DateUtils.getTime()); datasetVo.setVersionDesc(datasetVo.getDescription()); - datasetVo.setUsage("
\n" +
+        datasetVo.setUsage("
" +
                 "# 克隆数据集配置文件与存储参数到本地\n" +
                 "git clone -b " + branchName + " "+ projectUrl + "\n" +
                 "# 远程拉取配置文件\n" +
                 "dvc pull\n" +
-                "
\n");
+                "
"); datasetVo.setIdentifier(repositoryName); datasetVo.setId(gitlinIid); datasetVo.setOwner((String) userInfo.get("login")); @@ -135,82 +134,65 @@ public class NewDatasetServiceImpl implements NewDatasetService { return "新增数据集成功"; } - @Override - public CompletableFuture newCreateVersion(NewDatasetVo datasetVo) { - return CompletableFuture.supplyAsync(() -> { - try { - Jedis jedis = new Jedis(redisHost); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - String token = jedis.get(ci4sUsername + "_gitToken"); - String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); - String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); - // 创建分支 - String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); - String repositoryName = datasetVo.getIdentifier(); - if (StringUtils.equals(branchName, "master")) { - gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master"); - } - // 得到项目地址 - String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + public String newCreateVersion(NewDatasetVo datasetVo) throws Exception { + Jedis jedis = new Jedis(redisHost); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + String token = jedis.get(ci4sUsername + "_gitToken"); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + // 创建分支 + String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); + String repositoryName = datasetVo.getIdentifier(); + String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); + String localPath = localPathlocal + datasetVo.getId() + "/" + datasetVo.getName(); + String sourcePath = url.substring(0, url.lastIndexOf("/")); + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; - // 得到用户操作的路径 - String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String localPath = localPathlocal + loginUser.getUsername() + "/" + "datasets" + "/" + datasetVo.getName(); - String sourcePath = url.substring(0, url.lastIndexOf("/")); - // 命令行操作 git clone 项目地址 - if (FileUtil.checkDirectoryExists(localPath)) { - DVCUtils.gitFetch(localPath, gitLinkUsername, gitLinkPassword); - DVCUtils.gitCheckoutBranch(localPath, branchName); - } else { - DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); - } + // 创建本地分支 + DVCUtils.createLocalBranchBasedOnMaster(localPath, branchName); - String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName; + //dvc checkout + DVCUtils.dvcCheckout(localPath); - DVCUtils.moveFiles(sourcePath, localPath); - // 拼接生产的元数据后写入yaml文件 - datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); - datasetVo.setUpdateTime(DateUtils.getTime()); - datasetVo.setVersionDesc(datasetVo.getDescription()); - datasetVo.setUsage("
\n" +
-                        "# 克隆数据集配置文件与存储参数到本地\n" +
-                        "git clone -b " + branchName + " "+ projectUrl + "\n" +
-                        "# 远程拉取配置文件\n" +
-                        "dvc pull\n" +
-                        "
\n"); - datasetVo.setIdentifier(repositoryName); - datasetVo.setId(datasetVo.getId()); - datasetVo.setOwner((String) userInfo.get("login")); - if (StringUtils.isEmpty(datasetVo.getDatasetSource())){ - datasetVo.setDatasetSource("用户上传"); - } - YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); + // 准备数据 + String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName; - // dvc init 初始化 - DVCUtils.dvcInit(localPath); - // 配置远程S3地址 - DVCUtils.dvcRemoteAdd(localPath, s3Path); - DVCUtils.dvcConfigS3Credentials(localPath, endpoint); - DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); - DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); - // dvc 跟踪 - DVCUtils.dvcAdd(localPath, "data"); - // git commit - DVCUtils.gitAdd(localPath, "."); - DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); - DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword); - // dvc push 到远程S3 - DVCUtils.dvcPush(localPath); - return "新增数据集成功"; - } catch (Exception e) { - throw new RuntimeException(e); - } - }); + DVCUtils.moveFiles(sourcePath, localPath); + // 拼接生产的元数据后写入yaml文件 + datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); + datasetVo.setUpdateTime(DateUtils.getTime()); + datasetVo.setVersionDesc(datasetVo.getDescription()); + datasetVo.setUsage("
" +
+                "# 克隆数据集配置文件与存储参数到本地\n" +
+                "git clone -b " + branchName + " "+ projectUrl + "\n" +
+                "# 远程拉取配置文件\n" +
+                "dvc pull\n" +
+                "
"); + datasetVo.setIdentifier(repositoryName); + datasetVo.setId(datasetVo.getId()); + datasetVo.setOwner((String) userInfo.get("login")); + if (StringUtils.isEmpty(datasetVo.getDatasetSource())){ + datasetVo.setDatasetSource("用户上传"); + } + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); + //dvc数据跟踪 + // 配置远程S3地址 + DVCUtils.dvcRemoteAdd(localPath, s3Path); + DVCUtils.dvcConfigS3Credentials(localPath, endpoint); + DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); + DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); + // dvc 跟踪 + DVCUtils.dvcAdd(localPath, "data"); + DVCUtils.pushNewBranchToRemote(localPath,gitLinkUsername,gitLinkPassword,branchName); + //dvc push 到远程S3 + DVCUtils.dvcPush(localPath); + return "新增数据集成功"; } + @Override public Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception { Jedis jedis = new Jedis(redisHost); @@ -273,7 +255,6 @@ public class NewDatasetServiceImpl implements NewDatasetService { } version = (String) versionList.get(0).get("name"); } - List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +id, repositoryName, version, "data",gitLinkUsername, gitLinkPassword); // 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据 Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + id + "\\" + repositoryName + "\\" + "dataset.yaml"); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java index c69f446..46fa7b5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java @@ -2,22 +2,29 @@ package com.ruoyi.platform.utils; import org.eclipse.jgit.api.*; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.RepositoryState; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; -import org.eclipse.jgit.transport.CredentialsProvider; -import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.transport.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; -import java.nio.file.*; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class DVCUtils { - + private static final ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); private static final Logger log = LoggerFactory.getLogger(DVCUtils.class); private static void runCommand(String command, String workingDir) throws Exception { @@ -27,7 +34,7 @@ public class DVCUtils { BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); String line; while ((line = reader.readLine()) != null) { - System.out.println(line); + log.info(line); } int exitCode = process.waitFor(); if (exitCode != 0) { @@ -56,6 +63,16 @@ public class DVCUtils { cloneCommand.call(); } + public static void gitClone(String localPath, String repoUrl, String username, String password) throws GitAPIException { + CloneCommand cloneCommand = Git.cloneRepository() + .setURI(repoUrl) + .setDirectory(new java.io.File(localPath)) + .setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password)); + + cloneCommand.call(); + } + + public static void gitAdd(String localPath, String filePath) throws IOException, GitAPIException { FileRepositoryBuilder builder = new FileRepositoryBuilder(); Repository repository = builder.setGitDir(new File(localPath, ".git")) @@ -103,6 +120,356 @@ public class DVCUtils { } } + + /** + * 更新本地仓库中的所有分支 + * + * @param localPath 本地仓库路径 + * @param username 远程仓库用户名 + * @param password 远程仓库密码 + */ + public static void updateAllBranches(String localPath, String username, String password) { + try (Git git = Git.open(new File(localPath))) { + // 设置凭证，用于远程仓库的认证 + UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); + + // 获取远程分支列表 + List remoteBranches = git.branchList().setListMode(org.eclipse.jgit.api.ListBranchCommand.ListMode.REMOTE).call(); + + // 遍历远程分支并更新本地对应的分支 + for (Ref remoteBranch : remoteBranches) { + // 获取分支名称，去掉前缀 "refs/remotes/origin/" + String branchName = remoteBranch.getName().replace("refs/remotes/origin/", ""); + log.info("Updating branch: " + branchName); + + // 检查本地是否存在该分支，如果不存在则创建 + if (!branchExistsLocally(git, branchName)) { + git.branchCreate().setName(branchName).setStartPoint("origin/" + branchName).call(); + } + + // 切换到对应的分支 + git.checkout().setName(branchName).call(); + + // 拉取远程分支的最新内容 + PullCommand pull = git.pull().setRemoteBranchName(branchName).setCredentialsProvider(credentialsProvider); + pull.call(); + } + + log.info("All branches updated successfully."); + + } catch (IOException | GitAPIException e) { + log.error("Error occurred while updating all branches", e); + } + } + /** + * 更新本地仓库中的指定分支 + * + * @param localPath 本地仓库路径 + * @param username 远程仓库用户名 + * @param password 远程仓库密码 + * @param branchName 需要更新的分支名称 + */ + public static void updateBranch(String localPath, String username, String password, String branchName) { + try (Git git = Git.open(new File(localPath))) { + // 设置凭证，用于远程仓库的认证 + UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); + + // 检查本地是否存在该分支，如果不存在则创建 + if (!branchExistsLocally(git, branchName)) { + git.branchCreate().setName(branchName).setStartPoint("origin/" + branchName).call(); + } + + // 切换到对应的分支 + git.checkout().setName(branchName).call(); + + // 拉取远程分支的最新内容 + PullCommand pull = git.pull().setRemoteBranchName(branchName).setCredentialsProvider(credentialsProvider); + pull.call(); + + log.info("Branch " + branchName + " updated successfully."); + + } catch (IOException | GitAPIException e) { + log.error("Error occurred while updating branche", e); + } + } + + + /** + * 创建本地分支 + * + * @param localPath 本地仓库路径 + * @param branchName 要创建的分支名称 + */ + public static void createLocalBranch(String localPath, String branchName) { + try (Git git = Git.open(new File(localPath))) { + // 创建本地分支 + git.branchCreate().setName(branchName).call(); + log.info("本地分支 " + branchName + " 创建成功。"); + + } catch (IOException | GitAPIException e) { + log.error("Error occurred while creating local branch", e); + } + } + + /** + * 基于 master 创建本地分支 + * + * @param localPath 本地仓库路径 + * @param branchName 要创建的分支名称 + */ + public static void createLocalBranchBasedOnMaster(String localPath, String branchName) { + try (Git git = Git.open(new File(localPath))) { + // 切换到 master 分支 + git.checkout().setName("master").call(); + + // 创建新的本地分支 + git.branchCreate().setName(branchName).call(); + log.info("基于 master 的本地分支 " + branchName + " 创建成功。"); + + git.checkout().setName(branchName).call(); + + } catch (IOException | GitAPIException e) { + log.error("Exception occurred while creating local branch based on master",e); + } + } + + /** + * 将本地分支推送到远程仓库 + * + * @param localPath 本地仓库路径 + * @param username 远程仓库用户名 + * @param password 远程仓库密码 + * @param branchName 要推送的分支名称 + */ + public static void pushLocalBranchToRemote(String localPath, String username, String password, String branchName) { + try (Git git = Git.open(new File(localPath))) { + // 设置凭证，用于远程仓库的认证 + UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); + + // 切换到要推送的分支 + git.checkout().setName(branchName).call(); + + // 添加所有更改到暂存区 + git.add().addFilepattern(".").call(); + + // 提交更改 + git.commit().setMessage("commit from ci4s").call(); + + // 推送本地分支到远程仓库 + git.push() + .setRemote("origin") + .add(branchName) + .setCredentialsProvider(credentialsProvider) + .call(); + + log.info("本地分支 " + branchName + " 推送成功。"); + + } catch (IOException | GitAPIException e) { + log.error("Error occurred while pushing local branch to remote", e); + } + } + + /** + * 将本地分支推送到远程（新增远程分支） + * + * @param localPath 本地仓库路径 + * @param username 用户名 + * @param password 密码 + * @param branchName 要推送的分支名称 + * @throws IOException 如果仓库路径无效 + * @throws GitAPIException 如果Git操作失败 + */ + public static void pushNewBranchToRemote(String localPath, String username, String password, String branchName) throws IOException, GitAPIException { + try (Git git = Git.open(new File(localPath))) { + // 设置凭证，用于远程仓库的认证 + UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); + + // 切换到要推送的分支，确保分支已存在，并且切换至该分支 + git.checkout().setName(branchName).call(); + + // 创建一个新的 RefSpec + RefSpec refSpec = new RefSpec(branchName + ":" + branchName); + + // 添加所有更改到暂存区 + git.add().addFilepattern(".").call(); + + // 提交更改 + git.commit().setMessage("commit from ci4s").call(); + // 推送本地分支到远程端（如果远程无该分支，则新增） + Iterable pushResults = git.push() + .setRemote("origin") + .setRefSpecs(refSpec) + .setCredentialsProvider(credentialsProvider) + .call(); + + // 打印结果 + pushResults.forEach(pr ->log.info(pr.getMessages())); + + log.info("分支 " + branchName + " 已成功推送到远程仓库。"); + } + } + + /** + * 检查本地是否存在指定分支 + * + * @param git Git 实例 + * @param branchName 分支名称 + * @return 如果本地存在该分支返回 true，否则返回 false + */ + private static boolean branchExistsLocally(Git git, String branchName) throws GitAPIException { + List localBranches = git.branchList().call(); + for (Ref localBranch : localBranches) { + if (localBranch.getName().endsWith("/" + branchName)) { + return true; + } + } + return false; + } + + + public static void refreshRemoteBranches(String localPath, String username, String password, String branch) throws IOException, GitAPIException { + long startTime = System.currentTimeMillis(); + + try (Repository repository = new FileRepositoryBuilder() + .setGitDir(new File(localPath + "/.git")) + .readEnvironment() + .findGitDir() + .build()) { + + Git git = new Git(repository); + + // 检查仓库状态 + if (repository.getRepositoryState().equals(RepositoryState.MERGING) || + repository.getRepositoryState().equals(RepositoryState.MERGING_RESOLVED)) { + log.info("Repository is in a merging state, please resolve conflicts manually."); + return; + } + + // 设置凭证提供者 + UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); + + // 获取远程分支 + long fetchStartTime = System.currentTimeMillis(); + FetchResult fetchResult = git.fetch() + .setRemote("origin") + .setCredentialsProvider(credentialsProvider) + .call(); + long fetchEndTime = System.currentTimeMillis(); + log.info("Fetch time: " + (fetchEndTime - fetchStartTime) + " ms"); + + // 打印获取的远程分支 + Collection fetchedRefs = fetchResult.getAdvertisedRefs(); + for (Ref ref : fetchedRefs) { + log.info("Fetched branch: " + ref.getName()); + } + + // 更新本地分支信息 + long branchListStartTime = System.currentTimeMillis(); + git.branchList() + .setListMode(org.eclipse.jgit.api.ListBranchCommand.ListMode.REMOTE) + .call() + .forEach(ref -> executorService.submit(() -> { + String fullBranchName = ref.getName(); + String branchName = fullBranchName.replace("refs/remotes/origin/", ""); + + try { + processBranch(git, repository, credentialsProvider, fullBranchName, branchName); + } catch (Exception e) { + log.error("Failed to process branch: " + branchName, e); + } + })); + + executorService.shutdown(); + try { + executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + log.error("Executor service interrupted", e); + } + + long branchListEndTime = System.currentTimeMillis(); + log.info("Branch list and update time: " + (branchListEndTime - branchListStartTime) + " ms"); + + log.info("远程分支刷新到本地完成。"); + + // 切换分支 + gitCheckoutBranch(localPath, branch); + dvcCheckout(localPath); + + } catch (Exception e) { + throw new RuntimeException(e); + } + + long endTime = System.currentTimeMillis(); + log.info("Total execution time: " + (endTime - startTime) + " ms"); + } + + private static void processBranch(Git git, Repository repository, UsernamePasswordCredentialsProvider credentialsProvider, String fullBranchName, String branchName) throws Exception { + // 检查本地分支是否存在 + Ref localRef = repository.findRef("refs/heads/" + branchName); + if (localRef != null) { + // 如果存在，检查是否已经关联到远程分支 + if (!isBranchUpstreamSet(repository, branchName)) { + // 如果没有关联，设置上游分支 + long upstreamStartTime = System.currentTimeMillis(); + git.branchCreate() + .setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.SET_UPSTREAM) + .setStartPoint(fullBranchName) + .setName(branchName) + .call(); + long upstreamEndTime = System.currentTimeMillis(); + log.info("Set upstream for branch: " + branchName + ", time: " + (upstreamEndTime - upstreamStartTime) + " ms"); + } + } else { + // 如果不存在，创建本地分支并设置上游分支 + long createBranchStartTime = System.currentTimeMillis(); + git.branchCreate() + .setName(branchName) + .setStartPoint(fullBranchName) + .call(); + long createBranchEndTime = System.currentTimeMillis(); + log.info("Created local branch: " + branchName + ", time: " + (createBranchEndTime - createBranchStartTime) + " ms"); + + // 设置上游分支 + long setUpstreamStartTime = System.currentTimeMillis(); + git.branchCreate() + .setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.SET_UPSTREAM) + .setStartPoint(fullBranchName) + .setName(branchName) + .call(); + long setUpstreamEndTime = System.currentTimeMillis(); + log.info("Set upstream for branch: " + branchName + ", time: " + (setUpstreamEndTime - setUpstreamStartTime) + " ms"); + } + + // 执行 git pull + long pullStartTime = System.currentTimeMillis(); + PullCommand pullCommand = git.pull() + .setCredentialsProvider(credentialsProvider); + pullCommand.call(); + long pullEndTime = System.currentTimeMillis(); + log.info("Updated local branch: " + branchName + ", time: " + (pullEndTime - pullStartTime) + " ms"); + } + + /** + * 检查本地分支是否已经关联到远程分支 + * + * @param repository 仓库对象 + * @param branchName 分支名称 + * @return 如果已经关联，返回 true；否则返回 false + */ + private static boolean isBranchUpstreamSet(Repository repository, String branchName) { + try { + org.eclipse.jgit.lib.Config config = repository.getConfig(); + String upstreamBranch = config.getString("branch", branchName, "merge"); + return upstreamBranch != null && !upstreamBranch.isEmpty(); + } catch (Exception e) { + log.error("Failed to check upstream for branch: " + branchName, e); + return false; + } + } + + + + public static void gitFetch(String localPath, String username, String password) throws IOException, GitAPIException { FileRepositoryBuilder builder = new FileRepositoryBuilder(); Repository repository = builder.setGitDir(new File(localPath, ".git")) @@ -174,7 +541,7 @@ public class DVCUtils { } public static void dvcPush(String localPath) throws Exception { - String command = "dvc push"; + String command = "dvc push -v"; runCommand(command, localPath); } @@ -185,6 +552,13 @@ public class DVCUtils { } + // 方法 + public static void dvcCheckout(String localPath) throws Exception { + String command = "dvc checkout"; + runCommand(command, localPath); + } + + /** * 切换到指定分支并执行git pull，然后获取data文件夹下所有文件的路径、名称和大小 * @@ -197,15 +571,9 @@ public class DVCUtils { List> fileInfoList = new ArrayList<>(); try { - // 切换到指定目录 - Path repoPath = Paths.get(localPath, repoFolder, branch); - //刷新 TODO -// gitFetch(localPath, username, password); -// // 切换到指定分支 -// gitCheckoutBranch(localPath, branch); -// // 执行git pull -// gitPull(localPath, username, password); + //刷新 + refreshRemoteBranches(localPath+"/"+repoFolder, username, password,branch); // 读取data文件夹中的文件列表 String path = localPath + "\\" + repoFolder + "\\" + filePath; Path dataPath = Paths.get(path);