Merge remote-tracking branch 'origin/dev' into dev-czh
This commit is contained in:
chenzhihang
commit
92e77bcfe9
|
|
@ -11,14 +11,12 @@ import org.springframework.web.multipart.MultipartFile;
|
|||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
public interface NewDatasetService {
|
||||
|
||||
|
||||
String newCreateDataset(NewDatasetVo datasetVo) throws Exception;
|
||||
CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo)throws Exception;
|
||||
|
||||
String newCreateVersion(NewDatasetVo datasetVo) throws Exception;
|
||||
List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
|
||||
ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception;
|
||||
ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception;
|
||||
|
|
|
|||
|
|
@ -53,10 +53,15 @@ public class ExperimentInsServiceImpl implements ExperimentInsService {
|
|||
private String argoWorkflowPodLog;
|
||||
@Value("${argo.ins.logsLines}")
|
||||
private int logsLines;
|
||||
@Value("${minio.endpoint}")
|
||||
private String minioEndpoint;
|
||||
@Resource
|
||||
private K8sClientUtil k8sClientUtil;
|
||||
@Resource
|
||||
private MinioUtil minioUtil;
|
||||
private final MinioUtil minioUtil;
|
||||
public ExperimentInsServiceImpl(MinioUtil minioUtil) {
|
||||
this.minioUtil = minioUtil;
|
||||
}
|
||||
|
||||
|
||||
public enum ExperimentInsStatus {
|
||||
Running,
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ public class GitServiceImpl implements GitService {
|
|||
resMap.put("new_branch_name", branchName);
|
||||
resMap.put("old_branch_name", oldBranchName);
|
||||
String req = HttpUtils.sendPostWithToken(createBranchUrl, JsonUtils.objectToJson(resMap), token);
|
||||
System.out.println(req);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ import java.util.ArrayList;
|
|||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
|
@ -104,12 +103,12 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getTime());
|
||||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("<pre><code>\n" +
|
||||
datasetVo.setUsage("<pre><code>" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " "+ projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"</code></pre>\n");
|
||||
"</code></pre>");
|
||||
datasetVo.setIdentifier(repositoryName);
|
||||
datasetVo.setId(gitlinIid);
|
||||
datasetVo.setOwner((String) userInfo.get("login"));
|
||||
|
|
@ -135,10 +134,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
return "新增数据集成功";
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo) {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
public String newCreateVersion(NewDatasetVo datasetVo) throws Exception {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
|
|
@ -150,24 +146,18 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
// 创建分支
|
||||
String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
|
||||
String repositoryName = datasetVo.getIdentifier();
|
||||
if (StringUtils.equals(branchName, "master")) {
|
||||
gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
}
|
||||
// 得到项目地址
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal + datasetVo.getId() + "/" + datasetVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal + loginUser.getUsername() + "/" + "datasets" + "/" + datasetVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
if (FileUtil.checkDirectoryExists(localPath)) {
|
||||
DVCUtils.gitFetch(localPath, gitLinkUsername, gitLinkPassword);
|
||||
DVCUtils.gitCheckoutBranch(localPath, branchName);
|
||||
} else {
|
||||
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
|
||||
}
|
||||
// 创建本地分支
|
||||
DVCUtils.createLocalBranchBasedOnMaster(localPath, branchName);
|
||||
|
||||
//dvc checkout
|
||||
DVCUtils.dvcCheckout(localPath);
|
||||
|
||||
// 准备数据
|
||||
String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath);
|
||||
|
|
@ -175,12 +165,12 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getTime());
|
||||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("<pre><code>\n" +
|
||||
datasetVo.setUsage("<pre><code>" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " "+ projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"</code></pre>\n");
|
||||
"</code></pre>");
|
||||
datasetVo.setIdentifier(repositoryName);
|
||||
datasetVo.setId(datasetVo.getId());
|
||||
datasetVo.setOwner((String) userInfo.get("login"));
|
||||
|
|
@ -188,9 +178,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
datasetVo.setDatasetSource("用户上传");
|
||||
}
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
|
||||
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
//dvc数据跟踪
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
|
|
@ -198,19 +186,13 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath, "data");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath, ".");
|
||||
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.pushNewBranchToRemote(localPath,gitLinkUsername,gitLinkPassword,branchName);
|
||||
//dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath);
|
||||
return "新增数据集成功";
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
|
|
@ -273,7 +255,6 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
}
|
||||
version = (String) versionList.get(0).get("name");
|
||||
}
|
||||
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +id, repositoryName, version, "data",gitLinkUsername, gitLinkPassword);
|
||||
// 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + id + "\\" + repositoryName + "\\" + "dataset.yaml");
|
||||
|
|
|
|||
|
|
@ -2,22 +2,29 @@ package com.ruoyi.platform.utils;
|
|||
|
||||
import org.eclipse.jgit.api.*;
|
||||
import org.eclipse.jgit.api.errors.GitAPIException;
|
||||
import org.eclipse.jgit.lib.Ref;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.lib.RepositoryState;
|
||||
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
|
||||
import org.eclipse.jgit.transport.CredentialsProvider;
|
||||
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
|
||||
import org.eclipse.jgit.transport.*;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class DVCUtils {
|
||||
|
||||
private static final ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
||||
private static final Logger log = LoggerFactory.getLogger(DVCUtils.class);
|
||||
|
||||
private static void runCommand(String command, String workingDir) throws Exception {
|
||||
|
|
@ -27,7 +34,7 @@ public class DVCUtils {
|
|||
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
System.out.println(line);
|
||||
log.info(line);
|
||||
}
|
||||
int exitCode = process.waitFor();
|
||||
if (exitCode != 0) {
|
||||
|
|
@ -56,6 +63,16 @@ public class DVCUtils {
|
|||
cloneCommand.call();
|
||||
}
|
||||
|
||||
public static void gitClone(String localPath, String repoUrl, String username, String password) throws GitAPIException {
|
||||
CloneCommand cloneCommand = Git.cloneRepository()
|
||||
.setURI(repoUrl)
|
||||
.setDirectory(new java.io.File(localPath))
|
||||
.setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password));
|
||||
|
||||
cloneCommand.call();
|
||||
}
|
||||
|
||||
|
||||
public static void gitAdd(String localPath, String filePath) throws IOException, GitAPIException {
|
||||
FileRepositoryBuilder builder = new FileRepositoryBuilder();
|
||||
Repository repository = builder.setGitDir(new File(localPath, ".git"))
|
||||
|
|
@ -103,6 +120,356 @@ public class DVCUtils {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 更新本地仓库中的所有分支
|
||||
*
|
||||
* @param localPath 本地仓库路径
|
||||
* @param username 远程仓库用户名
|
||||
* @param password 远程仓库密码
|
||||
*/
|
||||
public static void updateAllBranches(String localPath, String username, String password) {
|
||||
try (Git git = Git.open(new File(localPath))) {
|
||||
// 设置凭证，用于远程仓库的认证
|
||||
UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
|
||||
|
||||
// 获取远程分支列表
|
||||
List<Ref> remoteBranches = git.branchList().setListMode(org.eclipse.jgit.api.ListBranchCommand.ListMode.REMOTE).call();
|
||||
|
||||
// 遍历远程分支并更新本地对应的分支
|
||||
for (Ref remoteBranch : remoteBranches) {
|
||||
// 获取分支名称，去掉前缀 "refs/remotes/origin/"
|
||||
String branchName = remoteBranch.getName().replace("refs/remotes/origin/", "");
|
||||
log.info("Updating branch: " + branchName);
|
||||
|
||||
// 检查本地是否存在该分支，如果不存在则创建
|
||||
if (!branchExistsLocally(git, branchName)) {
|
||||
git.branchCreate().setName(branchName).setStartPoint("origin/" + branchName).call();
|
||||
}
|
||||
|
||||
// 切换到对应的分支
|
||||
git.checkout().setName(branchName).call();
|
||||
|
||||
// 拉取远程分支的最新内容
|
||||
PullCommand pull = git.pull().setRemoteBranchName(branchName).setCredentialsProvider(credentialsProvider);
|
||||
pull.call();
|
||||
}
|
||||
|
||||
log.info("All branches updated successfully.");
|
||||
|
||||
} catch (IOException | GitAPIException e) {
|
||||
log.error("Error occurred while updating all branches", e);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 更新本地仓库中的指定分支
|
||||
*
|
||||
* @param localPath 本地仓库路径
|
||||
* @param username 远程仓库用户名
|
||||
* @param password 远程仓库密码
|
||||
* @param branchName 需要更新的分支名称
|
||||
*/
|
||||
public static void updateBranch(String localPath, String username, String password, String branchName) {
|
||||
try (Git git = Git.open(new File(localPath))) {
|
||||
// 设置凭证，用于远程仓库的认证
|
||||
UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
|
||||
|
||||
// 检查本地是否存在该分支，如果不存在则创建
|
||||
if (!branchExistsLocally(git, branchName)) {
|
||||
git.branchCreate().setName(branchName).setStartPoint("origin/" + branchName).call();
|
||||
}
|
||||
|
||||
// 切换到对应的分支
|
||||
git.checkout().setName(branchName).call();
|
||||
|
||||
// 拉取远程分支的最新内容
|
||||
PullCommand pull = git.pull().setRemoteBranchName(branchName).setCredentialsProvider(credentialsProvider);
|
||||
pull.call();
|
||||
|
||||
log.info("Branch " + branchName + " updated successfully.");
|
||||
|
||||
} catch (IOException | GitAPIException e) {
|
||||
log.error("Error occurred while updating branche", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建本地分支
|
||||
*
|
||||
* @param localPath 本地仓库路径
|
||||
* @param branchName 要创建的分支名称
|
||||
*/
|
||||
public static void createLocalBranch(String localPath, String branchName) {
|
||||
try (Git git = Git.open(new File(localPath))) {
|
||||
// 创建本地分支
|
||||
git.branchCreate().setName(branchName).call();
|
||||
log.info("本地分支 " + branchName + " 创建成功。");
|
||||
|
||||
} catch (IOException | GitAPIException e) {
|
||||
log.error("Error occurred while creating local branch", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 基于 master 创建本地分支
|
||||
*
|
||||
* @param localPath 本地仓库路径
|
||||
* @param branchName 要创建的分支名称
|
||||
*/
|
||||
public static void createLocalBranchBasedOnMaster(String localPath, String branchName) {
|
||||
try (Git git = Git.open(new File(localPath))) {
|
||||
// 切换到 master 分支
|
||||
git.checkout().setName("master").call();
|
||||
|
||||
// 创建新的本地分支
|
||||
git.branchCreate().setName(branchName).call();
|
||||
log.info("基于 master 的本地分支 " + branchName + " 创建成功。");
|
||||
|
||||
git.checkout().setName(branchName).call();
|
||||
|
||||
} catch (IOException | GitAPIException e) {
|
||||
log.error("Exception occurred while creating local branch based on master",e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将本地分支推送到远程仓库
|
||||
*
|
||||
* @param localPath 本地仓库路径
|
||||
* @param username 远程仓库用户名
|
||||
* @param password 远程仓库密码
|
||||
* @param branchName 要推送的分支名称
|
||||
*/
|
||||
public static void pushLocalBranchToRemote(String localPath, String username, String password, String branchName) {
|
||||
try (Git git = Git.open(new File(localPath))) {
|
||||
// 设置凭证，用于远程仓库的认证
|
||||
UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
|
||||
|
||||
// 切换到要推送的分支
|
||||
git.checkout().setName(branchName).call();
|
||||
|
||||
// 添加所有更改到暂存区
|
||||
git.add().addFilepattern(".").call();
|
||||
|
||||
// 提交更改
|
||||
git.commit().setMessage("commit from ci4s").call();
|
||||
|
||||
// 推送本地分支到远程仓库
|
||||
git.push()
|
||||
.setRemote("origin")
|
||||
.add(branchName)
|
||||
.setCredentialsProvider(credentialsProvider)
|
||||
.call();
|
||||
|
||||
log.info("本地分支 " + branchName + " 推送成功。");
|
||||
|
||||
} catch (IOException | GitAPIException e) {
|
||||
log.error("Error occurred while pushing local branch to remote", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将本地分支推送到远程（新增远程分支）
|
||||
*
|
||||
* @param localPath 本地仓库路径
|
||||
* @param username 用户名
|
||||
* @param password 密码
|
||||
* @param branchName 要推送的分支名称
|
||||
* @throws IOException 如果仓库路径无效
|
||||
* @throws GitAPIException 如果Git操作失败
|
||||
*/
|
||||
public static void pushNewBranchToRemote(String localPath, String username, String password, String branchName) throws IOException, GitAPIException {
|
||||
try (Git git = Git.open(new File(localPath))) {
|
||||
// 设置凭证，用于远程仓库的认证
|
||||
UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
|
||||
|
||||
// 切换到要推送的分支，确保分支已存在，并且切换至该分支
|
||||
git.checkout().setName(branchName).call();
|
||||
|
||||
// 创建一个新的 RefSpec
|
||||
RefSpec refSpec = new RefSpec(branchName + ":" + branchName);
|
||||
|
||||
// 添加所有更改到暂存区
|
||||
git.add().addFilepattern(".").call();
|
||||
|
||||
// 提交更改
|
||||
git.commit().setMessage("commit from ci4s").call();
|
||||
// 推送本地分支到远程端（如果远程无该分支，则新增）
|
||||
Iterable<PushResult> pushResults = git.push()
|
||||
.setRemote("origin")
|
||||
.setRefSpecs(refSpec)
|
||||
.setCredentialsProvider(credentialsProvider)
|
||||
.call();
|
||||
|
||||
// 打印结果
|
||||
pushResults.forEach(pr ->log.info(pr.getMessages()));
|
||||
|
||||
log.info("分支 " + branchName + " 已成功推送到远程仓库。");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查本地是否存在指定分支
|
||||
*
|
||||
* @param git Git 实例
|
||||
* @param branchName 分支名称
|
||||
* @return 如果本地存在该分支返回 true，否则返回 false
|
||||
*/
|
||||
private static boolean branchExistsLocally(Git git, String branchName) throws GitAPIException {
|
||||
List<Ref> localBranches = git.branchList().call();
|
||||
for (Ref localBranch : localBranches) {
|
||||
if (localBranch.getName().endsWith("/" + branchName)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public static void refreshRemoteBranches(String localPath, String username, String password, String branch) throws IOException, GitAPIException {
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
try (Repository repository = new FileRepositoryBuilder()
|
||||
.setGitDir(new File(localPath + "/.git"))
|
||||
.readEnvironment()
|
||||
.findGitDir()
|
||||
.build()) {
|
||||
|
||||
Git git = new Git(repository);
|
||||
|
||||
// 检查仓库状态
|
||||
if (repository.getRepositoryState().equals(RepositoryState.MERGING) ||
|
||||
repository.getRepositoryState().equals(RepositoryState.MERGING_RESOLVED)) {
|
||||
log.info("Repository is in a merging state, please resolve conflicts manually.");
|
||||
return;
|
||||
}
|
||||
|
||||
// 设置凭证提供者
|
||||
UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
|
||||
|
||||
// 获取远程分支
|
||||
long fetchStartTime = System.currentTimeMillis();
|
||||
FetchResult fetchResult = git.fetch()
|
||||
.setRemote("origin")
|
||||
.setCredentialsProvider(credentialsProvider)
|
||||
.call();
|
||||
long fetchEndTime = System.currentTimeMillis();
|
||||
log.info("Fetch time: " + (fetchEndTime - fetchStartTime) + " ms");
|
||||
|
||||
// 打印获取的远程分支
|
||||
Collection<Ref> fetchedRefs = fetchResult.getAdvertisedRefs();
|
||||
for (Ref ref : fetchedRefs) {
|
||||
log.info("Fetched branch: " + ref.getName());
|
||||
}
|
||||
|
||||
// 更新本地分支信息
|
||||
long branchListStartTime = System.currentTimeMillis();
|
||||
git.branchList()
|
||||
.setListMode(org.eclipse.jgit.api.ListBranchCommand.ListMode.REMOTE)
|
||||
.call()
|
||||
.forEach(ref -> executorService.submit(() -> {
|
||||
String fullBranchName = ref.getName();
|
||||
String branchName = fullBranchName.replace("refs/remotes/origin/", "");
|
||||
|
||||
try {
|
||||
processBranch(git, repository, credentialsProvider, fullBranchName, branchName);
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to process branch: " + branchName, e);
|
||||
}
|
||||
}));
|
||||
|
||||
executorService.shutdown();
|
||||
try {
|
||||
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
|
||||
} catch (InterruptedException e) {
|
||||
log.error("Executor service interrupted", e);
|
||||
}
|
||||
|
||||
long branchListEndTime = System.currentTimeMillis();
|
||||
log.info("Branch list and update time: " + (branchListEndTime - branchListStartTime) + " ms");
|
||||
|
||||
log.info("远程分支刷新到本地完成。");
|
||||
|
||||
// 切换分支
|
||||
gitCheckoutBranch(localPath, branch);
|
||||
dvcCheckout(localPath);
|
||||
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
long endTime = System.currentTimeMillis();
|
||||
log.info("Total execution time: " + (endTime - startTime) + " ms");
|
||||
}
|
||||
|
||||
private static void processBranch(Git git, Repository repository, UsernamePasswordCredentialsProvider credentialsProvider, String fullBranchName, String branchName) throws Exception {
|
||||
// 检查本地分支是否存在
|
||||
Ref localRef = repository.findRef("refs/heads/" + branchName);
|
||||
if (localRef != null) {
|
||||
// 如果存在，检查是否已经关联到远程分支
|
||||
if (!isBranchUpstreamSet(repository, branchName)) {
|
||||
// 如果没有关联，设置上游分支
|
||||
long upstreamStartTime = System.currentTimeMillis();
|
||||
git.branchCreate()
|
||||
.setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.SET_UPSTREAM)
|
||||
.setStartPoint(fullBranchName)
|
||||
.setName(branchName)
|
||||
.call();
|
||||
long upstreamEndTime = System.currentTimeMillis();
|
||||
log.info("Set upstream for branch: " + branchName + ", time: " + (upstreamEndTime - upstreamStartTime) + " ms");
|
||||
}
|
||||
} else {
|
||||
// 如果不存在，创建本地分支并设置上游分支
|
||||
long createBranchStartTime = System.currentTimeMillis();
|
||||
git.branchCreate()
|
||||
.setName(branchName)
|
||||
.setStartPoint(fullBranchName)
|
||||
.call();
|
||||
long createBranchEndTime = System.currentTimeMillis();
|
||||
log.info("Created local branch: " + branchName + ", time: " + (createBranchEndTime - createBranchStartTime) + " ms");
|
||||
|
||||
// 设置上游分支
|
||||
long setUpstreamStartTime = System.currentTimeMillis();
|
||||
git.branchCreate()
|
||||
.setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.SET_UPSTREAM)
|
||||
.setStartPoint(fullBranchName)
|
||||
.setName(branchName)
|
||||
.call();
|
||||
long setUpstreamEndTime = System.currentTimeMillis();
|
||||
log.info("Set upstream for branch: " + branchName + ", time: " + (setUpstreamEndTime - setUpstreamStartTime) + " ms");
|
||||
}
|
||||
|
||||
// 执行 git pull
|
||||
long pullStartTime = System.currentTimeMillis();
|
||||
PullCommand pullCommand = git.pull()
|
||||
.setCredentialsProvider(credentialsProvider);
|
||||
pullCommand.call();
|
||||
long pullEndTime = System.currentTimeMillis();
|
||||
log.info("Updated local branch: " + branchName + ", time: " + (pullEndTime - pullStartTime) + " ms");
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查本地分支是否已经关联到远程分支
|
||||
*
|
||||
* @param repository 仓库对象
|
||||
* @param branchName 分支名称
|
||||
* @return 如果已经关联，返回 true；否则返回 false
|
||||
*/
|
||||
private static boolean isBranchUpstreamSet(Repository repository, String branchName) {
|
||||
try {
|
||||
org.eclipse.jgit.lib.Config config = repository.getConfig();
|
||||
String upstreamBranch = config.getString("branch", branchName, "merge");
|
||||
return upstreamBranch != null && !upstreamBranch.isEmpty();
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to check upstream for branch: " + branchName, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public static void gitFetch(String localPath, String username, String password) throws IOException, GitAPIException {
|
||||
FileRepositoryBuilder builder = new FileRepositoryBuilder();
|
||||
Repository repository = builder.setGitDir(new File(localPath, ".git"))
|
||||
|
|
@ -174,7 +541,7 @@ public class DVCUtils {
|
|||
}
|
||||
|
||||
public static void dvcPush(String localPath) throws Exception {
|
||||
String command = "dvc push";
|
||||
String command = "dvc push -v";
|
||||
runCommand(command, localPath);
|
||||
}
|
||||
|
||||
|
|
@ -185,6 +552,13 @@ public class DVCUtils {
|
|||
}
|
||||
|
||||
|
||||
// 方法
|
||||
public static void dvcCheckout(String localPath) throws Exception {
|
||||
String command = "dvc checkout";
|
||||
runCommand(command, localPath);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 切换到指定分支并执行git pull，然后获取data文件夹下所有文件的路径、名称和大小
|
||||
*
|
||||
|
|
@ -197,15 +571,9 @@ public class DVCUtils {
|
|||
List<Map<String, Object>> fileInfoList = new ArrayList<>();
|
||||
|
||||
try {
|
||||
// 切换到指定目录
|
||||
Path repoPath = Paths.get(localPath, repoFolder, branch);
|
||||
|
||||
//刷新 TODO
|
||||
// gitFetch(localPath, username, password);
|
||||
// // 切换到指定分支
|
||||
// gitCheckoutBranch(localPath, branch);
|
||||
// // 执行git pull
|
||||
// gitPull(localPath, username, password);
|
||||
//刷新
|
||||
refreshRemoteBranches(localPath+"/"+repoFolder, username, password,branch);
|
||||
// 读取data文件夹中的文件列表
|
||||
String path = localPath + "\\" + repoFolder + "\\" + filePath;
|
||||
Path dataPath = Paths.get(path);
|
||||
|
|
|
|||
Loading…
Reference in New Issue