异步执行创建
This commit is contained in:
chenzhihang
parent
522ed32ff5
commit
24a51f4ead
|
|
@ -42,6 +42,7 @@ import java.nio.file.Files;
|
|||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
|
@ -623,22 +624,6 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
YamlUtils.generateYamlFile(metaMap, metaPath, "metadata");
|
||||
String meta = JSON.toJSONString(modelMetaVo);
|
||||
|
||||
dvcUtils.dvcInit(rootPath);
|
||||
// 配置远程S3地址
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
|
||||
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
|
||||
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
|
||||
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
|
||||
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
|
||||
|
||||
// dvc 跟踪
|
||||
dvcUtils.dvcAdd(rootPath, "model");
|
||||
// git commit
|
||||
dvcUtils.gitAdd(rootPath, ".");
|
||||
dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername);
|
||||
dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword);
|
||||
dvcUtils.dvcPush(rootPath);
|
||||
|
||||
//保存模型依赖
|
||||
modelDependency.setRepoId(gitlinIid);
|
||||
modelDependency.setIdentifier(repositoryName);
|
||||
|
|
@ -648,6 +633,30 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
modelDependency.setOwner(gitLinkUsername);
|
||||
modelDependency.setMeta(meta);
|
||||
modelDependency1Dao.insert(modelDependency);
|
||||
|
||||
CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
dvcUtils.dvcInit(rootPath);
|
||||
// 配置远程S3地址
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
|
||||
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
|
||||
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
|
||||
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
|
||||
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
|
||||
|
||||
// dvc 跟踪
|
||||
dvcUtils.dvcAdd(rootPath, "model");
|
||||
// git commit
|
||||
dvcUtils.gitAdd(rootPath, ".");
|
||||
dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername);
|
||||
dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword);
|
||||
dvcUtils.dvcPush(rootPath);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
return "新增模型成功";
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage());
|
||||
|
|
@ -719,18 +728,6 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
YamlUtils.generateYamlFile(metaMap, metaPath, "metadata");
|
||||
String meta = JSON.toJSONString(modelMetaVo);
|
||||
|
||||
// 配置远程S3地址
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName;
|
||||
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
|
||||
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
|
||||
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
|
||||
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
dvcUtils.dvcAdd(rootPath, "model");
|
||||
dvcUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName);
|
||||
//dvc push 到远程S3
|
||||
dvcUtils.dvcPush(rootPath);
|
||||
|
||||
//保存模型依赖
|
||||
modelDependency.setRepoId(modelsVo.getId());
|
||||
modelDependency.setIdentifier(repositoryName);
|
||||
|
|
@ -741,6 +738,24 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
modelDependency.setMeta(meta);
|
||||
modelDependency1Dao.insert(modelDependency);
|
||||
|
||||
CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
// 配置远程S3地址
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName;
|
||||
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
|
||||
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
|
||||
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
|
||||
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
dvcUtils.dvcAdd(rootPath, "model");
|
||||
dvcUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName);
|
||||
//dvc push 到远程S3
|
||||
dvcUtils.dvcPush(rootPath);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
return "新增模型版本成功";
|
||||
}
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import com.ruoyi.platform.service.GitService;
|
|||
import com.ruoyi.platform.service.NewDatasetService;
|
||||
import com.ruoyi.platform.utils.*;
|
||||
import com.ruoyi.platform.vo.GitProjectVo;
|
||||
import com.ruoyi.platform.vo.ModelsVo;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import com.ruoyi.platform.vo.VersionVo;
|
||||
import com.ruoyi.system.api.model.LoginUser;
|
||||
|
|
@ -37,6 +36,7 @@ import java.util.ArrayList;
|
|||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
|
@ -69,10 +69,11 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
private DatasetTempStorageService datasetTempStorageService;
|
||||
@Resource
|
||||
private DVCUtils dvcUtils;
|
||||
|
||||
@Override
|
||||
public String newCreateDataset(NewDatasetVo datasetVo) throws Exception {
|
||||
String token = gitService.checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost,redisPort);
|
||||
Jedis jedis = new Jedis(redisHost, redisPort);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
|
|
@ -90,25 +91,25 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
gitProjectVo.setPrivate(!datasetVo.getIsPublic());
|
||||
gitProjectVo.setUserId(userId);
|
||||
// 创建项目
|
||||
Map project = gitService.createProject(token,gitProjectVo);
|
||||
Map project = gitService.createProject(token, gitProjectVo);
|
||||
Integer gitlinIid = (Integer) project.get("id");
|
||||
if (gitlinIid == null){
|
||||
throw new Exception("创建数据集失败:"+project.get("message"));
|
||||
if (gitlinIid == null) {
|
||||
throw new Exception("创建数据集失败:" + project.get("message"));
|
||||
}
|
||||
// 创建分支
|
||||
String branchName = datasetVo.getVersion();
|
||||
gitService.createBranch(token,(String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
// 定义标签 标签1：ci4s-dataset 标签2：DataTag 标签3：DataType
|
||||
gitService.createTopic(token,gitlinIid, "ci4s-dataset");
|
||||
gitService.createTopic(token,gitlinIid, "datatag-" + datasetVo.getDataTag());
|
||||
gitService.createTopic(token, gitlinIid, "ci4s-dataset");
|
||||
gitService.createTopic(token, gitlinIid, "datatag-" + datasetVo.getDataTag());
|
||||
gitService.createTopic(token, gitlinIid, "datatype-" + datasetVo.getDataType());
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/"+ repositoryName + ".git";
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
|
||||
// 得到用户操作的路径
|
||||
String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String relatePath = ci4sUsername+"/datasets/" + gitlinIid + "/" + repositoryName;
|
||||
String localPath = localPathlocal+ relatePath;
|
||||
String relatePath = ci4sUsername + "/datasets/" + gitlinIid + "/" + repositoryName;
|
||||
String localPath = localPathlocal + relatePath;
|
||||
String datasetPath = localPath + "/dataset";
|
||||
|
||||
// 命令行操作 git clone 项目地址
|
||||
|
|
@ -122,7 +123,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("<pre><code>" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " "+ projectUrl + "\n" +
|
||||
"git clone -b " + branchName + " " + projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"</code></pre>");
|
||||
|
|
@ -134,26 +135,34 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
addDatasetSourceToDataVo(datasetVo);
|
||||
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
|
||||
// dvc init 初始化
|
||||
dvcUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
dvcUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
dvcUtils.dvcAdd(localPath, "dataset");
|
||||
// git commit
|
||||
dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
dvcUtils.dvcPush(localPath);
|
||||
|
||||
CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
// dvc init 初始化
|
||||
dvcUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
dvcUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
dvcUtils.dvcAdd(localPath, "dataset");
|
||||
// git commit
|
||||
dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
dvcUtils.dvcPush(localPath);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
return "新增数据集成功";
|
||||
}
|
||||
|
||||
public String newCreateVersion(NewDatasetVo datasetVo) throws Exception {
|
||||
String token = gitService.checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost,redisPort);
|
||||
Jedis jedis = new Jedis(redisHost, redisPort);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
|
|
@ -164,7 +173,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
|
||||
String repositoryName = datasetVo.getIdentifier();
|
||||
String sourcePath = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String relatePath = ci4sUsername +"/datasets/"+ datasetVo.getId() + "/" + repositoryName;
|
||||
String relatePath = ci4sUsername + "/datasets/" + datasetVo.getId() + "/" + repositoryName;
|
||||
String localPath = localPathlocal + relatePath;
|
||||
String datasetPath = localPath + "/dataset";
|
||||
String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
|
|
@ -188,13 +197,13 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
datasetVo.setUpdateTime(DateUtils.getTime());
|
||||
datasetVo.setUsage("<pre><code>" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " "+ projectUrl + "\n" +
|
||||
"git clone -b " + branchName + " " + projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"</code></pre>");
|
||||
datasetVo.setIdentifier(repositoryName);
|
||||
datasetVo.setOwner((String) userInfo.get("login"));
|
||||
if (StringUtils.isEmpty(datasetVo.getDatasetSource())){
|
||||
if (StringUtils.isEmpty(datasetVo.getDatasetSource())) {
|
||||
datasetVo.setDatasetSource("用户上传");
|
||||
}
|
||||
datasetVo.setDescription(newDatasetVo.getDescription());
|
||||
|
|
@ -205,17 +214,25 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
addDatasetSourceToDataVo(datasetVo);
|
||||
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
|
||||
//dvc数据跟踪
|
||||
// 配置远程S3地址
|
||||
dvcUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
dvcUtils.dvcAdd(localPath, "dataset");
|
||||
dvcUtils.pushNewBranchToRemote(localPath,gitLinkUsername,gitLinkPassword,branchName);
|
||||
//dvc push 到远程S3
|
||||
dvcUtils.dvcPush(localPath);
|
||||
|
||||
CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
//dvc数据跟踪
|
||||
// 配置远程S3地址
|
||||
dvcUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
dvcUtils.dvcAdd(localPath, "dataset");
|
||||
dvcUtils.pushNewBranchToRemote(localPath, gitLinkUsername, gitLinkPassword, branchName);
|
||||
//dvc push 到远程S3
|
||||
dvcUtils.dvcPush(localPath);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
return "新增数据集成功";
|
||||
}
|
||||
|
||||
|
|
@ -223,7 +240,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
@Override
|
||||
public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
String token = gitService.checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost,redisPort);
|
||||
Jedis jedis = new Jedis(redisHost, redisPort);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
|
|
@ -232,8 +249,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?category=manage";
|
||||
|
||||
String name = dataset.getName();
|
||||
if (StringUtils.isNotEmpty(name)){
|
||||
url =url + "&search=" + name;
|
||||
if (StringUtils.isNotEmpty(name)) {
|
||||
url = url + "&search=" + name;
|
||||
}
|
||||
String req = httpUtils.sendGetWithToken(url, null, token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
|
|
@ -248,7 +265,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
@Override
|
||||
public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
String token = gitService.checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost,redisPort);
|
||||
Jedis jedis = new Jedis(redisHost, redisPort);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
|
|
@ -258,8 +275,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&sort_by=praises_count";
|
||||
|
||||
String name = dataset.getName();
|
||||
if (StringUtils.isNotEmpty(name)){
|
||||
url =url + "&search=" + name;
|
||||
if (StringUtils.isNotEmpty(name)) {
|
||||
url = url + "&search=" + name;
|
||||
}
|
||||
|
||||
String req = httpUtils.sendGetWithToken(url, null, token);
|
||||
|
|
@ -287,14 +304,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
}
|
||||
version = (String) versionList.get(0).get("name");
|
||||
}
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, repo, version, "dataset",gitLinkUsername, gitLinkPassword);
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPathlocal + loginUser.getUsername() + "/datasets/" + id, repo, version, "dataset", gitLinkUsername, gitLinkPassword);
|
||||
// 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
|
||||
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!fileDetailsAfterGitPull:{}"+fileDetailsAfterGitPull);
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + repo + "/" + "dataset.yaml");
|
||||
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!fileDetailsAfterGitPull:{}" + fileDetailsAfterGitPull);
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + repo + "/" + "dataset.yaml");
|
||||
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
|
||||
List<VersionVo> versionVos = new ArrayList<VersionVo>();
|
||||
if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
|
||||
for(Map<String, Object> fileDetail : fileDetailsAfterGitPull){
|
||||
if (fileDetailsAfterGitPull != null && fileDetailsAfterGitPull.size() > 0) {
|
||||
for (Map<String, Object> fileDetail : fileDetailsAfterGitPull) {
|
||||
VersionVo versionVo = new VersionVo();
|
||||
versionVo.setUrl((String) fileDetail.get("filePath"));
|
||||
versionVo.setFileName((String) fileDetail.get("fileName"));
|
||||
|
|
@ -310,7 +327,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
@Override
|
||||
public List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception {
|
||||
String token = gitService.checkoutToken();
|
||||
List<Map<String, Object>> brancheList = gitService.getBrancheList(token,owner, repo);
|
||||
List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repo);
|
||||
return brancheList.stream()
|
||||
.filter(branch -> !"master".equals(branch.get("name")))
|
||||
.collect(Collectors.toList());
|
||||
|
|
@ -319,13 +336,13 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
@Override
|
||||
public void deleteDatasetNew(String repo, String owner) throws Exception {
|
||||
String token = gitService.checkoutToken();
|
||||
gitService.deleteProject(token,owner, repo);
|
||||
gitService.deleteProject(token, owner, repo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteDatasetVersionNew(String repo, String owner, String version, String relativePath) throws Exception {
|
||||
String token = gitService.checkoutToken();
|
||||
gitService.deleteBranch(token,owner, repo, version, localPathlocal + relativePath);
|
||||
gitService.deleteBranch(token, owner, repo, version, localPathlocal + relativePath);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -378,14 +395,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name,Integer id, String version) throws Exception {
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name, Integer id, String version) throws Exception {
|
||||
// 命令行操作 git clone 项目地址
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Jedis jedis = new Jedis(redisHost,redisPort);
|
||||
Jedis jedis = new Jedis(redisHost, redisPort);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+id+"/"+name;
|
||||
String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + id + "/" + name;
|
||||
|
||||
// 打包 data 文件夹
|
||||
String dataFolderPath = localPath + "/data";
|
||||
|
|
@ -412,7 +429,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+name+"_"+version+"_data.zip")
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + name + "_" + version + "_data.zip")
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.contentLength(zipFile.length())
|
||||
.body(resource);
|
||||
|
|
@ -420,7 +437,6 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
}
|
||||
|
||||
|
||||
|
||||
public List<NewDatasetVo> convert(List<Map<String, Object>> lst, String datasetTopic, String datasetTagName, String datasetTypeName) {
|
||||
if (lst != null && lst.size() > 0) {
|
||||
List<NewDatasetVo> result = new ArrayList<>();
|
||||
|
|
@ -460,12 +476,12 @@ public class NewDatasetServiceImpl implements NewDatasetService {
|
|||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
private void addDatasetSourceToDataVo(NewDatasetVo datasetVo){
|
||||
private void addDatasetSourceToDataVo(NewDatasetVo datasetVo) {
|
||||
DatasetTempStorage queryDatasetTempStorage = new DatasetTempStorage();
|
||||
queryDatasetTempStorage.setName(datasetVo.getName());
|
||||
queryDatasetTempStorage.setVersion(datasetVo.getVersion());
|
||||
DatasetTempStorage datasetTempStorage = datasetTempStorageService.queryByDatasetTempStorage(queryDatasetTempStorage);
|
||||
if (datasetTempStorage != null){
|
||||
if (datasetTempStorage != null) {
|
||||
String datasetSourceString = datasetTempStorage.getSource();
|
||||
Map<String, Object> datasetSourceMap = JacksonUtil.parseJSONStr2Map(datasetSourceString);
|
||||
String preprocessCode = (String) datasetSourceMap.get("preprocess_code");
|
||||
|
|
|
|||
Loading…
Reference in New Issue