ci4s
/
ci4sManagement-cloud
41
0
Fork 1
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

修改路径

This commit is contained in:
fanshuai 2024-09-12 09:07:59 +08:00
parent 2386d0d9d6
commit 2dd0102f34
3 changed files with 21 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java
View File

@ -19,11 +19,11 @@ public interface NewDatasetService {
String newCreateVersion(NewDatasetVo datasetVo) throws Exception;
List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception;
ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName,Integer id, String version) throws IOException, Exception;
ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name,Integer id, String version) throws IOException, Exception;
Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName,String repo, String owner, String version)throws Exception;
NewDatasetVo getNewDatasetDesc(Integer repoId,String name,String repo, String owner, String version)throws Exception;
List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception;

2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java
View File

@ -288,6 +288,8 @@ public class ExperimentServiceImpl implements ExperimentService {
if (dependendcy != null && trainInfo != null){
insertModelDependency(dependendcy,trainInfo,insert.getId(),experiment.getName());
}
//暂存数据集元数据{}
}catch (Exception e){
throw new RuntimeException(e);
}

28
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java
View File

@ -61,12 +61,14 @@ public class NewDatasetServiceImpl implements NewDatasetService {
@Override
public String newCreateDataset(NewDatasetVo datasetVo) throws Exception {
Jedis jedis = new Jedis(redisHost);
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
// 得到用户操作的路径
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
Integer userId = (Integer) userInfo.get("user_id");
// 拼接project
@ -80,6 +82,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
// 创建项目
Map project = gitService.createProject(gitProjectVo);
Integer gitlinIid = (Integer) project.get("id");
//相对路径!!!
String relativePath = ci4sUsername + "/" + "datasets" + "/" + gitlinIid + "/" + datasetVo.getName() + "/";
if (gitlinIid == null){
throw new Exception("创建数据集失败:"+project.get("message"));
}
@ -99,7 +103,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String sourcePath = url.substring(0, url.lastIndexOf("/"));
// 命令行操作 git clone 项目地址
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + gitlinIid + "/" + repositoryName + "/" + branchName;
String s3Path = "management-platform-files" + "/" + relativePath + branchName;
DVCUtils.moveFiles(sourcePath, localPath);
// 拼接生产的元数据后写入yaml文件
@ -115,7 +119,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
datasetVo.setIdentifier(repositoryName);
datasetVo.setId(gitlinIid);
datasetVo.setOwner((String) userInfo.get("login"));
datasetVo.setRelativePaths(ci4sUsername+"/" + gitlinIid + "/" + datasetVo.getName());
datasetVo.setRelativePaths(relativePath);
if (StringUtils.isEmpty(datasetVo.getDatasetSource())){
datasetVo.setDatasetSource("用户上传");
@ -152,7 +156,9 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
String repositoryName = datasetVo.getIdentifier();
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
String localPath = localPathlocal + ci4sUsername +"/datasets/"+ datasetVo.getId() + "/" + datasetVo.getName();
//相对路径!!!
String relativePath = ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + datasetVo.getName() + "/";
String localPath = localPathlocal + relativePath;
String sourcePath = url.substring(0, url.lastIndexOf("/"));
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
@ -167,7 +173,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
DVCUtils.dvcCheckout(localPath);
// 准备数据
String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName;
String s3Path = "management-platform-files" + "/" + relativePath + branchName;
DVCUtils.moveFiles(sourcePath, localPath);
// 拼接生产的元数据后写入yaml文件
@ -188,7 +194,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
datasetVo.setDescription(newDatasetVo.getDescription());
datasetVo.setDataTag(newDatasetVo.getDataTag());
datasetVo.setDataType(newDatasetVo.getDataType());
datasetVo.setRelativePaths(ci4sUsername + "/" + datasetVo.getId() + "/" + datasetVo.getName());
datasetVo.setRelativePaths(relativePath);
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
//dvc数据跟踪
// 配置远程S3地址
@ -264,7 +270,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}
@Override
public NewDatasetVo getNewDatasetDesc(Integer id, String repositoryName, String repo, String owner, String version) throws Exception {
public NewDatasetVo getNewDatasetDesc(Integer id, String name, String repo, String owner, String version) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
@ -276,9 +282,9 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}
version = (String) versionList.get(0).get("name");
}
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, repositoryName, version, "data",gitLinkUsername, gitLinkPassword);
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, name, version, "data",gitLinkUsername, gitLinkPassword);
// 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + repositoryName + "/" + "dataset.yaml");
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + name + "/" + "dataset.yaml");
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
List<VersionVo> versionVos = new ArrayList<VersionVo>();
if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
@ -375,7 +381,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}
@Override
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName,Integer id, String version) throws Exception {
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String name,Integer id, String version) throws Exception {
// 命令行操作 git clone 项目地址
LoginUser loginUser = SecurityUtils.getLoginUser();
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
@ -384,7 +390,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+id+"/"+repositoryName;
String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+id+"/"+name;
//不需要，没意义，我查看详情已经是最新的了
// if(folder.exists() && folder.isDirectory()){
// //切换分支
@ -422,7 +428,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+repositoryName+"_"+version+"_data.zip")
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+name+"_"+version+"_data.zip")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.contentLength(zipFile.length())
.body(resource);