diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java index 8fa97d0..71e96cc 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/NewDatasetService.java @@ -19,11 +19,11 @@ public interface NewDatasetService { String newCreateVersion(NewDatasetVo datasetVo) throws Exception; List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception; ResponseEntity downloadDatasetlocal(String filePath) throws Exception; - ResponseEntity downloadAllDatasetFilesNew(String repositoryName,Integer id, String version) throws IOException, Exception; + ResponseEntity downloadAllDatasetFilesNew(String name,Integer id, String version) throws IOException, Exception; Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; Page newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; - NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName,String repo, String owner, String version)throws Exception; + NewDatasetVo getNewDatasetDesc(Integer repoId,String name,String repo, String owner, String version)throws Exception; List> getVersionList(String repo, String owner) throws Exception; diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java index 388155d..293b782 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java @@ -288,6 +288,8 @@ public class ExperimentServiceImpl implements ExperimentService { if (dependendcy != null && trainInfo != null){ insertModelDependency(dependendcy,trainInfo,insert.getId(),experiment.getName()); } + //暂存数据集元数据{} + }catch (Exception e){ throw new RuntimeException(e); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java index 40c3189..235f1b9 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java @@ -61,12 +61,14 @@ public class NewDatasetServiceImpl implements NewDatasetService { @Override public String newCreateDataset(NewDatasetVo datasetVo) throws Exception { + Jedis jedis = new Jedis(redisHost); LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + // 得到用户操作的路径 Map userInfo = JsonUtils.jsonToMap(userReq); Integer userId = (Integer) userInfo.get("user_id"); // 拼接project @@ -80,6 +82,8 @@ public class NewDatasetServiceImpl implements NewDatasetService { // 创建项目 Map project = gitService.createProject(gitProjectVo); Integer gitlinIid = (Integer) project.get("id"); + //相对路径!!! + String relativePath = ci4sUsername + "/" + "datasets" + "/" + gitlinIid + "/" + datasetVo.getName() + "/"; if (gitlinIid == null){ throw new Exception("创建数据集失败:"+project.get("message")); } @@ -99,7 +103,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { String sourcePath = url.substring(0, url.lastIndexOf("/")); // 命令行操作 git clone 项目地址 DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); - String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + gitlinIid + "/" + repositoryName + "/" + branchName; + String s3Path = "management-platform-files" + "/" + relativePath + branchName; DVCUtils.moveFiles(sourcePath, localPath); // 拼接生产的元数据后写入yaml文件 @@ -115,7 +119,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { datasetVo.setIdentifier(repositoryName); datasetVo.setId(gitlinIid); datasetVo.setOwner((String) userInfo.get("login")); - datasetVo.setRelativePaths(ci4sUsername+"/" + gitlinIid + "/" + datasetVo.getName()); + datasetVo.setRelativePaths(relativePath); if (StringUtils.isEmpty(datasetVo.getDatasetSource())){ datasetVo.setDatasetSource("用户上传"); @@ -152,7 +156,9 @@ public class NewDatasetServiceImpl implements NewDatasetService { String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); String repositoryName = datasetVo.getIdentifier(); String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String localPath = localPathlocal + ci4sUsername +"/datasets/"+ datasetVo.getId() + "/" + datasetVo.getName(); + //相对路径!!! + String relativePath = ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + datasetVo.getName() + "/"; + String localPath = localPathlocal + relativePath; String sourcePath = url.substring(0, url.lastIndexOf("/")); String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; @@ -167,7 +173,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { DVCUtils.dvcCheckout(localPath); // 准备数据 - String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName; + String s3Path = "management-platform-files" + "/" + relativePath + branchName; DVCUtils.moveFiles(sourcePath, localPath); // 拼接生产的元数据后写入yaml文件 @@ -188,7 +194,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { datasetVo.setDescription(newDatasetVo.getDescription()); datasetVo.setDataTag(newDatasetVo.getDataTag()); datasetVo.setDataType(newDatasetVo.getDataType()); - datasetVo.setRelativePaths(ci4sUsername + "/" + datasetVo.getId() + "/" + datasetVo.getName()); + datasetVo.setRelativePaths(relativePath); YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset"); //dvc数据跟踪 // 配置远程S3地址 @@ -264,7 +270,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { } @Override - public NewDatasetVo getNewDatasetDesc(Integer id, String repositoryName, String repo, String owner, String version) throws Exception { + public NewDatasetVo getNewDatasetDesc(Integer id, String name, String repo, String owner, String version) throws Exception { LoginUser loginUser = SecurityUtils.getLoginUser(); String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); @@ -276,9 +282,9 @@ public class NewDatasetServiceImpl implements NewDatasetService { } version = (String) versionList.get(0).get("name"); } - List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, repositoryName, version, "data",gitLinkUsername, gitLinkPassword); + List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, name, version, "data",gitLinkUsername, gitLinkPassword); // 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据 - Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + repositoryName + "/" + "dataset.yaml"); + Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + name + "/" + "dataset.yaml"); NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class); List versionVos = new ArrayList(); if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){ @@ -375,7 +381,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { } @Override - public ResponseEntity downloadAllDatasetFilesNew(String repositoryName,Integer id, String version) throws Exception { + public ResponseEntity downloadAllDatasetFilesNew(String name,Integer id, String version) throws Exception { // 命令行操作 git clone 项目地址 LoginUser loginUser = SecurityUtils.getLoginUser(); String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); @@ -384,7 +390,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { Jedis jedis = new Jedis(redisHost); String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); Map userInfo = JsonUtils.jsonToMap(userReq); - String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+id+"/"+repositoryName; + String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+id+"/"+name; //不需要,没意义,我查看详情已经是最新的了 // if(folder.exists() && folder.isDirectory()){ // //切换分支 @@ -422,7 +428,7 @@ public class NewDatasetServiceImpl implements NewDatasetService { InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile)); return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+repositoryName+"_"+version+"_data.zip") + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+name+"_"+version+"_data.zip") .contentType(MediaType.APPLICATION_OCTET_STREAM) .contentLength(zipFile.length()) .body(resource);