From 16332d4d0b0b1ded3ffce07159d507adc483e569 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Thu, 5 Sep 2024 16:33:03 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=96=B0=E6=A8=A1=E5=9E=8Bdvc=E7=AE=A1?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/NewModelFromGitController.java | 48 ++- .../platform/domain/ModelDependency1.java | 25 ++ .../com/ruoyi/platform/domain/Models.java | 27 +- .../platform/mapper/ModelDependency1Dao.java | 17 ++ .../ruoyi/platform/service/ModelsService.java | 11 +- .../service/impl/DatasetServiceImpl.java | 118 ++++---- .../service/impl/ModelsServiceImpl.java | 281 ++++++++++++------ .../com/ruoyi/platform/utils/ConvertUtil.java | 17 +- .../com/ruoyi/platform/utils/DVCUtils.java | 10 +- .../com/ruoyi/platform/utils/YamlUtils.java | 10 +- .../platform/vo/ModelDependency1TreeVo.java | 18 ++ .../com/ruoyi/platform/vo/ModelMetaVo.java | 67 +++++ .../java/com/ruoyi/platform/vo/ModelsVo.java | 35 +-- .../ModelDependency1DaoMapper.xml | 29 ++ .../managementPlatform/ModelsDaoMapper.xml | 4 +- 15 files changed, 501 insertions(+), 216 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependency1TreeVo.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java create mode 100644 ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java index 2966d2b..14f53e8 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java @@ -9,6 +9,7 @@ import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.PageRequest; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; @@ -20,16 +21,17 @@ public class NewModelFromGitController { @Resource private ModelsService modelsService; - @PostMapping("/addModelAndVersion") - @ApiOperation("添加模型和版本") + @PostMapping("/addModel") + @ApiOperation("添加模型") public AjaxResult addModelAndVersion(@RequestBody ModelsVo modelsVo) throws Exception { return AjaxResult.success(this.modelsService.newCreateModel(modelsVo)); } - @PostMapping("/addVersion") - @ApiOperation("添加版本") - public AjaxResult addVersion(@RequestBody ModelsVo modelsVo) throws Exception { - return AjaxResult.success(this.modelsService.newCreateVersion(modelsVo)); + @CrossOrigin(origins = "*", allowedHeaders = "*") + @PostMapping("/upload") + @ApiOperation(value = "上传模型", notes = "根据模型id上传模型文件，并将信息存入数据库。") + public AjaxResult uploadModel(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception { + return AjaxResult.success(this.modelsService.uploadModelLocal(files, uuid)); } /** @@ -61,15 +63,15 @@ public class NewModelFromGitController { return modelsService.downloadModels(model_version_id); } - @GetMapping("/queryDatasets") + @GetMapping("/queryModels") @ApiOperation("模型广场公开模型分页查询,根据model_type,model_tag筛选，true公开false私有") - public AjaxResult queryDatasets(@RequestParam(value = "page") int page, - @RequestParam(value = "size") int size, - @RequestParam(value = "is_public") Boolean isPublic, - @RequestParam(value = "model_type", required = false) String modelType, - @RequestParam(value = "model_tag", required = false) String modelTag, - @RequestParam(value = "git_link_username") String gitLinkUsername, - @RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception { + public AjaxResult queryModels(@RequestParam(value = "page") int page, + @RequestParam(value = "size") int size, + @RequestParam(value = "is_public") Boolean isPublic, + @RequestParam(value = "model_type", required = false) String modelType, + @RequestParam(value = "model_tag", required = false) String modelTag, + @RequestParam(value = "git_link_username") String gitLinkUsername, + @RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception { PageRequest pageRequest = PageRequest.of(page, size); ModelsVo modelsVo = new ModelsVo(); modelsVo.setModelType(modelType); @@ -83,4 +85,22 @@ public class NewModelFromGitController { } } + @GetMapping("/getModelVersions/{modelName}") + @ApiOperation(value = "获取模型所有版本号") + public AjaxResult getModelVersions(@PathVariable("modelName") String modelName) throws Exception { + return AjaxResult.success(this.modelsService.getNewModelVersion(modelName)); + } + + + @GetMapping("/getModelMeta") + @ApiOperation(value = "获取模型元数据") + public AjaxResult getModelMeta(@RequestParam("modelName") String modelName, @RequestParam("version") String version) throws Exception { + return AjaxResult.success(this.modelsService.getModelMeta(modelName, version)); + } + + @GetMapping("/getModelDependencyTree") + @ApiOperation(value = "获取模型依赖关系树") + public AjaxResult getModelDependencyTree(@RequestParam("modelName") String modelName, @RequestParam("version") String version) throws Exception { + return AjaxResult.success(this.modelsService.getModelDependencyTree(modelName, version)); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java new file mode 100644 index 0000000..2cd3406 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java @@ -0,0 +1,25 @@ +package com.ruoyi.platform.domain; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import lombok.Data; + +import java.io.Serializable; + +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +@Data +public class ModelDependency1 implements Serializable { + private Long id; + + private String parentModel; + + private String modelName; + + private String version; + + private Integer repoId; + + private String repoName; + + private Integer state; +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/Models.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/Models.java index 449ae7f..db7f6bb 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/Models.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/Models.java @@ -6,8 +6,8 @@ import com.ruoyi.platform.annotations.CheckDuplicate; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import java.util.Date; import java.io.Serializable; +import java.util.Date; /** * (Models)实体类 @@ -53,17 +53,18 @@ public class Models implements Serializable { @ApiModelProperty(value = "状态，0失效，1生效") private Integer state; + @ApiModelProperty(value = "模型gitLink仓库id") + private Integer repoId; + + @ApiModelProperty(value = "模型gitLink仓库名称") + private String repoName; @ApiModelProperty(value = "模型类型名") private String modelTypeName; - - @ApiModelProperty(value = "模型tag名") private String modelTagName; - - public Integer getId() { return id; } @@ -170,6 +171,22 @@ public class Models implements Serializable { this.state = state; } + public Integer getRepoId() { + return repoId; + } + + public void setRepoId(Integer repoId) { + this.repoId = repoId; + } + + public String getRepoName() { + return repoName; + } + + public void setRepoName(String repoName) { + this.repoName = repoName; + } + public String getModelTagName() { return modelTagName; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java new file mode 100644 index 0000000..cb54b28 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java @@ -0,0 +1,17 @@ +package com.ruoyi.platform.mapper; + +import com.ruoyi.platform.domain.ModelDependency1; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +public interface ModelDependency1Dao { + + int insert(ModelDependency1 modelDependency1); + + List queryByModelName(@Param("modelName") String modelName); + + ModelDependency1 queryByModelNameAndVersion(@Param("modelName") String modelName, @Param("version") String version); + + List queryByParentModel(@Param("parentModel") String parentModel); +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java index 8502fd9..aea4559 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java @@ -3,6 +3,8 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.domain.Models; import com.ruoyi.platform.domain.ModelsVersion; +import com.ruoyi.platform.vo.ModelDependency1TreeVo; +import com.ruoyi.platform.vo.ModelMetaVo; import com.ruoyi.platform.vo.ModelsVo; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; @@ -86,14 +88,19 @@ public interface ModelsService { List> exportModels(String path, String uuid) throws Exception; - CompletableFuture newCreateModel(ModelsVo modelsVo) throws Exception; - CompletableFuture newCreateVersion(ModelsVo modelsVo); + List> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception; ResponseEntity downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception; Page newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; Page newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception; + + List getNewModelVersion(String modelName); + + ModelMetaVo getModelMeta(String modelName, String version) throws Exception; + + ModelDependency1TreeVo getModelDependencyTree(String modelName, String version)throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java index 4233d62..887b3ed 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java @@ -79,7 +79,7 @@ public class DatasetServiceImpl implements DatasetService { private String bucketName; @Resource - private MinioUtil minioUtil; + private MinioUtil minioUtil; @Value("${spring.redis.host}") private String redisHost; @@ -130,8 +130,8 @@ public class DatasetServiceImpl implements DatasetService { /** * 分页查询 * - * @param dataset 筛选条件 - * @param pageRequest 分页对象 + * @param dataset 筛选条件 + * @param pageRequest 分页对象 * @return 查询结果 */ @Override @@ -171,7 +171,7 @@ public class DatasetServiceImpl implements DatasetService { @Override public Dataset update(Dataset dataset) { int currentState = dataset.getState(); - if (currentState == 0){ + if (currentState == 0) { throw new RuntimeException("数据集已被删除，无法更新。"); } LoginUser loginUser = SecurityUtils.getLoginUser(); @@ -195,7 +195,7 @@ public class DatasetServiceImpl implements DatasetService { @Override public String removeById(Integer id) throws Exception { Dataset dataset = this.datasetDao.queryById(id); - if (dataset == null){ + if (dataset == null) { throw new Exception("数据集不存在"); } @@ -203,15 +203,15 @@ public class DatasetServiceImpl implements DatasetService { LoginUser loginUser = SecurityUtils.getLoginUser(); String username = loginUser.getUsername(); String createdBy = dataset.getCreateBy(); - if (!(StringUtils.equals(username,"admin") || StringUtils.equals(username,createdBy))){ + if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createdBy))) { throw new Exception("无权限删除该数据集"); } - if (datasetVersionService.queryByDatasetId(id).size()>0){ + if (datasetVersionService.queryByDatasetId(id).size() > 0) { throw new Exception("请先删除该数据集下的版本文件"); } dataset.setState(0); - return this.datasetDao.update(dataset)>0?"删除数据集成功":"删除数据集失败"; + return this.datasetDao.update(dataset) > 0 ? "删除数据集成功" : "删除数据集失败"; } /** @@ -230,14 +230,14 @@ public class DatasetServiceImpl implements DatasetService { // 从数据库中获取存储路径（即MinIO中的对象名称） String objectName = datasetVersion.getUrl(); - if(objectName == null || objectName.isEmpty() ){ + if (objectName == null || objectName.isEmpty()) { throw new Exception("未找到该版本数据集文件"); } try { // 使用ByteArrayOutputStream来捕获下载的数据 ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - minioUtil.downloadObject(bucketName,objectName,outputStream); + minioUtil.downloadObject(bucketName, objectName, outputStream); ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); InputStreamResource resource = new InputStreamResource(inputStream); @@ -250,20 +250,20 @@ public class DatasetServiceImpl implements DatasetService { e.printStackTrace(); throw new Exception("下载数据集文件错误"); } - } + } /** * 上传数据集 * * @param files 文件 - * @param uuid 唯一标识 + * @param uuid 唯一标识 * @return 是否成功 */ @Override public List> uploadDataset(MultipartFile[] files, String uuid) throws Exception { List> results = new ArrayList<>(); - for (MultipartFile file:files){ + for (MultipartFile file : files) { // 构建objectName String username = SecurityUtils.getLoginUser().getUsername(); String fileName = file.getOriginalFilename(); @@ -296,7 +296,7 @@ public class DatasetServiceImpl implements DatasetService { datasetVersion.setUpdateTime(date); datasetVersion.setState(1); datasetVersionDao.insert(datasetVersion); - }else { + } else { //改表 BeansUtils.copyPropertiesIgnoreNull(datasetVersion, version); Date createTime = version.getCreateTime(); @@ -306,7 +306,7 @@ public class DatasetServiceImpl implements DatasetService { datasetVersionService.update(version); } Map result = new HashMap(); - result.put("url",url); + result.put("url", url); return result; } @@ -330,7 +330,7 @@ public class DatasetServiceImpl implements DatasetService { @Transactional public String insertDatasetAndVersion(DatasetVo datasetVo) throws Exception { List datasetVersionVos = datasetVo.getDatasetVersionVos(); - if (datasetVersionVos==null || datasetVersionVos.isEmpty()){ + if (datasetVersionVos == null || datasetVersionVos.isEmpty()) { throw new Exception("数据集版本信息错误"); } @@ -342,11 +342,11 @@ public class DatasetServiceImpl implements DatasetService { dataset.setDataType(datasetVo.getDataType()); dataset.setDataTag(datasetVo.getDataTag()); Dataset datasetInsert = this.insert(dataset); - if (datasetInsert == null){ + if (datasetInsert == null) { throw new Exception("新增数据集失败"); } - for (VersionVo datasetVersionVo : datasetVersionVos){ + for (VersionVo datasetVersionVo : datasetVersionVos) { DatasetVersion datasetVersion = new DatasetVersion(); datasetVersion.setDatasetId(datasetInsert.getId()); datasetVersion.setVersion(datasetVo.getVersion()); @@ -388,7 +388,7 @@ public class DatasetServiceImpl implements DatasetService { } @Override - public ResponseEntity downloadAllDatasetFiles(Integer datasetId, String version) throws Exception { + public ResponseEntity downloadAllDatasetFiles(Integer datasetId, String version) throws Exception { // 根据数据集id查数据名 Dataset dataset = this.datasetDao.queryById(datasetId); String datasetName = dataset.getName(); @@ -420,7 +420,7 @@ public class DatasetServiceImpl implements DatasetService { // 设置响应 return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + datasetName + "_" + version + ".zip\"") + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + datasetName + "_" + version + ".zip\"") .contentType(MediaType.APPLICATION_OCTET_STREAM) .body(resource); } catch (Exception e) { @@ -442,7 +442,7 @@ public class DatasetServiceImpl implements DatasetService { minioUtil.copyDirectory(srcBucketName, srcDir, bucketName, targetDir); List movedItems = minioUtil.getAllObjectsByPrefix(bucketName, targetDir, true); for (Item movedItem : movedItems) { - if(!movedItem.isDir() && movedItem.size() > 0){ // 检查是否为非目录且文件大小大于0 + if (!movedItem.isDir() && movedItem.size() > 0) { // 检查是否为非目录且文件大小大于0 Map result = new HashMap<>(); String url = movedItem.objectName(); String fileName = extractFileName(url); @@ -459,7 +459,6 @@ public class DatasetServiceImpl implements DatasetService { } - @Override public CompletableFuture newCreateDataset(NewDatasetVo datasetVo) { return CompletableFuture.supplyAsync(() -> { @@ -493,19 +492,19 @@ public class DatasetServiceImpl implements DatasetService { gitService.createTopic(token, repoId, "DataTag_" + datasetVo.getDataTag()); gitService.createTopic(token, repoId, "DataType_" + datasetVo.getDataType()); // 得到项目地址 - String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git"; + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; // 得到用户操作的路径 String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String localPath = localPathlocal + "/" + repoId+"/"+datasetVo.getName(); + String localPath = localPathlocal + ci4sUsername + "/datasets/" + datasetVo.getName() + "/" + branchName; String sourcePath = url.substring(0, url.lastIndexOf("/")); // 命令行操作 git clone 项目地址 DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); - String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+repoId+"/"+ repositoryName + "/" + branchName; + String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + repoId + "/" + repositoryName + "/" + branchName; - DVCUtils.moveFiles(sourcePath, localPath); + DVCUtils.moveFiles(sourcePath, localPath + "/data"); //拼接生产的元数据后写入yaml文件 - datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login"))); + datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); datasetVo.setUpdateTime(DateUtils.getTime()); datasetVo.setVersionDesc(datasetVo.getDescription()); datasetVo.setUsage("```bash\n" + @@ -514,7 +513,7 @@ public class DatasetServiceImpl implements DatasetService { "# 远程拉取配置文件\n" + "dvc pull\n" + "```"); - YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),localPath, "dataset"); + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath + "/metadata", "metadata"); // dvc init 初始化 DVCUtils.dvcInit(localPath); // 配置远程S3地址 @@ -550,31 +549,31 @@ public class DatasetServiceImpl implements DatasetService { String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); Map userInfo = JsonUtils.jsonToMap(userReq); // 创建分支 - String branchName = StringUtils.isEmpty(datasetVo.getVersion())? "master" : datasetVo.getVersion(); + String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion(); String repositoryName = datasetVo.getIdentifier(); if (StringUtils.equals(branchName, "master")) { gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); } // 得到项目地址 - String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git"; + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; // 得到用户操作的路径 String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+ datasetVo.getName(); + String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + datasetVo.getName() + "/" + branchName; String sourcePath = url.substring(0, url.lastIndexOf("/")); // 命令行操作 git clone 项目地址 - if(FileUtil.checkDirectoryExists(localPath)){ + if (FileUtil.checkDirectoryExists(localPath)) { DVCUtils.gitFetch(localPath); - DVCUtils.gitCheckoutBranch(localPath,branchName); - }else { + DVCUtils.gitCheckoutBranch(localPath, branchName); + } else { DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword); } - String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+ datasetVo.getRepoId()+"/"+ repositoryName + "/" + branchName; + String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + datasetVo.getRepoId() + "/" + repositoryName + "/" + branchName; - DVCUtils.moveFiles(sourcePath, localPath); + DVCUtils.moveFiles(sourcePath, localPath + "/data"); //拼接生产的元数据后写入yaml文件 - datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login"))); + datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login"))); datasetVo.setUpdateTime(DateUtils.getTime()); datasetVo.setVersionDesc(datasetVo.getDescription()); datasetVo.setUsage("```bash\n" + @@ -583,7 +582,7 @@ public class DatasetServiceImpl implements DatasetService { "# 远程拉取配置文件\n" + "dvc pull\n" + "```"); - YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),localPath, "dataset"); + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath + "/metadata", "metadata"); // dvc init 初始化 DVCUtils.dvcInit(localPath); @@ -622,10 +621,10 @@ public class DatasetServiceImpl implements DatasetService { String datasetTagName = dataset.getDatasetTagName(); String datasetTypeName = dataset.getDatasetTypeName(); String topic_name = "ci4s_dataset"; - topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName; - topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName; - String url = gitendpoint + "/api/users/"+(String) userInfo.get("login")+"/projects.json?page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&category=manage&topic_name="+topic_name; - String req = HttpUtils.sendGetWithToken(url,null,token); + topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName; + topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName; + String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name; + String req = HttpUtils.sendGetWithToken(url, null, token); Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); Integer total = (Integer) stringObjectMap.get("count"); List> projects = (List>) stringObjectMap.get("projects"); @@ -647,11 +646,11 @@ public class DatasetServiceImpl implements DatasetService { String datasetTagName = dataset.getDatasetTagName(); String datasetTypeName = dataset.getDatasetTypeName(); String topic_name = "ci4s_dataset"; - topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName; - topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName; + topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName; + topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName; - String url = gitendpoint + "/api/projects.json?user_id="+userId+"&page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&sort_by=praises_count&topic_name="+topic_name; - String req = HttpUtils.sendGetWithToken(url,null,token); + String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name; + String req = HttpUtils.sendGetWithToken(url, null, token); Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); Integer total = (Integer) stringObjectMap.get("total_count"); List> projects = (List>) stringObjectMap.get("projects"); @@ -660,15 +659,17 @@ public class DatasetServiceImpl implements DatasetService { } @Override - public NewDatasetVo getNewDatasetDesc(Integer repoId,String repositoryName, String version) { + public NewDatasetVo getNewDatasetDesc(Integer repoId, String repositoryName, String version) { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); // cd到 localPathlocal/repoId/下面还有一个文件夹，然后做git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP - List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal+repoId, repositoryName, version); + List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal + ci4sUsername + "/datasets/", repositoryName, version, "data"); //在localPathlocal+repoId+"/"+repositoryName目录下的dataset.yaml中取到元数据 - Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + repoId + "/" + repositoryName + "/dataset.yaml"); + Map stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + ci4sUsername + "/datasets/" + repositoryName + "/" + version + "/metadata/metadata.yaml"); NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class); List versionVos = new ArrayList(); - if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){ - for(Map fileDetail : fileDetailsAfterGitPull){ + if (fileDetailsAfterGitPull != null && fileDetailsAfterGitPull.size() > 0) { + for (Map fileDetail : fileDetailsAfterGitPull) { VersionVo versionVo = new VersionVo(); versionVo.setUrl((String) fileDetail.get("filePath")); versionVo.setFileName((String) fileDetail.get("fileName")); @@ -685,11 +686,11 @@ public class DatasetServiceImpl implements DatasetService { public List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception { List> results = new ArrayList<>(); - for (MultipartFile file:files){ + for (MultipartFile file : files) { // 构建objectName String username = SecurityUtils.getLoginUser().getUsername(); String fileName = file.getOriginalFilename(); - String path = "/temp/"+ username +"/datasets/"+ uuid + "/"+"/data/" + fileName; + String path = "/temp/" + username + "/datasets/" + uuid + "/" + "/data/" + fileName; long sizeInBytes = file.getSize(); String formattedSize = FileUtil.formatFileSize(sizeInBytes); File targetFile = new File(path, file.getOriginalFilename()); @@ -720,17 +721,17 @@ public class DatasetServiceImpl implements DatasetService { String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); Map userInfo = JsonUtils.jsonToMap(userReq); Integer userId = (Integer) userInfo.get("user_id"); - String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git"; - String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName; + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + repositoryName; File folder = new File(localPath); - if(folder.exists() && folder.isDirectory()){ + if (folder.exists() && folder.isDirectory()) { //切换分支 DVCUtils.gitCheckoutBranch(localPath, version); //pull - DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword); + DVCUtils.gitPull(localPath, gitLinkUsername, gitLinkPassword); //dvc pull DVCUtils.dvcPull(localPath); - }else { + } else { DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword); } @@ -767,7 +768,6 @@ public class DatasetServiceImpl implements DatasetService { } - public List convert(List> lst) { if (lst != null && lst.size() > 0) { List newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index caef5b1..92038a0 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -4,20 +4,22 @@ import com.ruoyi.common.core.utils.DateUtils; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.annotations.CheckDuplicate; import com.ruoyi.platform.domain.AssetIcon; +import com.ruoyi.platform.domain.ModelDependency1; import com.ruoyi.platform.domain.Models; import com.ruoyi.platform.domain.ModelsVersion; +import com.ruoyi.platform.mapper.ModelDependency1Dao; import com.ruoyi.platform.mapper.ModelsDao; import com.ruoyi.platform.mapper.ModelsVersionDao; import com.ruoyi.platform.service.*; import com.ruoyi.platform.utils.*; -import com.ruoyi.platform.vo.GitProjectVo; -import com.ruoyi.platform.vo.ModelsVo; -import com.ruoyi.platform.vo.VersionVo; +import com.ruoyi.platform.vo.*; import com.ruoyi.system.api.model.LoginUser; import io.minio.messages.Item; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; @@ -63,6 +65,9 @@ public class ModelsServiceImpl implements ModelsService { @Resource private ModelsVersionService modelsVersionService; + @Resource + private ModelDependency1Dao modelDependency1Dao; + @Resource private MinioService minioService; @@ -376,7 +381,7 @@ public class ModelsServiceImpl implements ModelsService { @Override @Transactional public String insertModelAndVersion(ModelsVo modelsVo) throws Exception { - List modelsVersionVos = modelsVo.getModelsVersionVos(); + List modelsVersionVos = modelsVo.getModelVersionVos(); if (modelsVersionVos == null || modelsVersionVos.isEmpty()) { throw new Exception("模型版本信息错误"); @@ -514,7 +519,6 @@ public class ModelsServiceImpl implements ModelsService { public CompletableFuture newCreateModel(ModelsVo modelsVo) throws Exception { return CompletableFuture.supplyAsync(() -> { try { - String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); Jedis jedis = new Jedis(redisHost); @@ -522,55 +526,45 @@ public class ModelsServiceImpl implements ModelsService { Map userInfo = JsonUtils.jsonToMap(userReq); Integer userId = (Integer) userInfo.get("user_id"); - // 拼接project - String repositoryName = ci4sUsername + "_model_" + DateUtils.dateTimeNow(); - GitProjectVo gitProjectVo = new GitProjectVo(); - gitProjectVo.setRepositoryName(repositoryName); - gitProjectVo.setName(modelsVo.getName()); - gitProjectVo.setDescription(modelsVo.getDescription()); - gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0); - gitProjectVo.setUserId(userId); + String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + String repositoryName = modelsVo.getRepositoryName() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getRepositoryName(); + ModelDependency1 modelDependency = new ModelDependency1(); + List oldModelDependencys = modelDependency1Dao.queryByModelName(modelsVo.getName()); - // 创建项目 - Map project = gitService.createProject(token, gitProjectVo); + if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) { + if (oldModelDependencys.stream().map(ModelDependency1::getVersion).anyMatch(version -> version.equals(modelsVo.getVersion()))) { + throw new RuntimeException("模型版本已存在，请勿重复创建"); + } - // 创建分支 - String branchName = modelsVo.getVersion(); - gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); - // 定义标签 标签1：ci4s_model 标签2：ModelTag 标签3：ModelType - gitService.createTopic(token, (Integer) project.get("id"), "ci4s_model"); - gitService.createTopic(token, (Integer) project.get("id"), "ModelTag_" + modelsVo.getModelTag()); - gitService.createTopic(token, (Integer) project.get("id"), "ModelType_" + modelsVo.getModelType()); - // 得到项目地址 - String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; - // 得到用户操作的路径 - String url = modelsVo.getModelsVersionVos().get(0).getUrl(); - String localPath1 = localPath + modelsVo.getName(); - String sourcePath = url.substring(0, url.lastIndexOf("/")); - // 命令行操作 git clone 项目地址 - DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - String s3Path = "management-platform-files/" + ci4sUsername + "/models/" + repositoryName + "/" + branchName; + //新建版本 + commonDvc(token, userInfo, ci4sUsername, modelsVo.getRepositoryName(), modelsVo, "CreateModelFromPipeline"); + modelDependency.setRepoId(modelsVo.getRepoId()); + } else { + //新建模型 + // 拼接project + GitProjectVo gitProjectVo = new GitProjectVo(); + gitProjectVo.setRepositoryName(repositoryName); + gitProjectVo.setName(modelsVo.getName()); + gitProjectVo.setDescription(modelsVo.getDescription()); + gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0); + gitProjectVo.setUserId(userId); - // 拼接生产的元数据后写入yaml文件 - YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "model"); + // 创建项目 + Map project = gitService.createProject(token, gitProjectVo); + modelDependency.setRepoId((Integer) project.get("id")); + // 定义标签 标签1：ci4s_model 标签2：ModelTag 标签3：ModelType + gitService.createTopic(token, (Integer) project.get("id"), "ci4s_model"); + gitService.createTopic(token, (Integer) project.get("id"), "modeltag_" + modelsVo.getModelTag()); + gitService.createTopic(token, (Integer) project.get("id"), "modeltype_" + modelsVo.getModelType()); + commonDvc(token, userInfo, ci4sUsername, repositoryName, modelsVo, "createModel"); + } - DVCUtils.moveFiles(sourcePath, localPath1 + "/model"); - - // dvc init 初始化 - DVCUtils.dvcInit(localPath1); - // 配置远程S3地址 - DVCUtils.dvcRemoteAdd(localPath1, s3Path); - DVCUtils.dvcConfigS3Credentials(localPath1, endpoint); - DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId); - DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey); - // dvc 跟踪 - DVCUtils.dvcAdd(localPath1, "model"); - // git commit - DVCUtils.gitAdd(localPath1, "."); - DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername); - DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - // dvc push 到远程S3 - DVCUtils.dvcPush(localPath1); + //保存模型依赖 + modelDependency.setRepoName(repositoryName); + modelDependency.setModelName(modelsVo.getName()); + modelDependency.setVersion(modelsVo.getVersion()); + modelDependency.setParentModel(modelsVo.getParentModel()); + modelDependency1Dao.insert(modelDependency); return "新增模型成功"; } catch (Exception e) { logger.error(e.getMessage()); @@ -579,53 +573,32 @@ public class ModelsServiceImpl implements ModelsService { }); } - @Override - public CompletableFuture newCreateVersion(ModelsVo modelsVo) { - return CompletableFuture.supplyAsync(() -> { - try { - String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - LoginUser loginUser = SecurityUtils.getLoginUser(); - String ci4sUsername = loginUser.getUsername(); - Jedis jedis = new Jedis(redisHost); - String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); - Map userInfo = JsonUtils.jsonToMap(userReq); - // 创建分支 - String branchName = modelsVo.getVersion(); - String repositoryName = modelsVo.getRepositoryName(); - gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); - // 得到项目地址 - String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; - // 得到用户操作的路径 - String url = modelsVo.getModelVersionVos().get(0).getUrl(); - String localPath1 = localPath + ci4sUsername + "/model/" + modelsVo.getName(); - String sourcePath = url.substring(0, url.lastIndexOf("/")); - // 命令行操作 git clone 项目地址 - DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + repositoryName + "/" + branchName; - //拼接生产的元数据后写入yaml文件 - YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "dataset"); - DVCUtils.moveFiles(sourcePath, localPath1 + "/model"); - // dvc init 初始化 - DVCUtils.dvcInit(localPath1); - // 配置远程S3地址 - DVCUtils.dvcRemoteAdd(localPath1, s3Path); - DVCUtils.dvcConfigS3Credentials(localPath1, endpoint); - DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId); - DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey); - // dvc 跟踪 - DVCUtils.dvcAdd(localPath1, "model"); - // git commit - DVCUtils.gitAdd(localPath1, "."); - DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername); - DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); - // dvc push 到远程S3 - DVCUtils.dvcPush(localPath1); - return "新增模型成功"; - } catch (Exception e) { - throw new RuntimeException(e); - } - }); + @Override + public List> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception { + List> results = new ArrayList<>(); + + for (MultipartFile file : files) { + // 构建objectName + String username = SecurityUtils.getLoginUser().getUsername(); + String fileName = file.getOriginalFilename(); + String path = "/temp/" + username + "/models/" + uuid + "/model/"; + long sizeInBytes = file.getSize(); + String formattedSize = FileUtil.formatFileSize(sizeInBytes); + File targetFile = new File(path, file.getOriginalFilename()); + // 确保目录存在 + targetFile.getParentFile().mkdirs(); + // 保存文件到目标路径 + FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile); + // 返回上传文件的路径 + String absolutePath = targetFile.getAbsolutePath(); + Map result = new HashMap<>(); + result.put("fileName", fileName); + result.put("url", absolutePath); // objectName根据实际情况定义 + result.put("fileSize", formattedSize); + results.add(result); + } + return results; } @Override @@ -684,7 +657,7 @@ public class ModelsServiceImpl implements ModelsService { @Override public Page newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { - LoginUser loginUser = SecurityUtils.getLoginUser(); + LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); Jedis jedis = new Jedis(redisHost); @@ -728,6 +701,90 @@ public class ModelsServiceImpl implements ModelsService { return new PageImpl<>(convert(projects), pageRequest, total); } + @Override + public List getNewModelVersion(String modelName) { + List modelDependency1List = modelDependency1Dao.queryByModelName(modelName); + return modelDependency1List.stream().map(ModelDependency1::getVersion).collect(Collectors.toList()); + } + + @Override + public ModelMetaVo getModelMeta(String modelName, String version) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + // git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP + List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/", modelName, version, "model"); + + Map stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + modelName + "/" + version + "/metadata/metadata.yaml"); + return ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class); + } + + @Override + public ModelDependency1TreeVo getModelDependencyTree(String modelName, String version) throws Exception { + ModelDependency1 modelDependency1 = modelDependency1Dao.queryByModelNameAndVersion(modelName, version); + ModelDependency1TreeVo modelDependency1TreeVo = new ModelDependency1TreeVo(); + BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo); + + //递归查询父模型 + List parentModelList = new ArrayList<>(); + getParentModel(parentModelList, modelDependency1); + modelDependency1TreeVo.setPatrentModelList(parentModelList); + + //递归查询子模型 + getChildModel(modelDependency1TreeVo); + + return modelDependency1TreeVo; + } + + void commonDvc(String token, Map userInfo, String username, String repositoryName, ModelsVo modelsVo, String type) throws Exception { + String branchName = modelsVo.getVersion(); + + // 创建分支 + gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); + + String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git"; + String url = modelsVo.getModelVersionVos().get(0).getUrl(); + String sourcePath = url.substring(0, url.lastIndexOf("/")); + String rootPath = localPath + username + "/model/" + modelsVo.getName() + "/" + branchName; + String modelPath = rootPath + "/model"; + String metaPath = rootPath + "/metadata"; + + + if (type.equals("CreateModelFromPipeline") && FileUtil.checkDirectoryExists(rootPath)) { + DVCUtils.gitFetch(rootPath); + DVCUtils.gitCheckoutBranch(rootPath, branchName); + } else { + DVCUtils.gitClone(rootPath, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + } + + DVCUtils.moveFiles(sourcePath, modelPath); + + //拼接生产的元数据后写入yaml文件 + ModelMetaVo modelMetaVo = new ModelMetaVo(); + BeanUtils.copyProperties(modelsVo, modelMetaVo); + YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelMetaVo), metaPath, "metadata"); + + // dvc init 初始化 + DVCUtils.dvcInit(rootPath); + + // 配置远程S3地址 + String s3Path = "management-platform-files/" + username + "/model/" + repositoryName + "/" + branchName; + DVCUtils.dvcRemoteAdd(rootPath, s3Path); + DVCUtils.dvcConfigS3Credentials(rootPath, endpoint); + DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId); + DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey); + + // dvc 跟踪 + DVCUtils.dvcAdd(rootPath, "model"); + + // git commit + DVCUtils.gitAdd(rootPath, "."); + DVCUtils.gitCommit(rootPath, "commit from ci4s with " + username); + DVCUtils.gitPush(rootPath, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + + // dvc push 到远程S3 + DVCUtils.dvcPush(rootPath); + } + public List convert(List> lst) { if (lst != null && lst.size() > 0) { List newModelVos = ConvertUtil.convertListMapToObjectList(lst, ModelsVo.class); @@ -754,10 +811,38 @@ public class ModelsServiceImpl implements ModelsService { } } } - return newModelVos; } return new ArrayList<>(); } + void getParentModel(List modelList, ModelDependency1 modelDependency1) { + if (modelDependency1.getParentModel() != null) { + String[] split = modelDependency1.getParentModel().split(":"); + String parentModelName = split[0]; + String parentModelVersion = split[1]; + ModelDependency1 parentModel = modelDependency1Dao.queryByModelNameAndVersion(parentModelName, parentModelVersion); + modelList.add(parentModel); + if (parentModel.getParentModel() != null) { + getParentModel(modelList, parentModel); + } + } + } + + void getChildModel(ModelDependency1TreeVo modelDependency1TreeVo) { + List childModelList = new ArrayList<>(); + String model = modelDependency1TreeVo.getModelName() + ":" + modelDependency1TreeVo.getVersion(); + + List children = modelDependency1Dao.queryByParentModel(model); + if (children != null && !children.isEmpty()) { + for (ModelDependency1 modelDependency1 : children) { + ModelDependency1TreeVo modelDependency1TreeVo1 = new ModelDependency1TreeVo(); + BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo1); + getChildModel(modelDependency1TreeVo1); + childModelList.add(modelDependency1TreeVo1); + } + } + modelDependency1TreeVo.setChildModelList(childModelList); + } + } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java index 281664c..24cede7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java @@ -1,10 +1,11 @@ package com.ruoyi.platform.utils; +import org.apache.commons.beanutils.PropertyUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; -import org.apache.commons.beanutils.PropertyUtils; + import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; @@ -23,7 +24,7 @@ public class ConvertUtil { public static final Logger logger = LoggerFactory.getLogger(ConvertUtil.class); - /** + /** * 将实体对象转换为VO对象 * @param source 源实体对象 * @param target 目标VO对象的类 @@ -90,7 +91,8 @@ public class ConvertUtil { } } return targetObject; - } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | + NoSuchMethodException e) { e.printStackTrace(); } return null; @@ -98,8 +100,13 @@ public class ConvertUtil { private static boolean hasProperty(Class clazz, String propertyName) { try { - Field field = clazz.getDeclaredField(propertyName); - return field != null; + Class superClazz = clazz.getSuperclass(); + if(superClazz != Object.class){ + return hasProperty(superClazz, propertyName); + } else { + Field field = clazz.getDeclaredField(propertyName); + return field != null; + } } catch (NoSuchFieldException e) { return false; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java index e364dcb..1a6dd3b 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java @@ -186,12 +186,12 @@ public class DVCUtils { * @param branch 分支名称 * @return 包含文件路径、名称和大小的List> */ - public static List> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch) { + public static List> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch, String filePath) { List> fileInfoList = new ArrayList<>(); try { // 切换到指定目录 - Path repoPath = Paths.get(localPath, repoFolder); + Path repoPath = Paths.get(localPath, repoFolder, branch); // 切换到指定分支 ProcessBuilder pb = new ProcessBuilder("git", "checkout", branch); @@ -206,19 +206,19 @@ public class DVCUtils { process.waitFor(); // 读取data文件夹中的文件列表 - Path dataPath = Paths.get(repoPath.toString(), "data"); + Path dataPath = Paths.get(repoPath.toString(), filePath); File[] files = dataPath.toFile().listFiles(); if (files != null) { for (File file : files) { if (file.isFile()) { long size = Files.size(file.toPath()); - String filePath = file.getAbsolutePath(); + String absoluteFilePath = file.getAbsolutePath(); String fileName = file.getName(); Map fileDetails = new HashMap<>(); fileDetails.put("size", size); - fileDetails.put("filePath", filePath); + fileDetails.put("filePath", absoluteFilePath); fileDetails.put("fileName", fileName); fileInfoList.add(fileDetails); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java index 03ec45b..c18081f 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java @@ -16,8 +16,16 @@ public class YamlUtils { */ public static void generateYamlFile(Map data, String path, String fileName) { Yaml yaml = new Yaml(); - String fullPath = path + "/" + fileName + ".yaml"; + File directory = new File(path); + if (!directory.exists()) { + boolean isCreated = directory.mkdirs(); + if (!isCreated) { + throw new RuntimeException("创建路径失败: " + path); + } + } + + String fullPath = path + "/" + fileName + ".yaml"; try (FileWriter writer = new FileWriter(fullPath)) { yaml.dump(data, writer); } catch (IOException e) { diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependency1TreeVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependency1TreeVo.java new file mode 100644 index 0000000..f550544 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelDependency1TreeVo.java @@ -0,0 +1,18 @@ +package com.ruoyi.platform.vo; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import com.ruoyi.platform.domain.ModelDependency1; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +public class ModelDependency1TreeVo extends ModelDependency1 implements Serializable { + + List patrentModelList; + + List childModelList; +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java new file mode 100644 index 0000000..2716bf5 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelMetaVo.java @@ -0,0 +1,67 @@ +package com.ruoyi.platform.vo; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +public class ModelMetaVo implements Serializable { + @ApiModelProperty(name = "模型名称") + private String name; + + @ApiModelProperty(name = "版本") + private String version; + + @ApiModelProperty(name = "版本描述") + private String versionDescription; + + @ApiModelProperty(value = "创建人") + private String createBy; + + @ApiModelProperty(value = "更新时间") + private Date updateTime; + + @ApiModelProperty(value = "训练镜像") + private String image; + + @ApiModelProperty(value = "训练代码") + private String code; + + @ApiModelProperty(value = "模型来源") + private String modelSource; + + @ApiModelProperty(value = "训练数据集") + private List trainDatasets; + + @ApiModelProperty(value = "测试数据集") + private List testDatasets; + + @ApiModelProperty(value = "参数") + private HashMap params; + + @ApiModelProperty(value = "指标") + private HashMap metrics; + + @ApiModelProperty(value = "训练任务") + private String trainTask; + + @ApiModelProperty(value = "模型标签（模型能力）") + private String modelTag; + + @ApiModelProperty(value = "模型类型(模型框架)") + private String modelType; + + @ApiModelProperty(value = "模型描述") + private String description; + + @ApiModelProperty(value = "示例用法") + private String examples; + +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java index 1189da7..a57184c 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/ModelsVo.java @@ -10,15 +10,9 @@ import java.util.List; @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) @Data -public class ModelsVo implements Serializable { +public class ModelsVo extends ModelMetaVo implements Serializable { - - @ApiModelProperty(name = "name") - private String name; - - // private String version; - @ApiModelProperty(name = "description") - private String description; + private Integer id; /** * 模型可见范围 @@ -26,22 +20,6 @@ public class ModelsVo implements Serializable { @ApiModelProperty(name = "available_range") private int availableRange; - // private String url; - @ApiModelProperty(name = "model_type") - private String modelType; - - - @ApiModelProperty(name = "model_tag") - private String modelTag; - /** - * 版本 - */ - @ApiModelProperty(name = "version") - private String version; - - @ApiModelProperty(name = "model_version_vos") - private List modelVersionVos; - /** * 状态 */ @@ -53,14 +31,19 @@ public class ModelsVo implements Serializable { private String gitLinkPassword; @ApiModelProperty(name = "models_version_vos") - private List modelsVersionVos; + private List modelVersionVos; /** - * 数据集仓库名称 + * 模型仓库名称 */ @ApiModelProperty(name = "repository_name") private String repositoryName; @ApiModelProperty(name = "repo_id") private Integer repoId; + + /** + * 父模型 + */ + private String parentModel; } diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml new file mode 100644 index 0000000..64bedb3 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml @@ -0,0 +1,29 @@ + + + + + + + insert into model_dependency1(parent_model, model_name, version, repo_id, repo_name) + values (#{parentModel}, #{modelName}, #{version}, #{repoId}, #{repoName}) + + + + + + + + \ No newline at end of file diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelsDaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelsDaoMapper.xml index 07a9276..1b57c81 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelsDaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelsDaoMapper.xml @@ -14,12 +14,14 @@ + + From 4ec2ee52b365f146ec62ed44fc179abffd514e4a Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Thu, 5 Sep 2024 16:58:13 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/ModelsServiceImpl.java | 24 ++++++++++++------- .../com/ruoyi/platform/utils/DVCUtils.java | 2 +- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index 92038a0..5d7d4ad 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -521,12 +521,14 @@ public class ModelsServiceImpl implements ModelsService { try { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); Jedis jedis = new Jedis(redisHost); String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); Map userInfo = JsonUtils.jsonToMap(userReq); Integer userId = (Integer) userInfo.get("user_id"); - String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + String token = gitService.login(gitLinkUsername, gitLinkPassword); String repositoryName = modelsVo.getRepositoryName() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getRepositoryName(); ModelDependency1 modelDependency = new ModelDependency1(); List oldModelDependencys = modelDependency1Dao.queryByModelName(modelsVo.getName()); @@ -537,7 +539,7 @@ public class ModelsServiceImpl implements ModelsService { } //新建版本 - commonDvc(token, userInfo, ci4sUsername, modelsVo.getRepositoryName(), modelsVo, "CreateModelFromPipeline"); + commonDvc(token, userInfo, ci4sUsername, modelsVo.getRepositoryName(), modelsVo, gitLinkUsername, gitLinkPassword, "CreateModelFromPipeline"); modelDependency.setRepoId(modelsVo.getRepoId()); } else { //新建模型 @@ -556,7 +558,7 @@ public class ModelsServiceImpl implements ModelsService { gitService.createTopic(token, (Integer) project.get("id"), "ci4s_model"); gitService.createTopic(token, (Integer) project.get("id"), "modeltag_" + modelsVo.getModelTag()); gitService.createTopic(token, (Integer) project.get("id"), "modeltype_" + modelsVo.getModelType()); - commonDvc(token, userInfo, ci4sUsername, repositoryName, modelsVo, "createModel"); + commonDvc(token, userInfo, ci4sUsername, repositoryName, modelsVo, gitLinkUsername, gitLinkPassword, "createModel"); } //保存模型依赖 @@ -659,7 +661,9 @@ public class ModelsServiceImpl implements ModelsService { public Page newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); - String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + String token = gitService.login(gitLinkUsername, gitLinkPassword); Jedis jedis = new Jedis(redisHost); String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); Map userInfo = JsonUtils.jsonToMap(userReq); @@ -681,7 +685,9 @@ public class ModelsServiceImpl implements ModelsService { public Page newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); - String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); + String token = gitService.login(gitLinkUsername, gitLinkPassword); Jedis jedis = new Jedis(redisHost); String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); Map userInfo = JsonUtils.jsonToMap(userReq); @@ -711,8 +717,10 @@ public class ModelsServiceImpl implements ModelsService { public ModelMetaVo getModelMeta(String modelName, String version) throws Exception { LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); + String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername(); + String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword(); // git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP - List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/", modelName, version, "model"); + List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/", modelName, version, "model", gitLinkUsername, gitLinkPassword); Map stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + modelName + "/" + version + "/metadata/metadata.yaml"); return ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class); @@ -735,7 +743,7 @@ public class ModelsServiceImpl implements ModelsService { return modelDependency1TreeVo; } - void commonDvc(String token, Map userInfo, String username, String repositoryName, ModelsVo modelsVo, String type) throws Exception { + void commonDvc(String token, Map userInfo, String username, String repositoryName, ModelsVo modelsVo, String gitLinkUsername, String gitLinkPassword, String type) throws Exception { String branchName = modelsVo.getVersion(); // 创建分支 @@ -750,7 +758,7 @@ public class ModelsServiceImpl implements ModelsService { if (type.equals("CreateModelFromPipeline") && FileUtil.checkDirectoryExists(rootPath)) { - DVCUtils.gitFetch(rootPath); + DVCUtils.gitFetch(rootPath, gitLinkUsername, gitLinkPassword); DVCUtils.gitCheckoutBranch(rootPath, branchName); } else { DVCUtils.gitClone(rootPath, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword()); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java index 2d7d10a..1cbcfd4 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java @@ -193,7 +193,7 @@ public class DVCUtils { * @param branch 分支名称 * @return 包含文件路径、名称和大小的List> */ - public static List> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch,String username, String password) { + public static List> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch, String filePath , String username, String password) { List> fileInfoList = new ArrayList<>(); try { From e8cea9df8317c31946991518ef5cc811e0c4c652 Mon Sep 17 00:00:00 2001 From: chenzhihang <709011834@qq.com> Date: Thu, 5 Sep 2024 17:45:32 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=A0=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/NewModelFromGitController.java | 12 +++++------- .../platform/domain/ModelDependency1.java | 4 +++- .../platform/mapper/ModelDependency1Dao.java | 2 +- .../ruoyi/platform/service/ModelsService.java | 2 +- .../service/impl/ModelsServiceImpl.java | 19 +++++++++++-------- .../ModelDependency1DaoMapper.xml | 6 +++--- 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java index 14f53e8..92491e5 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/model/NewModelFromGitController.java @@ -67,17 +67,15 @@ public class NewModelFromGitController { @ApiOperation("模型广场公开模型分页查询,根据model_type,model_tag筛选，true公开false私有") public AjaxResult queryModels(@RequestParam(value = "page") int page, @RequestParam(value = "size") int size, - @RequestParam(value = "is_public") Boolean isPublic, + @RequestParam(value = "is_public", required = false) Boolean isPublic, @RequestParam(value = "model_type", required = false) String modelType, @RequestParam(value = "model_tag", required = false) String modelTag, - @RequestParam(value = "git_link_username") String gitLinkUsername, - @RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception { + @RequestParam(value = "name", required = false) String name) throws Exception { PageRequest pageRequest = PageRequest.of(page, size); ModelsVo modelsVo = new ModelsVo(); modelsVo.setModelType(modelType); modelsVo.setModelTag(modelTag); - modelsVo.setGitLinkUsername(gitLinkUsername); - modelsVo.setGitLinkPassword(gitLinkPassword); + modelsVo.setName(name); if (isPublic) { return AjaxResult.success(this.modelsService.newPubilcQueryByPage(modelsVo, pageRequest)); } else { @@ -100,7 +98,7 @@ public class NewModelFromGitController { @GetMapping("/getModelDependencyTree") @ApiOperation(value = "获取模型依赖关系树") - public AjaxResult getModelDependencyTree(@RequestParam("modelName") String modelName, @RequestParam("version") String version) throws Exception { - return AjaxResult.success(this.modelsService.getModelDependencyTree(modelName, version)); + public AjaxResult getModelDependencyTree(@RequestParam("repoId")Integer repoId, @RequestParam("modelName") String modelName, @RequestParam("version") String version) throws Exception { + return AjaxResult.success(this.modelsService.getModelDependencyTree(repoId, modelName, version)); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java index 2cd3406..60bedb7 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency1.java @@ -19,7 +19,9 @@ public class ModelDependency1 implements Serializable { private Integer repoId; - private String repoName; + private String identifier; + + private String owner; private Integer state; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java index cb54b28..2d1de12 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/ModelDependency1Dao.java @@ -11,7 +11,7 @@ public interface ModelDependency1Dao { List queryByModelName(@Param("modelName") String modelName); - ModelDependency1 queryByModelNameAndVersion(@Param("modelName") String modelName, @Param("version") String version); + ModelDependency1 queryByModelNameAndVersion(@Param("repoId") Integer repoId, @Param("modelName") String modelName, @Param("version") String version); List queryByParentModel(@Param("parentModel") String parentModel); } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java index aea4559..c437aa0 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/ModelsService.java @@ -102,5 +102,5 @@ public interface ModelsService { ModelMetaVo getModelMeta(String modelName, String version) throws Exception; - ModelDependency1TreeVo getModelDependencyTree(String modelName, String version)throws Exception; + ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version)throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java index 5d7d4ad..6e4c0f9 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java @@ -562,10 +562,11 @@ public class ModelsServiceImpl implements ModelsService { } //保存模型依赖 - modelDependency.setRepoName(repositoryName); + modelDependency.setIdentifier(repositoryName); modelDependency.setModelName(modelsVo.getName()); modelDependency.setVersion(modelsVo.getVersion()); modelDependency.setParentModel(modelsVo.getParentModel()); + modelDependency.setOwner(ci4sUsername); modelDependency1Dao.insert(modelDependency); return "新增模型成功"; } catch (Exception e) { @@ -723,12 +724,13 @@ public class ModelsServiceImpl implements ModelsService { List> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/", modelName, version, "model", gitLinkUsername, gitLinkPassword); Map stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + modelName + "/" + version + "/metadata/metadata.yaml"); + assert stringObjectMap != null; return ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class); } @Override - public ModelDependency1TreeVo getModelDependencyTree(String modelName, String version) throws Exception { - ModelDependency1 modelDependency1 = modelDependency1Dao.queryByModelNameAndVersion(modelName, version); + public ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version) throws Exception { + ModelDependency1 modelDependency1 = modelDependency1Dao.queryByModelNameAndVersion(repoId, modelName, version); ModelDependency1TreeVo modelDependency1TreeVo = new ModelDependency1TreeVo(); BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo); @@ -827,11 +829,12 @@ public class ModelsServiceImpl implements ModelsService { void getParentModel(List modelList, ModelDependency1 modelDependency1) { if (modelDependency1.getParentModel() != null) { String[] split = modelDependency1.getParentModel().split(":"); - String parentModelName = split[0]; - String parentModelVersion = split[1]; - ModelDependency1 parentModel = modelDependency1Dao.queryByModelNameAndVersion(parentModelName, parentModelVersion); + Integer parentRepoId = Integer.valueOf(split[0]); + String parentModelName = split[1]; + String parentModelVersion = split[2]; + ModelDependency1 parentModel = modelDependency1Dao.queryByModelNameAndVersion(parentRepoId, parentModelName, parentModelVersion); modelList.add(parentModel); - if (parentModel.getParentModel() != null) { + if (parentModel != null && parentModel.getParentModel() != null) { getParentModel(modelList, parentModel); } } @@ -839,7 +842,7 @@ public class ModelsServiceImpl implements ModelsService { void getChildModel(ModelDependency1TreeVo modelDependency1TreeVo) { List childModelList = new ArrayList<>(); - String model = modelDependency1TreeVo.getModelName() + ":" + modelDependency1TreeVo.getVersion(); + String model = modelDependency1TreeVo.getRepoId() + ":" + modelDependency1TreeVo.getModelName() + ":" + modelDependency1TreeVo.getVersion(); List children = modelDependency1Dao.queryByParentModel(model); if (children != null && !children.isEmpty()) { diff --git a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml index 64bedb3..e7967bc 100644 --- a/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml +++ b/ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/ModelDependency1DaoMapper.xml @@ -4,8 +4,8 @@ - insert into model_dependency1(parent_model, model_name, version, repo_id, repo_name) - values (#{parentModel}, #{modelName}, #{version}, #{repoId}, #{repoName}) + insert into model_dependency1(parent_model, model_name, version, repo_id, identifier, owner) + values (#{parentModel}, #{modelName}, #{version}, #{repoId}, #{identifier}, #{owner}) select * from model_dependency1 - where model_name = #{modelName} and version = #{version} and state = 1 + where repo_id = #{repoId} and model_name = #{modelName} and version = #{version} and state = 1 select * from model_dependency1 - where model_name = #{modelName} and state = 1 + where model_name = #{modelName} + and state = 1 + + + update model_dependency1 + set state = 0 + where repo_id = #{repoId} and identifier = #{identifier} and owner = #{owner} + + and version = #{version} + + + + + update model_dependency1 + set parent_model = null + where parent_model like concat(#{repoId}, ':', #{identifier}, + + ':', #{version}, + + '%') + \ No newline at end of file