Merge remote-tracking branch 'origin/dev-czh' into dev
# Conflicts: # ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java
This commit is contained in:
commit
ee25849a36
|
@ -9,6 +9,7 @@ import org.springframework.core.io.InputStreamResource;
|
|||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
|
@ -20,16 +21,17 @@ public class NewModelFromGitController {
|
|||
@Resource
|
||||
private ModelsService modelsService;
|
||||
|
||||
@PostMapping("/addModelAndVersion")
|
||||
@ApiOperation("添加模型和版本")
|
||||
@PostMapping("/addModel")
|
||||
@ApiOperation("添加模型")
|
||||
public AjaxResult addModelAndVersion(@RequestBody ModelsVo modelsVo) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.newCreateModel(modelsVo));
|
||||
}
|
||||
|
||||
@PostMapping("/addVersion")
|
||||
@ApiOperation("添加版本")
|
||||
public AjaxResult addVersion(@RequestBody ModelsVo modelsVo) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.newCreateVersion(modelsVo));
|
||||
@CrossOrigin(origins = "*", allowedHeaders = "*")
|
||||
@PostMapping("/upload")
|
||||
@ApiOperation(value = "上传模型", notes = "根据模型id上传模型文件,并将信息存入数据库。")
|
||||
public AjaxResult uploadModel(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.uploadModelLocal(files, uuid));
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -40,11 +42,9 @@ public class NewModelFromGitController {
|
|||
*/
|
||||
@GetMapping("/downloadAllFiles")
|
||||
@ApiOperation(value = "下载同一版本下所有模型,并打包")
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam(value = "repository_name") String repositoryName,
|
||||
@RequestParam(value = "version") String version,
|
||||
@RequestParam(value = "git_link_username") String gitLinkUsername,
|
||||
@RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception {
|
||||
return modelsService.downloadAllModelFilesNew(repositoryName, version, gitLinkUsername, gitLinkPassword);
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("identifier") String identifier,
|
||||
@RequestParam(value = "version") String version) throws Exception {
|
||||
return modelsService.downloadAllModelFilesNew(identifier, version);
|
||||
}
|
||||
|
||||
|
||||
|
@ -61,21 +61,19 @@ public class NewModelFromGitController {
|
|||
return modelsService.downloadModels(model_version_id);
|
||||
}
|
||||
|
||||
@GetMapping("/queryDatasets")
|
||||
@GetMapping("/queryModels")
|
||||
@ApiOperation("模型广场公开模型分页查询,根据model_type,model_tag筛选,true公开false私有")
|
||||
public AjaxResult queryDatasets(@RequestParam(value = "page") int page,
|
||||
@RequestParam(value = "size") int size,
|
||||
@RequestParam(value = "is_public") Boolean isPublic,
|
||||
@RequestParam(value = "model_type", required = false) String modelType,
|
||||
@RequestParam(value = "model_tag", required = false) String modelTag,
|
||||
@RequestParam(value = "git_link_username") String gitLinkUsername,
|
||||
@RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception {
|
||||
public AjaxResult queryModels(@RequestParam(value = "page") int page,
|
||||
@RequestParam(value = "size") int size,
|
||||
@RequestParam(value = "is_public", required = false) Boolean isPublic,
|
||||
@RequestParam(value = "model_type", required = false) String modelType,
|
||||
@RequestParam(value = "model_tag", required = false) String modelTag,
|
||||
@RequestParam(value = "name", required = false) String name) throws Exception {
|
||||
PageRequest pageRequest = PageRequest.of(page, size);
|
||||
ModelsVo modelsVo = new ModelsVo();
|
||||
modelsVo.setModelType(modelType);
|
||||
modelsVo.setModelTag(modelTag);
|
||||
modelsVo.setGitLinkUsername(gitLinkUsername);
|
||||
modelsVo.setGitLinkPassword(gitLinkPassword);
|
||||
modelsVo.setName(name);
|
||||
if (isPublic) {
|
||||
return AjaxResult.success(this.modelsService.newPubilcQueryByPage(modelsVo, pageRequest));
|
||||
} else {
|
||||
|
@ -83,4 +81,36 @@ public class NewModelFromGitController {
|
|||
}
|
||||
}
|
||||
|
||||
@GetMapping("/getModelVersions/{model_name}")
|
||||
@ApiOperation(value = "获取模型所有版本号")
|
||||
public AjaxResult getModelVersions(@PathVariable("model_name") String modelName) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.getNewModelVersion(modelName));
|
||||
}
|
||||
|
||||
|
||||
@GetMapping("/getModelDetail")
|
||||
@ApiOperation(value = "获取模型详细信息")
|
||||
public AjaxResult getModelDetail(@RequestParam("model_name") String modelName, @RequestParam("version") String version) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.getModelDetail(modelName, version));
|
||||
}
|
||||
|
||||
@GetMapping("/getModelDependencyTree")
|
||||
@ApiOperation(value = "获取模型依赖关系树")
|
||||
public AjaxResult getModelDependencyTree(@RequestParam("repoId") Integer repoId, @RequestParam("model_name") String modelName, @RequestParam("version") String version) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.getModelDependencyTree(repoId, modelName, version));
|
||||
}
|
||||
|
||||
@DeleteMapping("/delete")
|
||||
@ApiOperation(value = "删除模型")
|
||||
public AjaxResult deleteModel(@RequestParam("repoId")Integer repoId, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception {
|
||||
this.modelsService.deleteModel(repoId,identifier, owner);
|
||||
return AjaxResult.success();
|
||||
}
|
||||
|
||||
@DeleteMapping("/deleteVersion")
|
||||
@ApiOperation(value = "删除模型版本")
|
||||
public AjaxResult deleteVersion(@RequestParam("repoId")Integer repoId, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception {
|
||||
this.modelsService.deleteVersion(repoId, identifier, owner, version);
|
||||
return AjaxResult.success();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,27 @@
|
|||
package com.ruoyi.platform.domain;
|
||||
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
|
||||
@Data
|
||||
public class ModelDependency1 implements Serializable {
|
||||
private Long id;
|
||||
|
||||
private String parentModel;
|
||||
|
||||
private String modelName;
|
||||
|
||||
private String version;
|
||||
|
||||
private Integer repoId;
|
||||
|
||||
private String identifier;
|
||||
|
||||
private String owner;
|
||||
|
||||
private Integer state;
|
||||
}
|
|
@ -6,8 +6,8 @@ import com.ruoyi.platform.annotations.CheckDuplicate;
|
|||
import io.swagger.annotations.ApiModel;
|
||||
import io.swagger.annotations.ApiModelProperty;
|
||||
|
||||
import java.util.Date;
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* (Models)实体类
|
||||
|
@ -53,17 +53,18 @@ public class Models implements Serializable {
|
|||
@ApiModelProperty(value = "状态,0失效,1生效")
|
||||
private Integer state;
|
||||
|
||||
@ApiModelProperty(value = "模型gitLink仓库id")
|
||||
private Integer repoId;
|
||||
|
||||
@ApiModelProperty(value = "模型gitLink仓库名称")
|
||||
private String repoName;
|
||||
|
||||
@ApiModelProperty(value = "模型类型名")
|
||||
private String modelTypeName;
|
||||
|
||||
|
||||
|
||||
@ApiModelProperty(value = "模型tag名")
|
||||
private String modelTagName;
|
||||
|
||||
|
||||
|
||||
public Integer getId() {
|
||||
return id;
|
||||
}
|
||||
|
@ -170,6 +171,22 @@ public class Models implements Serializable {
|
|||
this.state = state;
|
||||
}
|
||||
|
||||
public Integer getRepoId() {
|
||||
return repoId;
|
||||
}
|
||||
|
||||
public void setRepoId(Integer repoId) {
|
||||
this.repoId = repoId;
|
||||
}
|
||||
|
||||
public String getRepoName() {
|
||||
return repoName;
|
||||
}
|
||||
|
||||
public void setRepoName(String repoName) {
|
||||
this.repoName = repoName;
|
||||
}
|
||||
|
||||
public String getModelTagName() {
|
||||
return modelTagName;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,21 @@
|
|||
package com.ruoyi.platform.mapper;
|
||||
|
||||
import com.ruoyi.platform.domain.ModelDependency1;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ModelDependency1Dao {
|
||||
|
||||
int insert(ModelDependency1 modelDependency1);
|
||||
|
||||
List<ModelDependency1> queryByModelName(@Param("modelName") String modelName);
|
||||
|
||||
ModelDependency1 queryByModelNameAndVersion(@Param("repoId") Integer repoId, @Param("modelName") String modelName, @Param("version") String version);
|
||||
|
||||
List<ModelDependency1> queryByParentModel(@Param("parentModel") String parentModel);
|
||||
|
||||
int deleteModel(@Param("repoId") Integer repoId, @Param("identifier")String identifier, @Param("owner") String owner, @Param("version") String version);
|
||||
|
||||
int deleteModelDependency(@Param("repoId") Integer repoId, @Param("identifier")String identifier, @Param("owner") String owner, @Param("version") String version);
|
||||
}
|
|
@ -3,6 +3,8 @@ package com.ruoyi.platform.service;
|
|||
|
||||
import com.ruoyi.platform.domain.Models;
|
||||
import com.ruoyi.platform.domain.ModelsVersion;
|
||||
import com.ruoyi.platform.vo.ModelDependency1TreeVo;
|
||||
import com.ruoyi.platform.vo.ModelMetaVo;
|
||||
import com.ruoyi.platform.vo.ModelsVo;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.data.domain.Page;
|
||||
|
@ -86,14 +88,23 @@ public interface ModelsService {
|
|||
|
||||
List<Map<String, String>> exportModels(String path, String uuid) throws Exception;
|
||||
|
||||
|
||||
CompletableFuture<String> newCreateModel(ModelsVo modelsVo) throws Exception;
|
||||
|
||||
CompletableFuture<String> newCreateVersion(ModelsVo modelsVo);
|
||||
List<Map<String, String>> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception;
|
||||
|
||||
ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception;
|
||||
ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String identifier, String version) throws IOException, Exception;
|
||||
|
||||
Page<ModelsVo> newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception;
|
||||
|
||||
Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception;
|
||||
|
||||
List<String> getNewModelVersion(String modelName);
|
||||
|
||||
ModelsVo getModelDetail(String modelName, String version) throws Exception;
|
||||
|
||||
ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version)throws Exception;
|
||||
|
||||
void deleteModel(Integer repoId, String identifier, String owner) throws Exception;
|
||||
|
||||
void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception;
|
||||
}
|
||||
|
|
|
@ -79,7 +79,7 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
private String bucketName;
|
||||
|
||||
@Resource
|
||||
private MinioUtil minioUtil;
|
||||
private MinioUtil minioUtil;
|
||||
|
||||
@Value("${spring.redis.host}")
|
||||
private String redisHost;
|
||||
|
@ -130,8 +130,8 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
/**
|
||||
* 分页查询
|
||||
*
|
||||
* @param dataset 筛选条件
|
||||
* @param pageRequest 分页对象
|
||||
* @param dataset 筛选条件
|
||||
* @param pageRequest 分页对象
|
||||
* @return 查询结果
|
||||
*/
|
||||
@Override
|
||||
|
@ -171,7 +171,7 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
@Override
|
||||
public Dataset update(Dataset dataset) {
|
||||
int currentState = dataset.getState();
|
||||
if (currentState == 0){
|
||||
if (currentState == 0) {
|
||||
throw new RuntimeException("数据集已被删除,无法更新。");
|
||||
}
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
|
@ -195,7 +195,7 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
@Override
|
||||
public String removeById(Integer id) throws Exception {
|
||||
Dataset dataset = this.datasetDao.queryById(id);
|
||||
if (dataset == null){
|
||||
if (dataset == null) {
|
||||
throw new Exception("数据集不存在");
|
||||
}
|
||||
|
||||
|
@ -203,7 +203,7 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String username = loginUser.getUsername();
|
||||
String createdBy = dataset.getCreateBy();
|
||||
if (!(StringUtils.equals(username,"admin") || StringUtils.equals(username,createdBy))){
|
||||
if (!(StringUtils.equals(username, "admin") || StringUtils.equals(username, createdBy))) {
|
||||
throw new Exception("无权限删除该数据集");
|
||||
}
|
||||
if (datasetVersionService.queryByDatasetId(id).size()>0){
|
||||
|
@ -497,13 +497,13 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal + "/" + repoId+"/"+datasetVo.getName();
|
||||
String localPath = localPathlocal + ci4sUsername + "/datasets/" + datasetVo.getName() + "/" + branchName;
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+repoId+"/"+ repositoryName + "/" + branchName;
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + repoId + "/" + repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath);
|
||||
DVCUtils.moveFiles(sourcePath, localPath + "/data");
|
||||
//拼接生产的元数据后写入yaml文件
|
||||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getNowDate());
|
||||
|
@ -514,7 +514,7 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"```");
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),localPath, "dataset");
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath + "/metadata", "metadata");
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
|
@ -561,7 +561,7 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+ datasetVo.getName();
|
||||
String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + datasetVo.getName() + "/" + branchName;
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
if(FileUtil.checkDirectoryExists(localPath)){
|
||||
|
@ -573,7 +573,7 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+ datasetVo.getRepoId()+"/"+ repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath);
|
||||
DVCUtils.moveFiles(sourcePath, localPath + "/data");
|
||||
//拼接生产的元数据后写入yaml文件
|
||||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getNowDate());
|
||||
|
@ -584,7 +584,7 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"```");
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),localPath, "dataset");
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath + "/metadata", "metadata");
|
||||
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
|
@ -663,6 +663,7 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
@Override
|
||||
public NewDatasetVo getNewDatasetDesc(Integer repoId,String repositoryName,String repo, String owner,String version) throws Exception{
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
// cd到 localPathlocal/repoId/下面还有一个文件夹,然后做git pull操作,然后读取里面的文件列表,列出每个文件的大小和名称,封装成MAP
|
||||
|
@ -674,9 +675,9 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
version = (String) versionList.get(0).get("name");
|
||||
}
|
||||
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal+repoId, repositoryName, version,gitLinkUsername, gitLinkPassword);
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal + ci4sUsername + "/datasets/", repositoryName, version, "data",gitLinkUsername, gitLinkPassword);//在localPathlocal+repoId+"/"+repositoryName目录下的dataset.yaml中取到元数据
|
||||
//在localPathlocal+repoId+"/"+repositoryName目录下的dataset.yaml中取到元数据
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + repoId + "/" + repositoryName + "/dataset.yaml");
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + ci4sUsername + "/datasets/" + repositoryName + "/" + version + "/metadata/metadata.yaml");
|
||||
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
|
||||
List<VersionVo> versionVos = new ArrayList<VersionVo>();
|
||||
if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
|
||||
|
|
|
@ -4,20 +4,22 @@ import com.ruoyi.common.core.utils.DateUtils;
|
|||
import com.ruoyi.common.security.utils.SecurityUtils;
|
||||
import com.ruoyi.platform.annotations.CheckDuplicate;
|
||||
import com.ruoyi.platform.domain.AssetIcon;
|
||||
import com.ruoyi.platform.domain.ModelDependency1;
|
||||
import com.ruoyi.platform.domain.Models;
|
||||
import com.ruoyi.platform.domain.ModelsVersion;
|
||||
import com.ruoyi.platform.mapper.ModelDependency1Dao;
|
||||
import com.ruoyi.platform.mapper.ModelsDao;
|
||||
import com.ruoyi.platform.mapper.ModelsVersionDao;
|
||||
import com.ruoyi.platform.service.*;
|
||||
import com.ruoyi.platform.utils.*;
|
||||
import com.ruoyi.platform.vo.GitProjectVo;
|
||||
import com.ruoyi.platform.vo.ModelsVo;
|
||||
import com.ruoyi.platform.vo.VersionVo;
|
||||
import com.ruoyi.platform.vo.*;
|
||||
import com.ruoyi.system.api.model.LoginUser;
|
||||
import io.minio.messages.Item;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.data.domain.Page;
|
||||
|
@ -63,6 +65,9 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
@Resource
|
||||
private ModelsVersionService modelsVersionService;
|
||||
|
||||
@Resource
|
||||
private ModelDependency1Dao modelDependency1Dao;
|
||||
|
||||
@Resource
|
||||
private MinioService minioService;
|
||||
|
||||
|
@ -376,7 +381,7 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
@Override
|
||||
@Transactional
|
||||
public String insertModelAndVersion(ModelsVo modelsVo) throws Exception {
|
||||
List<VersionVo> modelsVersionVos = modelsVo.getModelsVersionVos();
|
||||
List<VersionVo> modelsVersionVos = modelsVo.getModelVersionVos();
|
||||
|
||||
if (modelsVersionVos == null || modelsVersionVos.isEmpty()) {
|
||||
throw new Exception("模型版本信息错误");
|
||||
|
@ -511,66 +516,56 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> newCreateModel(ModelsVo modelsVo) throws Exception {
|
||||
public CompletableFuture<String> newCreateModel(ModelsVo modelsVo) {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
|
||||
// 拼接project
|
||||
String repositoryName = ci4sUsername + "_model_" + DateUtils.dateTimeNow();
|
||||
GitProjectVo gitProjectVo = new GitProjectVo();
|
||||
gitProjectVo.setRepositoryName(repositoryName);
|
||||
gitProjectVo.setName(modelsVo.getName());
|
||||
gitProjectVo.setDescription(modelsVo.getDescription());
|
||||
gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0);
|
||||
gitProjectVo.setUserId(userId);
|
||||
String token = gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
String repositoryName = modelsVo.getRepositoryName() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getRepositoryName();
|
||||
ModelDependency1 modelDependency = new ModelDependency1();
|
||||
List<ModelDependency1> oldModelDependencys = modelDependency1Dao.queryByModelName(modelsVo.getName());
|
||||
|
||||
// 创建项目
|
||||
Map project = gitService.createProject(token, gitProjectVo);
|
||||
if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) {
|
||||
if (oldModelDependencys.stream().map(ModelDependency1::getVersion).anyMatch(version -> version.equals(modelsVo.getVersion()))) {
|
||||
throw new RuntimeException("模型版本已存在,请勿重复创建");
|
||||
}
|
||||
|
||||
// 创建分支
|
||||
String branchName = modelsVo.getVersion();
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
// 定义标签 标签1:ci4s_model 标签2:ModelTag 标签3:ModelType
|
||||
gitService.createTopic(token, (Integer) project.get("id"), "ci4s_model");
|
||||
gitService.createTopic(token, (Integer) project.get("id"), "ModelTag_" + modelsVo.getModelTag());
|
||||
gitService.createTopic(token, (Integer) project.get("id"), "ModelType_" + modelsVo.getModelType());
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
// 得到用户操作的路径
|
||||
String url = modelsVo.getModelsVersionVos().get(0).getUrl();
|
||||
String localPath1 = localPath + modelsVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/models/" + repositoryName + "/" + branchName;
|
||||
//新建版本
|
||||
commonDvc(token, userInfo, ci4sUsername, modelsVo.getRepositoryName(), modelsVo, gitLinkUsername, gitLinkPassword, "CreateModelFromPipeline");
|
||||
modelDependency.setRepoId(modelsVo.getRepoId());
|
||||
} else {
|
||||
//新建模型
|
||||
// 拼接project
|
||||
GitProjectVo gitProjectVo = new GitProjectVo();
|
||||
gitProjectVo.setRepositoryName(repositoryName);
|
||||
gitProjectVo.setName(modelsVo.getName());
|
||||
gitProjectVo.setDescription(modelsVo.getDescription());
|
||||
gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0);
|
||||
gitProjectVo.setUserId(userId);
|
||||
|
||||
// 拼接生产的元数据后写入yaml文件
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "model");
|
||||
// 创建项目
|
||||
Map project = gitService.createProject(token, gitProjectVo);
|
||||
modelDependency.setRepoId((Integer) project.get("id"));
|
||||
// 定义标签 标签1:ci4s_model 标签2:ModelTag 标签3:ModelType
|
||||
gitService.createTopic(token, (Integer) project.get("id"), "ci4s_model");
|
||||
gitService.createTopic(token, (Integer) project.get("id"), "modeltag_" + modelsVo.getModelTag());
|
||||
gitService.createTopic(token, (Integer) project.get("id"), "modeltype_" + modelsVo.getModelType());
|
||||
commonDvc(token, userInfo, ci4sUsername, repositoryName, modelsVo, gitLinkUsername, gitLinkPassword, "createModel");
|
||||
}
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath1 + "/model");
|
||||
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath1);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath1, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath1, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath1, "model");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath1, ".");
|
||||
DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername);
|
||||
DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath1);
|
||||
//保存模型依赖
|
||||
modelDependency.setIdentifier(repositoryName);
|
||||
modelDependency.setModelName(modelsVo.getName());
|
||||
modelDependency.setVersion(modelsVo.getVersion());
|
||||
modelDependency.setParentModel(modelsVo.getParentModel());
|
||||
modelDependency.setOwner(ci4sUsername);
|
||||
modelDependency1Dao.insert(modelDependency);
|
||||
return "新增模型成功";
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage());
|
||||
|
@ -579,67 +574,45 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> newCreateVersion(ModelsVo modelsVo) {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
// 创建分支
|
||||
String branchName = modelsVo.getVersion();
|
||||
String repositoryName = modelsVo.getRepositoryName();
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
// 得到用户操作的路径
|
||||
String url = modelsVo.getModelVersionVos().get(0).getUrl();
|
||||
String localPath1 = localPath + ci4sUsername + "/model/" + modelsVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + repositoryName + "/" + branchName;
|
||||
//拼接生产的元数据后写入yaml文件
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "dataset");
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath1 + "/model");
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath1);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath1, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath1, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath1, "model");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath1, ".");
|
||||
DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername);
|
||||
DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath1);
|
||||
return "新增模型成功";
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
@Override
|
||||
public List<Map<String, String>> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception {
|
||||
List<Map<String, String>> results = new ArrayList<>();
|
||||
|
||||
for (MultipartFile file : files) {
|
||||
// 构建objectName
|
||||
String username = SecurityUtils.getLoginUser().getUsername();
|
||||
String fileName = file.getOriginalFilename();
|
||||
String path = "/temp/" + username + "/models/" + uuid + "/model/";
|
||||
long sizeInBytes = file.getSize();
|
||||
String formattedSize = FileUtil.formatFileSize(sizeInBytes);
|
||||
File targetFile = new File(path, file.getOriginalFilename());
|
||||
// 确保目录存在
|
||||
targetFile.getParentFile().mkdirs();
|
||||
// 保存文件到目标路径
|
||||
FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
|
||||
// 返回上传文件的路径
|
||||
String absolutePath = targetFile.getAbsolutePath();
|
||||
Map<String, String> result = new HashMap<>();
|
||||
result.put("fileName", fileName);
|
||||
result.put("url", absolutePath); // objectName根据实际情况定义
|
||||
result.put("fileSize", formattedSize);
|
||||
results.add(result);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception {
|
||||
public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String identifier, String version) throws Exception {
|
||||
// 命令行操作 git clone 项目地址
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
String localPath1 = localPath + ci4sUsername + "/model/" + repositoryName;
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
|
||||
|
||||
String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + identifier + ".git";
|
||||
String localPath1 = localPath + ci4sUsername + "/model/" + identifier;
|
||||
File folder = new File(localPath1);
|
||||
if (folder.exists() && folder.isDirectory()) {
|
||||
//切换分支
|
||||
|
@ -684,19 +657,22 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
|
||||
@Override
|
||||
public Page<ModelsVo> newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String token = gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
|
||||
|
||||
//拼接查询url
|
||||
String modelTagName = modelsVo.getModelTag();
|
||||
String modelTypeName = modelsVo.getModelType();
|
||||
String search = modelsVo.getName();
|
||||
String topic_name = "ci4s_model";
|
||||
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTypeName;
|
||||
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltype_" + modelTypeName;
|
||||
String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name;
|
||||
|
||||
String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name + "&search=" + search;
|
||||
String req = HttpUtils.sendGetWithToken(url, null, token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("count");
|
||||
|
@ -708,19 +684,21 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
public Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String token = gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
|
||||
//拼接查询url
|
||||
String modelTagName = modelsVo.getModelTag();
|
||||
String modelTypeName = modelsVo.getModelType();
|
||||
String search = modelsVo.getName();
|
||||
String topic_name = "ci4s_model";
|
||||
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTagName;
|
||||
topic_name = StringUtils.isEmpty(modelTypeName) ? topic_name : topic_name + ",modeltype_" + modelTypeName;
|
||||
|
||||
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name;
|
||||
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name + "&search=" + search;
|
||||
String req = HttpUtils.sendGetWithToken(url, null, token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("total_count");
|
||||
|
@ -728,6 +706,138 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
return new PageImpl<>(convert(projects), pageRequest, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getNewModelVersion(String modelName) {
|
||||
List<ModelDependency1> modelDependency1List = modelDependency1Dao.queryByModelName(modelName);
|
||||
return modelDependency1List.stream().map(ModelDependency1::getVersion).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ModelsVo getModelDetail(String modelName, String version) {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
|
||||
// git pull操作,然后读取里面的文件列表,列出每个文件的大小和名称,封装成MAP
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/", modelName, version, "model", gitLinkUsername, gitLinkPassword);
|
||||
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + modelName + "/" + version + "/metadata/metadata.yaml");
|
||||
assert stringObjectMap != null;
|
||||
|
||||
ModelMetaVo modelMetaVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class);
|
||||
ModelsVo modelsVo = new ModelsVo();
|
||||
BeanUtils.copyProperties(modelMetaVo, modelsVo);
|
||||
|
||||
List<VersionVo> versionVos = new ArrayList<VersionVo>();
|
||||
if (!fileDetailsAfterGitPull.isEmpty()) {
|
||||
for (Map<String, Object> fileDetail : fileDetailsAfterGitPull) {
|
||||
VersionVo versionVo = new VersionVo();
|
||||
versionVo.setUrl((String) fileDetail.get("filePath"));
|
||||
versionVo.setFileName((String) fileDetail.get("fileName"));
|
||||
long size = (long) fileDetail.get("size");
|
||||
versionVo.setFileSize(FileUtil.formatFileSize(size));
|
||||
versionVos.add(versionVo);
|
||||
}
|
||||
}
|
||||
modelsVo.setModelVersionVos(versionVos);
|
||||
|
||||
return modelsVo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version) throws Exception {
|
||||
ModelDependency1 modelDependency1 = modelDependency1Dao.queryByModelNameAndVersion(repoId, modelName, version);
|
||||
ModelDependency1TreeVo modelDependency1TreeVo = new ModelDependency1TreeVo();
|
||||
BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo);
|
||||
|
||||
//递归查询父模型
|
||||
List<ModelDependency1> parentModelList = new ArrayList<>();
|
||||
getParentModel(parentModelList, modelDependency1);
|
||||
modelDependency1TreeVo.setPatrentModelList(parentModelList);
|
||||
|
||||
//递归查询子模型
|
||||
getChildModel(modelDependency1TreeVo);
|
||||
|
||||
return modelDependency1TreeVo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteModel(Integer repoId, String identifier, String owner) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String token = gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
gitService.deleteProject(token, owner, identifier);
|
||||
|
||||
//删除模型依赖
|
||||
modelDependency1Dao.deleteModel(repoId, identifier, owner, null);
|
||||
modelDependency1Dao.deleteModelDependency(repoId, identifier, owner, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String token = gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
gitService.deleteBranch(token, owner, identifier, version);
|
||||
|
||||
//删除模型依赖
|
||||
modelDependency1Dao.deleteModel(repoId, identifier, owner, version);
|
||||
modelDependency1Dao.deleteModelDependency(repoId, identifier, owner, version);
|
||||
}
|
||||
|
||||
void commonDvc(String token, Map<String, Object> userInfo, String username, String repositoryName, ModelsVo modelsVo, String gitLinkUsername, String gitLinkPassword, String type) throws Exception {
|
||||
String branchName = modelsVo.getVersion();
|
||||
|
||||
// 创建分支
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
String url = modelsVo.getModelVersionVos().get(0).getUrl();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
String rootPath = localPath + username + "/model/" + modelsVo.getName() + "/" + branchName;
|
||||
String modelPath = rootPath + "/model";
|
||||
String metaPath = rootPath + "/metadata";
|
||||
|
||||
|
||||
if (type.equals("CreateModelFromPipeline") && FileUtil.checkDirectoryExists(rootPath)) {
|
||||
DVCUtils.gitFetch(rootPath, gitLinkUsername, gitLinkPassword);
|
||||
DVCUtils.gitCheckoutBranch(rootPath, branchName);
|
||||
} else {
|
||||
DVCUtils.gitClone(rootPath, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
}
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, modelPath);
|
||||
|
||||
//拼接生产的元数据后写入yaml文件
|
||||
ModelMetaVo modelMetaVo = new ModelMetaVo();
|
||||
BeanUtils.copyProperties(modelsVo, modelMetaVo);
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelMetaVo), metaPath, "metadata");
|
||||
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(rootPath);
|
||||
|
||||
// 配置远程S3地址
|
||||
String s3Path = "management-platform-files/" + username + "/model/" + repositoryName + "/" + branchName;
|
||||
DVCUtils.dvcRemoteAdd(rootPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(rootPath, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
|
||||
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(rootPath, "model");
|
||||
|
||||
// git commit
|
||||
DVCUtils.gitAdd(rootPath, ".");
|
||||
DVCUtils.gitCommit(rootPath, "commit from ci4s with " + username);
|
||||
DVCUtils.gitPush(rootPath, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(rootPath);
|
||||
}
|
||||
|
||||
public List<ModelsVo> convert(List<Map<String, Object>> lst) {
|
||||
if (lst != null && lst.size() > 0) {
|
||||
List<ModelsVo> newModelVos = ConvertUtil.convertListMapToObjectList(lst, ModelsVo.class);
|
||||
|
@ -754,10 +864,44 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
return newModelVos;
|
||||
}
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
void getParentModel(List<ModelDependency1> modelList, ModelDependency1 modelDependency1) {
|
||||
if (modelDependency1.getParentModel() != null) {
|
||||
String[] split = modelDependency1.getParentModel().split(":");
|
||||
Integer parentRepoId = Integer.valueOf(split[0]);
|
||||
String parentModelName = split[1];
|
||||
String parentModelVersion = split[2];
|
||||
ModelDependency1 parentModel = modelDependency1Dao.queryByModelNameAndVersion(parentRepoId, parentModelName, parentModelVersion);
|
||||
modelList.add(parentModel);
|
||||
if (parentModel != null && parentModel.getParentModel() != null) {
|
||||
getParentModel(modelList, parentModel);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void getChildModel(ModelDependency1TreeVo modelDependency1TreeVo) {
|
||||
List<ModelDependency1TreeVo> childModelList = new ArrayList<>();
|
||||
String model = modelDependency1TreeVo.getRepoId() + ":" + modelDependency1TreeVo.getModelName() + ":" + modelDependency1TreeVo.getVersion();
|
||||
|
||||
List<ModelDependency1> children = modelDependency1Dao.queryByParentModel(model);
|
||||
if (children != null && !children.isEmpty()) {
|
||||
for (ModelDependency1 modelDependency1 : children) {
|
||||
ModelDependency1TreeVo modelDependency1TreeVo1 = new ModelDependency1TreeVo();
|
||||
BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo1);
|
||||
getChildModel(modelDependency1TreeVo1);
|
||||
childModelList.add(modelDependency1TreeVo1);
|
||||
}
|
||||
}
|
||||
modelDependency1TreeVo.setChildModelList(childModelList);
|
||||
}
|
||||
|
||||
Map<String, Object> getUserInfo(String ci4sUsername) throws IOException {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
return JsonUtils.jsonToMap(userReq);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,10 +1,11 @@
|
|||
package com.ruoyi.platform.utils;
|
||||
|
||||
|
||||
import org.apache.commons.beanutils.PropertyUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.apache.commons.beanutils.PropertyUtils;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.util.ArrayList;
|
||||
|
@ -23,7 +24,7 @@ public class ConvertUtil {
|
|||
public static final Logger logger = LoggerFactory.getLogger(ConvertUtil.class);
|
||||
|
||||
|
||||
/**
|
||||
/**
|
||||
* 将实体对象转换为VO对象
|
||||
* @param source 源实体对象
|
||||
* @param target 目标VO对象的类
|
||||
|
@ -90,7 +91,8 @@ public class ConvertUtil {
|
|||
}
|
||||
}
|
||||
return targetObject;
|
||||
} catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
|
||||
} catch (InstantiationException | IllegalAccessException | InvocationTargetException |
|
||||
NoSuchMethodException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
|
@ -98,8 +100,13 @@ public class ConvertUtil {
|
|||
|
||||
private static boolean hasProperty(Class<?> clazz, String propertyName) {
|
||||
try {
|
||||
Field field = clazz.getDeclaredField(propertyName);
|
||||
return field != null;
|
||||
Class<?> superClazz = clazz.getSuperclass();
|
||||
if(superClazz != Object.class){
|
||||
return hasProperty(superClazz, propertyName);
|
||||
} else {
|
||||
Field field = clazz.getDeclaredField(propertyName);
|
||||
return field != null;
|
||||
}
|
||||
} catch (NoSuchFieldException e) {
|
||||
return false;
|
||||
}
|
||||
|
|
|
@ -193,12 +193,12 @@ public class DVCUtils {
|
|||
* @param branch 分支名称
|
||||
* @return 包含文件路径、名称和大小的List<Map<String, Object>>
|
||||
*/
|
||||
public static List<Map<String, Object>> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch,String username, String password) {
|
||||
public static List<Map<String, Object>> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch, String filePath , String username, String password) {
|
||||
List<Map<String, Object>> fileInfoList = new ArrayList<>();
|
||||
|
||||
try {
|
||||
// 切换到指定目录
|
||||
Path repoPath = Paths.get(localPath, repoFolder);
|
||||
Path repoPath = Paths.get(localPath, repoFolder, branch);
|
||||
|
||||
//刷新
|
||||
gitFetch(localPath, username, password);
|
||||
|
@ -207,19 +207,19 @@ public class DVCUtils {
|
|||
// 执行git pull
|
||||
gitPull(localPath, username, password);
|
||||
// 读取data文件夹中的文件列表
|
||||
Path dataPath = Paths.get(repoPath.toString(), "data");
|
||||
Path dataPath = Paths.get(repoPath.toString(), filePath);
|
||||
File[] files = dataPath.toFile().listFiles();
|
||||
|
||||
if (files != null) {
|
||||
for (File file : files) {
|
||||
if (file.isFile()) {
|
||||
long size = Files.size(file.toPath());
|
||||
String filePath = file.getAbsolutePath();
|
||||
String absoluteFilePath = file.getAbsolutePath();
|
||||
String fileName = file.getName();
|
||||
|
||||
Map<String, Object> fileDetails = new HashMap<>();
|
||||
fileDetails.put("size", size);
|
||||
fileDetails.put("filePath", filePath);
|
||||
fileDetails.put("filePath", absoluteFilePath);
|
||||
fileDetails.put("fileName", fileName);
|
||||
|
||||
fileInfoList.add(fileDetails);
|
||||
|
|
|
@ -16,8 +16,16 @@ public class YamlUtils {
|
|||
*/
|
||||
public static void generateYamlFile(Map<String, Object> data, String path, String fileName) {
|
||||
Yaml yaml = new Yaml();
|
||||
String fullPath = path + "/" + fileName + ".yaml";
|
||||
|
||||
File directory = new File(path);
|
||||
if (!directory.exists()) {
|
||||
boolean isCreated = directory.mkdirs();
|
||||
if (!isCreated) {
|
||||
throw new RuntimeException("创建路径失败: " + path);
|
||||
}
|
||||
}
|
||||
|
||||
String fullPath = path + "/" + fileName + ".yaml";
|
||||
try (FileWriter writer = new FileWriter(fullPath)) {
|
||||
yaml.dump(data, writer);
|
||||
} catch (IOException e) {
|
||||
|
|
|
@ -0,0 +1,18 @@
|
|||
package com.ruoyi.platform.vo;
|
||||
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import com.ruoyi.platform.domain.ModelDependency1;
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
|
||||
public class ModelDependency1TreeVo extends ModelDependency1 implements Serializable {
|
||||
|
||||
List<ModelDependency1> patrentModelList;
|
||||
|
||||
List<ModelDependency1TreeVo> childModelList;
|
||||
}
|
|
@ -0,0 +1,67 @@
|
|||
package com.ruoyi.platform.vo;
|
||||
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import io.swagger.annotations.ApiModelProperty;
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
|
||||
public class ModelMetaVo implements Serializable {
|
||||
@ApiModelProperty(name = "模型名称")
|
||||
private String name;
|
||||
|
||||
@ApiModelProperty(name = "版本")
|
||||
private String version;
|
||||
|
||||
@ApiModelProperty(name = "版本描述")
|
||||
private String versionDescription;
|
||||
|
||||
@ApiModelProperty(value = "创建人")
|
||||
private String createBy;
|
||||
|
||||
@ApiModelProperty(value = "更新时间")
|
||||
private Date updateTime;
|
||||
|
||||
@ApiModelProperty(value = "训练镜像")
|
||||
private String image;
|
||||
|
||||
@ApiModelProperty(value = "训练代码")
|
||||
private String code;
|
||||
|
||||
@ApiModelProperty(value = "模型来源")
|
||||
private String modelSource;
|
||||
|
||||
@ApiModelProperty(value = "训练数据集")
|
||||
private List<String> trainDatasets;
|
||||
|
||||
@ApiModelProperty(value = "测试数据集")
|
||||
private List<String> testDatasets;
|
||||
|
||||
@ApiModelProperty(value = "参数")
|
||||
private HashMap<String, String> params;
|
||||
|
||||
@ApiModelProperty(value = "指标")
|
||||
private HashMap<String, String> metrics;
|
||||
|
||||
@ApiModelProperty(value = "训练任务")
|
||||
private String trainTask;
|
||||
|
||||
@ApiModelProperty(value = "模型标签(模型能力)")
|
||||
private String modelTag;
|
||||
|
||||
@ApiModelProperty(value = "模型类型(模型框架)")
|
||||
private String modelType;
|
||||
|
||||
@ApiModelProperty(value = "模型描述")
|
||||
private String description;
|
||||
|
||||
@ApiModelProperty(value = "示例用法")
|
||||
private String examples;
|
||||
|
||||
}
|
|
@ -10,15 +10,9 @@ import java.util.List;
|
|||
|
||||
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
|
||||
@Data
|
||||
public class ModelsVo implements Serializable {
|
||||
public class ModelsVo extends ModelMetaVo implements Serializable {
|
||||
|
||||
|
||||
@ApiModelProperty(name = "name")
|
||||
private String name;
|
||||
|
||||
// private String version;
|
||||
@ApiModelProperty(name = "description")
|
||||
private String description;
|
||||
private Integer id;
|
||||
|
||||
/**
|
||||
* 模型可见范围
|
||||
|
@ -26,22 +20,6 @@ public class ModelsVo implements Serializable {
|
|||
@ApiModelProperty(name = "available_range")
|
||||
private int availableRange;
|
||||
|
||||
// private String url;
|
||||
@ApiModelProperty(name = "model_type")
|
||||
private String modelType;
|
||||
|
||||
|
||||
@ApiModelProperty(name = "model_tag")
|
||||
private String modelTag;
|
||||
/**
|
||||
* 版本
|
||||
*/
|
||||
@ApiModelProperty(name = "version")
|
||||
private String version;
|
||||
|
||||
@ApiModelProperty(name = "model_version_vos")
|
||||
private List<VersionVo> modelVersionVos;
|
||||
|
||||
/**
|
||||
* 状态
|
||||
*/
|
||||
|
@ -53,14 +31,19 @@ public class ModelsVo implements Serializable {
|
|||
private String gitLinkPassword;
|
||||
|
||||
@ApiModelProperty(name = "models_version_vos")
|
||||
private List<VersionVo> modelsVersionVos;
|
||||
private List<VersionVo> modelVersionVos;
|
||||
|
||||
/**
|
||||
* 数据集仓库名称
|
||||
* 模型仓库名称
|
||||
*/
|
||||
@ApiModelProperty(name = "repository_name")
|
||||
private String repositoryName;
|
||||
|
||||
@ApiModelProperty(name = "repo_id")
|
||||
private Integer repoId;
|
||||
|
||||
/**
|
||||
* 父模型
|
||||
*/
|
||||
private String parentModel;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,52 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.ruoyi.platform.mapper.ModelDependency1Dao">
|
||||
|
||||
|
||||
<insert id="insert" keyProperty="id" useGeneratedKeys="true">
|
||||
insert into model_dependency1(parent_model, model_name, version, repo_id, identifier, owner)
|
||||
values (#{parentModel}, #{modelName}, #{version}, #{repoId}, #{identifier}, #{owner})
|
||||
</insert>
|
||||
|
||||
<select id="queryByModelName" resultType="com.ruoyi.platform.domain.ModelDependency1">
|
||||
select *
|
||||
from model_dependency1
|
||||
where model_name = #{modelName}
|
||||
and state = 1
|
||||
</select>
|
||||
|
||||
<select id="queryByModelNameAndVersion" resultType="com.ruoyi.platform.domain.ModelDependency1">
|
||||
select *
|
||||
from model_dependency1
|
||||
where repo_id = #{repoId}
|
||||
and model_name = #{modelName}
|
||||
and version = #{version}
|
||||
and state = 1
|
||||
</select>
|
||||
|
||||
<select id="queryByParentModel" resultType="com.ruoyi.platform.domain.ModelDependency1">
|
||||
select *
|
||||
from model_dependency1
|
||||
where parent_model = #{parentModel}
|
||||
and state = 1
|
||||
</select>
|
||||
|
||||
<update id="deleteModel">
|
||||
update model_dependency1
|
||||
set state = 0
|
||||
where repo_id = #{repoId} and identifier = #{identifier} and owner = #{owner}
|
||||
<if test="version != null and version != ''">
|
||||
and version = #{version}
|
||||
</if>
|
||||
</update>
|
||||
|
||||
<update id="deleteModelDependency">
|
||||
update model_dependency1
|
||||
set parent_model = null
|
||||
where parent_model like concat(#{repoId}, ':', #{identifier},
|
||||
<if test="version != null and version != ''">
|
||||
':', #{version},
|
||||
</if>
|
||||
'%')
|
||||
</update>
|
||||
</mapper>
|
|
@ -14,12 +14,14 @@
|
|||
<result property="updateBy" column="update_by" jdbcType="VARCHAR"/>
|
||||
<result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
|
||||
<result property="state" column="state" jdbcType="INTEGER"/>
|
||||
<result property="repoId" column="repo_id" jdbcType="INTEGER"/>
|
||||
<result property="repoName" column="repo_name" jdbcType="INTEGER"/>
|
||||
</resultMap>
|
||||
|
||||
<!--查询单个-->
|
||||
<select id="queryById" resultMap="ModelsMap">
|
||||
select
|
||||
id, name, description,available_range, model_type,model_tag, create_by, create_time, update_by, update_time, state
|
||||
id, name, description,available_range, model_type,model_tag, create_by, create_time, update_by, update_time, state, repo_id, repo_name
|
||||
from models
|
||||
where id = #{id} and state = 1
|
||||
</select>
|
||||
|
|
Loading…
Reference in New Issue