ci4s
/
ci4sManagement-cloud
41
0
Fork 1
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

记录依赖关系~

This commit is contained in:
fanshuai 2024-05-30 16:06:38 +08:00
parent 305e6382de
commit b8dc8f64b6
5 changed files with 127 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/ModelDependency.java
View File

@ -2,6 +2,7 @@ package com.ruoyi.platform.domain;
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Data;
import java.util.Date;
import java.io.Serializable;
@ -12,6 +13,7 @@ import java.io.Serializable;
* @author Xidaray
* @since 2024-05-29 13:51:23
*/
@Data
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
public class ModelDependency implements Serializable {
private static final long serialVersionUID = -86753423714028539L;
@ -23,6 +25,8 @@ public class ModelDependency implements Serializable {
* 当前模型id
*/
private Integer currentModelId;
private String version;
/**
* 父模型
*/
@ -71,110 +75,5 @@ public class ModelDependency implements Serializable {
*/
private Integer state;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public Integer getCurrentModelId() {
return currentModelId;
}
public void setCurrentModelId(Integer currentModelId) {
this.currentModelId = currentModelId;
}
public String getParentModels() {
return parentModels;
}
public void setParentModels(String parentModels) {
this.parentModels = parentModels;
}
public String getRefItem() {
return refItem;
}
public void setRefItem(String refItem) {
this.refItem = refItem;
}
public String getTrainTask() {
return trainTask;
}
public void setTrainTask(String trainTask) {
this.trainTask = trainTask;
}
public String getTrainDataset() {
return trainDataset;
}
public void setTrainDataset(String trainDataset) {
this.trainDataset = trainDataset;
}
public String getTestDataset() {
return testDataset;
}
public void setTestDataset(String testDataset) {
this.testDataset = testDataset;
}
public String getProjectDependency() {
return projectDependency;
}
public void setProjectDependency(String projectDependency) {
this.projectDependency = projectDependency;
}
public String getCreateBy() {
return createBy;
}
public void setCreateBy(String createBy) {
this.createBy = createBy;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public String getUpdateBy() {
return updateBy;
}
public void setUpdateBy(String updateBy) {
this.updateBy = updateBy;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
public Integer getState() {
return state;
}
public void setState(Integer state) {
this.state = state;
}
}

11
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/ProjectDepency.java Normal file
View File

@ -0,0 +1,11 @@
package com.ruoyi.platform.domain.dependencydomain;
import lombok.Data;
import java.io.Serializable;
@Data
public class ProjectDepency implements Serializable {
private String url;
private String name;
private String branch;
}

14
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/dependencydomain/TrainTaskDepency.java Normal file
View File

@ -0,0 +1,14 @@
package com.ruoyi.platform.domain.dependencydomain;
import lombok.Data;
import java.io.Serializable;
@Data
public class TrainTaskDepency implements Serializable {
//训练任务名
private String name;
//实例id
private Integer insId;
//节点Id
private String taskId;
}

107
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java
View File

@ -2,14 +2,12 @@ package com.ruoyi.platform.service.impl;
import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.annotations.CheckDuplicate;
import com.ruoyi.platform.domain.Experiment;
import com.ruoyi.platform.domain.ExperimentIns;
import com.ruoyi.platform.domain.Workflow;
import com.ruoyi.platform.domain.*;
import com.ruoyi.platform.domain.dependencydomain.ProjectDepency;
import com.ruoyi.platform.domain.dependencydomain.TrainTaskDepency;
import com.ruoyi.platform.mapper.ExperimentDao;
import com.ruoyi.platform.mapper.ExperimentInsDao;
import com.ruoyi.platform.service.ExperimentInsService;
import com.ruoyi.platform.service.ExperimentService;
import com.ruoyi.platform.service.WorkflowService;
import com.ruoyi.platform.service.*;
import com.ruoyi.platform.utils.HttpUtils;
import com.ruoyi.platform.utils.JacksonUtil;
import com.ruoyi.platform.utils.JsonUtils;
@ -42,8 +40,12 @@ public class ExperimentServiceImpl implements ExperimentService {
@Resource
private ExperimentInsDao experimentInsDao;
@Resource
private ModelsService modelsService;
@Resource
private DatasetService datasetService;
@Resource
private ModelDependencyService modelDependencyService;
@Resource
@Lazy
@ -224,6 +226,8 @@ public class ExperimentServiceImpl implements ExperimentService {
throw new RuntimeException("转换流水线失败");
}
Map<String, Object> converMap = JsonUtils.jsonToMap(convertRes);
//得到dependendcy
Map<String ,Object> dependendcy = (Map<String, Object>)converMap.get("model_dependency");
// 组装运行接口json
Map<String, Object> runReqMap = new HashMap<>();
runReqMap.put("data", converMap.get("data"));
@ -238,6 +242,7 @@ public class ExperimentServiceImpl implements ExperimentService {
Map<String ,Object> output = (Map<String, Object>) converMap.get("output");
// 调argo运行接口
String runRes = HttpUtils.sendPost(argoUrl + argoWorkflowRun, JsonUtils.mapToJson(runReqMap));
if (runRes == null || StringUtils.isEmpty(runRes)) {
throw new RuntimeException("Failed to run workflow.");
}
@ -261,7 +266,11 @@ public class ExperimentServiceImpl implements ExperimentService {
String outputString = JsonUtils.mapToJson(output);
experimentIns.setNodesResult(outputString.replace("{{workflow.name}}", (String) metadata.get("name")));
//插入ExperimentIns表中
experimentInsService.insert(experimentIns);
ExperimentIns insert = experimentInsService.insert(experimentIns);
//插入到模型依赖关系表
insertModelDependendcy(dependendcy,insert.getId(),experiment.getName());
}catch (Exception e){
throw new RuntimeException(e);
}
@ -270,6 +279,86 @@ public class ExperimentServiceImpl implements ExperimentService {
return experiment;
}
private void insertModelDependendcy(Map<String ,Object> dependendcy,Integer experimentInsId,String expermentName)throws Exception {
Iterator<Map.Entry<String, Object>> dependendcyIterator = dependendcy.entrySet().iterator();
while (dependendcyIterator.hasNext()) {
ModelDependency modelDependency = new ModelDependency();
Map.Entry<String, Object> entry = dependendcyIterator.next();
String key = entry.getKey();
Map<String, Object> modelDel = (Map<String, Object>) entry.getValue();
//处理project数据
Map<String, String> projectMap = (Map<String, String>) modelDel.get("project");
ProjectDepency projectDepency = new ProjectDepency();
projectDepency.setBranch(projectMap.get("branch"));
String projectUrl = projectMap.get("url");
projectDepency.setUrl(projectUrl);
projectDepency.setName(projectUrl.substring(projectUrl.lastIndexOf('/') + 1, projectUrl.length() - 4));
//依赖项目
modelDependency.setProjectDependency(JsonUtils.objectToJson(projectDepency));
//处理source数据
Map<String, Object> sourceMap = (Map<String, Object>) modelDel.get("source");
List<Map<String, Object>> modelsList = (List<Map<String, Object>>) sourceMap.get("models");
for(int i=0;i<modelsList.size();i++){
Map<String, Object> model = modelsList.get(i);
Models models = modelsService.queryById((Integer) model.get("model_id"));
if (models == null){
throw new Exception("源模型不存在");
}
model.put("model_name", models.getName());
}
//父模型
modelDependency.setParentModels(JsonUtils.objectToJson(modelsList));
List<Map<String, Object>> datasetsList = (List<Map<String, Object>>) sourceMap.get("datasets");
for(int i=0;i<datasetsList.size();i++){
Map<String, Object> datasets = datasetsList.get(i);
Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id"));
if (dataset == null){
throw new Exception("源数据集不存在");
}
datasets.put("dataset_name", dataset.getName());
}
//训练数据集
modelDependency.setTrainDataset(JsonUtils.objectToJson(datasetsList));
TrainTaskDepency trainTaskDepency = new TrainTaskDepency();
trainTaskDepency.setTaskId(key);
trainTaskDepency.setInsId(experimentInsId);
trainTaskDepency.setName(expermentName);
//训练任务
modelDependency.setTrainTask(JsonUtils.objectToJson(trainTaskDepency));
//处理test数据
List<Map<String, Object>> testDatasetsList = (List<Map<String, Object>>) modelDel.get("test");
for(int i=0;i<testDatasetsList.size();i++){
Map<String, Object> datasets = testDatasetsList.get(i);
Dataset dataset = datasetService.queryById((Integer) datasets.get("dataset_id"));
if (dataset == null){
throw new Exception("源数据集不存在");
}
datasets.put("dataset_name", dataset.getName());
}
//测试数据集
modelDependency.setTestDataset(JsonUtils.objectToJson(testDatasetsList));
//检查是否存在target，如果存在说明在流水线用了节点导入，如果没有说明没有导入，等待手动push
List<Map<String, Object>> modelTargetList = (List<Map<String, Object>>) modelDel.get("target");
if (modelTargetList==null||modelTargetList.size()==0){
modelDependency.setState(0);
}else {
modelDependency.setState(1);
for(int i=0;i<modelTargetList.size();i++){
Map<String, Object> model = modelTargetList.get(i);
modelDependency.setVersion((String) model.get("model_version"));
modelDependency.setCurrentModelId((Integer) model.get("model_id"));
//因为可能有多成果模型，多次插入
modelDependencyService.insert(modelDependency);
}
}
}
}
@Override
public Experiment addAndRunExperiment(Experiment experiment) throws Exception {
// 第一步: 调用add方法插入实验记录到数据库

1
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelDependencyServiceImpl.java
View File

@ -64,7 +64,6 @@ public class ModelDependencyServiceImpl implements ModelDependencyService {
modelDependency.setUpdateBy(loginUser.getUsername());
modelDependency.setUpdateTime(new Date());
modelDependency.setCreateTime(new Date());
modelDependency.setState(1);
this.modelDependencyDao.insert(modelDependency);
return modelDependency;
}