ci4s
/
ci4sManagement-cloud
41
0
Fork 1
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

适配流水线接口

This commit is contained in:
fanshuai 2024-09-12 14:41:05 +08:00
parent 2dd0102f34
commit 5752c50bf2
8 changed files with 691 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetTempStorageController.java Normal file
View File

@ -0,0 +1,84 @@
package com.ruoyi.platform.controller.dataset;
import com.ruoyi.platform.domain.DatasetTempStorage;
import com.ruoyi.platform.service.DatasetTempStorageService;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
/**
* 数据集暂存的数据(DatasetTempStorage)表控制层
*
* @author makejava
* @since 2024-09-12 09:42:09
*/
@RestController
@RequestMapping("datasetTempStorage")
public class DatasetTempStorageController {
/**
* 服务对象
*/
@Resource
private DatasetTempStorageService datasetTempStorageService;
/**
* 分页查询
*
* @param datasetTempStorage 筛选条件
* @param pageRequest 分页对象
* @return 查询结果
*/
@GetMapping
public ResponseEntity<Page<DatasetTempStorage>> queryByPage(DatasetTempStorage datasetTempStorage, PageRequest pageRequest) {
return ResponseEntity.ok(this.datasetTempStorageService.queryByPage(datasetTempStorage, pageRequest));
}
/**
* 通过主键查询单条数据
*
* @param id 主键
* @return 单条数据
*/
@GetMapping("{id}")
public ResponseEntity<DatasetTempStorage> queryById(@PathVariable("id") Integer id) {
return ResponseEntity.ok(this.datasetTempStorageService.queryById(id));
}
/**
* 新增数据
*
* @param datasetTempStorage 实体
* @return 新增结果
*/
@PostMapping
public ResponseEntity<DatasetTempStorage> add(DatasetTempStorage datasetTempStorage) {
return ResponseEntity.ok(this.datasetTempStorageService.insert(datasetTempStorage));
}
/**
* 编辑数据
*
* @param datasetTempStorage 实体
* @return 编辑结果
*/
@PutMapping
public ResponseEntity<DatasetTempStorage> edit(DatasetTempStorage datasetTempStorage) {
return ResponseEntity.ok(this.datasetTempStorageService.update(datasetTempStorage));
}
/**
* 删除数据
*
* @param id 主键
* @return 删除是否成功
*/
@DeleteMapping
public ResponseEntity<Boolean> deleteById(Integer id) {
return ResponseEntity.ok(this.datasetTempStorageService.deleteById(id));
}
}

109
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/domain/DatasetTempStorage.java Normal file
View File

@ -0,0 +1,109 @@
package com.ruoyi.platform.domain;
import java.util.Date;
import java.io.Serializable;
/**
* 数据集暂存的数据(DatasetTempStorage)实体类
*
* @author makejava
* @since 2024-09-12 09:42:09
*/
public class DatasetTempStorage implements Serializable {
private static final long serialVersionUID = -44025176874790480L;
private Integer id;
private String name;
private String version;
private String source;
/**
* 为0失效，1生效
*/
private Integer state;
private String createBy;
private Date createTime;
private String updateBy;
private Date updateTime;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public Integer getState() {
return state;
}
public void setState(Integer state) {
this.state = state;
}
public String getCreateBy() {
return createBy;
}
public void setCreateBy(String createBy) {
this.createBy = createBy;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public String getUpdateBy() {
return updateBy;
}
public void setUpdateBy(String updateBy) {
this.updateBy = updateBy;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
}

84
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/mapper/DatasetTempStorageDao.java Normal file
View File

@ -0,0 +1,84 @@
package com.ruoyi.platform.mapper;
import com.ruoyi.platform.domain.DatasetTempStorage;
import org.apache.ibatis.annotations.Param;
import org.springframework.data.domain.Pageable;
import java.util.List;
/**
* 数据集暂存的数据(DatasetTempStorage)表数据库访问层
*
* @author makejava
* @since 2024-09-12 09:42:09
*/
public interface DatasetTempStorageDao {
/**
* 通过ID查询单条数据
*
* @param id 主键
* @return 实例对象
*/
DatasetTempStorage queryById(Integer id);
/**
* 查询指定行数据
*
* @param datasetTempStorage 查询条件
* @param pageable 分页对象
* @return 对象列表
*/
List<DatasetTempStorage> queryAllByLimit(DatasetTempStorage datasetTempStorage, @Param("pageable") Pageable pageable);
/**
* 统计总行数
*
* @param datasetTempStorage 查询条件
* @return 总行数
*/
long count(DatasetTempStorage datasetTempStorage);
/**
* 新增数据
*
* @param datasetTempStorage 实例对象
* @return 影响行数
*/
int insert(DatasetTempStorage datasetTempStorage);
/**
* 批量新增数据（MyBatis原生foreach方法）
*
* @param entities List<DatasetTempStorage> 实例对象列表
* @return 影响行数
*/
int insertBatch(@Param("entities") List<DatasetTempStorage> entities);
/**
* 批量新增或按主键更新数据（MyBatis原生foreach方法）
*
* @param entities List<DatasetTempStorage> 实例对象列表
* @return 影响行数
* @throws org.springframework.jdbc.BadSqlGrammarException 入参是空List的时候会抛SQL语句错误的异常，请自行校验入参
*/
int insertOrUpdateBatch(@Param("entities") List<DatasetTempStorage> entities);
/**
* 修改数据
*
* @param datasetTempStorage 实例对象
* @return 影响行数
*/
int update(DatasetTempStorage datasetTempStorage);
/**
* 通过主键删除数据
*
* @param id 主键
* @return 影响行数
*/
int deleteById(Integer id);
DatasetTempStorage queryByDatasetTempStorage(DatasetTempStorage datasetTempStorage);
}

57
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetTempStorageService.java Normal file
View File

@ -0,0 +1,57 @@
package com.ruoyi.platform.service;
import com.ruoyi.platform.domain.DatasetTempStorage;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
/**
* 数据集暂存的数据(DatasetTempStorage)表服务接口
*
* @author makejava
* @since 2024-09-12 09:42:10
*/
public interface DatasetTempStorageService {
/**
* 通过ID查询单条数据
*
* @param id 主键
* @return 实例对象
*/
DatasetTempStorage queryById(Integer id);
/**
* 分页查询
*
* @param datasetTempStorage 筛选条件
* @param pageRequest 分页对象
* @return 查询结果
*/
Page<DatasetTempStorage> queryByPage(DatasetTempStorage datasetTempStorage, PageRequest pageRequest);
DatasetTempStorage queryByDatasetTempStorage(DatasetTempStorage datasetTempStorage);
/**
* 新增数据
*
* @param datasetTempStorage 实例对象
* @return 实例对象
*/
DatasetTempStorage insert(DatasetTempStorage datasetTempStorage);
/**
* 修改数据
*
* @param datasetTempStorage 实例对象
* @return 实例对象
*/
DatasetTempStorage update(DatasetTempStorage datasetTempStorage);
/**
* 通过主键删除数据
*
* @param id 主键
* @return 是否成功
*/
boolean deleteById(Integer id);
}

87
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetTempStorageServiceImpl.java Normal file
View File

@ -0,0 +1,87 @@
package com.ruoyi.platform.service.impl;
import com.ruoyi.platform.domain.DatasetTempStorage;
import com.ruoyi.platform.mapper.DatasetTempStorageDao;
import com.ruoyi.platform.service.DatasetTempStorageService;
import org.springframework.stereotype.Service;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import javax.annotation.Resource;
/**
* 数据集暂存的数据(DatasetTempStorage)表服务实现类
*
* @author makejava
* @since 2024-09-12 09:42:10
*/
@Service("datasetTempStorageService")
public class DatasetTempStorageServiceImpl implements DatasetTempStorageService {
@Resource
private DatasetTempStorageDao datasetTempStorageDao;
/**
* 通过ID查询单条数据
*
* @param id 主键
* @return 实例对象
*/
@Override
public DatasetTempStorage queryById(Integer id) {
return this.datasetTempStorageDao.queryById(id);
}
/**
* 分页查询
*
* @param datasetTempStorage 筛选条件
* @param pageRequest 分页对象
* @return 查询结果
*/
@Override
public Page<DatasetTempStorage> queryByPage(DatasetTempStorage datasetTempStorage, PageRequest pageRequest) {
long total = this.datasetTempStorageDao.count(datasetTempStorage);
return new PageImpl<>(this.datasetTempStorageDao.queryAllByLimit(datasetTempStorage, pageRequest), pageRequest, total);
}
@Override
public DatasetTempStorage queryByDatasetTempStorage(DatasetTempStorage datasetTempStorage) {
return this.datasetTempStorageDao.queryByDatasetTempStorage(datasetTempStorage);
}
/**
* 新增数据
*
* @param datasetTempStorage 实例对象
* @return 实例对象
*/
@Override
public DatasetTempStorage insert(DatasetTempStorage datasetTempStorage) {
this.datasetTempStorageDao.insert(datasetTempStorage);
return datasetTempStorage;
}
/**
* 修改数据
*
* @param datasetTempStorage 实例对象
* @return 实例对象
*/
@Override
public DatasetTempStorage update(DatasetTempStorage datasetTempStorage) {
this.datasetTempStorageDao.update(datasetTempStorage);
return this.queryById(datasetTempStorage.getId());
}
/**
* 通过主键删除数据
*
* @param id 主键
* @return 是否成功
*/
@Override
public boolean deleteById(Integer id) {
return this.datasetTempStorageDao.deleteById(id) > 0;
}
}

49
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ExperimentServiceImpl.java
View File

@ -46,7 +46,8 @@ public class ExperimentServiceImpl implements ExperimentService {
private DatasetService datasetService;
@Resource
private ModelDependencyService modelDependencyService;
@Resource
private DatasetTempStorageService datasetTempStorageService;
@Resource
@Lazy
private WorkflowService workflowService;
@ -288,7 +289,13 @@ public class ExperimentServiceImpl implements ExperimentService {
if (dependendcy != null && trainInfo != null){
insertModelDependency(dependendcy,trainInfo,insert.getId(),experiment.getName());
}
Map<String ,Object> datasetDependendcy = (Map<String, Object>)converMap2.get("dataset_dependency");
//暂存数据集元数据{}
if (datasetDependendcy != null && trainInfo != null){
insertDatasetTempStorage(datasetDependendcy,trainInfo,experiment.getId(),insert.getId(),experiment.getName());
}
}catch (Exception e){
throw new RuntimeException(e);
@ -297,6 +304,7 @@ public class ExperimentServiceImpl implements ExperimentService {
experiment.setExperimentInsList(updatedExperimentInsList);
return experiment;
}
private void addDatesetToMetric(Map<String, Object> metricRecord, Map<String, Object> trainInfo) {
processMetricPart(metricRecord, trainInfo, "train", "model_train");
processMetricPart(metricRecord, trainInfo, "evaluate", "model_evaluate");
@ -437,6 +445,45 @@ public class ExperimentServiceImpl implements ExperimentService {
}
}
/**
* 存储数据集元数据到临时表
*
*
*/
private void insertDatasetTempStorage(Map<String, Object> datasetDependendcy, Map<String, Object> trainInfo, Integer experimentId,Integer experimentInsId, String experimentName) {
DatasetTempStorage datasetTempStorage = new DatasetTempStorage();
Iterator<Map.Entry<String, Object>> dependendcyIterator = datasetDependendcy.entrySet().iterator();
Map<String, Object> datasetExport = (Map<String, Object>) trainInfo.get("dataset_export");
Map<String, Object> datasetPreprocess = (Map<String, Object>) trainInfo.get("dataset_preprocess");
while (dependendcyIterator.hasNext()) {
Map.Entry<String, Object> entry = dependendcyIterator.next();
Map<String, Object> modelDel = (Map<String, Object>) entry.getValue();
Map<String, Object> source = (Map<String, Object>) modelDel.get("source"); //被处理数据集
// List<Map<String, Object>> test = (List<Map<String, Object>>) modelDel.get("test");
List<Map<String, Object>> target = (List<Map<String, Object>>) modelDel.get("target"); //导出的数据集
String sourceTaskId = (String) source.get("task_id");
Map<String, Object> datasetPreprocessMap = (Map<String, Object>)datasetPreprocess.get(sourceTaskId);
//处理project数据
Map<String, Object> projectMap = (Map<String, Object>) datasetPreprocessMap.get("project");
Map<String, Object> datasets = (Map<String, Object>) datasetPreprocessMap.get("datasets");
datasetTempStorage.setName((String) datasets.get("dataset_identifier"));
datasetTempStorage.setVersion((String) datasets.get("dataset_version"));
// 拼接需要的参数
Map<String, Object> sourceParams = new HashMap<>();
sourceParams.put("experiment_name", experimentName);
sourceParams.put("experiment_ins_id", experimentInsId);
sourceParams.put("experiment_id", experimentId);
sourceParams.put("train_name",sourceTaskId);
sourceParams.put("preprocess_code",projectMap);
datasetTempStorage.setSource(JacksonUtil.toJSONString(sourceParams));
datasetTempStorage.setState(1);
datasetTempStorageService.insert(datasetTempStorage);
}
}
/**
* 被废弃的旧JSON
* @param experiment

29
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java
View File

@ -3,6 +3,8 @@ package com.ruoyi.platform.service.impl;
import com.ruoyi.common.core.utils.DateUtils;
import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.domain.Dataset;
import com.ruoyi.platform.domain.DatasetTempStorage;
import com.ruoyi.platform.service.DatasetTempStorageService;
import com.ruoyi.platform.service.DvcService;
import com.ruoyi.platform.service.GitService;
import com.ruoyi.platform.service.NewDatasetService;
@ -58,7 +60,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String gitendpoint;
@Value("${git.localPath}")
String localPathlocal;
@Resource
private DatasetTempStorageService datasetTempStorageService;
@Override
public String newCreateDataset(NewDatasetVo datasetVo) throws Exception {
@ -121,9 +124,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
datasetVo.setOwner((String) userInfo.get("login"));
datasetVo.setRelativePaths(relativePath);
if (StringUtils.isEmpty(datasetVo.getDatasetSource())){
datasetVo.setDatasetSource("用户上传");
}
addDatasetSourceToDataVo(datasetVo);
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
// dvc init 初始化
DVCUtils.dvcInit(localPath);
@ -195,6 +197,9 @@ public class NewDatasetServiceImpl implements NewDatasetService {
datasetVo.setDataTag(newDatasetVo.getDataTag());
datasetVo.setDataType(newDatasetVo.getDataType());
datasetVo.setRelativePaths(relativePath);
addDatasetSourceToDataVo(datasetVo);
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
//dvc数据跟踪
// 配置远程S3地址
@ -473,5 +478,21 @@ public class NewDatasetServiceImpl implements NewDatasetService {
return new ArrayList<>();
}
private void addDatasetSourceToDataVo(NewDatasetVo datasetVo){
DatasetTempStorage queryDatasetTempStorage = new DatasetTempStorage();
queryDatasetTempStorage.setName(datasetVo.getName());
queryDatasetTempStorage.setVersion(datasetVo.getVersion());
DatasetTempStorage datasetTempStorage = datasetTempStorageService.queryByDatasetTempStorage(queryDatasetTempStorage);
if (datasetTempStorage == null){
String datasetSourceString = datasetTempStorage.getSource();
Map<String, Object> datasetSourceMap = JacksonUtil.parseJSONStr2Map(datasetSourceString);
String preprocessCode = (String) datasetSourceMap.get("preprocess_code");
datasetSourceMap.remove("preprocess_code");
datasetVo.setProcessingCode(preprocessCode);
datasetVo.setDatasetSource(JacksonUtil.toJSONString(datasetSourceMap));
datasetTempStorage.setState(0);
datasetTempStorageService.update(datasetTempStorage);
}
}
}

197
ruoyi-modules/management-platform/src/main/resources/mapper/managementPlatform/DatasetTempStorageDao.xml Normal file
View File

@ -0,0 +1,197 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ruoyi.platform.mapper.DatasetTempStorageDao">
<resultMap type="com.ruoyi.platform.domain.DatasetTempStorage" id="DatasetTempStorageMap">
<result property="id" column="id" jdbcType="INTEGER"/>
<result property="name" column="name" jdbcType="VARCHAR"/>
<result property="version" column="version" jdbcType="VARCHAR"/>
<result property="source" column="source" jdbcType="VARCHAR"/>
<result property="state" column="state" jdbcType="INTEGER"/>
<result property="createBy" column="create_by" jdbcType="VARCHAR"/>
<result property="createTime" column="create_time" jdbcType="TIMESTAMP"/>
<result property="updateBy" column="update_by" jdbcType="VARCHAR"/>
<result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
</resultMap>
<!--查询单个-->
<select id="queryById" resultMap="DatasetTempStorageMap">
select idnameversionsourcestatecreate_bycreate_timeupdate_byupdate_time
from dataset_temp_storage
where id = #{id}
</select>
<select id="queryByDatasetTempStorage" resultMap="DatasetTempStorageMap">
select
idnameversionsourcestatecreate_bycreate_timeupdate_byupdate_time
from dataset_temp_storage
<where>
<if test="id != null">
and id = #{id}
</if>
<if test="name != null and name != ''">
and name = #{name}
</if>
<if test="version != null and version != ''">
and version = #{version}
</if>
<if test="source != null and source != ''">
and source = #{source}
</if>
<if test="state != null">
and state = #{state}
</if>
<if test="createBy != null and createBy != ''">
and create_by = #{createBy}
</if>
<if test="createTime != null">
and create_time = #{createTime}
</if>
<if test="updateBy != null and updateBy != ''">
and update_by = #{updateBy}
</if>
<if test="updateTime != null">
and update_time = #{updateTime}
</if>
</where>
limit 1
</select>
<!--查询指定行数据-->
<select id="queryAllByLimit" resultMap="DatasetTempStorageMap">
select
idnameversionsourcestatecreate_bycreate_timeupdate_byupdate_time
from dataset_temp_storage
<where>
<if test="id != null">
and id = #{id}
</if>
<if test="name != null and name != ''">
and name = #{name}
</if>
<if test="version != null and version != ''">
and version = #{version}
</if>
<if test="source != null and source != ''">
and source = #{source}
</if>
<if test="state != null">
and state = #{state}
</if>
<if test="createBy != null and createBy != ''">
and create_by = #{createBy}
</if>
<if test="createTime != null">
and create_time = #{createTime}
</if>
<if test="updateBy != null and updateBy != ''">
and update_by = #{updateBy}
</if>
<if test="updateTime != null">
and update_time = #{updateTime}
</if>
</where>
limit #{pageable.offset}, #{pageable.pageSize}
</select>
<!--统计总行数-->
<select id="count" resultType="java.lang.Long">
select count(1)
from dataset_temp_storage
<where>
<if test="id != null">
and id = #{id}
</if>
<if test="name != null and name != ''">
and name = #{name}
</if>
<if test="version != null and version != ''">
and version = #{version}
</if>
<if test="source != null and source != ''">
and source = #{source}
</if>
<if test="state != null">
and state = #{state}
</if>
<if test="createBy != null and createBy != ''">
and create_by = #{createBy}
</if>
<if test="createTime != null">
and create_time = #{createTime}
</if>
<if test="updateBy != null and updateBy != ''">
and update_by = #{updateBy}
</if>
<if test="updateTime != null">
and update_time = #{updateTime}
</if>
</where>
</select>
<!--新增所有列-->
<insert id="insert" keyProperty="id" useGeneratedKeys="true">
insert into dataset_temp_storage(nameversionsourcestatecreate_bycreate_timeupdate_byupdate_time)
values (#{name}#{version}#{source}#{state}#{createBy}#{createTime}#{updateBy}#{updateTime})
</insert>
<insert id="insertBatch" keyProperty="id" useGeneratedKeys="true">
insert into dataset_temp_storage(nameversionsourcestatecreate_bycreate_timeupdate_byupdate_time)
values
<foreach collection="entities" item="entity" separator=",">
(#{entity.name}#{entity.version}#{entity.source}#{entity.state}#{entity.createBy}#{entity.createTime}#{entity.updateBy}#{entity.updateTime})
</foreach>
</insert>
<insert id="insertOrUpdateBatch" keyProperty="id" useGeneratedKeys="true">
insert into dataset_temp_storage(nameversionsourcestatecreate_bycreate_timeupdate_byupdate_time)
values
<foreach collection="entities" item="entity" separator=",">
(#{entity.name}#{entity.version}#{entity.source}#{entity.state}#{entity.createBy}#{entity.createTime}#{entity.updateBy}#{entity.updateTime})
</foreach>
on duplicate key update
name = values(name)version = values(version)source = values(source)state = values(state)create_by =
values(create_by)create_time = values(create_time)update_by = values(update_by)update_time = values(update_time)
</insert>
<!--通过主键修改数据-->
<update id="update">
update dataset_temp_storage
<set>
<if test="name != null and name != ''">
name = #{name},
</if>
<if test="version != null and version != ''">
version = #{version},
</if>
<if test="source != null and source != ''">
source = #{source},
</if>
<if test="state != null">
state = #{state},
</if>
<if test="createBy != null and createBy != ''">
create_by = #{createBy},
</if>
<if test="createTime != null">
create_time = #{createTime},
</if>
<if test="updateBy != null and updateBy != ''">
update_by = #{updateBy},
</if>
<if test="updateTime != null">
update_time = #{updateTime},
</if>
</set>
where id = #{id}
</update>
<!--通过主键删除-->
<delete id="deleteById">
delete
from dataset_temp_storage
where id = #{id}
</delete>
</mapper>