ci4s
/
ci4sManagement-cloud
41
0
Fork 1
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Merge remote-tracking branch 'origin/dev' into dev-czh

This commit is contained in:
chenzhihang 2024-09-18 13:39:59 +08:00
parent a5ce1a56fe d429a915eb
commit 223f6ba92b
35 changed files with 211 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
k8s/build-java.sh Normal file → Executable file
View File

0
k8s/build-node.sh Normal file → Executable file
View File

0
k8s/build.sh Normal file → Executable file
View File

0
k8s/build_and_deploy.sh Normal file → Executable file
View File

0
k8s/deploy.sh Normal file → Executable file
View File

4
k8s/dockerfiles/managent-dockerfile
View File

@ -1,6 +1,6 @@
# 基础镜像
FROM 172.20.32.187/ci4s/openjdk:8u162
#FROM 172.20.32.187/ci4s/openjdk-dvc:2024829
#FROM 172.20.32.187/ci4s/openjdk:8u162
FROM 172.20.32.187/ci4s/openjdk-dvc:2024829
# author
MAINTAINER ruoyi

8
k8s/template-yaml/deploy/k8s-7management.yaml
View File

@ -15,10 +15,14 @@ spec:
spec:
containers:
- name: ci4s-management-platform
image: 172.20.32.187/ci4s/managent:202406121003
image: 172.20.32.187/ci4s/ci4s-managent:202409181159
ports:
- containerPort: 9213
env:
- name: http_proxy
value: "http://172.20.32.253:3128"
- name: https_proxy
value: "http://172.20.32.253:3128"
---
apiVersion: v1
kind: Service

5
k8s/template-yaml/k8s-7management.yaml
View File

@ -18,11 +18,6 @@ spec:
image: ${k8s-7management-image}
ports:
- containerPort: 9213
env:
- name: http_proxy
value: "http://172.20.32.253:3128"
- name: https_proxy
value: "http://172.20.32.253:3128"
---
apiVersion: v1
kind: Service

2
react-ui/src/components/KFEmpty/index.tsx
View File

@ -46,7 +46,7 @@ function KFEmpty({
return (
<div className={classNames('kf-empty', className)} style={style}>
<img className="kf-empty__image" src={image} />
<img className="kf-empty__image" src={image} draggable={false} alt="" />
<div className="kf-empty__title">{title}</div>
<div className="kf-empty__content">{content}</div>
{hasFooter && (

2
react-ui/src/components/ModalTitle/index.tsx
View File

@ -18,7 +18,7 @@ type ModalTitleProps = {
function ModalTitle({ title, image, style, className }: ModalTitleProps) {
return (
<div className={classNames('kf-modal-title', className)} style={style}>
{image && <img className={'kf-modal-title__image'} src={image} alt="" />}
{image && <img className={'kf-modal-title__image'} src={image} draggable={false} alt="" />}
{title}
</div>
);

2
react-ui/src/components/SubAreaTitle/index.tsx
View File

@ -17,7 +17,7 @@ type SubAreaTitleProps = {
function SubAreaTitle({ title, image, style, className }: SubAreaTitleProps) {
return (
<div className={classNames('kf-subarea-title', className)} style={style}>
<img src={image} width={14} />
<img src={image} width={14} draggable={false} alt="" />
<span style={{ marginLeft: '8px' }}>{title}</span>
</div>
);

9
react-ui/src/pages/CodeConfig/components/CodeConfigItem/index.tsx
View File

@ -63,11 +63,16 @@ function CodeConfigItem({ item, onClick, onEdit, onRemove }: CodeConfigItemProps
</div>
<Flex justify="space-between">
<div className={styles['code-config-item__time']}>
<img style={{ width: '17px', marginRight: '6px' }} src={creatByImg} alt="" />
<img
style={{ width: '17px', marginRight: '6px' }}
src={creatByImg}
alt=""
draggable={false}
/>
<span>{item.create_by}</span>
</div>
<div className={styles['code-config-item__time']}>
<img style={{ width: '12px', marginRight: '5px' }} src={clock} alt="" />
<img style={{ width: '12px', marginRight: '5px' }} src={clock} alt="" draggable={false} />
<span>最近更新: {formatDate(item.update_time, 'YYYY-MM-DD')}</span>
</div>
</Flex>

2
react-ui/src/pages/Dataset/components/CategoryItem/index.tsx
View File

@ -23,12 +23,14 @@ function CategoryItem({ resourceType, item, isSelected, onClick }: CategoryItemP
style={{ width: '22px' }}
src={`/assets/images/${config.prefix}/${item.path}.png`}
alt=""
draggable={false}
/>
<img
className={styles['category-item__active-icon']}
style={{ width: '22px' }}
src={`/assets/images/${config.prefix}/${item.path}-hover.png`}
alt=""
draggable={false}
/>
<span className={styles['category-item__name']}>{item.name}</span>
</div>

9
react-ui/src/pages/Dataset/components/ResourceItem/index.tsx
View File

@ -39,11 +39,16 @@ function ResourceItem({ item, isPublic, onClick, onRemove }: ResourceItemProps)
<div className={styles['resource-item__description']}>{item.description}</div>
<Flex justify="space-between">
<div className={styles['resource-item__time']}>
<img style={{ width: '17px', marginRight: '6px' }} src={creatByImg} alt="" />
<img
style={{ width: '17px', marginRight: '6px' }}
src={creatByImg}
draggable={false}
alt=""
/>
<span>{item.create_by ?? ''}</span>
</div>
<div className={styles['resource-item__time']}>
<img style={{ width: '12px', marginRight: '5px' }} src={clock} alt="" />
<img style={{ width: '12px', marginRight: '5px' }} src={clock} draggable={false} alt="" />
<span>
{'最近更新: '}
{item.update_time ? formatDate(item.update_time, 'YYYY-MM-DD') : item.time_ago ?? ''}

8
react-ui/src/pages/Dataset/components/ResourcePage/index.tsx
View File

@ -18,8 +18,8 @@ function ResourcePage({ resourceType }: ResourcePageProps) {
const [activeTab, setActiveTab] = useState<string>(cacheState?.activeTab ?? CommonTabKeys.Public);
const [typeList, setTypeList] = useState<CategoryData[]>([]);
const [tagList, setTagList] = useState<CategoryData[]>([]);
const [activeType, setActiveType] = useState<number | undefined>(cacheState?.activeType);
const [activeTag, setActiveTag] = useState<number | undefined>(cacheState?.activeTag);
const [activeType, setActiveType] = useState<string | undefined>(cacheState?.activeType);
const [activeTag, setActiveTag] = useState<string | undefined>(cacheState?.activeTag);
const dataListRef = useRef<ResourceListRef>(null);
const config = resourceConfig[resourceType];
@ -34,12 +34,12 @@ function ResourcePage({ resourceType }: ResourcePageProps) {
// 选择类型
const chooseType = (record: CategoryData) => {
setActiveType((prev) => (prev === record.id ? undefined : record.id));
setActiveType((prev) => (prev === record.name ? undefined : record.name));
};
// 选择 Tag
const chooseTag = (record: CategoryData) => {
setActiveTag((prev) => (prev === record.id ? undefined : record.id));
setActiveTag((prev) => (prev === record.name ? undefined : record.name));
};
// 获取分类

2
react-ui/src/pages/Experiment/components/ExperimentInstance/index.tsx
View File

@ -116,6 +116,8 @@ function ExperimentInstanceComponent({
<img
style={{ width: '17px', marginRight: '7px' }}
src={experimentStatusInfo[item.status as ExperimentStatus]?.icon}
draggable={false}
alt=""
/>
<span
style={{ color: experimentStatusInfo[item.status as ExperimentStatus]?.color }}

7
react-ui/src/pages/Experiment/components/ExperimentStatusCell/index.tsx
View File

@ -14,7 +14,12 @@ function ExperimentStatusCell(status?: ExperimentStatus | null) {
}
return (
<div className={styles['experiment-status-cell']}>
<img style={{ width: '17px', marginRight: '7px' }} src={statusInfo[status]?.icon} />
<img
style={{ width: '17px', marginRight: '7px' }}
src={statusInfo[status]?.icon}
draggable={false}
alt=""
/>
<span
style={{ color: statusInfo[status]?.color }}
className={styles['experiment-status-cell__label']}

2
react-ui/src/pages/Experiment/components/TensorBoardStatus/index.tsx
View File

@ -70,6 +70,8 @@ function TensorBoardStatusCell({
className={styles['tensorBoard-status__icon']}
src={statusConfig[status].icon}
onClick={onClick}
draggable={false}
alt=""
/>
)}
</>

2
react-ui/src/pages/Experiment/index.jsx
View File

@ -385,6 +385,8 @@ function Experiment() {
style={{ width: '17px', marginRight: '6px' }}
key={index}
src={experimentStatusInfo[item].icon}
draggable={false}
alt=""
/>
);
})

1
react-ui/src/pages/Pipeline/components/ModelMenu/index.tsx
View File

@ -48,6 +48,7 @@ const ModelMenu = ({ onComponentDragEnd }: ModelMenuProps) => {
<img
style={{ height: '16px', marginRight: '15px' }}
src={`/assets/images/${ele.icon_path}.png`}
draggable={false}
alt=""
/>
)}

7
react-ui/src/pages/User/Center/components/AvatarCropper/index.tsx
View File

@ -116,7 +116,12 @@ const AvatarCropperForm: React.FC<AvatarCropperProps> = (props) => {
</Col>
<Col span={12} order={2}>
<div className={styles.avatarPreview}>
<img src={previewData} style={{ height: '100%', width: '100%' }} />
<img
src={previewData}
style={{ height: '100%', width: '100%' }}
draggable={false}
alt=""
/>
</div>
</Col>
</Row>

2
react-ui/src/pages/User/Center/index.tsx
View File

@ -141,7 +141,7 @@ const Center: React.FC = () => {
setCropperModalOpen(true);
}}
>
<img alt="" src={currentUser.avatar} />
<img src={currentUser.avatar} draggable={false} alt="" />
</div>
{renderUserInfo(currentUser)}
<Divider dashed />

5
react-ui/src/pages/User/Login/index.tsx
View File

@ -11,7 +11,7 @@ import styles from './login.less';
const LoginInputPrefix = ({ icon }: { icon: string }) => {
return (
<div className={styles['login-input-prefix']}>
<img className={styles['login-input-prefix__icon']} src={icon} alt="" />
<img className={styles['login-input-prefix__icon']} src={icon} alt="" draggable={false} />
<div className={styles['login-input-prefix__line']}></div>
</div>
);
@ -98,6 +98,7 @@ const Login = () => {
<img
src={require('@/assets/img/logo.png')}
style={{ width: '32px', marginRight: '12px' }}
draggable={false}
alt=""
/>
智能材料科研平台
@ -107,6 +108,7 @@ const Login = () => {
<img
src={require('@/assets/img/login-ai-logo.png')}
className={styles['user-login__left__title__img']}
draggable={false}
alt=""
/>
</div>
@ -116,6 +118,7 @@ const Login = () => {
<img
className={styles['user-login__left__bottom-img']}
src={require('@/assets/img/login-left-image.png')}
draggable={false}
alt=""
/>
</div>

2
react-ui/src/pages/Workspace/components/ExperimentTable/index.tsx
View File

@ -31,6 +31,8 @@ function ExperimentTable({ tableData = [], style }: ExperimentTableProps) {
src={experimentStatusInfo[item.status as ExperimentStatus]?.icon}
width={17}
height={17}
draggable={false}
alt=""
/>
</div>
<div className={styles['experiment-table__duration']}>

1
react-ui/src/pages/Workspace/components/QuickStart/WorkArrow.tsx
View File

@ -44,6 +44,7 @@ function WorkArrow({
<img
className={styles['work-arrow__img']}
src={require('@/assets/img/blue-triangle.png')}
draggable={false}
alt=""
width={10}
height={9}

2
react-ui/src/pages/Workspace/components/TotalStatistics/index.tsx
View File

@ -10,7 +10,7 @@ type TotalStatisticsProps = {
function TotalStatistics({ icon = '', title = '', count = 0, style }: TotalStatisticsProps) {
return (
<div className={styles['total-statistics']} style={style}>
<img className={styles['total-statistics__icon']} src={icon} />
<img className={styles['total-statistics__icon']} src={icon} draggable={false} alt="" />
<div>
<div className={styles['total-statistics__title']}>
<span>{title}</span>

20
react-ui/src/pages/Workspace/components/UserSpace/index.tsx
View File

@ -20,7 +20,15 @@ function UserSpace({ users = [] }: UserSpaceProps) {
className={styles['user-space__avatar']}
src={currentUser?.avatar}
alt=""
icon={<img src={require('@/assets/img/avatar-default.png')} width={56} height={56} />}
icon={
<img
src={require('@/assets/img/avatar-default.png')}
width={56}
height={56}
draggable={false}
alt=""
/>
}
></Avatar>
<div className={styles['user-space__name']}>{currentUser?.nickName}</div>
<div className={styles['user-space__role']}>{currentUser?.roleNames?.[0]?.roleName}</div>
@ -34,7 +42,7 @@ function UserSpace({ users = [] }: UserSpaceProps) {
<div className={styles['user-space__participant__count']}>8</div>
</Space>
<Flex align="center" gap={12} wrap="wrap">
{users?.map((item, index) => {
{users?.map((_item, index) => {
return (
<Avatar
key={index}
@ -42,7 +50,13 @@ function UserSpace({ users = [] }: UserSpaceProps) {
src={require(`@/assets/img/user-avatar/${index + 1}.png`)}
alt=""
icon={
<img src={require('@/assets/img/avatar-default.png')} width={36} height={36} />
<img
src={require('@/assets/img/avatar-default.png')}
width={36}
height={36}
draggable={false}
alt=""
/>
}
></Avatar>
);

18
react-ui/src/pages/Workspace/components/WorkspaceIntro/index.tsx
View File

@ -15,7 +15,13 @@ function WorkspaceIntro() {
type="primary"
style={{ marginRight: '20px' }}
icon={
<img src={require('@/assets/img/functional-material.png')} width={19} height={19} />
<img
src={require('@/assets/img/functional-material.png')}
width={19}
height={19}
draggable={false}
alt=""
/>
}
>
功能材料自主实验系统
@ -23,7 +29,13 @@ function WorkspaceIntro() {
<Button
type="default"
icon={
<img src={require('@/assets/img/molecular-material.png')} width={19} height={19} />
<img
src={require('@/assets/img/molecular-material.png')}
width={19}
height={19}
draggable={false}
alt=""
/>
}
>
分子材料自主实验系统
@ -34,6 +46,8 @@ function WorkspaceIntro() {
<img
className={styles['workspace-intro__icon']}
src={require('@/assets/img/workspace-intro-icon.png')}
draggable={false}
alt=""
/>
</div>
</div>

1
react-ui/src/pages/Workspace/index.less
View File

@ -53,6 +53,7 @@
width: 64px;
height: 64px;
background-color: white;
border-radius: 10px;
cursor: pointer;
}
}

3
react-ui/src/pages/Workspace/index.tsx
View File

@ -76,7 +76,8 @@ function Workspace() {
className={styles['workspace__robot-img']}
src={require('@/assets/img/robot.png')}
onClick={handleClick}
onDragStart={(e) => e.preventDefault()}
draggable={false}
alt=""
></img>
</Draggable>

1
react-ui/src/utils/ui.tsx
View File

@ -21,6 +21,7 @@ export function modalConfirm({ title, content, onOk, ...rest }: ModalFuncProps)
<img
src={require('@/assets/img/delete-icon.png')}
style={{ width: '120px', marginBottom: '24px' }}
draggable={false}
alt=""
/>
<div style={{ color: themes.textColor, fontSize: '16px', fontWeight: 500 }}>{title}</div>

16
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DvcServiceImpl.java
View File

@ -7,6 +7,8 @@ import com.ruoyi.system.api.model.LoginUser;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
@Service
public class DvcServiceImpl implements DvcService {
@Value("${minio.accessKey}")
@ -15,20 +17,22 @@ public class DvcServiceImpl implements DvcService {
String secretAccessKey;
@Value("${minio.endpoint}")
String endpoint;
@Resource
private DVCUtils dvcUtils;
@Override
public void initaddpushDvc(String localPath) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
// dvc init 初始化
DVCUtils.dvcInit(localPath);
dvcUtils.dvcInit(localPath);
// 配置远程S3地址
DVCUtils.dvcRemoteAdd(localPath,"");
// DVCUtils.dvcConfigS3Credentials(localPath,endpoint, accessKeyId, secretAccessKey);
dvcUtils.dvcRemoteAdd(localPath,"");
// dvcUtils.dvcConfigS3Credentials(localPath,endpoint, accessKeyId, secretAccessKey);
// dvc 跟踪
DVCUtils.dvcAdd(localPath , "data");
dvcUtils.dvcAdd(localPath , "data");
// git commit
DVCUtils.gitCommit(localPath, "commit from ci4s with "+loginUser.getUsername());
dvcUtils.gitCommit(localPath, "commit from ci4s with "+loginUser.getUsername());
// dvc push 到远程S3
DVCUtils.dvcPush(localPath);
dvcUtils.dvcPush(localPath);
}
}

48
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/ModelsServiceImpl.java
View File

@ -78,6 +78,8 @@ public class ModelsServiceImpl implements ModelsService {
private AssetIconService assetIconService;
@Resource
private NewHttpUtils httpUtils;
@Resource
private DVCUtils dvcUtils;
// 固定存储桶名
@Value("${minio.dataReleaseBucketName}")
@ -593,8 +595,8 @@ public class ModelsServiceImpl implements ModelsService {
String modelPath = rootPath + "/model";
String metaPath = rootPath + "/metadata";
DVCUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
DVCUtils.moveFiles(sourcePath, modelPath);
dvcUtils.gitClone(rootPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
dvcUtils.moveFiles(sourcePath, modelPath);
//拼接生产的元数据后写入yaml文件
ModelMetaVo modelMetaVo = new ModelMetaVo();
@ -621,21 +623,21 @@ public class ModelsServiceImpl implements ModelsService {
YamlUtils.generateYamlFile(metaMap, metaPath, "metadata");
String meta = JSON.toJSONString(modelMetaVo);
DVCUtils.dvcInit(rootPath);
dvcUtils.dvcInit(rootPath);
// 配置远程S3地址
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + gitlinIid + "/" + repositoryName + "/" + branchName;
DVCUtils.dvcRemoteAdd(rootPath, s3Path);
DVCUtils.dvcConfigS3Credentials(rootPath, endpoint);
DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
// dvc 跟踪
DVCUtils.dvcAdd(rootPath, "model");
dvcUtils.dvcAdd(rootPath, "model");
// git commit
DVCUtils.gitAdd(rootPath, ".");
DVCUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername);
DVCUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword);
DVCUtils.dvcPush(rootPath);
dvcUtils.gitAdd(rootPath, ".");
dvcUtils.gitCommit(rootPath, "commit from ci4s with " + ci4sUsername);
dvcUtils.gitPush(rootPath, gitLinkUsername, gitLinkPassword);
dvcUtils.dvcPush(rootPath);
//保存模型依赖
modelDependency.setRepoId(gitlinIid);
@ -686,10 +688,10 @@ public class ModelsServiceImpl implements ModelsService {
ModelMetaVo oldModelVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class);
// 创建本地分支
DVCUtils.createLocalBranchBasedOnMaster(rootPath, branchName);
dvcUtils.createLocalBranchBasedOnMaster(rootPath, branchName);
//dvc checkout
DVCUtils.dvcCheckout(rootPath);
DVCUtils.moveFiles(sourcePath, modelPath);
dvcUtils.dvcCheckout(rootPath);
dvcUtils.moveFiles(sourcePath, modelPath);
//拼接生产的元数据后写入yaml文件
ModelMetaVo modelMetaVo = new ModelMetaVo();
@ -719,15 +721,15 @@ public class ModelsServiceImpl implements ModelsService {
// 配置远程S3地址
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + modelsVo.getId() + "/" + repositoryName + "/" + branchName;
DVCUtils.dvcRemoteAdd(rootPath, s3Path);
DVCUtils.dvcConfigS3Credentials(rootPath, endpoint);
DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
dvcUtils.dvcRemoteAdd(rootPath, s3Path);
dvcUtils.dvcConfigS3Credentials(rootPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
// dvc 跟踪
DVCUtils.dvcAdd(rootPath, "model");
DVCUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName);
dvcUtils.dvcAdd(rootPath, "model");
dvcUtils.pushNewBranchToRemote(rootPath, gitLinkUsername, gitLinkPassword, branchName);
//dvc push 到远程S3
DVCUtils.dvcPush(rootPath);
dvcUtils.dvcPush(rootPath);
//保存模型依赖
modelDependency.setRepoId(modelsVo.getId());
@ -895,7 +897,7 @@ public class ModelsServiceImpl implements ModelsService {
}
// git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/" + id, name, version, "model", gitLinkUsername, gitLinkPassword);
List<Map<String, Object>> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/" + id, name, version, "model", gitLinkUsername, gitLinkPassword);
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + id + "/" + name + "/metadata/metadata.yaml");

47
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/NewDatasetServiceImpl.java
View File

@ -67,6 +67,8 @@ public class NewDatasetServiceImpl implements NewDatasetService {
private NewHttpUtils httpUtils;
@Resource
private DatasetTempStorageService datasetTempStorageService;
@Resource
private DVCUtils dvcUtils;
@Override
public String newCreateDataset(NewDatasetVo datasetVo) throws Exception {
String token = gitService.checkoutToken();
@ -110,10 +112,10 @@ public class NewDatasetServiceImpl implements NewDatasetService {
String datasetPath = localPath + "/dataset";
// 命令行操作 git clone 项目地址
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
dvcUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
String s3Path = "management-platform-files" + "/" + relatePath + "/" + branchName;
DVCUtils.moveFiles(sourcePath, datasetPath);
dvcUtils.moveFiles(sourcePath, datasetPath);
// 拼接生产的元数据后写入yaml文件
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
datasetVo.setUpdateTime(DateUtils.getTime());
@ -133,20 +135,19 @@ public class NewDatasetServiceImpl implements NewDatasetService {
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
// dvc init 初始化
DVCUtils.dvcInit(localPath);
dvcUtils.dvcInit(localPath);
// 配置远程S3地址
DVCUtils.dvcRemoteAdd(localPath, s3Path);
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
dvcUtils.dvcRemoteAdd(localPath, s3Path);
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
// dvc 跟踪
DVCUtils.dvcAdd(localPath, "dataset");
dvcUtils.dvcAdd(localPath, "dataset");
// git commit
DVCUtils.gitAdd(localPath, ".");
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
dvcUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
dvcUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
// dvc push 到远程S3
DVCUtils.dvcPush(localPath);
dvcUtils.dvcPush(localPath);
return "新增数据集成功";
}
@ -173,15 +174,15 @@ public class NewDatasetServiceImpl implements NewDatasetService {
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
// 创建本地分支
DVCUtils.createLocalBranchBasedOnMaster(localPath, branchName);
dvcUtils.createLocalBranchBasedOnMaster(localPath, branchName);
//dvc checkout
DVCUtils.dvcCheckout(localPath);
dvcUtils.dvcCheckout(localPath);
// 准备数据
String s3Path = "management-platform-files" + "/" + relatePath + "/" + branchName;
DVCUtils.moveFiles(sourcePath, datasetPath);
dvcUtils.moveFiles(sourcePath, datasetPath);
// 拼接生产的元数据后写入yaml文件
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
datasetVo.setUpdateTime(DateUtils.getTime());
@ -206,15 +207,15 @@ public class NewDatasetServiceImpl implements NewDatasetService {
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
//dvc数据跟踪
// 配置远程S3地址
DVCUtils.dvcRemoteAdd(localPath, s3Path);
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
dvcUtils.dvcRemoteAdd(localPath, s3Path);
dvcUtils.dvcConfigS3Credentials(localPath, endpoint);
dvcUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
dvcUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
// dvc 跟踪
DVCUtils.dvcAdd(localPath, "dataset");
DVCUtils.pushNewBranchToRemote(localPath,gitLinkUsername,gitLinkPassword,branchName);
dvcUtils.dvcAdd(localPath, "dataset");
dvcUtils.pushNewBranchToRemote(localPath,gitLinkUsername,gitLinkPassword,branchName);
//dvc push 到远程S3
DVCUtils.dvcPush(localPath);
dvcUtils.dvcPush(localPath);
return "新增数据集成功";
}
@ -292,7 +293,7 @@ public class NewDatasetServiceImpl implements NewDatasetService {
}
version = (String) versionList.get(0).get("name");
}
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, name, version, "data",gitLinkUsername, gitLinkPassword);
List<Map<String, Object>> fileDetailsAfterGitPull = dvcUtils.getFileDetailsAfterGitPull(localPathlocal +loginUser.getUsername()+"/datasets/" + id, name, version, "data",gitLinkUsername, gitLinkPassword);
// 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal +loginUser.getUsername()+"/datasets/" + id + "/" + name + "/" + "dataset.yaml");
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);

83
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java
View File

@ -9,6 +9,8 @@ import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import org.eclipse.jgit.transport.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.File;
@ -24,10 +26,32 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@Component
public class DVCUtils {
private static final ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
private static final Logger log = LoggerFactory.getLogger(DVCUtils.class);
private static final ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<>(executorService);
@Value("${proxy.useProxy:false}")
private boolean useProxy;
@Value("${proxy.host}")
private String host;
@Value("${proxy.port}")
private Integer port;
private class ProxyConfigCallback implements TransportConfigCallback {
@Override
public void configure(Transport transport) {
if (useProxy) {
System.setProperty("http.proxyHost", host);
System.setProperty("http.proxyPort", String.valueOf(port));
System.setProperty("https.proxyHost", host);
System.setProperty("https.proxyPort", String.valueOf(port));
log.info("Proxy configured: {}:{}", host, port);
}
}
}
private static void runCommand(String command, String workingDir) throws Exception {
ProcessBuilder processBuilder = new ProcessBuilder(command.split(" "));
processBuilder.directory(new File(workingDir));
@ -43,7 +67,7 @@ public class DVCUtils {
}
}
public static void moveFiles(String sourcePath, String targetPath) throws Exception {
public void moveFiles(String sourcePath, String targetPath) throws Exception {
Path sourceDir = Paths.get(sourcePath);
Path targetDir = Paths.get(targetPath);
@ -54,17 +78,18 @@ public class DVCUtils {
Files.move(sourceDir, targetDir, StandardCopyOption.REPLACE_EXISTING);
}
public static void gitClone(String localPath, String repoUrl, String branch, String username, String password) throws GitAPIException {
public void gitClone(String localPath, String repoUrl, String branch, String username, String password) throws GitAPIException {
CloneCommand cloneCommand = Git.cloneRepository()
.setURI(repoUrl)
.setBranch(branch)
.setDirectory(new java.io.File(localPath))
.setTransportConfigCallback(new ProxyConfigCallback())
.setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password));
cloneCommand.call();
}
public static void gitClone(String localPath, String repoUrl, String username, String password) throws GitAPIException {
public void gitClone(String localPath, String repoUrl, String username, String password) throws GitAPIException {
CloneCommand cloneCommand = Git.cloneRepository()
.setURI(repoUrl)
.setDirectory(new java.io.File(localPath))
@ -74,7 +99,7 @@ public class DVCUtils {
}
public static void gitAdd(String localPath, String filePath) throws IOException, GitAPIException {
public void gitAdd(String localPath, String filePath) throws IOException, GitAPIException {
FileRepositoryBuilder builder = new FileRepositoryBuilder();
Repository repository = builder.setGitDir(new File(localPath, ".git"))
.readEnvironment()
@ -87,7 +112,7 @@ public class DVCUtils {
}
}
public static void gitCommit(String localPath, String commitMessage) throws IOException, GitAPIException {
public void gitCommit(String localPath, String commitMessage) throws IOException, GitAPIException {
FileRepositoryBuilder builder = new FileRepositoryBuilder();
Repository repository = builder.setGitDir(new File(localPath, ".git"))
.readEnvironment()
@ -105,7 +130,7 @@ public class DVCUtils {
}
}
public static void gitPush(String localPath, String username, String password) throws IOException, GitAPIException {
public void gitPush(String localPath, String username, String password) throws IOException, GitAPIException {
FileRepositoryBuilder builder = new FileRepositoryBuilder();
Repository repository = builder.setGitDir(new File(localPath, ".git"))
.readEnvironment()
@ -116,6 +141,7 @@ public class DVCUtils {
CredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
PushCommand pushCommand = git.push();
pushCommand.setCredentialsProvider(credentialsProvider)
.setTransportConfigCallback(new ProxyConfigCallback())
.setForce(true)
.call();
}
@ -129,7 +155,7 @@ public class DVCUtils {
* @param username 远程仓库用户名
* @param password 远程仓库密码
*/
public static void updateAllBranches(String localPath, String username, String password) {
public void updateAllBranches(String localPath, String username, String password) {
try (Git git = Git.open(new File(localPath))) {
// 设置凭证，用于远程仓库的认证
UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
@ -171,7 +197,7 @@ public class DVCUtils {
* @param password 远程仓库密码
* @param branchName 需要更新的分支名称
*/
public static void updateBranch(String localPath, String username, String password, String branchName) {
public void updateBranch(String localPath, String username, String password, String branchName) {
try (Git git = Git.open(new File(localPath))) {
// 设置凭证，用于远程仓库的认证
UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
@ -202,7 +228,7 @@ public class DVCUtils {
* @param localPath 本地仓库路径
* @param branchName 要创建的分支名称
*/
public static void createLocalBranch(String localPath, String branchName) {
public void createLocalBranch(String localPath, String branchName) {
try (Git git = Git.open(new File(localPath))) {
// 创建本地分支
git.branchCreate().setName(branchName).call();
@ -219,7 +245,7 @@ public class DVCUtils {
* @param localPath 本地仓库路径
* @param branchName 要创建的分支名称
*/
public static void createLocalBranchBasedOnMaster(String localPath, String branchName) {
public void createLocalBranchBasedOnMaster(String localPath, String branchName) {
try (Git git = Git.open(new File(localPath))) {
// 切换到 master 分支
git.checkout()
@ -241,7 +267,7 @@ public class DVCUtils {
* @param password 远程仓库密码
* @param branchName 要推送的分支名称
*/
public static void pushLocalBranchToRemote(String localPath, String username, String password, String branchName) {
public void pushLocalBranchToRemote(String localPath, String username, String password, String branchName) {
try (Git git = Git.open(new File(localPath))) {
// 设置凭证，用于远程仓库的认证
UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
@ -279,7 +305,7 @@ public class DVCUtils {
* @throws IOException 如果仓库路径无效
* @throws GitAPIException 如果Git操作失败
*/
public static void pushNewBranchToRemote(String localPath, String username, String password, String branchName) throws IOException, GitAPIException {
public void pushNewBranchToRemote(String localPath, String username, String password, String branchName) throws IOException, GitAPIException {
try (Git git = Git.open(new File(localPath))) {
// 设置凭证，用于远程仓库的认证
UsernamePasswordCredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
@ -300,6 +326,7 @@ public class DVCUtils {
.setRemote("origin")
.setRefSpecs(refSpec)
.setCredentialsProvider(credentialsProvider)
.setTransportConfigCallback(new ProxyConfigCallback())
.call();
// 打印结果
@ -326,7 +353,7 @@ public class DVCUtils {
return false;
}
public static void refreshRemoteBranches(String localPath, String username, String password, String branch) throws Exception {
public void refreshRemoteBranches(String localPath, String username, String password, String branch) throws Exception {
try (Repository repository = new FileRepositoryBuilder()
.setGitDir(new File(localPath + "/.git"))
.readEnvironment()
@ -349,6 +376,7 @@ public class DVCUtils {
FetchResult fetchResult = git.fetch()
.setRemote("origin")
.setCredentialsProvider(credentialsProvider)
.setTransportConfigCallback(new ProxyConfigCallback()) // 设置代理
.call();
// 打印获取的远程分支
@ -394,7 +422,7 @@ public class DVCUtils {
dvcCheckout(localPath);
}
private static void processBranch(Git git, Repository repository, UsernamePasswordCredentialsProvider credentialsProvider, String fullBranchName, String branchName) throws Exception {
private void processBranch(Git git, Repository repository, UsernamePasswordCredentialsProvider credentialsProvider, String fullBranchName, String branchName) throws Exception {
// 检查本地分支是否存在
Ref localRef = repository.findRef("refs/heads/" + branchName);
if (localRef != null) {
@ -422,6 +450,7 @@ public class DVCUtils {
}
// 执行 git pull
PullCommand pullCommand = git.pull()
.setTransportConfigCallback(new ProxyConfigCallback())
.setCredentialsProvider(credentialsProvider);
pullCommand.call();
}
@ -447,7 +476,7 @@ public class DVCUtils {
public static void gitFetch(String localPath, String username, String password) throws IOException, GitAPIException {
public void gitFetch(String localPath, String username, String password) throws IOException, GitAPIException {
FileRepositoryBuilder builder = new FileRepositoryBuilder();
Repository repository = builder.setGitDir(new File(localPath, ".git"))
.readEnvironment()
@ -460,7 +489,7 @@ public class DVCUtils {
fetchCommand.setCredentialsProvider(credentialsProvider).call();
}
}
public static void gitCheckoutBranch(String localPath, String branchName) throws IOException, GitAPIException {
public void gitCheckoutBranch(String localPath, String branchName) throws IOException, GitAPIException {
FileRepositoryBuilder builder = new FileRepositoryBuilder();
Repository repository = builder.setGitDir(new File(localPath, ".git"))
.readEnvironment()
@ -473,7 +502,7 @@ public class DVCUtils {
}
}
public static void gitPull(String localPath, String username, String password) throws IOException, GitAPIException {
public void gitPull(String localPath, String username, String password) throws IOException, GitAPIException {
FileRepositoryBuilder builder = new FileRepositoryBuilder();
Repository repository = builder.setGitDir(new File(localPath, ".git"))
.readEnvironment()
@ -487,50 +516,50 @@ public class DVCUtils {
}
}
public static void dvcInit(String localPath) throws Exception {
public void dvcInit(String localPath) throws Exception {
String command = "dvc init";
runCommand(command, localPath);
}
public static void dvcAdd(String localPath, String filePath) throws Exception {
public void dvcAdd(String localPath, String filePath) throws Exception {
String command = "dvc add " + filePath;
runCommand(command, localPath);
}
public static void dvcRemoteAdd(String localPath, String s3RemoteUrl) throws Exception {
public void dvcRemoteAdd(String localPath, String s3RemoteUrl) throws Exception {
String command = "dvc remote add -d myremote s3://" + s3RemoteUrl;
runCommand(command, localPath);
}
public static void dvcConfigS3Credentials(String localPath, String endpointurl) throws Exception {
public void dvcConfigS3Credentials(String localPath, String endpointurl) throws Exception {
String command = "dvc remote modify myremote endpointurl " + endpointurl;
runCommand(command, localPath);
}
public static void dvcConfigS3Credentials2(String localPath, String accessKeyId) throws Exception {
public void dvcConfigS3Credentials2(String localPath, String accessKeyId) throws Exception {
String command = "dvc remote modify myremote access_key_id " + accessKeyId;
runCommand(command, localPath);
}
public static void dvcConfigS3Credentials3(String localPath, String secretAccessKey) throws Exception {
public void dvcConfigS3Credentials3(String localPath, String secretAccessKey) throws Exception {
String command = "dvc remote modify myremote secret_access_key " + secretAccessKey;
runCommand(command, localPath);
}
public static void dvcPush(String localPath) throws Exception {
public void dvcPush(String localPath) throws Exception {
String command = "dvc push -v";
runCommand(command, localPath);
}
// 更新的 dvcPull 方法
public static void dvcPull(String localPath) throws Exception {
public void dvcPull(String localPath) throws Exception {
String command = "dvc pull";
runCommand(command, localPath);
}
// 方法
public static void dvcCheckout(String localPath) throws Exception {
public void dvcCheckout(String localPath) throws Exception {
String command = "dvc checkout";
runCommand(command, localPath);
}
@ -544,7 +573,7 @@ public class DVCUtils {
* @param branch 分支名称
* @return 包含文件路径、名称和大小的List<Map<String, Object>>
*/
public static List<Map<String, Object>> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch, String filePath , String username, String password) {
public List<Map<String, Object>> getFileDetailsAfterGitPull(String localPath, String repoFolder, String branch, String filePath , String username, String password) {
List<Map<String, Object>> fileInfoList = new ArrayList<>();
try {