ci4s
/
ci4sManagement-cloud
41
0
Fork 1
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

dvc

This commit is contained in:
fanshuai 2024-08-30 10:03:07 +08:00
parent f6db5d28eb
commit df87d16d8f
14 changed files with 638 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
ruoyi-modules/management-platform/pom.xml
View File

@ -142,6 +142,11 @@
<artifactId>docker-java</artifactId>
<version>3.2.13</version>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.9.4</version>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-transport-httpclient5</artifactId>
@ -227,7 +232,16 @@
<version>2.6</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>5.13.0.202109080827-r</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>3.6.0</version>
</dependency>
</dependencies>

2
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetController.java
View File

@ -224,5 +224,7 @@ public class DatasetController {
public AjaxResult uploadDatasetPipeline(@RequestBody(required =false) DatasetVersion datasetVersion) throws Exception {
return AjaxResult.success(this.datasetService.uploadDatasetPipeline(datasetVersion));
}
}

110
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java Normal file
View File

@ -0,0 +1,110 @@
package com.ruoyi.platform.controller.dataset;
import com.ruoyi.common.core.web.domain.AjaxResult;
import com.ruoyi.platform.domain.Dataset;
import com.ruoyi.platform.service.DatasetService;
import com.ruoyi.platform.vo.DatasetVo;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
@RestController
@RequestMapping("newdataset")
@Api(value = "新数据集管理")
public class NewDatasetFromGitController {
/**
* 服务对象
*/
@Resource
private DatasetService datasetService;
/**
* 新增数据集与版本新
*
* @param datasetVo 实体
* @return 新增结果
*/
@PostMapping("/addDatasetAndVersion")
@ApiOperation("添加数据集和版本")
public AjaxResult addDatasetAndVersion(@RequestBody DatasetVo datasetVo) throws Exception {
return AjaxResult.success(this.datasetService.newCreateDataset(datasetVo));
}
/**
* 新增数据集与版本新
*
* @param datasetVo 实体
* @return 新增结果
*/
@PostMapping("/addVersion")
@ApiOperation("添加版本")
public AjaxResult addVersion(@RequestBody DatasetVo datasetVo) throws Exception {
return AjaxResult.success(this.datasetService.newCreateVersion(datasetVo));
}
/**
* 上传数据集
*
* @param files 上传的数据集文件
* @param uuid 上传唯一标识，构建url
* @return 上传结果
*/
@CrossOrigin(origins = "*", allowedHeaders = "*")
@PostMapping("/upload")
@ApiOperation(value = "上传数据集")
public AjaxResult uploadDataset(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception {
return AjaxResult.success(this.datasetService.uploadDatasetlocal(files,uuid));
}
/**
* 数据集打包下载
*
* @param version 数据集版本
* @return 单条数据
*/
@GetMapping("/downloadAllFiles")
@ApiOperation(value = "下载同一版本下所有数据集，并打包")
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("repository_name") String repositoryName, @RequestParam("version") String version) throws Exception {
return datasetService.downloadAllDatasetFilesNew(repositoryName, version);
}
/**
* 下载数据集
*
* @param dataset_version_id ps:这里的id是dataset_version表的主键
* @return 单条数据
*/
@GetMapping("/download/{dataset_version_id}")
@ApiOperation(value = "下载单个数据集文件", notes = "根据数据集版本表id下载单个数据集文件")
public ResponseEntity<InputStreamResource> downloadDataset(@PathVariable("dataset_version_id") Integer dataset_version_id) throws Exception {
return datasetService.downloadDataset(dataset_version_id);
}
@GetMapping("/queryDatasets")
@ApiOperation("数据集广场公开数据集分页查询,根据data_type,data_tag筛选，true公开false私有")
public AjaxResult queryDatasets(Dataset dataset, @RequestParam("page") int page,
@RequestParam("size") int size,
@RequestParam(value = "is_public") Boolean isPublic,
@RequestParam(value = "data_type", required = false) String dataType,
@RequestParam(value = "data_tag", required = false) String dataTag) throws Exception {
PageRequest pageRequest = PageRequest.of(page, size);
if(isPublic){
return AjaxResult.success(this.datasetService.newPubilcQueryByPage(dataset, pageRequest));
}else {
return AjaxResult.success(this.datasetService.newPersonalQueryByPage(dataset, pageRequest));
}
}
}

7
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java
View File

@ -90,7 +90,7 @@ public class JupyterController extends BaseController {
@GetMapping(value = "/testdvc")
public AjaxResult testdvc() throws Exception {
DatasetVo datasetVo = new DatasetVo();
datasetVo.setName("testdvc82sas155");
datasetVo.setName("testdassad23");
datasetVo.setDescription("sss");
datasetVo.setAvailableRange(0);
datasetVo.setDataTag("计算机视觉");
@ -98,10 +98,9 @@ public class JupyterController extends BaseController {
datasetVo.setVersion("dev");
List<VersionVo> datasetVersionVos = new ArrayList<>();
VersionVo versionVo = new VersionVo();
versionVo.setUrl("/home/xxx.txt");
versionVo.setUrl("E:/test/bb/data/xssa.doc");
datasetVersionVos.add(versionVo);
datasetVo.setDatasetVersionVos(datasetVersionVos);
datasetService.newCreateDataset(datasetVo);
return AjaxResult.success();
return AjaxResult.success(datasetService.newCreateDataset(datasetVo));
}
}

9
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java
View File

@ -3,12 +3,14 @@ package com.ruoyi.platform.service;
import com.ruoyi.platform.domain.Dataset;
import com.ruoyi.platform.domain.DatasetVersion;
import com.ruoyi.platform.vo.DatasetVo;
import com.ruoyi.platform.vo.NewDatasetVo;
import org.springframework.core.io.InputStreamResource;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.ResponseEntity;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
@ -91,6 +93,13 @@ DatasetService {
CompletableFuture<String> newCreateDataset(DatasetVo datasetVo) throws Exception;
CompletableFuture<String> newCreateVersion(DatasetVo datasetVo);
List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception;
Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
}

217
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java
View File

@ -1,6 +1,5 @@
package com.ruoyi.platform.service.impl;
import com.ruoyi.common.core.utils.DateUtils;
import com.ruoyi.common.security.utils.SecurityUtils;
import com.ruoyi.platform.annotations.CheckDuplicate;
@ -13,6 +12,7 @@ import com.ruoyi.platform.service.*;
import com.ruoyi.platform.utils.*;
import com.ruoyi.platform.vo.DatasetVo;
import com.ruoyi.platform.vo.GitProjectVo;
import com.ruoyi.platform.vo.NewDatasetVo;
import com.ruoyi.platform.vo.VersionVo;
import com.ruoyi.system.api.model.LoginUser;
import io.minio.messages.Item;
@ -33,10 +33,11 @@ import org.springframework.web.multipart.MultipartFile;
import redis.clients.jedis.Jedis;
import javax.annotation.Resource;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.*;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.CompletableFuture;
@ -461,9 +462,7 @@ public class DatasetServiceImpl implements DatasetService {
public CompletableFuture<String> newCreateDataset(DatasetVo datasetVo) {
return CompletableFuture.supplyAsync(() -> {
try {
String gitusername = datasetVo.getGitusername();
String gitpassword = datasetVo.getGitpassword();
String token = gitService.login(gitusername, gitpassword);
String token = gitService.login("fanshuai", "h1n2x3j4y5@");
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
Jedis jedis = new Jedis(redisHost);
@ -492,11 +491,67 @@ public class DatasetServiceImpl implements DatasetService {
// 得到用户操作的路径
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
String localPath = "/home/tempwork" + datasetVo.getName();
String localPath = "E:/test/" + datasetVo.getName();
String sourcePath = url.substring(0, url.lastIndexOf("/"));
// 命令行操作 git clone 项目地址
DVCUtils.gitClone(localPath, projectUrl, branchName, gitusername, gitpassword);
DVCUtils.gitClone(localPath, projectUrl, branchName, "fanshuai", "h1n2x3j4y5@");
String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + repositoryName + "/" + branchName;
//拼接生产的元数据后写入yaml文件
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),sourcePath, "dataset");
DVCUtils.moveFiles(sourcePath, localPath);
// dvc init 初始化
DVCUtils.dvcInit(localPath);
// 配置远程S3地址
DVCUtils.dvcRemoteAdd(localPath, s3Path);
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
// dvc 跟踪
DVCUtils.dvcAdd(localPath, "data");
// git commit
DVCUtils.gitAdd(localPath, ".");
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
DVCUtils.gitPush(localPath, "fanshuai", "h1n2x3j4y5@");
// dvc push 到远程S3
DVCUtils.dvcPush(localPath);
return "新增数据集成功";
} catch (Exception e) {
throw new RuntimeException(e);
}
});
}
@Override
public CompletableFuture<String> newCreateVersion(DatasetVo datasetVo) {
return CompletableFuture.supplyAsync(() -> {
try {
String token = gitService.login("fanshuai", "h1n2x3j4y5@");
LoginUser loginUser = SecurityUtils.getLoginUser();
String ci4sUsername = loginUser.getUsername();
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
// 创建分支
String branchName = datasetVo.getVersion();
String repositoryName = datasetVo.getRepositoryName();
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
// 得到项目地址
String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
// 得到用户操作的路径
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
String localPath = "E:/test/"+ loginUser.getUsername()+"/datasets/"+ datasetVo.getName();
String sourcePath = url.substring(0, url.lastIndexOf("/"));
// 命令行操作 git clone 项目地址
DVCUtils.gitClone(localPath, projectUrl, branchName, "fanshuai", "h1n2x3j4y5@");
String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + repositoryName + "/" + branchName;
//拼接生产的元数据后写入yaml文件
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),sourcePath, "dataset");
DVCUtils.moveFiles(sourcePath, localPath);
// dvc init 初始化
DVCUtils.dvcInit(localPath);
@ -510,7 +565,7 @@ public class DatasetServiceImpl implements DatasetService {
// git commit
DVCUtils.gitAdd(localPath, ".");
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
DVCUtils.gitPush(localPath, gitusername, gitpassword);
DVCUtils.gitPush(localPath, "fanshuai", "h1n2x3j4y5@");
// dvc push 到远程S3
DVCUtils.dvcPush(localPath);
return "新增数据集成功";
@ -520,6 +575,56 @@ public class DatasetServiceImpl implements DatasetService {
});
}
@Override
public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
String token = gitService.login("fanshuai", "h1n2x3j4y5@");
String ci4sUsername = loginUser.getUsername();
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
//拼接查询url
String datasetTagName = dataset.getDatasetTagName();
String datasetTypeName = dataset.getDatasetTypeName();
String topic_name = "ci4s_dataset";
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName;
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName;
String url = gitendpoint + "/api/users/"+(String) userInfo.get("login")+"/projects.json?page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&category=manage&topic_name="+topic_name;
String req = HttpUtils.sendGetWithToken(url,null,token);
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
Integer total = (Integer) stringObjectMap.get("count");
List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
return new PageImpl<>(convert(projects), pageRequest, total);
}
@Override
public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
LoginUser loginUser = SecurityUtils.getLoginUser();
String token = gitService.login("fanshuai", "h1n2x3j4y5@");
String ci4sUsername = loginUser.getUsername();
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
Integer userId = (Integer) userInfo.get("user_id");
//拼接查询url
String datasetTagName = dataset.getDatasetTagName();
String datasetTypeName = dataset.getDatasetTypeName();
String topic_name = "ci4s_dataset";
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName;
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName;
String url = gitendpoint + "/api/projects.json?user_id="+userId+"&page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&sort_by=praises_count&topic_name="+topic_name;
String req = HttpUtils.sendGetWithToken(url,null,token);
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
Integer total = (Integer) stringObjectMap.get("total_count");
List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
return new PageImpl<>(convert(projects), pageRequest, total);
}
@Override
public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
List<Map<String, String>> results = new ArrayList<>();
@ -527,7 +632,7 @@ public class DatasetServiceImpl implements DatasetService {
// 构建objectName
String username = SecurityUtils.getLoginUser().getUsername();
String fileName = file.getOriginalFilename();
String path = "/datasets/" + username + "/" + uuid + "/"+"/data/" + fileName;
String path = "/temp/"+ username +"/datasets/"+ uuid + "/"+"/data/" + fileName;
long sizeInBytes = file.getSize();
String formattedSize = FileUtil.formatFileSize(sizeInBytes);
File targetFile = new File(path, file.getOriginalFilename());
@ -546,4 +651,94 @@ public class DatasetServiceImpl implements DatasetService {
return results;
}
@Override
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception {
// 命令行操作 git clone 项目地址
LoginUser loginUser = SecurityUtils.getLoginUser();
String token = gitService.login("fanshuai", "h1n2x3j4y5@");
String ci4sUsername = loginUser.getUsername();
Jedis jedis = new Jedis(redisHost);
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
Integer userId = (Integer) userInfo.get("user_id");
String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
String localPath = "E:/test/"+ loginUser.getUsername()+"/datasets/" +repositoryName;
File folder = new File(localPath);
if(folder.exists() && folder.isDirectory()){
//切换分支
DVCUtils.gitCheckoutBranch(localPath, version);
//pull
DVCUtils.gitPull(localPath,"fanshuai", "h1n2x3j4y5@");
//dvc pull
DVCUtils.dvcPull(localPath);
}else {
DVCUtils.gitClone(localPath, projectUrl, version, "fanshuai", "h1n2x3j4y5@");
}
// 打包 data 文件夹
String dataFolderPath = localPath + "/data";
String zipFilePath = localPath + "/data.zip";
try (FileOutputStream fos = new FileOutputStream(zipFilePath);
ZipOutputStream zos = new ZipOutputStream(fos)) {
Path sourcePath = Paths.get(dataFolderPath);
Files.walk(sourcePath).forEach(path -> {
if (!Files.isDirectory(path)) {
ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString());
try {
zos.putNextEntry(zipEntry);
Files.copy(path, zos);
zos.closeEntry();
} catch (IOException e) {
throw new RuntimeException("Error while zipping: " + path, e);
}
}
});
}
// 返回压缩文件的输入流
File zipFile = new File(zipFilePath);
InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip")
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.contentLength(zipFile.length())
.body(resource);
}
public List<NewDatasetVo> convert(List<Map<String, Object>> lst) {
if (lst != null && lst.size() > 0) {
List<NewDatasetVo> newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class);
for (NewDatasetVo newDatasetVo : newDatasetVos) {
Map<String, Object> map = lst.stream()
.filter(m -> m.get("repo_id").equals(newDatasetVo.getRepoId()))
.findFirst()
.orElse(null);
if (map != null) {
List<Map<String, Object>> topics = (List<Map<String, Object>>) map.get("topics");
if (topics != null) {
topics.forEach(topic -> {
String name = (String) topic.get("name");
if (name != null) {
if (name.startsWith("datatag_")) {
newDatasetVo.setDataTag(name.substring("datatag_".length()));
} else if (name.startsWith("datatype_")) {
newDatasetVo.setDataType(name.substring("datatype_".length()));
}
}
});
}
}
}
return newDatasetVos;
}
return new ArrayList<>();
}
}

64
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java
View File

@ -4,10 +4,13 @@ package com.ruoyi.platform.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.apache.commons.beanutils.PropertyUtils;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
/**
@ -60,5 +63,64 @@ public class ConvertUtil {
return targetList;
}
public static <T> List<T> convertListMapToObjectList(List<Map<String, Object>> listMap, Class<T> targetClass) {
List<T> resultList = new ArrayList<>();
if (listMap != null) {
for (Map<String, Object> map : listMap) {
T targetObject = convertMapToObject(map, targetClass);
if (targetObject != null) {
resultList.add(targetObject);
}
}
}
return resultList;
}
private static <T> T convertMapToObject(Map<String, Object> map, Class<T> targetClass) {
try {
T targetObject = targetClass.newInstance();
for (Map.Entry<String, Object> entry : map.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
String camelCaseKey = toCamelCase(key);
if (hasProperty(targetClass, camelCaseKey)) {
PropertyUtils.setProperty(targetObject, camelCaseKey, value);
}
}
return targetObject;
} catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
e.printStackTrace();
}
return null;
}
private static boolean hasProperty(Class<?> clazz, String propertyName) {
try {
Field field = clazz.getDeclaredField(propertyName);
return field != null;
} catch (NoSuchFieldException e) {
return false;
}
}
private static String toCamelCase(String key) {
StringBuilder sb = new StringBuilder();
boolean nextUpperCase = false;
for (char c : key.toCharArray()) {
if (c == '_') {
nextUpperCase = true;
} else {
if (nextUpperCase) {
sb.append(Character.toUpperCase(c));
nextUpperCase = false;
} else {
sb.append(c);
}
}
}
return sb.toString();
}
}

32
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java
View File

@ -95,6 +95,33 @@ public class DVCUtils {
}
}
public static void gitCheckoutBranch(String localPath, String branchName) throws IOException, GitAPIException {
FileRepositoryBuilder builder = new FileRepositoryBuilder();
Repository repository = builder.setGitDir(new File(localPath, ".git"))
.readEnvironment()
.findGitDir()
.build();
try (Git git = new Git(repository)) {
CheckoutCommand checkoutCommand = git.checkout();
checkoutCommand.setName(branchName).call();
}
}
public static void gitPull(String localPath, String username, String password) throws IOException, GitAPIException {
FileRepositoryBuilder builder = new FileRepositoryBuilder();
Repository repository = builder.setGitDir(new File(localPath, ".git"))
.readEnvironment()
.findGitDir()
.build();
try (Git git = new Git(repository)) {
CredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password);
PullCommand pullCommand = git.pull();
pullCommand.setCredentialsProvider(credentialsProvider).call();
}
}
public static void dvcInit(String localPath) throws Exception {
String command = "dvc init";
runCommand(command, localPath);
@ -111,13 +138,15 @@ public class DVCUtils {
}
public static void dvcConfigS3Credentials(String localPath, String endpointurl) throws Exception {
String command = "dvc remote modify myremote endpointurl " + endpointurl ;
String command = "dvc remote modify myremote endpointurl " + endpointurl;
runCommand(command, localPath);
}
public static void dvcConfigS3Credentials2(String localPath, String accessKeyId) throws Exception {
String command = "dvc remote modify myremote access_key_id " + accessKeyId;
runCommand(command, localPath);
}
public static void dvcConfigS3Credentials3(String localPath, String secretAccessKey) throws Exception {
String command = "dvc remote modify myremote secret_access_key " + secretAccessKey;
runCommand(command, localPath);
@ -128,6 +157,7 @@ public class DVCUtils {
runCommand(command, localPath);
}
// 更新的 dvcPull 方法
public static void dvcPull(String localPath) throws Exception {
String command = "dvc pull";
runCommand(command, localPath);

57
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java
View File

@ -11,6 +11,7 @@ import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
@ -89,15 +90,61 @@ public class HttpUtils {
public static String sendGet(String url, String param) {
return sendGet(url, param, "UTF-8");
}
// /**
// * 向指定 URL 发送带token的GET方法的请求
// *
// * @param url 发送请求的 URL
// * @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
// * @return 所代表远程资源的响应结果
// */
// public static String sendGetWithToken(String url, String param,String token) {
// return sendGet(url, param, "UTF-8",token);
// }
/**
* 向指定 URL 发送带token的GET方法的请求
* 向指定 URL 发送带 token 的 GET 方法的请求，使用 Apache HttpClient
*
* @param url 发送请求的 URL
* @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
* @param url 发送请求的 URL
* @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
* @param token 认证 token
* @return 所代表远程资源的响应结果
*/
public static String sendGetWithToken(String url, String param,String token) {
return sendGet(url, param, "UTF-8",token);
public static String sendGetWithToken(String url, String param, String token) {
String result = "";
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
URIBuilder uriBuilder = new URIBuilder(url);
if (param != null && !param.isEmpty()) {
String[] pairs = param.split("&");
for (String pair : pairs) {
int idx = pair.indexOf("=");
if (idx != -1) {
uriBuilder.setParameter(pair.substring(0, idx), pair.substring(idx + 1));
}
}
}
URI uri = uriBuilder.build();
HttpGet httpGet = new HttpGet(uri);
httpGet.setHeader("Content-Type", "application/json");
httpGet.setHeader("Authorization", "Bearer " + token);
log.info("Executing request: " + httpGet.getRequestLine());
try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode == 200) {
result = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
} else {
throw new IOException("HTTP request failed with response code: " + statusCode);
}
log.info("Response: " + result);
}
} catch (URISyntaxException e) {
log.error("URISyntaxException, url=" + url + ", param=" + param, e);
} catch (IOException e) {
log.error("IOException, url=" + url + ", param=" + param, e);
}
return result;
}
/**
* 向指定 URL 发送带token的GET方法的请求

5
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/JsonUtils.java
View File

@ -51,4 +51,9 @@ public class JsonUtils {
return flatMap;
}
// 将Object转换为Map
public static Map<String, Object> objectToMap(Object object) throws IOException {
return objectMapper.convertValue(object, Map.class);
}
}

28
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java Normal file
View File

@ -0,0 +1,28 @@
package com.ruoyi.platform.utils;
import org.yaml.snakeyaml.Yaml;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Map;
public class YamlUtils {
/**
* 将Map对象转换为YAML格式并写入指定路径的文件中
*
* @param data Map对象
* @param path 文件路径
* @param fileName 文件名
*/
public static void generateYamlFile(Map<String, Object> data, String path, String fileName) {
Yaml yaml = new Yaml();
String fullPath = path + "/" + fileName + ".yaml";
try (FileWriter writer = new FileWriter(fullPath)) {
yaml.dump(data, writer);
} catch (IOException e) {
e.printStackTrace();
}
}
}

10
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/DatasetVo.java
View File

@ -28,10 +28,6 @@ public class DatasetVo implements Serializable {
@ApiModelProperty(name = "data_tag")
private String dataTag;
@ApiModelProperty(name = "gitusername")
private String gitusername;
@ApiModelProperty(name = "gitpassword")
private String gitpassword;
/**
* 版本
*/
@ -47,4 +43,10 @@ public class DatasetVo implements Serializable {
@ApiModelProperty(name = "available_cluster")
private String availableCluster;
/**
* 数据集仓库名称
*/
@ApiModelProperty(name = "repository_name")
private String repositoryName;
}

67
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/GitProjectVo.java Normal file
View File

@ -0,0 +1,67 @@
package com.ruoyi.platform.vo;
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.io.Serializable;
@ApiModel(description = "Git项目信息")
@Data
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
public class GitProjectVo implements Serializable {
@ApiModelProperty(value = "用户或组织ID", required = true)
private int userId;
@ApiModelProperty(value = "项目名称", required = true)
private String name;
@ApiModelProperty(value = "项目描述", required = false)
private String description;
@ApiModelProperty(value = "项目标识", required = true)
private String repositoryName;
// @ApiModelProperty(value = "项目分类ID", required = false)
// private int projectCategoryId;
//
// @ApiModelProperty(value = "项目语言ID", required = false)
// private int projectLanguageId;
// @ApiModelProperty(value = "项目忽略文件ID", required = false)
// private int ignoreId;
// @ApiModelProperty(value = "项目许可证ID", required = false)
// private int licenseId;
@ApiModelProperty(value = "项目是否为私有项目", required = false)
private boolean isPrivate;
// @ApiModelProperty(value = "是否为区块链激励项目", required = false)
// private boolean blockchain;
//
// @ApiModelProperty(value = "区块链激励金额总数", required = false)
// private int blockchainTokenAll;
//
// @Override
// public String toString() {
// return "GitProjectVo{" +
// "userId=" + userId +
// ", name='" + name + '\'' +
// ", description='" + description + '\'' +
// ", repositoryName='" + repositoryName + '\'' +
// ", projectCategoryId=" + projectCategoryId +
// ", projectLanguageId=" + projectLanguageId +
// ", ignoreId=" + ignoreId +
// ", licenseId=" + licenseId +
// ", isPrivate=" + isPrivate +
// ", blockchain=" + blockchain +
// ", blockchainTokenAll=" + blockchainTokenAll +
// '}';
// }
}

41
ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java Normal file
View File

@ -0,0 +1,41 @@
package com.ruoyi.platform.vo;
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.apache.ibatis.annotations.MapKey;
import java.io.Serializable;
import java.util.List;
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
@Data
public class NewDatasetVo implements Serializable {
@ApiModelProperty(name = "name")
private String name;
/**
* 数据集仓库名称
*/
@ApiModelProperty(name = "identifier")
private String identifier;
@ApiModelProperty(name = "description")
private String description;
/**
* 是否公开1公开，0私有
*/
@ApiModelProperty(name = "is_public")
private Boolean isPublic;
@ApiModelProperty(name = "data_type")
private String dataType;
@ApiModelProperty(name = "data_tag")
private String dataTag;
@ApiModelProperty(name = "time_ago")
private String timeAgo;
@ApiModelProperty(name = "repo_id")
private Integer repoId;
}