From df87d16d8f1684bd7effb689374f11df3ccf8168 Mon Sep 17 00:00:00 2001 From: fanshuai <1141904845@qq.com> Date: Fri, 30 Aug 2024 10:03:07 +0800 Subject: [PATCH] dvc --- ruoyi-modules/management-platform/pom.xml | 16 +- .../controller/dataset/DatasetController.java | 2 + .../dataset/NewDatasetFromGitController.java | 110 +++++++++ .../controller/jupyter/JupyterController.java | 7 +- .../platform/service/DatasetService.java | 9 + .../service/impl/DatasetServiceImpl.java | 217 +++++++++++++++++- .../com/ruoyi/platform/utils/ConvertUtil.java | 64 +++++- .../com/ruoyi/platform/utils/DVCUtils.java | 32 ++- .../com/ruoyi/platform/utils/HttpUtils.java | 57 ++++- .../com/ruoyi/platform/utils/JsonUtils.java | 5 + .../com/ruoyi/platform/utils/YamlUtils.java | 28 +++ .../java/com/ruoyi/platform/vo/DatasetVo.java | 10 +- .../com/ruoyi/platform/vo/GitProjectVo.java | 67 ++++++ .../com/ruoyi/platform/vo/NewDatasetVo.java | 41 ++++ 14 files changed, 638 insertions(+), 27 deletions(-) create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/GitProjectVo.java create mode 100644 ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java diff --git a/ruoyi-modules/management-platform/pom.xml b/ruoyi-modules/management-platform/pom.xml index c004f9a..e7c559f 100644 --- a/ruoyi-modules/management-platform/pom.xml +++ b/ruoyi-modules/management-platform/pom.xml @@ -142,6 +142,11 @@ docker-java 3.2.13 + + commons-beanutils + commons-beanutils + 1.9.4 + com.github.docker-java docker-java-transport-httpclient5 @@ -227,7 +232,16 @@ 2.6 compile - + + org.eclipse.jgit + org.eclipse.jgit + 5.13.0.202109080827-r + + + redis.clients + jedis + 3.6.0 + diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetController.java index cbd87f4..8e36471 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/DatasetController.java @@ -224,5 +224,7 @@ public class DatasetController { public AjaxResult uploadDatasetPipeline(@RequestBody(required =false) DatasetVersion datasetVersion) throws Exception { return AjaxResult.success(this.datasetService.uploadDatasetPipeline(datasetVersion)); } + + } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java new file mode 100644 index 0000000..0821d3a --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/dataset/NewDatasetFromGitController.java @@ -0,0 +1,110 @@ +package com.ruoyi.platform.controller.dataset; + +import com.ruoyi.common.core.web.domain.AjaxResult; +import com.ruoyi.platform.domain.Dataset; +import com.ruoyi.platform.service.DatasetService; +import com.ruoyi.platform.vo.DatasetVo; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.core.io.InputStreamResource; +import org.springframework.data.domain.PageRequest; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.Resource; +@RestController +@RequestMapping("newdataset") +@Api(value = "新数据集管理") +public class NewDatasetFromGitController { + /** + * 服务对象 + */ + @Resource + private DatasetService datasetService; + + + + /** + * 新增数据集与版本新 + * + * @param datasetVo 实体 + * @return 新增结果 + */ + @PostMapping("/addDatasetAndVersion") + @ApiOperation("添加数据集和版本") + public AjaxResult addDatasetAndVersion(@RequestBody DatasetVo datasetVo) throws Exception { + return AjaxResult.success(this.datasetService.newCreateDataset(datasetVo)); + + } + + + /** + * 新增数据集与版本新 + * + * @param datasetVo 实体 + * @return 新增结果 + */ + @PostMapping("/addVersion") + @ApiOperation("添加版本") + public AjaxResult addVersion(@RequestBody DatasetVo datasetVo) throws Exception { + return AjaxResult.success(this.datasetService.newCreateVersion(datasetVo)); + + } + + + /** + * 上传数据集 + * + * @param files 上传的数据集文件 + * @param uuid 上传唯一标识，构建url + * @return 上传结果 + */ + @CrossOrigin(origins = "*", allowedHeaders = "*") + @PostMapping("/upload") + @ApiOperation(value = "上传数据集") + public AjaxResult uploadDataset(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception { + return AjaxResult.success(this.datasetService.uploadDatasetlocal(files,uuid)); + } + + /** + * 数据集打包下载 + * + * @param version 数据集版本 + * @return 单条数据 + */ + @GetMapping("/downloadAllFiles") + @ApiOperation(value = "下载同一版本下所有数据集，并打包") + public ResponseEntity downloadAllDatasetFiles(@RequestParam("repository_name") String repositoryName, @RequestParam("version") String version) throws Exception { + return datasetService.downloadAllDatasetFilesNew(repositoryName, version); + } + + /** + * 下载数据集 + * + * @param dataset_version_id ps:这里的id是dataset_version表的主键 + * @return 单条数据 + */ + + @GetMapping("/download/{dataset_version_id}") + @ApiOperation(value = "下载单个数据集文件", notes = "根据数据集版本表id下载单个数据集文件") + public ResponseEntity downloadDataset(@PathVariable("dataset_version_id") Integer dataset_version_id) throws Exception { + return datasetService.downloadDataset(dataset_version_id); + } + + @GetMapping("/queryDatasets") + @ApiOperation("数据集广场公开数据集分页查询,根据data_type,data_tag筛选，true公开false私有") + public AjaxResult queryDatasets(Dataset dataset, @RequestParam("page") int page, + @RequestParam("size") int size, + @RequestParam(value = "is_public") Boolean isPublic, + @RequestParam(value = "data_type", required = false) String dataType, + @RequestParam(value = "data_tag", required = false) String dataTag) throws Exception { + PageRequest pageRequest = PageRequest.of(page, size); + if(isPublic){ + return AjaxResult.success(this.datasetService.newPubilcQueryByPage(dataset, pageRequest)); + }else { + return AjaxResult.success(this.datasetService.newPersonalQueryByPage(dataset, pageRequest)); + } + } + +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java index 080698c..02bff0d 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/controller/jupyter/JupyterController.java @@ -90,7 +90,7 @@ public class JupyterController extends BaseController { @GetMapping(value = "/testdvc") public AjaxResult testdvc() throws Exception { DatasetVo datasetVo = new DatasetVo(); - datasetVo.setName("testdvc82sas155"); + datasetVo.setName("testdassad23"); datasetVo.setDescription("sss"); datasetVo.setAvailableRange(0); datasetVo.setDataTag("计算机视觉"); @@ -98,10 +98,9 @@ public class JupyterController extends BaseController { datasetVo.setVersion("dev"); List datasetVersionVos = new ArrayList<>(); VersionVo versionVo = new VersionVo(); - versionVo.setUrl("/home/xxx.txt"); + versionVo.setUrl("E:/test/bb/data/xssa.doc"); datasetVersionVos.add(versionVo); datasetVo.setDatasetVersionVos(datasetVersionVos); - datasetService.newCreateDataset(datasetVo); - return AjaxResult.success(); + return AjaxResult.success(datasetService.newCreateDataset(datasetVo)); } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java index 148a1eb..a22e710 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/DatasetService.java @@ -3,12 +3,14 @@ package com.ruoyi.platform.service; import com.ruoyi.platform.domain.Dataset; import com.ruoyi.platform.domain.DatasetVersion; import com.ruoyi.platform.vo.DatasetVo; +import com.ruoyi.platform.vo.NewDatasetVo; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.http.ResponseEntity; import org.springframework.web.multipart.MultipartFile; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -91,6 +93,13 @@ DatasetService { CompletableFuture newCreateDataset(DatasetVo datasetVo) throws Exception; + CompletableFuture newCreateVersion(DatasetVo datasetVo); + + List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception; + + ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception; + Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; + Page newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java index 86a6002..9f59fbb 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/service/impl/DatasetServiceImpl.java @@ -1,6 +1,5 @@ package com.ruoyi.platform.service.impl; - import com.ruoyi.common.core.utils.DateUtils; import com.ruoyi.common.security.utils.SecurityUtils; import com.ruoyi.platform.annotations.CheckDuplicate; @@ -13,6 +12,7 @@ import com.ruoyi.platform.service.*; import com.ruoyi.platform.utils.*; import com.ruoyi.platform.vo.DatasetVo; import com.ruoyi.platform.vo.GitProjectVo; +import com.ruoyi.platform.vo.NewDatasetVo; import com.ruoyi.platform.vo.VersionVo; import com.ruoyi.system.api.model.LoginUser; import io.minio.messages.Item; @@ -33,10 +33,11 @@ import org.springframework.web.multipart.MultipartFile; import redis.clients.jedis.Jedis; import javax.annotation.Resource; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; +import java.io.*; import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -461,9 +462,7 @@ public class DatasetServiceImpl implements DatasetService { public CompletableFuture newCreateDataset(DatasetVo datasetVo) { return CompletableFuture.supplyAsync(() -> { try { - String gitusername = datasetVo.getGitusername(); - String gitpassword = datasetVo.getGitpassword(); - String token = gitService.login(gitusername, gitpassword); + String token = gitService.login("fanshuai", "h1n2x3j4y5@"); LoginUser loginUser = SecurityUtils.getLoginUser(); String ci4sUsername = loginUser.getUsername(); Jedis jedis = new Jedis(redisHost); @@ -492,11 +491,67 @@ public class DatasetServiceImpl implements DatasetService { // 得到用户操作的路径 String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); - String localPath = "/home/tempwork" + datasetVo.getName(); + String localPath = "E:/test/" + datasetVo.getName(); String sourcePath = url.substring(0, url.lastIndexOf("/")); // 命令行操作 git clone 项目地址 - DVCUtils.gitClone(localPath, projectUrl, branchName, gitusername, gitpassword); + DVCUtils.gitClone(localPath, projectUrl, branchName, "fanshuai", "h1n2x3j4y5@"); String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + repositoryName + "/" + branchName; + //拼接生产的元数据后写入yaml文件 + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),sourcePath, "dataset"); + + DVCUtils.moveFiles(sourcePath, localPath); + + + + // dvc init 初始化 + DVCUtils.dvcInit(localPath); + // 配置远程S3地址 + DVCUtils.dvcRemoteAdd(localPath, s3Path); + DVCUtils.dvcConfigS3Credentials(localPath, endpoint); + DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId); + DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey); + // dvc 跟踪 + DVCUtils.dvcAdd(localPath, "data"); + // git commit + DVCUtils.gitAdd(localPath, "."); + DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); + DVCUtils.gitPush(localPath, "fanshuai", "h1n2x3j4y5@"); + // dvc push 到远程S3 + DVCUtils.dvcPush(localPath); + return "新增数据集成功"; + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + @Override + public CompletableFuture newCreateVersion(DatasetVo datasetVo) { + return CompletableFuture.supplyAsync(() -> { + try { + String token = gitService.login("fanshuai", "h1n2x3j4y5@"); + LoginUser loginUser = SecurityUtils.getLoginUser(); + String ci4sUsername = loginUser.getUsername(); + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + // 创建分支 + String branchName = datasetVo.getVersion(); + String repositoryName = datasetVo.getRepositoryName(); + gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master"); + // 得到项目地址 + String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git"; + + // 得到用户操作的路径 + String url = datasetVo.getDatasetVersionVos().get(0).getUrl(); + String localPath = "E:/test/"+ loginUser.getUsername()+"/datasets/"+ datasetVo.getName(); + String sourcePath = url.substring(0, url.lastIndexOf("/")); + // 命令行操作 git clone 项目地址 + DVCUtils.gitClone(localPath, projectUrl, branchName, "fanshuai", "h1n2x3j4y5@"); + String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + repositoryName + "/" + branchName; + //拼接生产的元数据后写入yaml文件 + YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),sourcePath, "dataset"); + DVCUtils.moveFiles(sourcePath, localPath); // dvc init 初始化 DVCUtils.dvcInit(localPath); @@ -510,7 +565,7 @@ public class DatasetServiceImpl implements DatasetService { // git commit DVCUtils.gitAdd(localPath, "."); DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername()); - DVCUtils.gitPush(localPath, gitusername, gitpassword); + DVCUtils.gitPush(localPath, "fanshuai", "h1n2x3j4y5@"); // dvc push 到远程S3 DVCUtils.dvcPush(localPath); return "新增数据集成功"; @@ -520,6 +575,56 @@ public class DatasetServiceImpl implements DatasetService { }); } + + @Override + public Page newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception { + LoginUser loginUser = SecurityUtils.getLoginUser(); + String token = gitService.login("fanshuai", "h1n2x3j4y5@"); + String ci4sUsername = loginUser.getUsername(); + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + //拼接查询url + String datasetTagName = dataset.getDatasetTagName(); + String datasetTypeName = dataset.getDatasetTypeName(); + String topic_name = "ci4s_dataset"; + topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName; + topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName; + String url = gitendpoint + "/api/users/"+(String) userInfo.get("login")+"/projects.json?page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&category=manage&topic_name="+topic_name; + String req = HttpUtils.sendGetWithToken(url,null,token); + Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); + Integer total = (Integer) stringObjectMap.get("count"); + List> projects = (List>) stringObjectMap.get("projects"); + return new PageImpl<>(convert(projects), pageRequest, total); + } + + @Override + public Page newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception { + + LoginUser loginUser = SecurityUtils.getLoginUser(); + String token = gitService.login("fanshuai", "h1n2x3j4y5@"); + String ci4sUsername = loginUser.getUsername(); + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + Integer userId = (Integer) userInfo.get("user_id"); + //拼接查询url + String datasetTagName = dataset.getDatasetTagName(); + String datasetTypeName = dataset.getDatasetTypeName(); + String topic_name = "ci4s_dataset"; + topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName; + topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName; + + String url = gitendpoint + "/api/projects.json?user_id="+userId+"&page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&sort_by=praises_count&topic_name="+topic_name; + String req = HttpUtils.sendGetWithToken(url,null,token); + Map stringObjectMap = JacksonUtil.parseJSONStr2Map(req); + Integer total = (Integer) stringObjectMap.get("total_count"); + List> projects = (List>) stringObjectMap.get("projects"); + return new PageImpl<>(convert(projects), pageRequest, total); + + } + + @Override public List> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception { List> results = new ArrayList<>(); @@ -527,7 +632,7 @@ public class DatasetServiceImpl implements DatasetService { // 构建objectName String username = SecurityUtils.getLoginUser().getUsername(); String fileName = file.getOriginalFilename(); - String path = "/datasets/" + username + "/" + uuid + "/"+"/data/" + fileName; + String path = "/temp/"+ username +"/datasets/"+ uuid + "/"+"/data/" + fileName; long sizeInBytes = file.getSize(); String formattedSize = FileUtil.formatFileSize(sizeInBytes); File targetFile = new File(path, file.getOriginalFilename()); @@ -546,4 +651,94 @@ public class DatasetServiceImpl implements DatasetService { return results; } + @Override + public ResponseEntity downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception { + // 命令行操作 git clone 项目地址 + LoginUser loginUser = SecurityUtils.getLoginUser(); + String token = gitService.login("fanshuai", "h1n2x3j4y5@"); + String ci4sUsername = loginUser.getUsername(); + Jedis jedis = new Jedis(redisHost); + String userReq = jedis.get(ci4sUsername + "_gitUserInfo"); + Map userInfo = JsonUtils.jsonToMap(userReq); + Integer userId = (Integer) userInfo.get("user_id"); + String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git"; + String localPath = "E:/test/"+ loginUser.getUsername()+"/datasets/" +repositoryName; + File folder = new File(localPath); + if(folder.exists() && folder.isDirectory()){ + //切换分支 + DVCUtils.gitCheckoutBranch(localPath, version); + //pull + DVCUtils.gitPull(localPath,"fanshuai", "h1n2x3j4y5@"); + //dvc pull + DVCUtils.dvcPull(localPath); + }else { + DVCUtils.gitClone(localPath, projectUrl, version, "fanshuai", "h1n2x3j4y5@"); + } + + // 打包 data 文件夹 + String dataFolderPath = localPath + "/data"; + String zipFilePath = localPath + "/data.zip"; + try (FileOutputStream fos = new FileOutputStream(zipFilePath); + ZipOutputStream zos = new ZipOutputStream(fos)) { + Path sourcePath = Paths.get(dataFolderPath); + Files.walk(sourcePath).forEach(path -> { + if (!Files.isDirectory(path)) { + ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString()); + try { + zos.putNextEntry(zipEntry); + Files.copy(path, zos); + zos.closeEntry(); + } catch (IOException e) { + throw new RuntimeException("Error while zipping: " + path, e); + } + } + }); + } + + // 返回压缩文件的输入流 + File zipFile = new File(zipFilePath); + InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile)); + + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip") + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .contentLength(zipFile.length()) + .body(resource); + + } + + + + public List convert(List> lst) { + if (lst != null && lst.size() > 0) { + List newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class); + + for (NewDatasetVo newDatasetVo : newDatasetVos) { + Map map = lst.stream() + .filter(m -> m.get("repo_id").equals(newDatasetVo.getRepoId())) + .findFirst() + .orElse(null); + + if (map != null) { + List> topics = (List>) map.get("topics"); + if (topics != null) { + topics.forEach(topic -> { + String name = (String) topic.get("name"); + if (name != null) { + if (name.startsWith("datatag_")) { + newDatasetVo.setDataTag(name.substring("datatag_".length())); + } else if (name.startsWith("datatype_")) { + newDatasetVo.setDataType(name.substring("datatype_".length())); + } + } + }); + } + } + } + + return newDatasetVos; + } + return new ArrayList<>(); + } + } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java index bfc051a..ca13861 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/ConvertUtil.java @@ -4,10 +4,13 @@ package com.ruoyi.platform.utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; - +import org.apache.commons.beanutils.PropertyUtils; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; /** @@ -60,5 +63,64 @@ public class ConvertUtil { return targetList; } + public static List convertListMapToObjectList(List> listMap, Class targetClass) { + List resultList = new ArrayList<>(); + + if (listMap != null) { + for (Map map : listMap) { + T targetObject = convertMapToObject(map, targetClass); + if (targetObject != null) { + resultList.add(targetObject); + } + } + } + + return resultList; + } + + private static T convertMapToObject(Map map, Class targetClass) { + try { + T targetObject = targetClass.newInstance(); + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + String camelCaseKey = toCamelCase(key); + if (hasProperty(targetClass, camelCaseKey)) { + PropertyUtils.setProperty(targetObject, camelCaseKey, value); + } + } + return targetObject; + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + e.printStackTrace(); + } + return null; + } + + private static boolean hasProperty(Class clazz, String propertyName) { + try { + Field field = clazz.getDeclaredField(propertyName); + return field != null; + } catch (NoSuchFieldException e) { + return false; + } + } + + private static String toCamelCase(String key) { + StringBuilder sb = new StringBuilder(); + boolean nextUpperCase = false; + for (char c : key.toCharArray()) { + if (c == '_') { + nextUpperCase = true; + } else { + if (nextUpperCase) { + sb.append(Character.toUpperCase(c)); + nextUpperCase = false; + } else { + sb.append(c); + } + } + } + return sb.toString(); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java index ce726da..387f09a 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/DVCUtils.java @@ -95,6 +95,33 @@ public class DVCUtils { } } + public static void gitCheckoutBranch(String localPath, String branchName) throws IOException, GitAPIException { + FileRepositoryBuilder builder = new FileRepositoryBuilder(); + Repository repository = builder.setGitDir(new File(localPath, ".git")) + .readEnvironment() + .findGitDir() + .build(); + + try (Git git = new Git(repository)) { + CheckoutCommand checkoutCommand = git.checkout(); + checkoutCommand.setName(branchName).call(); + } + } + + public static void gitPull(String localPath, String username, String password) throws IOException, GitAPIException { + FileRepositoryBuilder builder = new FileRepositoryBuilder(); + Repository repository = builder.setGitDir(new File(localPath, ".git")) + .readEnvironment() + .findGitDir() + .build(); + + try (Git git = new Git(repository)) { + CredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(username, password); + PullCommand pullCommand = git.pull(); + pullCommand.setCredentialsProvider(credentialsProvider).call(); + } + } + public static void dvcInit(String localPath) throws Exception { String command = "dvc init"; runCommand(command, localPath); @@ -111,13 +138,15 @@ public class DVCUtils { } public static void dvcConfigS3Credentials(String localPath, String endpointurl) throws Exception { - String command = "dvc remote modify myremote endpointurl " + endpointurl ; + String command = "dvc remote modify myremote endpointurl " + endpointurl; runCommand(command, localPath); } + public static void dvcConfigS3Credentials2(String localPath, String accessKeyId) throws Exception { String command = "dvc remote modify myremote access_key_id " + accessKeyId; runCommand(command, localPath); } + public static void dvcConfigS3Credentials3(String localPath, String secretAccessKey) throws Exception { String command = "dvc remote modify myremote secret_access_key " + secretAccessKey; runCommand(command, localPath); @@ -128,6 +157,7 @@ public class DVCUtils { runCommand(command, localPath); } + // 更新的 dvcPull 方法 public static void dvcPull(String localPath) throws Exception { String command = "dvc pull"; runCommand(command, localPath); diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java index ccfea3c..504c4c6 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/HttpUtils.java @@ -11,6 +11,7 @@ import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultHttpClient; @@ -89,15 +90,61 @@ public class HttpUtils { public static String sendGet(String url, String param) { return sendGet(url, param, "UTF-8"); } +// /** +// * 向指定 URL 发送带token的GET方法的请求 +// * +// * @param url 发送请求的 URL +// * @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。 +// * @return 所代表远程资源的响应结果 +// */ +// public static String sendGetWithToken(String url, String param,String token) { +// return sendGet(url, param, "UTF-8",token); +// } /** - * 向指定 URL 发送带token的GET方法的请求 + * 向指定 URL 发送带 token 的 GET 方法的请求，使用 Apache HttpClient * - * @param url 发送请求的 URL - * @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。 + * @param url 发送请求的 URL + * @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。 + * @param token 认证 token * @return 所代表远程资源的响应结果 */ - public static String sendGetWithToken(String url, String param,String token) { - return sendGet(url, param, "UTF-8",token); + public static String sendGetWithToken(String url, String param, String token) { + String result = ""; + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + URIBuilder uriBuilder = new URIBuilder(url); + if (param != null && !param.isEmpty()) { + String[] pairs = param.split("&"); + for (String pair : pairs) { + int idx = pair.indexOf("="); + if (idx != -1) { + uriBuilder.setParameter(pair.substring(0, idx), pair.substring(idx + 1)); + } + } + } + + URI uri = uriBuilder.build(); + HttpGet httpGet = new HttpGet(uri); + + httpGet.setHeader("Content-Type", "application/json"); + httpGet.setHeader("Authorization", "Bearer " + token); + + log.info("Executing request: " + httpGet.getRequestLine()); + + try (CloseableHttpResponse response = httpClient.execute(httpGet)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + result = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + } else { + throw new IOException("HTTP request failed with response code: " + statusCode); + } + log.info("Response: " + result); + } + } catch (URISyntaxException e) { + log.error("URISyntaxException, url=" + url + ", param=" + param, e); + } catch (IOException e) { + log.error("IOException, url=" + url + ", param=" + param, e); + } + return result; } /** * 向指定 URL 发送带token的GET方法的请求 diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/JsonUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/JsonUtils.java index 78dab0c..2aa0ba0 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/JsonUtils.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/JsonUtils.java @@ -51,4 +51,9 @@ public class JsonUtils { return flatMap; } + + // 将Object转换为Map + public static Map objectToMap(Object object) throws IOException { + return objectMapper.convertValue(object, Map.class); + } } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java new file mode 100644 index 0000000..395e78b --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/utils/YamlUtils.java @@ -0,0 +1,28 @@ +package com.ruoyi.platform.utils; + +import org.yaml.snakeyaml.Yaml; + +import java.io.FileWriter; +import java.io.IOException; +import java.util.Map; + +public class YamlUtils { + + /** + * 将Map对象转换为YAML格式并写入指定路径的文件中 + * + * @param data Map对象 + * @param path 文件路径 + * @param fileName 文件名 + */ + public static void generateYamlFile(Map data, String path, String fileName) { + Yaml yaml = new Yaml(); + String fullPath = path + "/" + fileName + ".yaml"; + + try (FileWriter writer = new FileWriter(fullPath)) { + yaml.dump(data, writer); + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/DatasetVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/DatasetVo.java index 2253b10..2a1dcc0 100644 --- a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/DatasetVo.java +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/DatasetVo.java @@ -28,10 +28,6 @@ public class DatasetVo implements Serializable { @ApiModelProperty(name = "data_tag") private String dataTag; - @ApiModelProperty(name = "gitusername") - private String gitusername; - @ApiModelProperty(name = "gitpassword") - private String gitpassword; /** * 版本 */ @@ -47,4 +43,10 @@ public class DatasetVo implements Serializable { @ApiModelProperty(name = "available_cluster") private String availableCluster; + + /** + * 数据集仓库名称 + */ + @ApiModelProperty(name = "repository_name") + private String repositoryName; } diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/GitProjectVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/GitProjectVo.java new file mode 100644 index 0000000..a31a7cd --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/GitProjectVo.java @@ -0,0 +1,67 @@ +package com.ruoyi.platform.vo; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +@ApiModel(description = "Git项目信息") +@Data +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +public class GitProjectVo implements Serializable { + + @ApiModelProperty(value = "用户或组织ID", required = true) + private int userId; + + @ApiModelProperty(value = "项目名称", required = true) + private String name; + + @ApiModelProperty(value = "项目描述", required = false) + private String description; + + @ApiModelProperty(value = "项目标识", required = true) + private String repositoryName; + +// @ApiModelProperty(value = "项目分类ID", required = false) +// private int projectCategoryId; +// +// @ApiModelProperty(value = "项目语言ID", required = false) +// private int projectLanguageId; + +// @ApiModelProperty(value = "项目忽略文件ID", required = false) +// private int ignoreId; + +// @ApiModelProperty(value = "项目许可证ID", required = false) +// private int licenseId; + + @ApiModelProperty(value = "项目是否为私有项目", required = false) + private boolean isPrivate; + +// @ApiModelProperty(value = "是否为区块链激励项目", required = false) +// private boolean blockchain; +// +// @ApiModelProperty(value = "区块链激励金额总数", required = false) +// private int blockchainTokenAll; + +// +// @Override +// public String toString() { +// return "GitProjectVo{" + +// "userId=" + userId + +// ", name='" + name + '\'' + +// ", description='" + description + '\'' + +// ", repositoryName='" + repositoryName + '\'' + +// ", projectCategoryId=" + projectCategoryId + +// ", projectLanguageId=" + projectLanguageId + +// ", ignoreId=" + ignoreId + +// ", licenseId=" + licenseId + +// ", isPrivate=" + isPrivate + +// ", blockchain=" + blockchain + +// ", blockchainTokenAll=" + blockchainTokenAll + +// '}'; +// } +} + diff --git a/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java new file mode 100644 index 0000000..7ea7276 --- /dev/null +++ b/ruoyi-modules/management-platform/src/main/java/com/ruoyi/platform/vo/NewDatasetVo.java @@ -0,0 +1,41 @@ +package com.ruoyi.platform.vo; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.apache.ibatis.annotations.MapKey; + +import java.io.Serializable; +import java.util.List; + +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +@Data +public class NewDatasetVo implements Serializable { + + + @ApiModelProperty(name = "name") + private String name; + /** + * 数据集仓库名称 + */ + @ApiModelProperty(name = "identifier") + private String identifier; + @ApiModelProperty(name = "description") + private String description; + /** + * 是否公开1公开，0私有 + */ + @ApiModelProperty(name = "is_public") + private Boolean isPublic; + @ApiModelProperty(name = "data_type") + private String dataType; + @ApiModelProperty(name = "data_tag") + private String dataTag; + @ApiModelProperty(name = "time_ago") + private String timeAgo; + @ApiModelProperty(name = "repo_id") + private Integer repoId; + + +}