update dataset version
This commit is contained in:
parent
6fdb5bcf78
commit
156888782e
|
@ -2,10 +2,8 @@ package com.ruoyi.platform.controller.dataset;
|
|||
|
||||
import com.ruoyi.common.core.web.domain.AjaxResult;
|
||||
import com.ruoyi.platform.domain.Dataset;
|
||||
import com.ruoyi.platform.service.DatasetService;
|
||||
import com.ruoyi.platform.vo.DatasetVo;
|
||||
import com.ruoyi.platform.service.NewDatasetService;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiOperation;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
|
@ -22,7 +20,7 @@ public class NewDatasetFromGitController {
|
|||
* 服务对象
|
||||
*/
|
||||
@Resource
|
||||
private DatasetService datasetService;
|
||||
private NewDatasetService newDatasetService;
|
||||
|
||||
|
||||
|
||||
|
@ -35,7 +33,7 @@ public class NewDatasetFromGitController {
|
|||
@PostMapping("/addDatasetAndVersion")
|
||||
@ApiOperation("添加数据集和版本")
|
||||
public AjaxResult addDatasetAndVersion(@RequestBody NewDatasetVo datasetVo) throws Exception {
|
||||
return AjaxResult.success(this.datasetService.newCreateDataset(datasetVo));
|
||||
return AjaxResult.success(this.newDatasetService.newCreateDataset(datasetVo));
|
||||
|
||||
}
|
||||
|
||||
|
@ -49,7 +47,7 @@ public class NewDatasetFromGitController {
|
|||
@PostMapping("/addVersion")
|
||||
@ApiOperation("添加版本")
|
||||
public AjaxResult addVersion(@RequestBody NewDatasetVo datasetVo) throws Exception {
|
||||
return AjaxResult.success(this.datasetService.newCreateVersion(datasetVo));
|
||||
return AjaxResult.success(this.newDatasetService.newCreateVersion(datasetVo));
|
||||
|
||||
}
|
||||
|
||||
|
@ -62,39 +60,39 @@ public class NewDatasetFromGitController {
|
|||
@RequestParam(value = "data_tag", required = false) String dataTag) throws Exception {
|
||||
PageRequest pageRequest = PageRequest.of(page, size);
|
||||
if(isPublic){
|
||||
return AjaxResult.success(this.datasetService.newPubilcQueryByPage(dataset, pageRequest));
|
||||
return AjaxResult.success(this.newDatasetService.newPubilcQueryByPage(dataset, pageRequest));
|
||||
}else {
|
||||
return AjaxResult.success(this.datasetService.newPersonalQueryByPage(dataset, pageRequest));
|
||||
return AjaxResult.success(this.newDatasetService.newPersonalQueryByPage(dataset, pageRequest));
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping("/getVersionList")
|
||||
@ApiOperation(value = "获取分支列表")
|
||||
public AjaxResult getVersionList(@RequestParam("identifier") String repo,@RequestParam("owner")String owner) throws Exception {
|
||||
return AjaxResult.success(this.datasetService.getVersionList(repo,owner));
|
||||
return AjaxResult.success(this.newDatasetService.getVersionList(repo,owner));
|
||||
}
|
||||
|
||||
@GetMapping("/getDatasetDetail")
|
||||
@ApiOperation(value = "获取数据集详情")
|
||||
public AjaxResult getDatasetVersions(@RequestParam("name") String name,
|
||||
@RequestParam("repo_id") Integer repoId,
|
||||
@RequestParam("id") Integer id,
|
||||
@RequestParam(value ="owner",required = false)String owner,
|
||||
@RequestParam(value = "identifier",required = false) String repo,
|
||||
@RequestParam(value = "identifier",required = false) String identifier,
|
||||
@RequestParam(value = "version",required = false)String version) throws Exception {
|
||||
return AjaxResult.success(this.datasetService.getNewDatasetDesc(repoId,name,repo,owner,version));
|
||||
return AjaxResult.success(this.newDatasetService.getNewDatasetDesc(id,name,identifier,owner,version));
|
||||
}
|
||||
|
||||
@DeleteMapping("/deleteDataset")
|
||||
@ApiOperation(value = "删除数据集")
|
||||
public AjaxResult deleteDataset(@RequestParam("identifier") String repo,@RequestParam("owner")String owner) throws Exception {
|
||||
this.datasetService.deleteDatasetNew(repo,owner);
|
||||
this.newDatasetService.deleteDatasetNew(repo,owner);
|
||||
return AjaxResult.success();
|
||||
}
|
||||
|
||||
@DeleteMapping("/deleteDatasetVersion")
|
||||
@ApiOperation(value = "删除数据集版本")
|
||||
public AjaxResult deleteDatasetVersion(@RequestParam("identifier") String repo,@RequestParam("owner")String owner,@RequestParam("version")String version) throws Exception {
|
||||
this.datasetService.deleteDatasetVersionNew(repo,owner,version);
|
||||
this.newDatasetService.deleteDatasetVersionNew(repo,owner,version);
|
||||
return AjaxResult.success();
|
||||
}
|
||||
|
||||
|
@ -110,7 +108,7 @@ public class NewDatasetFromGitController {
|
|||
@PostMapping("/upload")
|
||||
@ApiOperation(value = "上传数据集")
|
||||
public AjaxResult uploadDataset(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception {
|
||||
return AjaxResult.success(this.datasetService.uploadDatasetlocal(files,uuid));
|
||||
return AjaxResult.success(this.newDatasetService.uploadDatasetlocal(files,uuid));
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -122,7 +120,7 @@ public class NewDatasetFromGitController {
|
|||
@GetMapping("/downloadAllFiles")
|
||||
@ApiOperation(value = "下载同一版本下所有数据集,并打包")
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("repository_name") String repositoryName, @RequestParam("version") String version) throws Exception {
|
||||
return datasetService.downloadAllDatasetFilesNew(repositoryName, version);
|
||||
return newDatasetService.downloadAllDatasetFilesNew(repositoryName, version);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -135,6 +133,6 @@ public class NewDatasetFromGitController {
|
|||
@GetMapping("/downloadSinggerFile")
|
||||
@ApiOperation(value = "下载单个数据集文件", notes = "根据数据集版本表id下载单个数据集文件")
|
||||
public ResponseEntity<InputStreamResource> downloadDataset(@RequestParam("url") String url) throws Exception {
|
||||
return datasetService.downloadDatasetlocal(url);
|
||||
return newDatasetService.downloadDatasetlocal(url);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,9 +4,11 @@ import com.ruoyi.common.core.web.controller.BaseController;
|
|||
import com.ruoyi.common.core.web.domain.AjaxResult;
|
||||
import com.ruoyi.common.core.web.domain.GenericsAjaxResult;
|
||||
import com.ruoyi.platform.domain.DevEnvironment;
|
||||
import com.ruoyi.platform.service.DatasetService;
|
||||
import com.ruoyi.platform.service.JupyterService;
|
||||
import com.ruoyi.platform.vo.*;
|
||||
import com.ruoyi.platform.service.NewDatasetService;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import com.ruoyi.platform.vo.PodStatusVo;
|
||||
import com.ruoyi.platform.vo.VersionVo;
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiOperation;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponse;
|
||||
|
@ -27,7 +29,7 @@ public class JupyterController extends BaseController {
|
|||
@Resource
|
||||
private JupyterService jupyterService;
|
||||
@Resource
|
||||
private DatasetService datasetService;
|
||||
private NewDatasetService newDatasetService;
|
||||
@GetMapping(value = "/getURL")
|
||||
@ApiOperation("得到访问地址")
|
||||
public GenericsAjaxResult<String> getURL() throws IOException {
|
||||
|
@ -101,6 +103,6 @@ public class JupyterController extends BaseController {
|
|||
datasetVo.setVersionDesc("this is a test");
|
||||
datasetVersionVos.add(versionVo);
|
||||
datasetVo.setDatasetVersionVos(datasetVersionVos);
|
||||
return AjaxResult.success(datasetService.newCreateDataset(datasetVo));
|
||||
return AjaxResult.success(newDatasetService.newCreateDataset(datasetVo));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -21,11 +21,7 @@ import java.util.concurrent.CompletableFuture;
|
|||
* @author Xidaray
|
||||
* @since 2023-11-28 11:51:22
|
||||
*/
|
||||
public interface
|
||||
DatasetService {
|
||||
|
||||
|
||||
|
||||
public interface DatasetService {
|
||||
|
||||
|
||||
/**
|
||||
|
@ -39,12 +35,12 @@ DatasetService {
|
|||
/**
|
||||
* 分页查询
|
||||
*
|
||||
* @param dataset 筛选条件
|
||||
* @param pageRequest 分页对象
|
||||
* @param dataset 筛选条件
|
||||
* @param pageRequest 分页对象
|
||||
* @return 查询结果
|
||||
*/
|
||||
Page<Dataset> queryByPage(Dataset dataset, PageRequest pageRequest);
|
||||
|
||||
|
||||
/**
|
||||
* 新增数据
|
||||
*
|
||||
|
@ -54,7 +50,6 @@ DatasetService {
|
|||
Dataset insert(Dataset dataset) throws Exception;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 修改数据
|
||||
*
|
||||
|
@ -82,30 +77,11 @@ DatasetService {
|
|||
List<String> getDatasetVersions(Integer datasetId) throws Exception;
|
||||
|
||||
String insertDatasetAndVersion(DatasetVo datasetVo) throws Exception;
|
||||
|
||||
void checkDeclaredName(Dataset insert) throws Exception;
|
||||
|
||||
ResponseEntity<InputStreamResource> downloadAllDatasetFiles(Integer datasetId, String version) throws Exception;
|
||||
|
||||
List<Map<String, String>> exportDataset(String path, String uuid) throws Exception;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
CompletableFuture<String> newCreateDataset(NewDatasetVo datasetVo) throws Exception;
|
||||
CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo);
|
||||
|
||||
List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
|
||||
ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception;
|
||||
ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception;
|
||||
Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
|
||||
Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
|
||||
|
||||
NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName,String repo, String owner, String version)throws Exception;
|
||||
|
||||
List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception;
|
||||
|
||||
void deleteDatasetNew(String repo, String owner) throws Exception;
|
||||
|
||||
void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,35 @@
|
|||
package com.ruoyi.platform.service;
|
||||
|
||||
import com.ruoyi.platform.domain.Dataset;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
public interface NewDatasetService {
|
||||
|
||||
|
||||
String newCreateDataset(NewDatasetVo datasetVo) throws Exception;
|
||||
CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo)throws Exception;
|
||||
|
||||
List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
|
||||
ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception;
|
||||
ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception;
|
||||
Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
|
||||
Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
|
||||
|
||||
NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName,String repo, String owner, String version)throws Exception;
|
||||
|
||||
List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception;
|
||||
|
||||
void deleteDatasetNew(String repo, String owner) throws Exception;
|
||||
|
||||
void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception;
|
||||
}
|
|
@ -1,6 +1,5 @@
|
|||
package com.ruoyi.platform.service.impl;
|
||||
|
||||
import com.ruoyi.common.core.utils.DateUtils;
|
||||
import com.ruoyi.common.security.utils.SecurityUtils;
|
||||
import com.ruoyi.platform.annotations.CheckDuplicate;
|
||||
import com.ruoyi.platform.domain.AssetIcon;
|
||||
|
@ -9,14 +8,13 @@ import com.ruoyi.platform.domain.DatasetVersion;
|
|||
import com.ruoyi.platform.mapper.DatasetDao;
|
||||
import com.ruoyi.platform.mapper.DatasetVersionDao;
|
||||
import com.ruoyi.platform.service.*;
|
||||
import com.ruoyi.platform.utils.*;
|
||||
import com.ruoyi.platform.utils.BeansUtils;
|
||||
import com.ruoyi.platform.utils.FileUtil;
|
||||
import com.ruoyi.platform.utils.MinioUtil;
|
||||
import com.ruoyi.platform.vo.DatasetVo;
|
||||
import com.ruoyi.platform.vo.GitProjectVo;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import com.ruoyi.platform.vo.VersionVo;
|
||||
import com.ruoyi.system.api.model.LoginUser;
|
||||
import io.minio.messages.Item;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
|
@ -30,17 +28,13 @@ import org.springframework.http.ResponseEntity;
|
|||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import redis.clients.jedis.Jedis;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.*;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.lang.reflect.Field;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
@ -68,12 +62,6 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
@Resource
|
||||
private MinioService minioService;
|
||||
|
||||
@Resource
|
||||
private GitService gitService;
|
||||
|
||||
@Resource
|
||||
private DvcService dvcService;
|
||||
|
||||
// 固定存储桶名
|
||||
@Value("${minio.dataReleaseBucketName}")
|
||||
private String bucketName;
|
||||
|
@ -81,19 +69,6 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
@Resource
|
||||
private MinioUtil minioUtil;
|
||||
|
||||
@Value("${spring.redis.host}")
|
||||
private String redisHost;
|
||||
@Value("${minio.accessKey}")
|
||||
String accessKeyId;
|
||||
@Value("${minio.secretKey}")
|
||||
String secretAccessKey;
|
||||
@Value("${minio.endpoint}")
|
||||
String endpoint;
|
||||
@Value("${git.endpoint}")
|
||||
String gitendpoint;
|
||||
@Value("${git.localPath}")
|
||||
String localPathlocal;
|
||||
|
||||
/**
|
||||
* 通过ID查询单条数据
|
||||
*
|
||||
|
@ -457,447 +432,4 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> newCreateDataset(NewDatasetVo datasetVo) {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
// 拼接project
|
||||
String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
|
||||
GitProjectVo gitProjectVo = new GitProjectVo();
|
||||
gitProjectVo.setRepositoryName(repositoryName);
|
||||
gitProjectVo.setName(datasetVo.getName());
|
||||
gitProjectVo.setDescription(datasetVo.getDescription());
|
||||
gitProjectVo.setPrivate(!datasetVo.getIsPublic());
|
||||
gitProjectVo.setUserId(userId);
|
||||
// 创建项目
|
||||
Map project = gitService.createProject(token, gitProjectVo);
|
||||
Integer repoId = (Integer) project.get("id");
|
||||
// 创建分支
|
||||
String branchName = datasetVo.getVersion();
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
// 定义标签 标签1:ci4s_dataset 标签2:DataTag 标签3:DataType
|
||||
gitService.createTopic(token, repoId, "ci4s_dataset");
|
||||
gitService.createTopic(token, repoId, "DataTag_" + datasetVo.getDataTag());
|
||||
gitService.createTopic(token, repoId, "DataType_" + datasetVo.getDataType());
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal + "/" + repoId+"/"+datasetVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+repoId+"/"+ repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath);
|
||||
//拼接生产的元数据后写入yaml文件
|
||||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getNowDate());
|
||||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("```bash\n" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " " + projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"```");
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),localPath, "dataset");
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath, "data");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath, ".");
|
||||
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath);
|
||||
return "新增数据集成功";
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo) {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
// 创建分支
|
||||
String branchName = StringUtils.isEmpty(datasetVo.getVersion())? "master" : datasetVo.getVersion();
|
||||
String repositoryName = datasetVo.getIdentifier();
|
||||
if (StringUtils.equals(branchName, "master")) {
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
}
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/"+ datasetVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
if(FileUtil.checkDirectoryExists(localPath)){
|
||||
DVCUtils.gitFetch(localPath,gitLinkUsername, gitLinkPassword);
|
||||
DVCUtils.gitCheckoutBranch(localPath,branchName);
|
||||
}else {
|
||||
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
|
||||
}
|
||||
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+ datasetVo.getRepoId()+"/"+ repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath);
|
||||
//拼接生产的元数据后写入yaml文件
|
||||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getNowDate());
|
||||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("```bash\n" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " " + projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"```");
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo),localPath, "dataset");
|
||||
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath, "data");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath, ".");
|
||||
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath);
|
||||
return "新增数据集成功";
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
//拼接查询url
|
||||
String datasetTagName = dataset.getDatasetTagName();
|
||||
String datasetTypeName = dataset.getDatasetTypeName();
|
||||
String topic_name = "ci4s_dataset";
|
||||
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName;
|
||||
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName;
|
||||
String url = gitendpoint + "/api/users/"+(String) userInfo.get("login")+"/projects.json?page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&category=manage&topic_name="+topic_name;
|
||||
String req = HttpUtils.sendGetWithToken(url,null,token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("count");
|
||||
List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
|
||||
return new PageImpl<>(convert(projects), pageRequest, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
//拼接查询url
|
||||
String datasetTagName = dataset.getDatasetTagName();
|
||||
String datasetTypeName = dataset.getDatasetTypeName();
|
||||
String topic_name = "ci4s_dataset";
|
||||
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName;
|
||||
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName;
|
||||
|
||||
String url = gitendpoint + "/api/projects.json?user_id="+userId+"&page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&sort_by=praises_count&topic_name="+topic_name;
|
||||
String req = HttpUtils.sendGetWithToken(url,null,token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("total_count");
|
||||
List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
|
||||
if (projects == null){
|
||||
return new PageImpl<>(new ArrayList<>(), pageRequest, 0);
|
||||
}
|
||||
return new PageImpl<>(convert(projects), pageRequest, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public NewDatasetVo getNewDatasetDesc(Integer repoId,String repositoryName,String repo, String owner,String version) throws Exception{
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
// cd到 localPathlocal/repoId/下面还有一个文件夹,然后做git pull操作,然后读取里面的文件列表,列出每个文件的大小和名称,封装成MAP
|
||||
if (StringUtils.isEmpty(version)){
|
||||
List<Map<String, Object>> versionList = this.getVersionList(repo, owner);
|
||||
if (versionList.size() == 0){
|
||||
throw new Exception("数据集文件不存在");
|
||||
}
|
||||
version = (String) versionList.get(0).get("name");
|
||||
}
|
||||
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal+repoId, repositoryName, version,gitLinkUsername, gitLinkPassword);
|
||||
//在localPathlocal+repoId+"/"+repositoryName目录下的dataset.yaml中取到元数据
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + repoId + "/" + repositoryName + "/dataset.yaml");
|
||||
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
|
||||
List<VersionVo> versionVos = new ArrayList<VersionVo>();
|
||||
if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
|
||||
for(Map<String, Object> fileDetail : fileDetailsAfterGitPull){
|
||||
VersionVo versionVo = new VersionVo();
|
||||
versionVo.setUrl((String) fileDetail.get("filePath"));
|
||||
versionVo.setFileName((String) fileDetail.get("fileName"));
|
||||
long size = (long) fileDetail.get("size");
|
||||
versionVo.setFileSize(FileUtil.formatFileSize(size));
|
||||
versionVos.add(versionVo);
|
||||
}
|
||||
}
|
||||
newDatasetVo.setDatasetVersionVos(versionVos);
|
||||
return newDatasetVo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repo);
|
||||
return brancheList.stream()
|
||||
.filter(branch -> "master".equals(branch.get("name")))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteDatasetNew(String repo, String owner) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
gitService.deleteProject(token, owner, repo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
gitService.deleteBranch(token, owner, repo, version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
|
||||
List<Map<String, String>> results = new ArrayList<>();
|
||||
|
||||
for (MultipartFile file:files){
|
||||
// 构建objectName
|
||||
String username = SecurityUtils.getLoginUser().getUsername();
|
||||
String fileName = file.getOriginalFilename();
|
||||
String path = "/temp/"+ username +"/datasets/"+ uuid + "/"+"/data/" + fileName;
|
||||
long sizeInBytes = file.getSize();
|
||||
String formattedSize = FileUtil.formatFileSize(sizeInBytes);
|
||||
File targetFile = new File(path, file.getOriginalFilename());
|
||||
// 确保目录存在
|
||||
targetFile.getParentFile().mkdirs();
|
||||
// 保存文件到目标路径
|
||||
FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
|
||||
// 返回上传文件的路径
|
||||
String absolutePath = targetFile.getAbsolutePath();
|
||||
Map<String, String> result = new HashMap<>();
|
||||
result.put("fileName", fileName);
|
||||
result.put("url", absolutePath); // objectName根据实际情况定义
|
||||
result.put("fileSize", formattedSize);
|
||||
results.add(result);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception {
|
||||
File file = new File(filePath);
|
||||
|
||||
if (!file.exists()) {
|
||||
throw new FileNotFoundException("File not found: " + filePath);
|
||||
}
|
||||
|
||||
InputStreamResource resource = new InputStreamResource(new FileInputStream(file));
|
||||
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
|
||||
headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length()));
|
||||
headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.headers(headers)
|
||||
.contentLength(file.length())
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.body(resource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception {
|
||||
// 命令行操作 git clone 项目地址
|
||||
checkoutToken();
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName;
|
||||
File folder = new File(localPath);
|
||||
if(folder.exists() && folder.isDirectory()){
|
||||
//切换分支
|
||||
DVCUtils.gitCheckoutBranch(localPath, version);
|
||||
//pull
|
||||
DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword);
|
||||
//dvc pull
|
||||
DVCUtils.dvcPull(localPath);
|
||||
}else {
|
||||
DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword);
|
||||
}
|
||||
|
||||
// 打包 data 文件夹
|
||||
String dataFolderPath = localPath + "/data";
|
||||
String zipFilePath = localPath + "/data.zip";
|
||||
try (FileOutputStream fos = new FileOutputStream(zipFilePath);
|
||||
ZipOutputStream zos = new ZipOutputStream(fos)) {
|
||||
Path sourcePath = Paths.get(dataFolderPath);
|
||||
Files.walk(sourcePath).forEach(path -> {
|
||||
if (!Files.isDirectory(path)) {
|
||||
ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString());
|
||||
try {
|
||||
zos.putNextEntry(zipEntry);
|
||||
Files.copy(path, zos);
|
||||
zos.closeEntry();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Error while zipping: " + path, e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 返回压缩文件的输入流
|
||||
File zipFile = new File(zipFilePath);
|
||||
InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip")
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.contentLength(zipFile.length())
|
||||
.body(resource);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
public List<NewDatasetVo> convert(List<Map<String, Object>> lst) {
|
||||
if (lst != null && lst.size() > 0) {
|
||||
List<NewDatasetVo> newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class);
|
||||
|
||||
for (NewDatasetVo newDatasetVo : newDatasetVos) {
|
||||
Map<String, Object> map = lst.stream()
|
||||
.filter(m -> m.get("repo_id").equals(newDatasetVo.getRepoId()))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
|
||||
if (map != null) {
|
||||
List<Map<String, Object>> topics = (List<Map<String, Object>>) map.get("topics");
|
||||
if (topics != null) {
|
||||
topics.forEach(topic -> {
|
||||
String name = (String) topic.get("name");
|
||||
if (name != null) {
|
||||
if (name.startsWith("datatag_")) {
|
||||
newDatasetVo.setDataTag(name.substring("datatag_".length()));
|
||||
} else if (name.startsWith("datatype_")) {
|
||||
newDatasetVo.setDataType(name.substring("datatype_".length()));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Map<String, Object> author = (Map<String, Object>) map.get("author");
|
||||
newDatasetVo.setCreateBy((String) author.get("name"));
|
||||
newDatasetVo.setOwner((String) author.get("login"));
|
||||
}
|
||||
}
|
||||
|
||||
return newDatasetVos;
|
||||
}
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
private String checkoutToken(){
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
|
||||
if(StringUtils.isEmpty(token)){
|
||||
gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
token = jedis.get(ci4sUsername+"_gitToken");
|
||||
}else {
|
||||
try {
|
||||
Map userInfo = gitService.getUserInfo(token);
|
||||
if (userInfo == null ||(userInfo.get("status")!=null&&401==(Integer) userInfo.get("status"))){
|
||||
gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
token = jedis.get(ci4sUsername+"_gitToken");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
}
|
||||
return token;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -122,6 +122,9 @@ public class GitServiceImpl implements GitService {
|
|||
@Override
|
||||
public Map getUserInfo(String token) throws Exception {
|
||||
String userReq = HttpUtils.sendGetWithToken("https://www.gitlink.org.cn/api/users/get_user_info.json",null, token);
|
||||
if (StringUtils.isEmpty(userReq)){
|
||||
return null;
|
||||
}
|
||||
Map<String, Object> runResMap = JsonUtils.jsonToMap(userReq);
|
||||
return runResMap;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,501 @@
|
|||
package com.ruoyi.platform.service.impl;
|
||||
|
||||
import com.ruoyi.common.core.utils.DateUtils;
|
||||
import com.ruoyi.common.security.utils.SecurityUtils;
|
||||
import com.ruoyi.platform.domain.Dataset;
|
||||
import com.ruoyi.platform.service.DvcService;
|
||||
import com.ruoyi.platform.service.GitService;
|
||||
import com.ruoyi.platform.service.NewDatasetService;
|
||||
import com.ruoyi.platform.utils.*;
|
||||
import com.ruoyi.platform.vo.GitProjectVo;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import com.ruoyi.platform.vo.VersionVo;
|
||||
import com.ruoyi.system.api.model.LoginUser;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageImpl;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import redis.clients.jedis.Jedis;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@Service
|
||||
public class NewDatasetServiceImpl implements NewDatasetService {
|
||||
|
||||
@Resource
|
||||
private GitService gitService;
|
||||
|
||||
@Resource
|
||||
private DvcService dvcService;
|
||||
@Value("${spring.redis.host}")
|
||||
private String redisHost;
|
||||
@Value("${minio.accessKey}")
|
||||
String accessKeyId;
|
||||
@Value("${minio.secretKey}")
|
||||
String secretAccessKey;
|
||||
@Value("${minio.endpoint}")
|
||||
String endpoint;
|
||||
@Value("${git.endpoint}")
|
||||
String gitendpoint;
|
||||
@Value("${git.localPath}")
|
||||
String localPathlocal;
|
||||
|
||||
@Override
|
||||
public String newCreateDataset(NewDatasetVo datasetVo) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername + "_gitToken");
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
// 拼接project
|
||||
String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
|
||||
GitProjectVo gitProjectVo = new GitProjectVo();
|
||||
gitProjectVo.setRepositoryName(repositoryName);
|
||||
gitProjectVo.setName(datasetVo.getName());
|
||||
gitProjectVo.setDescription(datasetVo.getDescription());
|
||||
gitProjectVo.setPrivate(!datasetVo.getIsPublic());
|
||||
gitProjectVo.setUserId(userId);
|
||||
// 创建项目
|
||||
Map project = gitService.createProject(token, gitProjectVo);
|
||||
Integer gitlinIid = (Integer) project.get("id");
|
||||
// 创建分支
|
||||
String branchName = datasetVo.getVersion();
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
// 定义标签 标签1:ci4s_dataset 标签2:DataTag 标签3:DataType
|
||||
gitService.createTopic(token, gitlinIid, "ci4s_dataset");
|
||||
gitService.createTopic(token, gitlinIid, "DataTag_" + datasetVo.getDataTag());
|
||||
gitService.createTopic(token, gitlinIid, "DataType_" + datasetVo.getDataType());
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/"+ repositoryName + ".git";
|
||||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal + gitlinIid + "/" + datasetVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
|
||||
String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + gitlinIid + "/" + repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath);
|
||||
// 拼接生产的元数据后写入yaml文件
|
||||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getTime());
|
||||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("<pre><code>\n" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " "+ projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"</code></pre>\n");
|
||||
datasetVo.setIdentifier(repositoryName);
|
||||
datasetVo.setId(gitlinIid);
|
||||
datasetVo.setOwner((String) userInfo.get("login"));
|
||||
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath, "data");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath, ".");
|
||||
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath);
|
||||
return "新增数据集成功";
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo) {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername + "_gitToken");
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
// 创建分支
|
||||
String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
|
||||
String repositoryName = datasetVo.getIdentifier();
|
||||
if (StringUtils.equals(branchName, "master")) {
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
}
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal + loginUser.getUsername() + "/" + "datasets" + "/" + datasetVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
if (FileUtil.checkDirectoryExists(localPath)) {
|
||||
DVCUtils.gitFetch(localPath, gitLinkUsername, gitLinkPassword);
|
||||
DVCUtils.gitCheckoutBranch(localPath, branchName);
|
||||
} else {
|
||||
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
|
||||
}
|
||||
|
||||
String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath);
|
||||
// 拼接生产的元数据后写入yaml文件
|
||||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getTime());
|
||||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("<pre><code>\n" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " "+ projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"</code></pre>\n");
|
||||
datasetVo.setIdentifier(repositoryName);
|
||||
datasetVo.setId(datasetVo.getId());
|
||||
datasetVo.setOwner((String) userInfo.get("login"));
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
|
||||
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath, "data");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath, ".");
|
||||
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath);
|
||||
return "新增数据集成功";
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername + "_gitToken");
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
// 拼接查询url
|
||||
String datasetTagName = dataset.getDatasetTagName();
|
||||
String datasetTypeName = dataset.getDatasetTypeName();
|
||||
String topic_name = "ci4s_dataset";
|
||||
topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName;
|
||||
topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName;
|
||||
String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name;
|
||||
String req = HttpUtils.sendGetWithToken(url, null, token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("count");
|
||||
List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
|
||||
return new PageImpl<>(convert(projects), pageRequest, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername + "_gitToken");
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
// 拼接查询url
|
||||
String datasetTagName = dataset.getDatasetTagName();
|
||||
String datasetTypeName = dataset.getDatasetTypeName();
|
||||
String topic_name = "ci4s_dataset";
|
||||
topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName;
|
||||
topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName;
|
||||
|
||||
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name;
|
||||
String req = HttpUtils.sendGetWithToken(url, null, token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("total_count");
|
||||
List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
|
||||
if (projects == null) {
|
||||
return new PageImpl<>(new ArrayList<>(), pageRequest, 0);
|
||||
}
|
||||
return new PageImpl<>(convert(projects), pageRequest, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public NewDatasetVo getNewDatasetDesc(Integer id, String repositoryName, String repo, String owner, String version) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
// cd到 localPathlocal/id/下面还有一个文件夹,然后做git pull操作,然后读取里面的文件列表,列出每个文件的大小和名称,封装成MAP
|
||||
if (StringUtils.isEmpty(version)) {
|
||||
List<Map<String, Object>> versionList = this.getVersionList(repo, owner);
|
||||
if (versionList.size() == 0) {
|
||||
throw new Exception("数据集文件不存在");
|
||||
}
|
||||
version = (String) versionList.get(0).get("name");
|
||||
}
|
||||
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull("E:\\test\\" +id, repositoryName, version, gitLinkUsername, gitLinkPassword);
|
||||
// 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile("E:\\test\\" + id + "\\" + repositoryName + "\\" + "dataset.yaml");
|
||||
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
|
||||
List<VersionVo> versionVos = new ArrayList<VersionVo>();
|
||||
if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
|
||||
for(Map<String, Object> fileDetail : fileDetailsAfterGitPull){
|
||||
VersionVo versionVo = new VersionVo();
|
||||
versionVo.setUrl((String) fileDetail.get("filePath"));
|
||||
versionVo.setFileName((String) fileDetail.get("fileName"));
|
||||
long size = (long) fileDetail.get("size");
|
||||
versionVo.setFileSize(FileUtil.formatFileSize(size));
|
||||
versionVos.add(versionVo);
|
||||
}
|
||||
}
|
||||
newDatasetVo.setDatasetVersionVos(versionVos);
|
||||
return newDatasetVo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
List<Map<String, Object>> brancheList = gitService.getBrancheList(token, owner, repo);
|
||||
return brancheList.stream()
|
||||
.filter(branch -> !"master".equals(branch.get("name")))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteDatasetNew(String repo, String owner) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
gitService.deleteProject(token, owner, repo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception {
|
||||
checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
gitService.deleteBranch(token, owner, repo, version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
|
||||
List<Map<String, String>> results = new ArrayList<>();
|
||||
|
||||
for (MultipartFile file : files) {
|
||||
// 构建objectName
|
||||
String username = SecurityUtils.getLoginUser().getUsername();
|
||||
String fileName = file.getOriginalFilename();
|
||||
String path = localPathlocal + "temp" + "/" + username + "/" + "datasets" + "/" + uuid + "/" + "data" + "/" + fileName;
|
||||
long sizeInBytes = file.getSize();
|
||||
String formattedSize = FileUtil.formatFileSize(sizeInBytes);
|
||||
File targetFile = new File(path);
|
||||
// 确保目录存在
|
||||
targetFile.getParentFile().mkdirs();
|
||||
// 保存文件到目标路径
|
||||
FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
|
||||
// 返回上传文件的路径
|
||||
String absolutePath = targetFile.getAbsolutePath();
|
||||
Map<String, String> result = new HashMap<>();
|
||||
result.put("fileName", fileName);
|
||||
result.put("url", path); // objectName根据实际情况定义
|
||||
result.put("fileSize", formattedSize);
|
||||
results.add(result);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception {
|
||||
File file = new File(filePath);
|
||||
|
||||
if (!file.exists()) {
|
||||
throw new FileNotFoundException("File not found: " + filePath);
|
||||
}
|
||||
|
||||
InputStreamResource resource = new InputStreamResource(new FileInputStream(file));
|
||||
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
|
||||
headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length()));
|
||||
headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.headers(headers)
|
||||
.contentLength(file.length())
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.body(resource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception {
|
||||
// 命令行操作 git clone 项目地址
|
||||
checkoutToken();
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName;
|
||||
File folder = new File(localPath);
|
||||
if(folder.exists() && folder.isDirectory()){
|
||||
//切换分支
|
||||
DVCUtils.gitCheckoutBranch(localPath, version);
|
||||
//pull
|
||||
DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword);
|
||||
//dvc pull
|
||||
DVCUtils.dvcPull(localPath);
|
||||
}else {
|
||||
DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword);
|
||||
}
|
||||
|
||||
// 打包 data 文件夹
|
||||
String dataFolderPath = localPath + "/data";
|
||||
String zipFilePath = localPath + "/data.zip";
|
||||
try (FileOutputStream fos = new FileOutputStream(zipFilePath);
|
||||
ZipOutputStream zos = new ZipOutputStream(fos)) {
|
||||
Path sourcePath = Paths.get(dataFolderPath);
|
||||
Files.walk(sourcePath).forEach(path -> {
|
||||
if (!Files.isDirectory(path)) {
|
||||
ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString());
|
||||
try {
|
||||
zos.putNextEntry(zipEntry);
|
||||
Files.copy(path, zos);
|
||||
zos.closeEntry();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Error while zipping: " + path, e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 返回压缩文件的输入流
|
||||
File zipFile = new File(zipFilePath);
|
||||
InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip")
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.contentLength(zipFile.length())
|
||||
.body(resource);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
public List<NewDatasetVo> convert(List<Map<String, Object>> lst) {
|
||||
if (lst != null && lst.size() > 0) {
|
||||
List<NewDatasetVo> newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class);
|
||||
|
||||
for (NewDatasetVo newDatasetVo : newDatasetVos) {
|
||||
Map<String, Object> map = lst.stream()
|
||||
.filter(m -> m.get("id").equals(newDatasetVo.getId()))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
|
||||
if (map != null) {
|
||||
List<Map<String, Object>> topics = (List<Map<String, Object>>) map.get("topics");
|
||||
if (topics != null) {
|
||||
topics.forEach(topic -> {
|
||||
String name = (String) topic.get("name");
|
||||
if (name != null) {
|
||||
if (name.startsWith("datatag_")) {
|
||||
newDatasetVo.setDataTag(name.substring("datatag_".length()));
|
||||
} else if (name.startsWith("datatype_")) {
|
||||
newDatasetVo.setDataType(name.substring("datatype_".length()));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Map<String, Object> author = (Map<String, Object>) map.get("author");
|
||||
newDatasetVo.setCreateBy((String) author.get("name"));
|
||||
newDatasetVo.setOwner((String) author.get("login"));
|
||||
}
|
||||
}
|
||||
|
||||
return newDatasetVos;
|
||||
}
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
private String checkoutToken(){
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
|
||||
if(StringUtils.isEmpty(token)){
|
||||
gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
token = jedis.get(ci4sUsername+"_gitToken");
|
||||
}else {
|
||||
try {
|
||||
Map userInfo = gitService.getUserInfo(token);
|
||||
if (userInfo == null ||(userInfo.get("status")!=null&&401==(Integer) userInfo.get("status"))){
|
||||
gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
token = jedis.get(ci4sUsername+"_gitToken");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
}
|
||||
return token;
|
||||
|
||||
}
|
||||
|
||||
}
|
|
@ -201,11 +201,11 @@ public class DVCUtils {
|
|||
Path repoPath = Paths.get(localPath, repoFolder);
|
||||
|
||||
//刷新
|
||||
gitFetch(localPath, username, password);
|
||||
// 切换到指定分支
|
||||
gitCheckoutBranch(localPath, branch);
|
||||
// 执行git pull
|
||||
gitPull(localPath, username, password);
|
||||
// gitFetch(localPath, username, password);
|
||||
// // 切换到指定分支
|
||||
// gitCheckoutBranch(localPath, branch);
|
||||
// // 执行git pull
|
||||
// gitPull(localPath, username, password);
|
||||
// 读取data文件夹中的文件列表
|
||||
Path dataPath = Paths.get(repoPath.toString(), "data");
|
||||
File[] files = dataPath.toFile().listFiles();
|
||||
|
|
|
@ -42,8 +42,8 @@ public class NewDatasetVo implements Serializable {
|
|||
|
||||
@ApiModelProperty(name = "time_ago",value = "更新时间")
|
||||
private String timeAgo;
|
||||
@ApiModelProperty(name = "repo_id",value = "数据集仓库id")
|
||||
private Integer repoId;
|
||||
@ApiModelProperty(name = "id",value = "数据集仓库id")
|
||||
private Integer id;
|
||||
@ApiModelProperty(name = "visits",value = "访问次数")
|
||||
private Integer visits;
|
||||
@ApiModelProperty(name = "create_by",value = "创建者")
|
||||
|
@ -53,7 +53,7 @@ public class NewDatasetVo implements Serializable {
|
|||
@ApiModelProperty(name = "usage",value = "使用示例")
|
||||
private String usage;
|
||||
@ApiModelProperty(name = "update_time",value = "更新时间")
|
||||
private Date updateTime;
|
||||
private String updateTime;
|
||||
@ApiModelProperty(name = "processing_code",value = "处理代码")
|
||||
private String processingCode;
|
||||
@ApiModelProperty(name = "owner",value = "项目所有者")
|
||||
|
|
Loading…
Reference in New Issue