Compare commits
2 Commits
77fadb29d9
...
53046a939a
| Author | SHA1 | Date |
|---|---|---|
 fanshuai
|
53046a939a | |
|
fanshuai
|
156888782e |
|
|
@ -2,10 +2,8 @@ package com.ruoyi.platform.controller.dataset;
|
|||
|
||||
import com.ruoyi.common.core.web.domain.AjaxResult;
|
||||
import com.ruoyi.platform.domain.Dataset;
|
||||
import com.ruoyi.platform.service.DatasetService;
|
||||
import com.ruoyi.platform.vo.DatasetVo;
|
||||
import com.ruoyi.platform.service.NewDatasetService;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiOperation;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
|
|
@ -22,7 +20,7 @@ public class NewDatasetFromGitController {
|
|||
* 服务对象
|
||||
*/
|
||||
@Resource
|
||||
private DatasetService datasetService;
|
||||
private NewDatasetService newDatasetService;
|
||||
|
||||
|
||||
|
||||
|
|
@ -35,7 +33,7 @@ public class NewDatasetFromGitController {
|
|||
@PostMapping("/addDatasetAndVersion")
|
||||
@ApiOperation("添加数据集和版本")
|
||||
public AjaxResult addDatasetAndVersion(@RequestBody NewDatasetVo datasetVo) throws Exception {
|
||||
return AjaxResult.success(this.datasetService.newCreateDataset(datasetVo));
|
||||
return AjaxResult.success(this.newDatasetService.newCreateDataset(datasetVo));
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -49,7 +47,7 @@ public class NewDatasetFromGitController {
|
|||
@PostMapping("/addVersion")
|
||||
@ApiOperation("添加版本")
|
||||
public AjaxResult addVersion(@RequestBody NewDatasetVo datasetVo) throws Exception {
|
||||
return AjaxResult.success(this.datasetService.newCreateVersion(datasetVo));
|
||||
return AjaxResult.success(this.newDatasetService.newCreateVersion(datasetVo));
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -62,39 +60,39 @@ public class NewDatasetFromGitController {
|
|||
@RequestParam(value = "data_tag", required = false) String dataTag) throws Exception {
|
||||
PageRequest pageRequest = PageRequest.of(page, size);
|
||||
if(isPublic){
|
||||
return AjaxResult.success(this.datasetService.newPubilcQueryByPage(dataset, pageRequest));
|
||||
return AjaxResult.success(this.newDatasetService.newPubilcQueryByPage(dataset, pageRequest));
|
||||
}else {
|
||||
return AjaxResult.success(this.datasetService.newPersonalQueryByPage(dataset, pageRequest));
|
||||
return AjaxResult.success(this.newDatasetService.newPersonalQueryByPage(dataset, pageRequest));
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping("/getVersionList")
|
||||
@ApiOperation(value = "获取分支列表")
|
||||
public AjaxResult getVersionList(@RequestParam("identifier") String repo,@RequestParam("owner")String owner) throws Exception {
|
||||
return AjaxResult.success(this.datasetService.getVersionList(repo,owner));
|
||||
return AjaxResult.success(this.newDatasetService.getVersionList(repo,owner));
|
||||
}
|
||||
|
||||
@GetMapping("/getDatasetDetail")
|
||||
@ApiOperation(value = "获取数据集详情")
|
||||
public AjaxResult getDatasetVersions(@RequestParam("name") String name,
|
||||
@RequestParam("repo_id") Integer repoId,
|
||||
@RequestParam("id") Integer id,
|
||||
@RequestParam(value ="owner",required = false)String owner,
|
||||
@RequestParam(value = "identifier",required = false) String repo,
|
||||
@RequestParam(value = "identifier",required = false) String identifier,
|
||||
@RequestParam(value = "version",required = false)String version) throws Exception {
|
||||
return AjaxResult.success(this.datasetService.getNewDatasetDesc(repoId,name,repo,owner,version));
|
||||
return AjaxResult.success(this.newDatasetService.getNewDatasetDesc(id,name,identifier,owner,version));
|
||||
}
|
||||
|
||||
@DeleteMapping("/deleteDataset")
|
||||
@ApiOperation(value = "删除数据集")
|
||||
public AjaxResult deleteDataset(@RequestParam("identifier") String repo,@RequestParam("owner")String owner) throws Exception {
|
||||
this.datasetService.deleteDatasetNew(repo,owner);
|
||||
this.newDatasetService.deleteDatasetNew(repo,owner);
|
||||
return AjaxResult.success();
|
||||
}
|
||||
|
||||
@DeleteMapping("/deleteDatasetVersion")
|
||||
@ApiOperation(value = "删除数据集版本")
|
||||
public AjaxResult deleteDatasetVersion(@RequestParam("identifier") String repo,@RequestParam("owner")String owner,@RequestParam("version")String version) throws Exception {
|
||||
this.datasetService.deleteDatasetVersionNew(repo,owner,version);
|
||||
this.newDatasetService.deleteDatasetVersionNew(repo,owner,version);
|
||||
return AjaxResult.success();
|
||||
}
|
||||
|
||||
|
|
@ -110,7 +108,7 @@ public class NewDatasetFromGitController {
|
|||
@PostMapping("/upload")
|
||||
@ApiOperation(value = "上传数据集")
|
||||
public AjaxResult uploadDataset(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception {
|
||||
return AjaxResult.success(this.datasetService.uploadDatasetlocal(files,uuid));
|
||||
return AjaxResult.success(this.newDatasetService.uploadDatasetlocal(files,uuid));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -122,7 +120,7 @@ public class NewDatasetFromGitController {
|
|||
@GetMapping("/downloadAllFiles")
|
||||
@ApiOperation(value = "下载同一版本下所有数据集，并打包")
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("repository_name") String repositoryName, @RequestParam("version") String version) throws Exception {
|
||||
return datasetService.downloadAllDatasetFilesNew(repositoryName, version);
|
||||
return newDatasetService.downloadAllDatasetFilesNew(repositoryName, version);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -135,6 +133,6 @@ public class NewDatasetFromGitController {
|
|||
@GetMapping("/downloadSinggerFile")
|
||||
@ApiOperation(value = "下载单个数据集文件", notes = "根据数据集版本表id下载单个数据集文件")
|
||||
public ResponseEntity<InputStreamResource> downloadDataset(@RequestParam("url") String url) throws Exception {
|
||||
return datasetService.downloadDatasetlocal(url);
|
||||
return newDatasetService.downloadDatasetlocal(url);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,9 +4,11 @@ import com.ruoyi.common.core.web.controller.BaseController;
|
|||
import com.ruoyi.common.core.web.domain.AjaxResult;
|
||||
import com.ruoyi.common.core.web.domain.GenericsAjaxResult;
|
||||
import com.ruoyi.platform.domain.DevEnvironment;
|
||||
import com.ruoyi.platform.service.DatasetService;
|
||||
import com.ruoyi.platform.service.JupyterService;
|
||||
import com.ruoyi.platform.vo.*;
|
||||
import com.ruoyi.platform.service.NewDatasetService;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import com.ruoyi.platform.vo.PodStatusVo;
|
||||
import com.ruoyi.platform.vo.VersionVo;
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiOperation;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponse;
|
||||
|
|
@ -27,7 +29,7 @@ public class JupyterController extends BaseController {
|
|||
@Resource
|
||||
private JupyterService jupyterService;
|
||||
@Resource
|
||||
private DatasetService datasetService;
|
||||
private NewDatasetService newDatasetService;
|
||||
@GetMapping(value = "/getURL")
|
||||
@ApiOperation("得到访问地址")
|
||||
public GenericsAjaxResult<String> getURL() throws IOException {
|
||||
|
|
@ -101,6 +103,6 @@ public class JupyterController extends BaseController {
|
|||
datasetVo.setVersionDesc("this is a test");
|
||||
datasetVersionVos.add(versionVo);
|
||||
datasetVo.setDatasetVersionVos(datasetVersionVos);
|
||||
return AjaxResult.success(datasetService.newCreateDataset(datasetVo));
|
||||
return AjaxResult.success(newDatasetService.newCreateDataset(datasetVo));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import org.springframework.core.io.InputStreamResource;
|
|||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
|
|
@ -21,17 +20,16 @@ public class NewModelFromGitController {
|
|||
@Resource
|
||||
private ModelsService modelsService;
|
||||
|
||||
@PostMapping("/addModel")
|
||||
@ApiOperation("添加模型")
|
||||
@PostMapping("/addModelAndVersion")
|
||||
@ApiOperation("添加模型和版本")
|
||||
public AjaxResult addModelAndVersion(@RequestBody ModelsVo modelsVo) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.newCreateModel(modelsVo));
|
||||
}
|
||||
|
||||
@CrossOrigin(origins = "*", allowedHeaders = "*")
|
||||
@PostMapping("/upload")
|
||||
@ApiOperation(value = "上传模型", notes = "根据模型id上传模型文件，并将信息存入数据库。")
|
||||
public AjaxResult uploadModel(@RequestParam("file") MultipartFile[] files, @RequestParam("uuid") String uuid) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.uploadModelLocal(files, uuid));
|
||||
@PostMapping("/addVersion")
|
||||
@ApiOperation("添加版本")
|
||||
public AjaxResult addVersion(@RequestBody ModelsVo modelsVo) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.newCreateVersion(modelsVo));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -42,9 +40,11 @@ public class NewModelFromGitController {
|
|||
*/
|
||||
@GetMapping("/downloadAllFiles")
|
||||
@ApiOperation(value = "下载同一版本下所有模型，并打包")
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam("identifier") String identifier,
|
||||
@RequestParam(value = "version") String version) throws Exception {
|
||||
return modelsService.downloadAllModelFilesNew(identifier, version);
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFiles(@RequestParam(value = "repository_name") String repositoryName,
|
||||
@RequestParam(value = "version") String version,
|
||||
@RequestParam(value = "git_link_username") String gitLinkUsername,
|
||||
@RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception {
|
||||
return modelsService.downloadAllModelFilesNew(repositoryName, version, gitLinkUsername, gitLinkPassword);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -61,19 +61,21 @@ public class NewModelFromGitController {
|
|||
return modelsService.downloadModels(model_version_id);
|
||||
}
|
||||
|
||||
@GetMapping("/queryModels")
|
||||
@GetMapping("/queryDatasets")
|
||||
@ApiOperation("模型广场公开模型分页查询,根据model_type,model_tag筛选，true公开false私有")
|
||||
public AjaxResult queryModels(@RequestParam(value = "page") int page,
|
||||
@RequestParam(value = "size") int size,
|
||||
@RequestParam(value = "is_public", required = false) Boolean isPublic,
|
||||
@RequestParam(value = "model_type", required = false) String modelType,
|
||||
@RequestParam(value = "model_tag", required = false) String modelTag,
|
||||
@RequestParam(value = "name", required = false) String name) throws Exception {
|
||||
public AjaxResult queryDatasets(@RequestParam(value = "page") int page,
|
||||
@RequestParam(value = "size") int size,
|
||||
@RequestParam(value = "is_public") Boolean isPublic,
|
||||
@RequestParam(value = "model_type", required = false) String modelType,
|
||||
@RequestParam(value = "model_tag", required = false) String modelTag,
|
||||
@RequestParam(value = "git_link_username") String gitLinkUsername,
|
||||
@RequestParam(value = "git_link_password") String gitLinkPassword) throws Exception {
|
||||
PageRequest pageRequest = PageRequest.of(page, size);
|
||||
ModelsVo modelsVo = new ModelsVo();
|
||||
modelsVo.setModelType(modelType);
|
||||
modelsVo.setModelTag(modelTag);
|
||||
modelsVo.setName(name);
|
||||
modelsVo.setGitLinkUsername(gitLinkUsername);
|
||||
modelsVo.setGitLinkPassword(gitLinkPassword);
|
||||
if (isPublic) {
|
||||
return AjaxResult.success(this.modelsService.newPubilcQueryByPage(modelsVo, pageRequest));
|
||||
} else {
|
||||
|
|
@ -81,36 +83,4 @@ public class NewModelFromGitController {
|
|||
}
|
||||
}
|
||||
|
||||
@GetMapping("/getVersionList")
|
||||
@ApiOperation(value = "获取模型分支列表")
|
||||
public AjaxResult getVersionList(@RequestParam("identifier") String identifier, @RequestParam("owner")String owner) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.getVersionList(identifier, owner));
|
||||
}
|
||||
|
||||
|
||||
@GetMapping("/getModelDetail")
|
||||
@ApiOperation(value = "获取模型详细信息")
|
||||
public AjaxResult getModelDetail(@RequestParam("model_name") String modelName, @RequestParam("identifier") String identifier,@RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.getModelDetail(modelName, identifier, owner, version));
|
||||
}
|
||||
|
||||
@GetMapping("/getModelDependencyTree")
|
||||
@ApiOperation(value = "获取模型依赖关系树")
|
||||
public AjaxResult getModelDependencyTree(@RequestParam("repoId") Integer repoId, @RequestParam("model_name") String modelName, @RequestParam("version") String version) throws Exception {
|
||||
return AjaxResult.success(this.modelsService.getModelDependencyTree(repoId, modelName, version));
|
||||
}
|
||||
|
||||
@DeleteMapping("/delete")
|
||||
@ApiOperation(value = "删除模型")
|
||||
public AjaxResult deleteModel(@RequestParam("repoId")Integer repoId, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner) throws Exception {
|
||||
this.modelsService.deleteModel(repoId,identifier, owner);
|
||||
return AjaxResult.success();
|
||||
}
|
||||
|
||||
@DeleteMapping("/deleteVersion")
|
||||
@ApiOperation(value = "删除模型版本")
|
||||
public AjaxResult deleteVersion(@RequestParam("repoId")Integer repoId, @RequestParam("identifier") String identifier, @RequestParam("owner") String owner, @RequestParam("version") String version) throws Exception {
|
||||
this.modelsService.deleteVersion(repoId, identifier, owner, version);
|
||||
return AjaxResult.success();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ import com.ruoyi.platform.annotations.CheckDuplicate;
|
|||
import io.swagger.annotations.ApiModel;
|
||||
import io.swagger.annotations.ApiModelProperty;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* (Models)实体类
|
||||
|
|
@ -53,18 +53,17 @@ public class Models implements Serializable {
|
|||
@ApiModelProperty(value = "状态，0失效，1生效")
|
||||
private Integer state;
|
||||
|
||||
@ApiModelProperty(value = "模型gitLink仓库id")
|
||||
private Integer repoId;
|
||||
|
||||
@ApiModelProperty(value = "模型gitLink仓库名称")
|
||||
private String repoName;
|
||||
|
||||
@ApiModelProperty(value = "模型类型名")
|
||||
private String modelTypeName;
|
||||
|
||||
|
||||
|
||||
@ApiModelProperty(value = "模型tag名")
|
||||
private String modelTagName;
|
||||
|
||||
|
||||
|
||||
public Integer getId() {
|
||||
return id;
|
||||
}
|
||||
|
|
@ -171,22 +170,6 @@ public class Models implements Serializable {
|
|||
this.state = state;
|
||||
}
|
||||
|
||||
public Integer getRepoId() {
|
||||
return repoId;
|
||||
}
|
||||
|
||||
public void setRepoId(Integer repoId) {
|
||||
this.repoId = repoId;
|
||||
}
|
||||
|
||||
public String getRepoName() {
|
||||
return repoName;
|
||||
}
|
||||
|
||||
public void setRepoName(String repoName) {
|
||||
this.repoName = repoName;
|
||||
}
|
||||
|
||||
public String getModelTagName() {
|
||||
return modelTagName;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,11 +21,7 @@ import java.util.concurrent.CompletableFuture;
|
|||
* @author Xidaray
|
||||
* @since 2023-11-28 11:51:22
|
||||
*/
|
||||
public interface
|
||||
DatasetService {
|
||||
|
||||
|
||||
|
||||
public interface DatasetService {
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -39,12 +35,12 @@ DatasetService {
|
|||
/**
|
||||
* 分页查询
|
||||
*
|
||||
* @param dataset 筛选条件
|
||||
* @param pageRequest 分页对象
|
||||
* @param dataset 筛选条件
|
||||
* @param pageRequest 分页对象
|
||||
* @return 查询结果
|
||||
*/
|
||||
Page<Dataset> queryByPage(Dataset dataset, PageRequest pageRequest);
|
||||
|
||||
|
||||
/**
|
||||
* 新增数据
|
||||
*
|
||||
|
|
@ -54,7 +50,6 @@ DatasetService {
|
|||
Dataset insert(Dataset dataset) throws Exception;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 修改数据
|
||||
*
|
||||
|
|
@ -82,30 +77,11 @@ DatasetService {
|
|||
List<String> getDatasetVersions(Integer datasetId) throws Exception;
|
||||
|
||||
String insertDatasetAndVersion(DatasetVo datasetVo) throws Exception;
|
||||
|
||||
void checkDeclaredName(Dataset insert) throws Exception;
|
||||
|
||||
ResponseEntity<InputStreamResource> downloadAllDatasetFiles(Integer datasetId, String version) throws Exception;
|
||||
|
||||
List<Map<String, String>> exportDataset(String path, String uuid) throws Exception;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
CompletableFuture<String> newCreateDataset(NewDatasetVo datasetVo) throws Exception;
|
||||
CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo);
|
||||
|
||||
List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
|
||||
ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception;
|
||||
ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception;
|
||||
Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
|
||||
Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
|
||||
|
||||
NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName,String repo, String owner, String version)throws Exception;
|
||||
|
||||
List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception;
|
||||
|
||||
void deleteDatasetNew(String repo, String owner) throws Exception;
|
||||
|
||||
void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception;
|
||||
}
|
||||
}
|
||||
|
|
@ -11,17 +11,17 @@ public interface GitService {
|
|||
String login(String username, String password);
|
||||
|
||||
//输入token，项目名，tag，创建新项目,返回项目地址
|
||||
Map createProject(String token, GitProjectVo gitProjectVo) throws Exception;
|
||||
Map createProject(GitProjectVo gitProjectVo) throws Exception;
|
||||
|
||||
void createBranch(String token,String owner, String projectName, String branchName, String oldBranchName) throws Exception;
|
||||
void createBranch(String owner, String projectName, String branchName, String oldBranchName) throws Exception;
|
||||
|
||||
void createTopic(String token, Integer id, String topicName) throws Exception;
|
||||
void createTopic(Integer id, String topicName) throws Exception;
|
||||
|
||||
List<Map<String, Object>> getBrancheList(String token, String owner, String projectName) throws Exception;
|
||||
List<Map<String, Object>> getBrancheList(String owner, String projectName) throws Exception;
|
||||
|
||||
void deleteProject(String token, String owner, String projectName) throws Exception;
|
||||
void deleteProject(String owner, String projectName) throws Exception;
|
||||
|
||||
void deleteBranch(String token, String owner,String projectName ,String branchName) throws Exception;
|
||||
void deleteBranch(String owner,String projectName ,String branchName) throws Exception;
|
||||
|
||||
Map getUserInfo(String token) throws Exception;
|
||||
Map getUserInfo() throws Exception;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package com.ruoyi.platform.service;
|
|||
|
||||
import com.ruoyi.platform.domain.Models;
|
||||
import com.ruoyi.platform.domain.ModelsVersion;
|
||||
import com.ruoyi.platform.vo.ModelDependency1TreeVo;
|
||||
import com.ruoyi.platform.vo.ModelsVo;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.data.domain.Page;
|
||||
|
|
@ -87,23 +86,14 @@ public interface ModelsService {
|
|||
|
||||
List<Map<String, String>> exportModels(String path, String uuid) throws Exception;
|
||||
|
||||
|
||||
CompletableFuture<String> newCreateModel(ModelsVo modelsVo) throws Exception;
|
||||
|
||||
List<Map<String, String>> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception;
|
||||
CompletableFuture<String> newCreateVersion(ModelsVo modelsVo);
|
||||
|
||||
ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String identifier, String version) throws IOException, Exception;
|
||||
ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception;
|
||||
|
||||
Page<ModelsVo> newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception;
|
||||
|
||||
Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception;
|
||||
|
||||
List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception;
|
||||
|
||||
ModelsVo getModelDetail(String modelName, String identifier, String owner, String version) throws Exception;
|
||||
|
||||
ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version) throws Exception;
|
||||
|
||||
void deleteModel(Integer repoId, String identifier, String owner) throws Exception;
|
||||
|
||||
void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
package com.ruoyi.platform.service;
|
||||
|
||||
import com.ruoyi.platform.domain.Dataset;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
public interface NewDatasetService {
|
||||
|
||||
|
||||
String newCreateDataset(NewDatasetVo datasetVo) throws Exception;
|
||||
CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo)throws Exception;
|
||||
|
||||
List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception;
|
||||
ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception;
|
||||
ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws IOException, Exception;
|
||||
Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
|
||||
Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception;
|
||||
|
||||
NewDatasetVo getNewDatasetDesc(Integer repoId,String RepositoryName,String repo, String owner, String version)throws Exception;
|
||||
|
||||
List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception;
|
||||
|
||||
void deleteDatasetNew(String repo, String owner) throws Exception;
|
||||
|
||||
void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception;
|
||||
}
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
package com.ruoyi.platform.service.impl;
|
||||
|
||||
import com.ruoyi.common.core.utils.DateUtils;
|
||||
import com.ruoyi.common.security.utils.SecurityUtils;
|
||||
import com.ruoyi.platform.annotations.CheckDuplicate;
|
||||
import com.ruoyi.platform.domain.AssetIcon;
|
||||
|
|
@ -9,14 +8,13 @@ import com.ruoyi.platform.domain.DatasetVersion;
|
|||
import com.ruoyi.platform.mapper.DatasetDao;
|
||||
import com.ruoyi.platform.mapper.DatasetVersionDao;
|
||||
import com.ruoyi.platform.service.*;
|
||||
import com.ruoyi.platform.utils.*;
|
||||
import com.ruoyi.platform.utils.BeansUtils;
|
||||
import com.ruoyi.platform.utils.FileUtil;
|
||||
import com.ruoyi.platform.utils.MinioUtil;
|
||||
import com.ruoyi.platform.vo.DatasetVo;
|
||||
import com.ruoyi.platform.vo.GitProjectVo;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import com.ruoyi.platform.vo.VersionVo;
|
||||
import com.ruoyi.system.api.model.LoginUser;
|
||||
import io.minio.messages.Item;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
|
|
@ -30,17 +28,13 @@ import org.springframework.http.ResponseEntity;
|
|||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import redis.clients.jedis.Jedis;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.*;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.lang.reflect.Field;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
|
@ -68,12 +62,6 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
@Resource
|
||||
private MinioService minioService;
|
||||
|
||||
@Resource
|
||||
private GitService gitService;
|
||||
|
||||
@Resource
|
||||
private DvcService dvcService;
|
||||
|
||||
// 固定存储桶名
|
||||
@Value("${minio.dataReleaseBucketName}")
|
||||
private String bucketName;
|
||||
|
|
@ -81,19 +69,6 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
@Resource
|
||||
private MinioUtil minioUtil;
|
||||
|
||||
@Value("${spring.redis.host}")
|
||||
private String redisHost;
|
||||
@Value("${minio.accessKey}")
|
||||
String accessKeyId;
|
||||
@Value("${minio.secretKey}")
|
||||
String secretAccessKey;
|
||||
@Value("${minio.endpoint}")
|
||||
String endpoint;
|
||||
@Value("${git.endpoint}")
|
||||
String gitendpoint;
|
||||
@Value("${git.localPath}")
|
||||
String localPathlocal;
|
||||
|
||||
/**
|
||||
* 通过ID查询单条数据
|
||||
*
|
||||
|
|
@ -458,413 +433,4 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
return results;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> newCreateDataset(NewDatasetVo datasetVo) {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
gitService.checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
// 拼接project
|
||||
String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
|
||||
GitProjectVo gitProjectVo = new GitProjectVo();
|
||||
gitProjectVo.setRepositoryName(repositoryName);
|
||||
gitProjectVo.setName(datasetVo.getName());
|
||||
gitProjectVo.setDescription(datasetVo.getDescription());
|
||||
gitProjectVo.setPrivate(!datasetVo.getIsPublic());
|
||||
gitProjectVo.setUserId(userId);
|
||||
// 创建项目
|
||||
Map project = gitService.createProject(token, gitProjectVo);
|
||||
Integer repoId = (Integer) project.get("id");
|
||||
// 创建分支
|
||||
String branchName = datasetVo.getVersion();
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
// 定义标签 标签1：ci4s_dataset 标签2：DataTag 标签3：DataType
|
||||
gitService.createTopic(token, repoId, "ci4s_dataset");
|
||||
gitService.createTopic(token, repoId, "DataTag_" + datasetVo.getDataTag());
|
||||
gitService.createTopic(token, repoId, "DataType_" + datasetVo.getDataType());
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal + ci4sUsername + "/datasets/" + datasetVo.getName() + "/" + branchName;
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/" + repoId + "/" + repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath + "/data");
|
||||
//拼接生产的元数据后写入yaml文件
|
||||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getNowDate());
|
||||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("```bash\n" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " " + projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"```");
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath + "/metadata", "metadata");
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath, "data");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath, ".");
|
||||
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath);
|
||||
return "新增数据集成功";
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo) {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
gitService.checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
// 创建分支
|
||||
String branchName = StringUtils.isEmpty(datasetVo.getVersion())? "master" : datasetVo.getVersion();
|
||||
String repositoryName = datasetVo.getIdentifier();
|
||||
if (StringUtils.equals(branchName, "master")) {
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
}
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal + loginUser.getUsername() + "/datasets/" + datasetVo.getName() + "/" + branchName;
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
if(FileUtil.checkDirectoryExists(localPath)){
|
||||
DVCUtils.gitFetch(localPath,gitLinkUsername, gitLinkPassword);
|
||||
DVCUtils.gitCheckoutBranch(localPath,branchName);
|
||||
}else {
|
||||
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
|
||||
}
|
||||
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/datasets/"+ datasetVo.getRepoId()+"/"+ repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath + "/data");
|
||||
//拼接生产的元数据后写入yaml文件
|
||||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String)userInfo.get("nickname"))? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getNowDate());
|
||||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("```bash\n" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " " + projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"```");
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath + "/metadata", "metadata");
|
||||
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath, "data");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath, ".");
|
||||
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath);
|
||||
return "新增数据集成功";
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
gitService.checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
//拼接查询url
|
||||
String datasetTagName = dataset.getDatasetTagName();
|
||||
String datasetTypeName = dataset.getDatasetTypeName();
|
||||
String topic_name = "ci4s_dataset";
|
||||
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName;
|
||||
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName;
|
||||
String url = gitendpoint + "/api/users/"+(String) userInfo.get("login")+"/projects.json?page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&category=manage&topic_name="+topic_name;
|
||||
String req = HttpUtils.sendGetWithToken(url,null,token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("count");
|
||||
List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
|
||||
return new PageImpl<>(convert(projects), pageRequest, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
gitService.checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
//拼接查询url
|
||||
String datasetTagName = dataset.getDatasetTagName();
|
||||
String datasetTypeName = dataset.getDatasetTypeName();
|
||||
String topic_name = "ci4s_dataset";
|
||||
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatag_" + datasetTagName;
|
||||
topic_name =StringUtils.isEmpty(datasetTagName)?topic_name : topic_name+",datatype_" + datasetTypeName;
|
||||
|
||||
String url = gitendpoint + "/api/projects.json?user_id="+userId+"&page="+pageRequest.getPageNumber()+"&limit="+pageRequest.getPageSize()+"&sort_by=praises_count&topic_name="+topic_name;
|
||||
String req = HttpUtils.sendGetWithToken(url,null,token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("total_count");
|
||||
List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
|
||||
if (projects == null){
|
||||
return new PageImpl<>(new ArrayList<>(), pageRequest, 0);
|
||||
}
|
||||
return new PageImpl<>(convert(projects), pageRequest, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public NewDatasetVo getNewDatasetDesc(Integer repoId,String repositoryName,String repo, String owner,String version) throws Exception{
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
// cd到 localPathlocal/repoId/下面还有一个文件夹，然后做git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP
|
||||
if (StringUtils.isEmpty(version)){
|
||||
List<Map<String, Object>> versionList = this.getVersionList(repo, owner);
|
||||
if (versionList.size() == 0){
|
||||
throw new Exception("数据集文件不存在");
|
||||
}
|
||||
version = (String) versionList.get(0).get("name");
|
||||
}
|
||||
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPathlocal + ci4sUsername + "/datasets/", repositoryName, version, "data",gitLinkUsername, gitLinkPassword);//在localPathlocal+repoId+"/"+repositoryName目录下的dataset.yaml中取到元数据
|
||||
//在localPathlocal+repoId+"/"+repositoryName目录下的dataset.yaml中取到元数据
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPathlocal + ci4sUsername + "/datasets/" + repositoryName + "/" + version + "/metadata/metadata.yaml");
|
||||
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
|
||||
List<VersionVo> versionVos = new ArrayList<VersionVo>();
|
||||
if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
|
||||
for(Map<String, Object> fileDetail : fileDetailsAfterGitPull){
|
||||
VersionVo versionVo = new VersionVo();
|
||||
versionVo.setUrl((String) fileDetail.get("filePath"));
|
||||
versionVo.setFileName((String) fileDetail.get("fileName"));
|
||||
long size = (long) fileDetail.get("size");
|
||||
versionVo.setFileSize(FileUtil.formatFileSize(size));
|
||||
versionVos.add(versionVo);
|
||||
}
|
||||
}
|
||||
newDatasetVo.setDatasetVersionVos(versionVos);
|
||||
return newDatasetVo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception {
|
||||
List<Map<String, Object>> brancheList = gitService.getBrancheList(owner, repo);
|
||||
return brancheList.stream()
|
||||
.filter(branch -> "master".equals(branch.get("name")))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteDatasetNew(String repo, String owner) throws Exception {
|
||||
gitService.checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
gitService.deleteProject(token, owner, repo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception {
|
||||
gitService.checkoutToken();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
gitService.deleteBranch(token, owner, repo, version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
|
||||
List<Map<String, String>> results = new ArrayList<>();
|
||||
|
||||
for (MultipartFile file:files){
|
||||
// 构建objectName
|
||||
String username = SecurityUtils.getLoginUser().getUsername();
|
||||
String fileName = file.getOriginalFilename();
|
||||
String path = "/temp/"+ username +"/datasets/"+ uuid + "/"+"/data/" + fileName;
|
||||
long sizeInBytes = file.getSize();
|
||||
String formattedSize = FileUtil.formatFileSize(sizeInBytes);
|
||||
File targetFile = new File(path, file.getOriginalFilename());
|
||||
// 确保目录存在
|
||||
targetFile.getParentFile().mkdirs();
|
||||
// 保存文件到目标路径
|
||||
FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
|
||||
// 返回上传文件的路径
|
||||
String absolutePath = targetFile.getAbsolutePath();
|
||||
Map<String, String> result = new HashMap<>();
|
||||
result.put("fileName", fileName);
|
||||
result.put("url", absolutePath); // objectName根据实际情况定义
|
||||
result.put("fileSize", formattedSize);
|
||||
results.add(result);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception {
|
||||
File file = new File(filePath);
|
||||
|
||||
if (!file.exists()) {
|
||||
throw new FileNotFoundException("File not found: " + filePath);
|
||||
}
|
||||
|
||||
InputStreamResource resource = new InputStreamResource(new FileInputStream(file));
|
||||
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
|
||||
headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length()));
|
||||
headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.headers(headers)
|
||||
.contentLength(file.length())
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.body(resource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception {
|
||||
// 命令行操作 git clone 项目地址
|
||||
gitService.checkoutToken();
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName;
|
||||
File folder = new File(localPath);
|
||||
if(folder.exists() && folder.isDirectory()){
|
||||
//切换分支
|
||||
DVCUtils.gitCheckoutBranch(localPath, version);
|
||||
//pull
|
||||
DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword);
|
||||
//dvc pull
|
||||
DVCUtils.dvcPull(localPath);
|
||||
}else {
|
||||
DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword);
|
||||
}
|
||||
|
||||
// 打包 data 文件夹
|
||||
String dataFolderPath = localPath + "/data";
|
||||
String zipFilePath = localPath + "/data.zip";
|
||||
try (FileOutputStream fos = new FileOutputStream(zipFilePath);
|
||||
ZipOutputStream zos = new ZipOutputStream(fos)) {
|
||||
Path sourcePath = Paths.get(dataFolderPath);
|
||||
Files.walk(sourcePath).forEach(path -> {
|
||||
if (!Files.isDirectory(path)) {
|
||||
ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString());
|
||||
try {
|
||||
zos.putNextEntry(zipEntry);
|
||||
Files.copy(path, zos);
|
||||
zos.closeEntry();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Error while zipping: " + path, e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 返回压缩文件的输入流
|
||||
File zipFile = new File(zipFilePath);
|
||||
InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip")
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.contentLength(zipFile.length())
|
||||
.body(resource);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
public List<NewDatasetVo> convert(List<Map<String, Object>> lst) {
|
||||
if (lst != null && lst.size() > 0) {
|
||||
List<NewDatasetVo> newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class);
|
||||
|
||||
for (NewDatasetVo newDatasetVo : newDatasetVos) {
|
||||
Map<String, Object> map = lst.stream()
|
||||
.filter(m -> m.get("repo_id").equals(newDatasetVo.getRepoId()))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
|
||||
if (map != null) {
|
||||
List<Map<String, Object>> topics = (List<Map<String, Object>>) map.get("topics");
|
||||
if (topics != null) {
|
||||
topics.forEach(topic -> {
|
||||
String name = (String) topic.get("name");
|
||||
if (name != null) {
|
||||
if (name.startsWith("datatag_")) {
|
||||
newDatasetVo.setDataTag(name.substring("datatag_".length()));
|
||||
} else if (name.startsWith("datatype_")) {
|
||||
newDatasetVo.setDataType(name.substring("datatype_".length()));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Map<String, Object> author = (Map<String, Object>) map.get("author");
|
||||
newDatasetVo.setCreateBy((String) author.get("name"));
|
||||
newDatasetVo.setOwner((String) author.get("login"));
|
||||
}
|
||||
}
|
||||
|
||||
return newDatasetVos;
|
||||
}
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,7 +68,6 @@ public class GitServiceImpl implements GitService {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String checkoutToken() {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
|
|
@ -82,7 +81,7 @@ public class GitServiceImpl implements GitService {
|
|||
token = jedis.get(ci4sUsername + "_gitToken");
|
||||
} else {
|
||||
try {
|
||||
Map userInfo = getUserInfo(token);
|
||||
Map userInfo = getUserInfo();
|
||||
if (userInfo == null || (userInfo.get("status") != null && 401 == (Integer) userInfo.get("status"))) {
|
||||
login(gitLinkUsername, gitLinkPassword);
|
||||
token = jedis.get(ci4sUsername + "_gitToken");
|
||||
|
|
@ -96,18 +95,20 @@ public class GitServiceImpl implements GitService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public Map createProject(String token, GitProjectVo gitProjectVo) throws Exception {
|
||||
public Map createProject(GitProjectVo gitProjectVo) throws Exception {
|
||||
String token = this.checkoutToken();
|
||||
String userReq = HttpUtils.sendPostWithToken("https://www.gitlink.org.cn/api/projects.json", JsonUtils.objectToJson(gitProjectVo), token);
|
||||
return JsonUtils.jsonToMap(userReq);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createBranch(String token, String owner, String projectName, String branchName, String oldBranchName) throws Exception {
|
||||
public void createBranch(String owner, String projectName, String branchName, String oldBranchName) throws Exception {
|
||||
//https://www.gitlink.org.cn/api/v1/fanshuai/testdssa8755/branches.json
|
||||
// {
|
||||
// "new_branch_name": "SsS",
|
||||
// "old_branch_name": "master"
|
||||
// }
|
||||
String token = this.checkoutToken();
|
||||
String createBranchUrl = "https://www.gitlink.org.cn/api/v1/" + owner + "/" + projectName + "/branches.json";
|
||||
Map<String, Object> resMap = new HashMap<>();
|
||||
resMap.put("new_branch_name", branchName);
|
||||
|
|
@ -116,8 +117,9 @@ public class GitServiceImpl implements GitService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void createTopic(String token, Integer id, String topicName) throws Exception {
|
||||
public void createTopic(Integer id, String topicName) throws Exception {
|
||||
// https://www.gitlink.org.cn/api/v1/project_topics.json
|
||||
String token = this.checkoutToken();
|
||||
Map<String, Object> resMap = new HashMap<>();
|
||||
resMap.put("project_id", id);
|
||||
resMap.put("name", topicName);
|
||||
|
|
@ -138,18 +140,24 @@ public class GitServiceImpl implements GitService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void deleteProject(String token, String owner, String projectName) throws Exception {
|
||||
public void deleteProject(String owner, String projectName) throws Exception {
|
||||
String token = this.checkoutToken();
|
||||
HttpUtils.sendDeleteRequest("https://www.gitlink.org.cn/api/" + owner + "/" + projectName + ".json", token);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteBranch(String token, String owner, String projectName, String branchName) throws Exception {
|
||||
public void deleteBranch(String owner, String projectName, String branchName) throws Exception {
|
||||
String token = this.checkoutToken();
|
||||
HttpUtils.sendDeleteRequest("https://www.gitlink.org.cn/api/v1/" + owner + "/" + projectName + "/branches/" + branchName + ".json", token);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map getUserInfo(String token) throws Exception {
|
||||
String userReq = HttpUtils.sendGetWithToken("https://www.gitlink.org.cn/api/users/get_user_info.json", null, token);
|
||||
public Map getUserInfo() throws Exception {
|
||||
String token = this.checkoutToken();
|
||||
String userReq = HttpUtils.sendGetWithToken("https://www.gitlink.org.cn/api/users/get_user_info.json",null, token);
|
||||
if (StringUtils.isEmpty(userReq)){
|
||||
return null;
|
||||
}
|
||||
Map<String, Object> runResMap = JsonUtils.jsonToMap(userReq);
|
||||
return runResMap;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,22 +4,20 @@ import com.ruoyi.common.core.utils.DateUtils;
|
|||
import com.ruoyi.common.security.utils.SecurityUtils;
|
||||
import com.ruoyi.platform.annotations.CheckDuplicate;
|
||||
import com.ruoyi.platform.domain.AssetIcon;
|
||||
import com.ruoyi.platform.domain.ModelDependency1;
|
||||
import com.ruoyi.platform.domain.Models;
|
||||
import com.ruoyi.platform.domain.ModelsVersion;
|
||||
import com.ruoyi.platform.mapper.ModelDependency1Dao;
|
||||
import com.ruoyi.platform.mapper.ModelsDao;
|
||||
import com.ruoyi.platform.mapper.ModelsVersionDao;
|
||||
import com.ruoyi.platform.service.*;
|
||||
import com.ruoyi.platform.utils.*;
|
||||
import com.ruoyi.platform.vo.*;
|
||||
import com.ruoyi.platform.vo.GitProjectVo;
|
||||
import com.ruoyi.platform.vo.ModelsVo;
|
||||
import com.ruoyi.platform.vo.VersionVo;
|
||||
import com.ruoyi.system.api.model.LoginUser;
|
||||
import io.minio.messages.Item;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.data.domain.Page;
|
||||
|
|
@ -65,9 +63,6 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
@Resource
|
||||
private ModelsVersionService modelsVersionService;
|
||||
|
||||
@Resource
|
||||
private ModelDependency1Dao modelDependency1Dao;
|
||||
|
||||
@Resource
|
||||
private MinioService minioService;
|
||||
|
||||
|
|
@ -381,7 +376,7 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
@Override
|
||||
@Transactional
|
||||
public String insertModelAndVersion(ModelsVo modelsVo) throws Exception {
|
||||
List<VersionVo> modelsVersionVos = modelsVo.getModelVersionVos();
|
||||
List<VersionVo> modelsVersionVos = modelsVo.getModelsVersionVos();
|
||||
|
||||
if (modelsVersionVos == null || modelsVersionVos.isEmpty()) {
|
||||
throw new Exception("模型版本信息错误");
|
||||
|
|
@ -516,59 +511,65 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> newCreateModel(ModelsVo modelsVo) {
|
||||
public CompletableFuture<String> newCreateModel(ModelsVo modelsVo) throws Exception {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
gitService.checkoutToken();
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
|
||||
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
String token = (String) userInfo.get("token");
|
||||
|
||||
// 拼接project
|
||||
String repositoryName = ci4sUsername + "_model_" + DateUtils.dateTimeNow();
|
||||
GitProjectVo gitProjectVo = new GitProjectVo();
|
||||
gitProjectVo.setRepositoryName(repositoryName);
|
||||
gitProjectVo.setName(modelsVo.getName());
|
||||
gitProjectVo.setDescription(modelsVo.getDescription());
|
||||
gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0);
|
||||
gitProjectVo.setUserId(userId);
|
||||
|
||||
String repositoryName = modelsVo.getRepositoryName() == null ? ci4sUsername + "_model_" + DateUtils.dateTimeNow() : modelsVo.getRepositoryName();
|
||||
ModelDependency1 modelDependency = new ModelDependency1();
|
||||
List<ModelDependency1> oldModelDependencys = modelDependency1Dao.queryByModelName(modelsVo.getName());
|
||||
// 创建项目
|
||||
Map project = gitService.createProject(gitProjectVo);
|
||||
|
||||
if (oldModelDependencys != null && !oldModelDependencys.isEmpty()) {
|
||||
if (oldModelDependencys.stream().map(ModelDependency1::getVersion).anyMatch(version -> version.equals(modelsVo.getVersion()))) {
|
||||
throw new RuntimeException("模型版本已存在，请勿重复创建");
|
||||
}
|
||||
// 创建分支
|
||||
String branchName = modelsVo.getVersion();
|
||||
gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
// 定义标签 标签1：ci4s_model 标签2：ModelTag 标签3：ModelType
|
||||
gitService.createTopic((Integer) project.get("id"), "ci4s_model");
|
||||
gitService.createTopic((Integer) project.get("id"), "ModelTag_" + modelsVo.getModelTag());
|
||||
gitService.createTopic((Integer) project.get("id"), "ModelType_" + modelsVo.getModelType());
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
// 得到用户操作的路径
|
||||
String url = modelsVo.getModelsVersionVos().get(0).getUrl();
|
||||
String localPath1 = localPath + modelsVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/models/" + repositoryName + "/" + branchName;
|
||||
|
||||
//新建版本
|
||||
commonDvc(token, userInfo, ci4sUsername, modelsVo.getRepositoryName(), modelsVo, gitLinkUsername, gitLinkPassword, "CreateModelFromPipeline");
|
||||
modelDependency.setRepoId(modelsVo.getRepoId());
|
||||
} else {
|
||||
//新建模型
|
||||
// 拼接project
|
||||
GitProjectVo gitProjectVo = new GitProjectVo();
|
||||
gitProjectVo.setRepositoryName(repositoryName);
|
||||
gitProjectVo.setName(modelsVo.getName());
|
||||
gitProjectVo.setDescription(modelsVo.getDescription());
|
||||
gitProjectVo.setPrivate(modelsVo.getAvailableRange() == 0);
|
||||
gitProjectVo.setUserId(userId);
|
||||
// 拼接生产的元数据后写入yaml文件
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "model");
|
||||
|
||||
// 创建项目
|
||||
Map project = gitService.createProject(token, gitProjectVo);
|
||||
modelDependency.setRepoId((Integer) project.get("id"));
|
||||
// 定义标签 标签1：ci4s_model 标签2：ModelTag 标签3：ModelType
|
||||
gitService.createTopic(token, (Integer) project.get("id"), "ci4s_model");
|
||||
gitService.createTopic(token, (Integer) project.get("id"), "modeltag_" + modelsVo.getModelTag());
|
||||
gitService.createTopic(token, (Integer) project.get("id"), "modeltype_" + modelsVo.getModelType());
|
||||
commonDvc(token, userInfo, ci4sUsername, repositoryName, modelsVo, gitLinkUsername, gitLinkPassword, "createModel");
|
||||
}
|
||||
DVCUtils.moveFiles(sourcePath, localPath1 + "/model");
|
||||
|
||||
//保存模型依赖
|
||||
modelDependency.setIdentifier(repositoryName);
|
||||
modelDependency.setModelName(modelsVo.getName());
|
||||
modelDependency.setVersion(modelsVo.getVersion());
|
||||
modelDependency.setParentModel(modelsVo.getParentModel());
|
||||
modelDependency.setOwner(ci4sUsername);
|
||||
modelDependency1Dao.insert(modelDependency);
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath1);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath1, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath1, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath1, "model");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath1, ".");
|
||||
DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername);
|
||||
DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath1);
|
||||
return "新增模型成功";
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage());
|
||||
|
|
@ -577,45 +578,67 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
});
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<Map<String, String>> uploadModelLocal(MultipartFile[] files, String uuid) throws Exception {
|
||||
List<Map<String, String>> results = new ArrayList<>();
|
||||
public CompletableFuture<String> newCreateVersion(ModelsVo modelsVo) {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
// 创建分支
|
||||
String branchName = modelsVo.getVersion();
|
||||
String repositoryName = modelsVo.getRepositoryName();
|
||||
gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
// 得到用户操作的路径
|
||||
String url = modelsVo.getModelVersionVos().get(0).getUrl();
|
||||
String localPath1 = localPath + ci4sUsername + "/model/" + modelsVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
DVCUtils.gitClone(localPath1, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
String s3Path = "management-platform-files/" + ci4sUsername + "/model/" + repositoryName + "/" + branchName;
|
||||
//拼接生产的元数据后写入yaml文件
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelsVo), sourcePath, "dataset");
|
||||
|
||||
for (MultipartFile file : files) {
|
||||
// 构建objectName
|
||||
String username = SecurityUtils.getLoginUser().getUsername();
|
||||
String fileName = file.getOriginalFilename();
|
||||
String path = "/temp/" + username + "/models/" + uuid + "/model/";
|
||||
long sizeInBytes = file.getSize();
|
||||
String formattedSize = FileUtil.formatFileSize(sizeInBytes);
|
||||
File targetFile = new File(path, file.getOriginalFilename());
|
||||
// 确保目录存在
|
||||
targetFile.getParentFile().mkdirs();
|
||||
// 保存文件到目标路径
|
||||
FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
|
||||
// 返回上传文件的路径
|
||||
String absolutePath = targetFile.getAbsolutePath();
|
||||
Map<String, String> result = new HashMap<>();
|
||||
result.put("fileName", fileName);
|
||||
result.put("url", absolutePath); // objectName根据实际情况定义
|
||||
result.put("fileSize", formattedSize);
|
||||
results.add(result);
|
||||
}
|
||||
return results;
|
||||
DVCUtils.moveFiles(sourcePath, localPath1 + "/model");
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath1);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath1, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath1, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath1, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath1, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath1, "model");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath1, ".");
|
||||
DVCUtils.gitCommit(localPath1, "commit from ci4s with " + ci4sUsername);
|
||||
DVCUtils.gitPush(localPath1, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath1);
|
||||
return "新增模型成功";
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String identifier, String version) throws Exception {
|
||||
public ResponseEntity<InputStreamResource> downloadAllModelFilesNew(String repositoryName, String version, String gitLinkUsername, String gitLinkPassword) throws IOException, Exception {
|
||||
// 命令行操作 git clone 项目地址
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
|
||||
|
||||
String projectUrl = gitendpoint + "/" + userInfo.get("login") + "/" + identifier + ".git";
|
||||
String localPath1 = localPath + ci4sUsername + "/model/" + identifier;
|
||||
String token = gitService.login(gitLinkUsername, gitLinkPassword);
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
String localPath1 = localPath + ci4sUsername + "/model/" + repositoryName;
|
||||
File folder = new File(localPath1);
|
||||
if (folder.exists() && folder.isDirectory()) {
|
||||
//切换分支
|
||||
|
|
@ -662,18 +685,17 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
public Page<ModelsVo> newPubilcQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
|
||||
String token = (String) userInfo.get("token");
|
||||
|
||||
String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
//拼接查询url
|
||||
String modelTagName = modelsVo.getModelTag();
|
||||
String modelTypeName = modelsVo.getModelType();
|
||||
String search = modelsVo.getName();
|
||||
String topic_name = "ci4s_model";
|
||||
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTypeName;
|
||||
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltype_" + modelTypeName;
|
||||
|
||||
String url = gitendpoint + "/api/users/" + userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name + "&search=" + search;
|
||||
String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name;
|
||||
String req = HttpUtils.sendGetWithToken(url, null, token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("count");
|
||||
|
|
@ -685,19 +707,19 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
public Page<ModelsVo> newPersonalQueryByPage(ModelsVo modelsVo, PageRequest pageRequest) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
|
||||
String token = gitService.login(modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
String token = (String) userInfo.get("token");
|
||||
|
||||
//拼接查询url
|
||||
String modelTagName = modelsVo.getModelTag();
|
||||
String modelTypeName = modelsVo.getModelType();
|
||||
String search = modelsVo.getName();
|
||||
String topic_name = "ci4s_model";
|
||||
topic_name = StringUtils.isEmpty(modelTagName) ? topic_name : topic_name + ",modeltag_" + modelTagName;
|
||||
topic_name = StringUtils.isEmpty(modelTypeName) ? topic_name : topic_name + ",modeltype_" + modelTypeName;
|
||||
|
||||
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name + "&search=" + search;
|
||||
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name;
|
||||
String req = HttpUtils.sendGetWithToken(url, null, token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("total_count");
|
||||
|
|
@ -705,149 +727,6 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
return new PageImpl<>(convert(projects), pageRequest, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, Object>> getVersionList(String identifier, String owner) throws Exception {
|
||||
List<Map<String, Object>> brancheList = gitService.getBrancheList(owner, identifier);
|
||||
return brancheList.stream()
|
||||
.filter(branch -> "master".equals(branch.get("name")))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ModelsVo getModelDetail(String modelName, String identifier, String owner, String version) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
|
||||
if (StringUtils.isEmpty(version)){
|
||||
List<Map<String, Object>> versionList = this.getVersionList(owner, identifier);
|
||||
if (versionList.size() == 0){
|
||||
throw new Exception("数据集文件不存在");
|
||||
}
|
||||
version = (String) versionList.get(0).get("name");
|
||||
}
|
||||
|
||||
// git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull(localPath + ci4sUsername + "/model/", modelName, version, "model", gitLinkUsername, gitLinkPassword);
|
||||
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile(localPath + ci4sUsername + "/model/" + modelName + "/" + version + "/metadata/metadata.yaml");
|
||||
assert stringObjectMap != null;
|
||||
|
||||
ModelMetaVo modelMetaVo = ConvertUtil.convertMapToObject(stringObjectMap, ModelMetaVo.class);
|
||||
ModelsVo modelsVo = new ModelsVo();
|
||||
BeanUtils.copyProperties(modelMetaVo, modelsVo);
|
||||
|
||||
List<VersionVo> versionVos = new ArrayList<>();
|
||||
if (!fileDetailsAfterGitPull.isEmpty()) {
|
||||
for (Map<String, Object> fileDetail : fileDetailsAfterGitPull) {
|
||||
VersionVo versionVo = new VersionVo();
|
||||
versionVo.setUrl((String) fileDetail.get("filePath"));
|
||||
versionVo.setFileName((String) fileDetail.get("fileName"));
|
||||
long size = (long) fileDetail.get("size");
|
||||
versionVo.setFileSize(FileUtil.formatFileSize(size));
|
||||
versionVos.add(versionVo);
|
||||
}
|
||||
}
|
||||
modelsVo.setModelVersionVos(versionVos);
|
||||
|
||||
return modelsVo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ModelDependency1TreeVo getModelDependencyTree(Integer repoId, String modelName, String version) throws Exception {
|
||||
ModelDependency1 modelDependency1 = modelDependency1Dao.queryByModelNameAndVersion(repoId, modelName, version);
|
||||
ModelDependency1TreeVo modelDependency1TreeVo = new ModelDependency1TreeVo();
|
||||
BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo);
|
||||
|
||||
//递归查询父模型
|
||||
List<ModelDependency1> parentModelList = new ArrayList<>();
|
||||
getParentModel(parentModelList, modelDependency1);
|
||||
modelDependency1TreeVo.setPatrentModelList(parentModelList);
|
||||
|
||||
//递归查询子模型
|
||||
getChildModel(modelDependency1TreeVo);
|
||||
|
||||
return modelDependency1TreeVo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteModel(Integer repoId, String identifier, String owner) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
|
||||
String token = (String) userInfo.get("token");
|
||||
|
||||
|
||||
gitService.deleteProject(token, owner, identifier);
|
||||
//删除模型依赖
|
||||
modelDependency1Dao.deleteModel(repoId, identifier, owner, null);
|
||||
modelDependency1Dao.deleteModelDependency(repoId, identifier, owner, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteVersion(Integer repoId, String identifier, String owner, String version) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Map<String, Object> userInfo = getUserInfo(ci4sUsername);
|
||||
String token = (String) userInfo.get("token");
|
||||
|
||||
gitService.deleteBranch(token, owner, identifier, version);
|
||||
//删除模型依赖
|
||||
modelDependency1Dao.deleteModel(repoId, identifier, owner, version);
|
||||
modelDependency1Dao.deleteModelDependency(repoId, identifier, owner, version);
|
||||
}
|
||||
|
||||
void commonDvc(String token, Map<String, Object> userInfo, String username, String repositoryName, ModelsVo modelsVo, String gitLinkUsername, String gitLinkPassword, String type) throws Exception {
|
||||
String branchName = modelsVo.getVersion();
|
||||
|
||||
// 创建分支
|
||||
gitService.createBranch(token, (String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
String url = modelsVo.getModelVersionVos().get(0).getUrl();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
String rootPath = localPath + username + "/model/" + modelsVo.getName() + "/" + branchName;
|
||||
String modelPath = rootPath + "/model";
|
||||
String metaPath = rootPath + "/metadata";
|
||||
|
||||
|
||||
if (type.equals("CreateModelFromPipeline") && FileUtil.checkDirectoryExists(rootPath)) {
|
||||
DVCUtils.gitFetch(rootPath, gitLinkUsername, gitLinkPassword);
|
||||
DVCUtils.gitCheckoutBranch(rootPath, branchName);
|
||||
} else {
|
||||
DVCUtils.gitClone(rootPath, projectUrl, branchName, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
}
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, modelPath);
|
||||
|
||||
//拼接生产的元数据后写入yaml文件
|
||||
ModelMetaVo modelMetaVo = new ModelMetaVo();
|
||||
BeanUtils.copyProperties(modelsVo, modelMetaVo);
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(modelMetaVo), metaPath, "metadata");
|
||||
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(rootPath);
|
||||
|
||||
// 配置远程S3地址
|
||||
String s3Path = "management-platform-files/" + username + "/model/" + repositoryName + "/" + branchName;
|
||||
DVCUtils.dvcRemoteAdd(rootPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(rootPath, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(rootPath, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(rootPath, secretAccessKey);
|
||||
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(rootPath, "model");
|
||||
|
||||
// git commit
|
||||
DVCUtils.gitAdd(rootPath, ".");
|
||||
DVCUtils.gitCommit(rootPath, "commit from ci4s with " + username);
|
||||
DVCUtils.gitPush(rootPath, modelsVo.getGitLinkUsername(), modelsVo.getGitLinkPassword());
|
||||
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(rootPath);
|
||||
}
|
||||
|
||||
public List<ModelsVo> convert(List<Map<String, Object>> lst) {
|
||||
if (lst != null && lst.size() > 0) {
|
||||
List<ModelsVo> newModelVos = ConvertUtil.convertListMapToObjectList(lst, ModelsVo.class);
|
||||
|
|
@ -874,48 +753,10 @@ public class ModelsServiceImpl implements ModelsService {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
return newModelVos;
|
||||
}
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
void getParentModel(List<ModelDependency1> modelList, ModelDependency1 modelDependency1) {
|
||||
if (modelDependency1.getParentModel() != null) {
|
||||
String[] split = modelDependency1.getParentModel().split(":");
|
||||
Integer parentRepoId = Integer.valueOf(split[0]);
|
||||
String parentModelName = split[1];
|
||||
String parentModelVersion = split[2];
|
||||
ModelDependency1 parentModel = modelDependency1Dao.queryByModelNameAndVersion(parentRepoId, parentModelName, parentModelVersion);
|
||||
modelList.add(parentModel);
|
||||
if (parentModel != null && parentModel.getParentModel() != null) {
|
||||
getParentModel(modelList, parentModel);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void getChildModel(ModelDependency1TreeVo modelDependency1TreeVo) {
|
||||
List<ModelDependency1TreeVo> childModelList = new ArrayList<>();
|
||||
String model = modelDependency1TreeVo.getRepoId() + ":" + modelDependency1TreeVo.getModelName() + ":" + modelDependency1TreeVo.getVersion();
|
||||
|
||||
List<ModelDependency1> children = modelDependency1Dao.queryByParentModel(model);
|
||||
if (children != null && !children.isEmpty()) {
|
||||
for (ModelDependency1 modelDependency1 : children) {
|
||||
ModelDependency1TreeVo modelDependency1TreeVo1 = new ModelDependency1TreeVo();
|
||||
BeanUtils.copyProperties(modelDependency1, modelDependency1TreeVo1);
|
||||
getChildModel(modelDependency1TreeVo1);
|
||||
childModelList.add(modelDependency1TreeVo1);
|
||||
}
|
||||
}
|
||||
modelDependency1TreeVo.setChildModelList(childModelList);
|
||||
}
|
||||
|
||||
Map<String, Object> getUserInfo(String ci4sUsername) throws IOException {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
|
||||
String token = gitService.checkoutToken();
|
||||
userInfo.put("token", token);
|
||||
return userInfo;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,466 @@
|
|||
package com.ruoyi.platform.service.impl;
|
||||
|
||||
import com.ruoyi.common.core.utils.DateUtils;
|
||||
import com.ruoyi.common.security.utils.SecurityUtils;
|
||||
import com.ruoyi.platform.domain.Dataset;
|
||||
import com.ruoyi.platform.service.DvcService;
|
||||
import com.ruoyi.platform.service.GitService;
|
||||
import com.ruoyi.platform.service.NewDatasetService;
|
||||
import com.ruoyi.platform.utils.*;
|
||||
import com.ruoyi.platform.vo.GitProjectVo;
|
||||
import com.ruoyi.platform.vo.NewDatasetVo;
|
||||
import com.ruoyi.platform.vo.VersionVo;
|
||||
import com.ruoyi.system.api.model.LoginUser;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageImpl;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import redis.clients.jedis.Jedis;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@Service
|
||||
public class NewDatasetServiceImpl implements NewDatasetService {
|
||||
|
||||
@Resource
|
||||
private GitService gitService;
|
||||
|
||||
@Resource
|
||||
private DvcService dvcService;
|
||||
@Value("${spring.redis.host}")
|
||||
private String redisHost;
|
||||
@Value("${minio.accessKey}")
|
||||
String accessKeyId;
|
||||
@Value("${minio.secretKey}")
|
||||
String secretAccessKey;
|
||||
@Value("${minio.endpoint}")
|
||||
String endpoint;
|
||||
@Value("${git.endpoint}")
|
||||
String gitendpoint;
|
||||
@Value("${git.localPath}")
|
||||
String localPathlocal;
|
||||
|
||||
@Override
|
||||
public String newCreateDataset(NewDatasetVo datasetVo) throws Exception {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
// 拼接project
|
||||
String repositoryName = ci4sUsername + "_dataset_" + DateUtils.dateTimeNow();
|
||||
GitProjectVo gitProjectVo = new GitProjectVo();
|
||||
gitProjectVo.setRepositoryName(repositoryName);
|
||||
gitProjectVo.setName(datasetVo.getName());
|
||||
gitProjectVo.setDescription(datasetVo.getDescription());
|
||||
gitProjectVo.setPrivate(!datasetVo.getIsPublic());
|
||||
gitProjectVo.setUserId(userId);
|
||||
// 创建项目
|
||||
Map project = gitService.createProject(gitProjectVo);
|
||||
Integer gitlinIid = (Integer) project.get("id");
|
||||
// 创建分支
|
||||
String branchName = datasetVo.getVersion();
|
||||
gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
// 定义标签 标签1：ci4s_dataset 标签2：DataTag 标签3：DataType
|
||||
gitService.createTopic(gitlinIid, "ci4s_dataset");
|
||||
gitService.createTopic(gitlinIid, "DataTag_" + datasetVo.getDataTag());
|
||||
gitService.createTopic( gitlinIid, "DataType_" + datasetVo.getDataType());
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/"+ repositoryName + ".git";
|
||||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal + gitlinIid + "/" + datasetVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
|
||||
String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + gitlinIid + "/" + repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath);
|
||||
// 拼接生产的元数据后写入yaml文件
|
||||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getTime());
|
||||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("<pre><code>\n" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " "+ projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"</code></pre>\n");
|
||||
datasetVo.setIdentifier(repositoryName);
|
||||
datasetVo.setId(gitlinIid);
|
||||
datasetVo.setOwner((String) userInfo.get("login"));
|
||||
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath, "data");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath, ".");
|
||||
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath);
|
||||
return "新增数据集成功";
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> newCreateVersion(NewDatasetVo datasetVo) {
|
||||
return CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername + "_gitToken");
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
// 创建分支
|
||||
String branchName = StringUtils.isEmpty(datasetVo.getVersion()) ? "master" : datasetVo.getVersion();
|
||||
String repositoryName = datasetVo.getIdentifier();
|
||||
if (StringUtils.equals(branchName, "master")) {
|
||||
gitService.createBranch((String) userInfo.get("login"), repositoryName, branchName, "master");
|
||||
}
|
||||
// 得到项目地址
|
||||
String projectUrl = gitendpoint + "/" + (String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
|
||||
// 得到用户操作的路径
|
||||
String url = datasetVo.getDatasetVersionVos().get(0).getUrl();
|
||||
String localPath = localPathlocal + loginUser.getUsername() + "/" + "datasets" + "/" + datasetVo.getName();
|
||||
String sourcePath = url.substring(0, url.lastIndexOf("/"));
|
||||
// 命令行操作 git clone 项目地址
|
||||
if (FileUtil.checkDirectoryExists(localPath)) {
|
||||
DVCUtils.gitFetch(localPath, gitLinkUsername, gitLinkPassword);
|
||||
DVCUtils.gitCheckoutBranch(localPath, branchName);
|
||||
} else {
|
||||
DVCUtils.gitClone(localPath, projectUrl, branchName, gitLinkUsername, gitLinkPassword);
|
||||
}
|
||||
|
||||
String s3Path = "management-platform-files" + "/" + ci4sUsername + "/" + "datasets" + "/" + datasetVo.getId() + "/" + repositoryName + "/" + branchName;
|
||||
|
||||
DVCUtils.moveFiles(sourcePath, localPath);
|
||||
// 拼接生产的元数据后写入yaml文件
|
||||
datasetVo.setCreateBy(String.valueOf(StringUtils.isNotEmpty((String) userInfo.get("nickname")) ? userInfo.get("nickname") : userInfo.get("login")));
|
||||
datasetVo.setUpdateTime(DateUtils.getTime());
|
||||
datasetVo.setVersionDesc(datasetVo.getDescription());
|
||||
datasetVo.setUsage("<pre><code>\n" +
|
||||
"# 克隆数据集配置文件与存储参数到本地\n" +
|
||||
"git clone -b " + branchName + " "+ projectUrl + "\n" +
|
||||
"# 远程拉取配置文件\n" +
|
||||
"dvc pull\n" +
|
||||
"</code></pre>\n");
|
||||
datasetVo.setIdentifier(repositoryName);
|
||||
datasetVo.setId(datasetVo.getId());
|
||||
datasetVo.setOwner((String) userInfo.get("login"));
|
||||
YamlUtils.generateYamlFile(JsonUtils.objectToMap(datasetVo), localPath, "dataset");
|
||||
|
||||
// dvc init 初始化
|
||||
DVCUtils.dvcInit(localPath);
|
||||
// 配置远程S3地址
|
||||
DVCUtils.dvcRemoteAdd(localPath, s3Path);
|
||||
DVCUtils.dvcConfigS3Credentials(localPath, endpoint);
|
||||
DVCUtils.dvcConfigS3Credentials2(localPath, accessKeyId);
|
||||
DVCUtils.dvcConfigS3Credentials3(localPath, secretAccessKey);
|
||||
// dvc 跟踪
|
||||
DVCUtils.dvcAdd(localPath, "data");
|
||||
// git commit
|
||||
DVCUtils.gitAdd(localPath, ".");
|
||||
DVCUtils.gitCommit(localPath, "commit from ci4s with " + loginUser.getUsername());
|
||||
DVCUtils.gitPush(localPath, gitLinkUsername, gitLinkPassword);
|
||||
// dvc push 到远程S3
|
||||
DVCUtils.dvcPush(localPath);
|
||||
return "新增数据集成功";
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<NewDatasetVo> newPersonalQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername + "_gitToken");
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
// 拼接查询url
|
||||
String datasetTagName = dataset.getDatasetTagName();
|
||||
String datasetTypeName = dataset.getDatasetTypeName();
|
||||
String topic_name = "ci4s_dataset";
|
||||
topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName;
|
||||
topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName;
|
||||
String url = gitendpoint + "/api/users/" + (String) userInfo.get("login") + "/projects.json?page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&category=manage&topic_name=" + topic_name;
|
||||
String req = HttpUtils.sendGetWithToken(url, null, token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("count");
|
||||
List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
|
||||
return new PageImpl<>(convert(projects), pageRequest, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<NewDatasetVo> newPubilcQueryByPage(Dataset dataset, PageRequest pageRequest) throws Exception {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername + "_gitToken");
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
// 拼接查询url
|
||||
String datasetTagName = dataset.getDatasetTagName();
|
||||
String datasetTypeName = dataset.getDatasetTypeName();
|
||||
String topic_name = "ci4s_dataset";
|
||||
topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatag_" + datasetTagName;
|
||||
topic_name = StringUtils.isEmpty(datasetTagName) ? topic_name : topic_name + ",datatype_" + datasetTypeName;
|
||||
|
||||
String url = gitendpoint + "/api/projects.json?user_id=" + userId + "&page=" + pageRequest.getPageNumber() + "&limit=" + pageRequest.getPageSize() + "&sort_by=praises_count&topic_name=" + topic_name;
|
||||
String req = HttpUtils.sendGetWithToken(url, null, token);
|
||||
Map<String, Object> stringObjectMap = JacksonUtil.parseJSONStr2Map(req);
|
||||
Integer total = (Integer) stringObjectMap.get("total_count");
|
||||
List<Map<String, Object>> projects = (List<Map<String, Object>>) stringObjectMap.get("projects");
|
||||
if (projects == null) {
|
||||
return new PageImpl<>(new ArrayList<>(), pageRequest, 0);
|
||||
}
|
||||
return new PageImpl<>(convert(projects), pageRequest, total);
|
||||
}
|
||||
|
||||
@Override
|
||||
public NewDatasetVo getNewDatasetDesc(Integer id, String repositoryName, String repo, String owner, String version) throws Exception {
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
// cd到 localPathlocal/id/下面还有一个文件夹，然后做git pull操作，然后读取里面的文件列表，列出每个文件的大小和名称，封装成MAP
|
||||
if (StringUtils.isEmpty(version)) {
|
||||
List<Map<String, Object>> versionList = this.getVersionList(repo, owner);
|
||||
if (versionList.size() == 0) {
|
||||
throw new Exception("数据集文件不存在");
|
||||
}
|
||||
version = (String) versionList.get(0).get("name");
|
||||
}
|
||||
|
||||
List<Map<String, Object>> fileDetailsAfterGitPull = DVCUtils.getFileDetailsAfterGitPull("E:\\test\\" +id, repositoryName, version, "data",gitLinkUsername, gitLinkPassword);
|
||||
// 在localPathlocal+id+"/"+repositoryName目录下的dataset.yaml中取到元数据
|
||||
Map<String, Object> stringObjectMap = YamlUtils.loadYamlFile("E:\\test\\" + id + "\\" + repositoryName + "\\" + "dataset.yaml");
|
||||
NewDatasetVo newDatasetVo = ConvertUtil.convertMapToObject(stringObjectMap, NewDatasetVo.class);
|
||||
List<VersionVo> versionVos = new ArrayList<VersionVo>();
|
||||
if (fileDetailsAfterGitPull!=null&&fileDetailsAfterGitPull.size()>0){
|
||||
for(Map<String, Object> fileDetail : fileDetailsAfterGitPull){
|
||||
VersionVo versionVo = new VersionVo();
|
||||
versionVo.setUrl((String) fileDetail.get("filePath"));
|
||||
versionVo.setFileName((String) fileDetail.get("fileName"));
|
||||
long size = (long) fileDetail.get("size");
|
||||
versionVo.setFileSize(FileUtil.formatFileSize(size));
|
||||
versionVos.add(versionVo);
|
||||
}
|
||||
}
|
||||
newDatasetVo.setDatasetVersionVos(versionVos);
|
||||
return newDatasetVo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, Object>> getVersionList(String repo, String owner) throws Exception {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
List<Map<String, Object>> brancheList = gitService.getBrancheList(owner, repo);
|
||||
return brancheList.stream()
|
||||
.filter(branch -> !"master".equals(branch.get("name")))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteDatasetNew(String repo, String owner) throws Exception {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
gitService.deleteProject(owner, repo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteDatasetVersionNew(String repo, String owner, String version) throws Exception {
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
String token = jedis.get(ci4sUsername+"_gitToken");
|
||||
gitService.deleteBranch(owner, repo, version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, String>> uploadDatasetlocal(MultipartFile[] files, String uuid) throws Exception {
|
||||
List<Map<String, String>> results = new ArrayList<>();
|
||||
|
||||
for (MultipartFile file : files) {
|
||||
// 构建objectName
|
||||
String username = SecurityUtils.getLoginUser().getUsername();
|
||||
String fileName = file.getOriginalFilename();
|
||||
String path = localPathlocal + "temp" + "/" + username + "/" + "datasets" + "/" + uuid + "/" + "data" + "/" + fileName;
|
||||
long sizeInBytes = file.getSize();
|
||||
String formattedSize = FileUtil.formatFileSize(sizeInBytes);
|
||||
File targetFile = new File(path);
|
||||
// 确保目录存在
|
||||
targetFile.getParentFile().mkdirs();
|
||||
// 保存文件到目标路径
|
||||
FileUtils.copyInputStreamToFile(file.getInputStream(), targetFile);
|
||||
// 返回上传文件的路径
|
||||
String absolutePath = targetFile.getAbsolutePath();
|
||||
Map<String, String> result = new HashMap<>();
|
||||
result.put("fileName", fileName);
|
||||
result.put("url", path); // objectName根据实际情况定义
|
||||
result.put("fileSize", formattedSize);
|
||||
results.add(result);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadDatasetlocal(String filePath) throws Exception {
|
||||
File file = new File(filePath);
|
||||
|
||||
if (!file.exists()) {
|
||||
throw new FileNotFoundException("File not found: " + filePath);
|
||||
}
|
||||
|
||||
InputStreamResource resource = new InputStreamResource(new FileInputStream(file));
|
||||
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
|
||||
headers.add(HttpHeaders.CONTENT_LENGTH, String.valueOf(file.length()));
|
||||
headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.headers(headers)
|
||||
.contentLength(file.length())
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.body(resource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ResponseEntity<InputStreamResource> downloadAllDatasetFilesNew(String repositoryName, String version) throws Exception {
|
||||
// 命令行操作 git clone 项目地址
|
||||
LoginUser loginUser = SecurityUtils.getLoginUser();
|
||||
String gitLinkUsername = loginUser.getSysUser().getGitLinkUsername();
|
||||
String gitLinkPassword = loginUser.getSysUser().getGitLinkPassword();
|
||||
String ci4sUsername = loginUser.getUsername();
|
||||
Jedis jedis = new Jedis(redisHost);
|
||||
String userReq = jedis.get(ci4sUsername + "_gitUserInfo");
|
||||
Map<String, Object> userInfo = JsonUtils.jsonToMap(userReq);
|
||||
Integer userId = (Integer) userInfo.get("user_id");
|
||||
String projectUrl = gitendpoint + "/" +(String) userInfo.get("login") + "/" + repositoryName + ".git";
|
||||
String localPath = localPathlocal+ loginUser.getUsername()+"/datasets/" +repositoryName;
|
||||
File folder = new File(localPath);
|
||||
if(folder.exists() && folder.isDirectory()){
|
||||
//切换分支
|
||||
DVCUtils.gitCheckoutBranch(localPath, version);
|
||||
//pull
|
||||
DVCUtils.gitPull(localPath,gitLinkUsername, gitLinkPassword);
|
||||
//dvc pull
|
||||
DVCUtils.dvcPull(localPath);
|
||||
}else {
|
||||
DVCUtils.gitClone(localPath, projectUrl, version, gitLinkUsername, gitLinkPassword);
|
||||
}
|
||||
|
||||
// 打包 data 文件夹
|
||||
String dataFolderPath = localPath + "/data";
|
||||
String zipFilePath = localPath + "/data.zip";
|
||||
try (FileOutputStream fos = new FileOutputStream(zipFilePath);
|
||||
ZipOutputStream zos = new ZipOutputStream(fos)) {
|
||||
Path sourcePath = Paths.get(dataFolderPath);
|
||||
Files.walk(sourcePath).forEach(path -> {
|
||||
if (!Files.isDirectory(path)) {
|
||||
ZipEntry zipEntry = new ZipEntry(sourcePath.relativize(path).toString());
|
||||
try {
|
||||
zos.putNextEntry(zipEntry);
|
||||
Files.copy(path, zos);
|
||||
zos.closeEntry();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Error while zipping: " + path, e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 返回压缩文件的输入流
|
||||
File zipFile = new File(zipFilePath);
|
||||
InputStreamResource resource = new InputStreamResource(new FileInputStream(zipFile));
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=data.zip")
|
||||
.contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.contentLength(zipFile.length())
|
||||
.body(resource);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
public List<NewDatasetVo> convert(List<Map<String, Object>> lst) {
|
||||
if (lst != null && lst.size() > 0) {
|
||||
List<NewDatasetVo> newDatasetVos = ConvertUtil.convertListMapToObjectList(lst, NewDatasetVo.class);
|
||||
|
||||
for (NewDatasetVo newDatasetVo : newDatasetVos) {
|
||||
Map<String, Object> map = lst.stream()
|
||||
.filter(m -> m.get("id").equals(newDatasetVo.getId()))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
|
||||
if (map != null) {
|
||||
List<Map<String, Object>> topics = (List<Map<String, Object>>) map.get("topics");
|
||||
if (topics != null) {
|
||||
topics.forEach(topic -> {
|
||||
String name = (String) topic.get("name");
|
||||
if (name != null) {
|
||||
if (name.startsWith("datatag_")) {
|
||||
newDatasetVo.setDataTag(name.substring("datatag_".length()));
|
||||
} else if (name.startsWith("datatype_")) {
|
||||
newDatasetVo.setDataType(name.substring("datatype_".length()));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Map<String, Object> author = (Map<String, Object>) map.get("author");
|
||||
newDatasetVo.setCreateBy((String) author.get("name"));
|
||||
newDatasetVo.setOwner((String) author.get("login"));
|
||||
}
|
||||
}
|
||||
|
||||
return newDatasetVos;
|
||||
}
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -201,11 +201,11 @@ public class DVCUtils {
|
|||
Path repoPath = Paths.get(localPath, repoFolder, branch);
|
||||
|
||||
//刷新
|
||||
gitFetch(localPath, username, password);
|
||||
// 切换到指定分支
|
||||
gitCheckoutBranch(localPath, branch);
|
||||
// 执行git pull
|
||||
gitPull(localPath, username, password);
|
||||
// gitFetch(localPath, username, password);
|
||||
// // 切换到指定分支
|
||||
// gitCheckoutBranch(localPath, branch);
|
||||
// // 执行git pull
|
||||
// gitPull(localPath, username, password);
|
||||
// 读取data文件夹中的文件列表
|
||||
Path dataPath = Paths.get(repoPath.toString(), filePath);
|
||||
File[] files = dataPath.toFile().listFiles();
|
||||
|
|
|
|||
|
|
@ -16,16 +16,8 @@ public class YamlUtils {
|
|||
*/
|
||||
public static void generateYamlFile(Map<String, Object> data, String path, String fileName) {
|
||||
Yaml yaml = new Yaml();
|
||||
|
||||
File directory = new File(path);
|
||||
if (!directory.exists()) {
|
||||
boolean isCreated = directory.mkdirs();
|
||||
if (!isCreated) {
|
||||
throw new RuntimeException("创建路径失败: " + path);
|
||||
}
|
||||
}
|
||||
|
||||
String fullPath = path + "/" + fileName + ".yaml";
|
||||
|
||||
try (FileWriter writer = new FileWriter(fullPath)) {
|
||||
yaml.dump(data, writer);
|
||||
} catch (IOException e) {
|
||||
|
|
|
|||
|
|
@ -10,9 +10,15 @@ import java.util.List;
|
|||
|
||||
@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class)
|
||||
@Data
|
||||
public class ModelsVo extends ModelMetaVo implements Serializable {
|
||||
public class ModelsVo implements Serializable {
|
||||
|
||||
private Integer id;
|
||||
|
||||
@ApiModelProperty(name = "name")
|
||||
private String name;
|
||||
|
||||
// private String version;
|
||||
@ApiModelProperty(name = "description")
|
||||
private String description;
|
||||
|
||||
/**
|
||||
* 模型可见范围
|
||||
|
|
@ -20,6 +26,22 @@ public class ModelsVo extends ModelMetaVo implements Serializable {
|
|||
@ApiModelProperty(name = "available_range")
|
||||
private int availableRange;
|
||||
|
||||
// private String url;
|
||||
@ApiModelProperty(name = "model_type")
|
||||
private String modelType;
|
||||
|
||||
|
||||
@ApiModelProperty(name = "model_tag")
|
||||
private String modelTag;
|
||||
/**
|
||||
* 版本
|
||||
*/
|
||||
@ApiModelProperty(name = "version")
|
||||
private String version;
|
||||
|
||||
@ApiModelProperty(name = "model_version_vos")
|
||||
private List<VersionVo> modelVersionVos;
|
||||
|
||||
/**
|
||||
* 状态
|
||||
*/
|
||||
|
|
@ -31,19 +53,14 @@ public class ModelsVo extends ModelMetaVo implements Serializable {
|
|||
private String gitLinkPassword;
|
||||
|
||||
@ApiModelProperty(name = "models_version_vos")
|
||||
private List<VersionVo> modelVersionVos;
|
||||
private List<VersionVo> modelsVersionVos;
|
||||
|
||||
/**
|
||||
* 模型仓库名称
|
||||
* 数据集仓库名称
|
||||
*/
|
||||
@ApiModelProperty(name = "repository_name")
|
||||
private String repositoryName;
|
||||
|
||||
@ApiModelProperty(name = "repo_id")
|
||||
private Integer repoId;
|
||||
|
||||
/**
|
||||
* 父模型
|
||||
*/
|
||||
private String parentModel;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,8 +42,8 @@ public class NewDatasetVo implements Serializable {
|
|||
|
||||
@ApiModelProperty(name = "time_ago",value = "更新时间")
|
||||
private String timeAgo;
|
||||
@ApiModelProperty(name = "repo_id",value = "数据集仓库id")
|
||||
private Integer repoId;
|
||||
@ApiModelProperty(name = "id",value = "数据集仓库id")
|
||||
private Integer id;
|
||||
@ApiModelProperty(name = "visits",value = "访问次数")
|
||||
private Integer visits;
|
||||
@ApiModelProperty(name = "create_by",value = "创建者")
|
||||
|
|
@ -53,7 +53,7 @@ public class NewDatasetVo implements Serializable {
|
|||
@ApiModelProperty(name = "usage",value = "使用示例")
|
||||
private String usage;
|
||||
@ApiModelProperty(name = "update_time",value = "更新时间")
|
||||
private Date updateTime;
|
||||
private String updateTime;
|
||||
@ApiModelProperty(name = "processing_code",value = "处理代码")
|
||||
private String processingCode;
|
||||
@ApiModelProperty(name = "owner",value = "项目所有者")
|
||||
|
|
|
|||
|
|
@ -14,14 +14,12 @@
|
|||
<result property="updateBy" column="update_by" jdbcType="VARCHAR"/>
|
||||
<result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
|
||||
<result property="state" column="state" jdbcType="INTEGER"/>
|
||||
<result property="repoId" column="repo_id" jdbcType="INTEGER"/>
|
||||
<result property="repoName" column="repo_name" jdbcType="INTEGER"/>
|
||||
</resultMap>
|
||||
|
||||
<!--查询单个-->
|
||||
<select id="queryById" resultMap="ModelsMap">
|
||||
select
|
||||
id, name, description,available_range, model_type,model_tag, create_by, create_time, update_by, update_time, state, repo_id, repo_name
|
||||
id, name, description,available_range, model_type,model_tag, create_by, create_time, update_by, update_time, state
|
||||
from models
|
||||
where id = #{id} and state = 1
|
||||
</select>
|
||||
|
|
@ -29,7 +27,7 @@
|
|||
<!--查询单个-->
|
||||
<select id="findByName" resultMap="ModelsMap">
|
||||
select
|
||||
id, name, description,available_range, model_type,model_tag, create_by, create_time, update_by, update_time, state
|
||||
id, name, description,available_range, model_type,model_tag, create_by, create_time, update_by, update_time, state
|
||||
from models
|
||||
where name = #{name} and state = 1 limit 1
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue