diff --git a/.github/workflows/image-build-and-push-nightly.yml b/.github/workflows/image-build-and-push-nightly.yml index a42c787f..7eeab252 100644 --- a/.github/workflows/image-build-and-push-nightly.yml +++ b/.github/workflows/image-build-and-push-nightly.yml @@ -70,3 +70,10 @@ jobs: tags: qiskit/quantum-serverless-notebook:nightly-py310 build-args: IMAGE_PY_VERSION=3.10 + - name: Build and push repository server + uses: docker/build-push-action@v3 + with: + context: . + file: ./infrastructure/docker/Dockerfile-repository-server + push: true + tags: qiskit/quantum-repository-server:${{ github.event.inputs.tag }} diff --git a/.github/workflows/image-build-and-push.yml b/.github/workflows/image-build-and-push.yml index 4f177255..a08905f9 100644 --- a/.github/workflows/image-build-and-push.yml +++ b/.github/workflows/image-build-and-push.yml @@ -72,3 +72,11 @@ jobs: tags: qiskit/quantum-serverless-notebook:${{ github.event.inputs.tag }}-py310 build-args: IMAGE_PY_VERSION=3.10 + - name: Build and push repository server + uses: docker/build-push-action@v3 + with: + context: . + file: ./infrastructure/docker/Dockerfile-repository-server + push: true + tags: qiskit/quantum-repository-server:${{ github.event.inputs.tag }} + diff --git a/.github/workflows/repository-verify.yaml b/.github/workflows/repository-verify.yaml new file mode 100644 index 00000000..cba3706b --- /dev/null +++ b/.github/workflows/repository-verify.yaml @@ -0,0 +1,44 @@ +name: Repository verify process + +on: + pull_request: + +jobs: + verify-client: + name: lint, test + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8'] + + defaults: + run: + working-directory: ./repository + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Set up tox environment + run: | + pver=${{ matrix.python-version }} + tox_env="-epy${pver/./}" + echo tox_env + echo TOX_ENV=$tox_env >> $GITHUB_ENV + + - name: Install tox + run: | + pip install tox + + - name: Run styles check + run: tox -elint + + - name: Test using tox environment + run: | + tox ${{ env.TOX_ENV }} diff --git a/Makefile b/Makefile index 4974c2ce..161b0eb3 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ repository=qiskit notebookImageName=$(repository)/quantum-serverless-notebook rayNodeImageName=$(repository)/quantum-serverless-ray-node +repositoryServerImageName=$(repository)/quantum-repository-server # ============= # Docker images @@ -14,8 +15,8 @@ rayNodeImageName=$(repository)/quantum-serverless-ray-node build-and-push: build-all push-all -build-all: build-notebook build-ray-node -push-all: push-notebook push-ray-node +build-all: build-notebook build-ray-node build-repository-server +push-all: push-notebook push-ray-node push-repository-server build-notebook: docker build -t $(notebookImageName):$(version) -f ./infrastructure/docker/Dockerfile-notebook . @@ -23,8 +24,14 @@ build-notebook: build-ray-node: docker build -t $(rayNodeImageName):$(version) -f ./infrastructure/docker/Dockerfile-ray-qiskit . +build-repository-server: + docker build -t $(repositoryServerImageName):$(version) -f ./infrastructure/docker/Dockerfile-repository-server . + push-notebook: docker push $(notebookImageName):$(version) push-ray-node: docker push $(rayNodeImageName):$(version) + +push-repository-server: + docker push $(repositoryServerImageName):$(version) diff --git a/client/quantum_serverless/core/constants.py b/client/quantum_serverless/core/constants.py index 8b12de4b..8a0ca487 100644 --- a/client/quantum_serverless/core/constants.py +++ b/client/quantum_serverless/core/constants.py @@ -22,5 +22,11 @@ OT_SPAN_DEFAULT_NAME = "entrypoint" OT_ATTRIBUTE_PREFIX = "qs" OT_LABEL_CALL_LOCATION = "qs.location" + +# repository +REPO_HOST_KEY = "REPO_HOST_KEY" +REPO_PORT_KEY = "REPO_PORT_KEY" + + # container image RAY_IMAGE = "qiskit/quantum-serverless-ray-node:latest-py39" diff --git a/client/quantum_serverless/core/program.py b/client/quantum_serverless/core/program.py index 91ad25a5..aeb18c81 100644 --- a/client/quantum_serverless/core/program.py +++ b/client/quantum_serverless/core/program.py @@ -26,9 +26,22 @@ Quantum serverless nested program Program """ +import dataclasses +import json +import logging +import os +import tarfile from abc import ABC -from typing import Optional, Dict, List, Any from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Dict, List, Any + +import requests + +from quantum_serverless.core.constants import ( + REPO_HOST_KEY, + REPO_PORT_KEY, +) from quantum_serverless.exception import QuantumServerlessException from quantum_serverless.utils.json import is_jsonable @@ -39,7 +52,7 @@ class Program: # pylint: disable=too-many-instance-attributes """Serverless programs. Args: - name: program name + title: program name entrypoint: is a script that will be executed as a job ex: job.py arguments: arguments for entrypoint script @@ -50,7 +63,7 @@ class Program: # pylint: disable=too-many-instance-attributes version: version of a program """ - name: str + title: str entrypoint: str working_dir: str = "./" arguments: Optional[Dict[str, Any]] = None @@ -58,6 +71,13 @@ class Program: # pylint: disable=too-many-instance-attributes dependencies: Optional[List[str]] = None description: Optional[str] = None version: Optional[str] = None + tags: Optional[List[str]] = None + + @classmethod + def from_json(cls, data: Dict[str, Any]): + """Reconstructs Program from dictionary.""" + field_names = set(f.name for f in dataclasses.fields(Program)) + return Program(**{k: v for k, v in data.items() if k in field_names}) def __post_init__(self): if self.arguments is not None and not is_jsonable(self.arguments): @@ -80,22 +100,130 @@ class ProgramStorage(ABC): """ raise NotImplementedError - def get_programs(self) -> List[str]: + def get_programs(self, **kwargs) -> List[str]: """Returns list of available programs to get. + Args: + kwargs: filtering criteria + Returns: List of names of programs """ raise NotImplementedError - def get_program(self, name: str, **kwargs) -> Program: - """Returns program by name of other query criterieas. + def get_program(self, title: str, **kwargs) -> Optional[Program]: + """Returns program by name of other query criteria. Args: - name: name of program + title: title of the program **kwargs: other args Returns: Program """ raise NotImplementedError + + +class ProgramRepository(ProgramStorage): + """ProgramRepository.""" + + def __init__( + self, + host: Optional[str] = None, + port: Optional[int] = None, + token: Optional[str] = None, + folder: Optional[str] = None, + ): + """Program repository implementation. + + Args: + host: host of backend + port: port of backend + token: authentication token + folder: path to directory where title files will be stored + """ + self.folder = folder or os.path.dirname(os.path.abspath(__file__)) + self._host = host or os.environ.get(REPO_HOST_KEY, "http://localhost") + self._port = port or os.environ.get(REPO_PORT_KEY, 80) + self._token = token + self._base_url = f"{self._host}:{self._port}/v1/api/nested-programs/" + + def save_program(self, program: Program) -> bool: + raise NotImplementedError("Not implemented yet.") + + def get_programs(self, **kwargs) -> List[str]: + result = [] + response = requests.get(url=self._base_url, params=kwargs, timeout=10) + if response.ok: + response_data = json.loads(response.text) + result = [entry.get("title") for entry in response_data.get("results", [])] + return result + + def get_program(self, title: str, **kwargs) -> Optional[Program]: + result = None + response = requests.get( + url=f"{self._base_url}", + params={"title": title}, + allow_redirects=True, + timeout=10, + ) + if response.ok: + response_data = json.loads(response.text) + results = response_data.get("results", []) + if len(results) > 0: + artifact = results[0].get("artifact") + result = Program.from_json(results[0]) + result.working_dir = download_and_unpack_artifact( + artifact_url=artifact, program=result, folder=self.folder + ) + else: + logging.warning("No entries were found for your request.") + return result + + +def download_and_unpack_artifact( + artifact_url: str, + program: Program, + folder: str, + headers: Optional[Dict[str, Any]] = None, +) -> str: + """Downloads and extract artifact files into destination folder. + + Args: + artifact_url: url to get artifact from + program: program object artifact belongs to + folder: root of programs folder a.k.a unpack destination + headers: optional headers needed for download requests + + Returns: + workdir for program + """ + program_folder_path = os.path.join(folder, program.title) + artifact_file_name = "artifact" + tarfile_path = os.path.join(program_folder_path, artifact_file_name) + + # check if program already exist on the disc + if os.path.exists(program_folder_path): + logging.warning("Program folder already exist. Will be overwritten.") + + # download file + response = requests.get(url=artifact_url, stream=True, headers=headers, timeout=100) + if not response.ok: + raise QuantumServerlessException( + f"Error during fetch of [{artifact_url}] file." + ) + + Path(program_folder_path).mkdir(parents=True, exist_ok=True) + + with open(tarfile_path, "wb") as file: + for data in response.iter_content(): + file.write(data) + + # unpack tarfile + with tarfile.open(tarfile_path, "r") as file_obj: + file_obj.extractall(program_folder_path) + + # delete artifact + if os.path.exists(tarfile_path): + os.remove(tarfile_path) + return program_folder_path diff --git a/client/quantum_serverless/core/provider.py b/client/quantum_serverless/core/provider.py index 6993dc77..9691dc4f 100644 --- a/client/quantum_serverless/core/provider.py +++ b/client/quantum_serverless/core/provider.py @@ -254,7 +254,7 @@ class Provider(JsonSerializable): entrypoint = f"python {program.entrypoint} {arguments}" # set program name so OT can use it as parent span name - env_vars = {**(program.env_vars or {}), **{OT_PROGRAM_NAME: program.name}} + env_vars = {**(program.env_vars or {}), **{OT_PROGRAM_NAME: program.title}} job_id = job_client.submit_job( entrypoint=entrypoint, diff --git a/client/tests/core/test_program.py b/client/tests/core/test_program.py index ca54a8cb..70ad6fd2 100644 --- a/client/tests/core/test_program.py +++ b/client/tests/core/test_program.py @@ -23,7 +23,7 @@ class TestProgram(TestCase): def test_arguments_validation(self): """Tests arguments validation.""" program = Program( - name="awesome_program", + title="awesome_program", entrypoint="awesome.py", arguments={"one": 1, "json": {"one": 1, "two": 2}}, ) @@ -31,7 +31,7 @@ class TestProgram(TestCase): with self.assertRaises(QuantumServerlessException): Program( - name="awesome_program", + title="awesome_program", entrypoint="awesome.py", arguments={"one": 1, "json": {"one": np.array([1]), "two": 2}}, ) @@ -64,7 +64,7 @@ def test_program(): wait_for_job_client(serverless) program = Program( - name="simple_job", + title="simple_job", entrypoint="job.py", working_dir=resources_path, description="description", diff --git a/client/tests/core/test_program_repository.py b/client/tests/core/test_program_repository.py new file mode 100644 index 00000000..0ff52ee0 --- /dev/null +++ b/client/tests/core/test_program_repository.py @@ -0,0 +1,144 @@ +"""Test program repository.""" +import json +import os.path +import shutil +from pathlib import Path +from unittest import TestCase, mock + +from quantum_serverless.core.program import ProgramRepository, Program + +responses = { + "http://localhost:80/v1/api/nested-programs/": { + "count": 2, + "results": [ + { + "id": "be7e5406-d111-4768-9a93-7cedf4d46608", + "created": "2023-02-15T14:38:41.120934Z", + "updated": "2023-02-15T14:38:41.120975Z", + "title": "hello_world", + "description": "", + "entrypoint": "hello_world.py", + "working_dir": "./", + "version": "0.0.0", + "dependencies": None, + "env_vars": None, + "arguments": None, + "tags": None, + "public": None, + "artifact": "urL_for_artifact", + }, + { + "id": "92d47ca1-adec-4407-b072-857265d0b02a", + "created": "2023-02-15T16:05:12.791987Z", + "updated": "2023-02-15T16:05:12.792044Z", + "title": "Test", + "description": "Test", + "entrypoint": "test.py", + "working_dir": "./", + "version": "0.0.0", + "dependencies": None, + "env_vars": {"DEBUG": "1"}, + "arguments": None, + "tags": None, + "public": None, + "artifact": "urL_for_artifact", + }, + ], + } +} + + +class MockResponse: + """MockResponse.""" + + def __init__(self, json_data): + """General mock response. + + Args: + json_data: data in response + """ + self.json_data = json_data + + @property + def text(self): + """Text of response.""" + return json.dumps(self.json_data) + + @property + def ok(self): # pylint: disable=invalid-name + """Status of response.""" + return True + + +class MockedStreamingResponse: + """MockedStreamingResponse.""" + + def __init__(self, file_path: str): + self.file_path = file_path + + def iter_content(self): + """Iterate through file content.""" + content = [] + with open(self.file_path, "rb") as file: + content.append(file.read()) + return content + + @property + def ok(self): # pylint: disable=invalid-name + """Status of response.""" + return True + + +def mocked_requests_get(**kwargs): + """Mock request side effect.""" + url = kwargs.get("url") + stream = kwargs.get("stream", False) + result = None + if not stream and url: + result = MockResponse(responses.get(url, {})) + if stream and url: + result = MockedStreamingResponse( + file_path=os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "resources", + "program.tar", + ) + ) + return result + + +class TestRepository(TestCase): + """Tests for repository.""" + + def setUp(self) -> None: + self.resources_folder = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "resources" + ) + self.programs_folder = os.path.join(self.resources_folder, "programs") + Path(self.programs_folder).mkdir(parents=True, exist_ok=True) + + def tearDown(self) -> None: + if os.path.exists(self.programs_folder): + shutil.rmtree(self.programs_folder) + + @mock.patch("requests.get", side_effect=mocked_requests_get) + def test_repository_get_programs(self, mock_get): + """Tests program repository.""" + + repository = ProgramRepository(host="http://localhost") + programs = repository.get_programs() + self.assertEqual(programs, ["hello_world", "Test"]) + self.assertEqual(len(mock_get.call_args_list), 1) + + @mock.patch("requests.get", side_effect=mocked_requests_get) + def test_repository_get_program(self, mock_get): + """Tests single program fetch.""" + repository = ProgramRepository( + host="http://localhost", folder=self.programs_folder + ) + program = repository.get_program("hello_world") + self.assertEqual(program.title, "hello_world") + self.assertEqual(program.version, "0.0.0") + self.assertIsInstance(program, Program) + self.assertEqual(len(mock_get.call_args_list), 2) diff --git a/client/tests/resources/program.tar b/client/tests/resources/program.tar new file mode 100644 index 00000000..a3a73711 Binary files /dev/null and b/client/tests/resources/program.tar differ diff --git a/infrastructure/docker/Dockerfile-repository-server b/infrastructure/docker/Dockerfile-repository-server new file mode 100644 index 00000000..beed7740 --- /dev/null +++ b/infrastructure/docker/Dockerfile-repository-server @@ -0,0 +1,16 @@ +FROM registry.access.redhat.com/ubi9/python-39@sha256:9d69d5f28b58f78b7b366783bc8785c59af37187c90e25b3ed582942c4c72919 + +WORKDIR /app + +USER 0 +COPY repository . +RUN chown -R 1001:0 /app +USER 1001 + +RUN pip3 install -r requirements.txt +RUN pip3 install -r requirements-dev.txt +RUN python manage.py migrate + +EXPOSE 8000 +CMD ["python3", "manage.py", "runserver", "0.0.0.0:8000"] + diff --git a/infrastructure/docker/README.md b/infrastructure/docker/README.md index 6ee3cd68..de53a83a 100644 --- a/infrastructure/docker/README.md +++ b/infrastructure/docker/README.md @@ -1,8 +1,9 @@ # Docker images -Docker images that the infrastructure requires to deploy. There are two main images: +Docker images that the infrastructure requires to deploy. There are three main images: - [Jupyter notebook](#ray-node-with-jupyter-notebook) - [Ray](#custom-ray) +- [Repository server](#repository-server) ## Custom ray @@ -49,3 +50,24 @@ docker build -f ./infrastructure/docker/Dockerfile-notebook -t . ### Versions - Python == 3.9 + + +## Repository Server + +An image of the repository server. + +### Build +To build image just run from the root of the project: + +```shell +make build-repository-server +``` + +or in case you want to customize as much as possible your build: + +```shell +docker build -f ./infrastructure/docker/Dockerfile-repository-server -t . +``` + +### Versions +- Python == 3.9 diff --git a/infrastructure/helm/quantumserverless/Chart.yaml b/infrastructure/helm/quantumserverless/Chart.yaml index 06600215..38658475 100644 --- a/infrastructure/helm/quantumserverless/Chart.yaml +++ b/infrastructure/helm/quantumserverless/Chart.yaml @@ -35,6 +35,9 @@ dependencies: condition: keycloakEnable version: 13.3.0 repository: https://charts.bitnami.com/bitnami + - name: repository + condition: repositoryEnable + version: 0.1.0 - name: prometheus condition: prometheusEnable version: 19.7.2 diff --git a/infrastructure/helm/quantumserverless/charts/repository/.helmignore b/infrastructure/helm/quantumserverless/charts/repository/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/infrastructure/helm/quantumserverless/charts/repository/Chart.yaml b/infrastructure/helm/quantumserverless/charts/repository/Chart.yaml new file mode 100644 index 00000000..2719dc83 --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: repository +description: Helm chart to deploy repository API in kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/infrastructure/helm/quantumserverless/charts/repository/templates/NOTES.txt b/infrastructure/helm/quantumserverless/charts/repository/templates/NOTES.txt new file mode 100644 index 00000000..b417432c --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "repository.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "repository.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "repository.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "repository.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/infrastructure/helm/quantumserverless/charts/repository/templates/_helpers.tpl b/infrastructure/helm/quantumserverless/charts/repository/templates/_helpers.tpl new file mode 100644 index 00000000..aa1e4f91 --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "repository.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "repository.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "repository.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "repository.labels" -}} +helm.sh/chart: {{ include "repository.chart" . }} +{{ include "repository.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "repository.selectorLabels" -}} +app.kubernetes.io/name: {{ include "repository.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "repository.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "repository.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/infrastructure/helm/quantumserverless/charts/repository/templates/deployment.yaml b/infrastructure/helm/quantumserverless/charts/repository/templates/deployment.yaml new file mode 100644 index 00000000..ba7631a8 --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/templates/deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "repository.fullname" . }} + labels: + {{- include "repository.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "repository.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "repository.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "repository.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + #livenessProbe: + # httpGet: + # path: / + # port: http + #readinessProbe: + # httpGet: + # path: / + # port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/infrastructure/helm/quantumserverless/charts/repository/templates/hpa.yaml b/infrastructure/helm/quantumserverless/charts/repository/templates/hpa.yaml new file mode 100644 index 00000000..981b9bc9 --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "repository.fullname" . }} + labels: + {{- include "repository.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "repository.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/infrastructure/helm/quantumserverless/charts/repository/templates/ingress.yaml b/infrastructure/helm/quantumserverless/charts/repository/templates/ingress.yaml new file mode 100644 index 00000000..74ec2ee8 --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "repository.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "repository.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/infrastructure/helm/quantumserverless/charts/repository/templates/service.yaml b/infrastructure/helm/quantumserverless/charts/repository/templates/service.yaml new file mode 100644 index 00000000..d9a7b402 --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "repository.fullname" . }} + labels: + {{- include "repository.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "repository.selectorLabels" . | nindent 4 }} diff --git a/infrastructure/helm/quantumserverless/charts/repository/templates/serviceaccount.yaml b/infrastructure/helm/quantumserverless/charts/repository/templates/serviceaccount.yaml new file mode 100644 index 00000000..c375abee --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "repository.serviceAccountName" . }} + labels: + {{- include "repository.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/infrastructure/helm/quantumserverless/charts/repository/templates/tests/test-connection.yaml b/infrastructure/helm/quantumserverless/charts/repository/templates/tests/test-connection.yaml new file mode 100644 index 00000000..84e95add --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "repository.fullname" . }}-test-connection" + labels: + {{- include "repository.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "repository.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/infrastructure/helm/quantumserverless/charts/repository/values.yaml b/infrastructure/helm/quantumserverless/charts/repository/values.yaml new file mode 100644 index 00000000..dbef1525 --- /dev/null +++ b/infrastructure/helm/quantumserverless/charts/repository/values.yaml @@ -0,0 +1,82 @@ +# Default values for repository. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: qiskit/quantum-repository-server + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: NodePort + port: 8000 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/infrastructure/helm/quantumserverless/values.yaml b/infrastructure/helm/quantumserverless/values.yaml index 611acb5e..4faa291d 100644 --- a/infrastructure/helm/quantumserverless/values.yaml +++ b/infrastructure/helm/quantumserverless/values.yaml @@ -258,6 +258,12 @@ keycloak: mountPath: /opt/bitnami/keycloak/data/import extraStartupArgs: "--import-realm" +# =================== +# Quantum Repository +# =================== + +repositoryEnable: true + # =================== # Prometheus # =================== @@ -265,4 +271,3 @@ keycloak: prometheusEnable: true # prometheus: # special configuration over the prometheus sub-chart - diff --git a/repository/.gitignore b/repository/.gitignore new file mode 100644 index 00000000..3742525f --- /dev/null +++ b/repository/.gitignore @@ -0,0 +1,138 @@ +# Django # +*.log +*.pot +*.pyc +__pycache__ +db.sqlite3 +media +.DS_Store + +# Backup files # +*.bak + +# If you are using PyCharm # +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# File-based project format +*.iws + +# IntelliJ +out/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Python # +*.py[cod] +*$py.class + +# Distribution / packaging +.Python build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Sublime Text # +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache +*.sublime-workspace +*.sublime-project + +# sftp configuration file +sftp-config.json + +# Package control specific files Package +Control.last-run +Control.ca-list +Control.ca-bundle +Control.system-ca-bundle +GitHub.sublime-settings + +# Visual Studio Code # +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history \ No newline at end of file diff --git a/repository/README.md b/repository/README.md new file mode 100644 index 00000000..2443c22b --- /dev/null +++ b/repository/README.md @@ -0,0 +1,40 @@ +[![Stability](https://img.shields.io/badge/stability-alpha-f4d03f.svg)](https://github.com/Qiskit-Extensions/quantum-serverless/releases) +[![Repository verify process](https://github.com/Qiskit-Extensions/quantum-serverless/actions/workflows/repository-verify.yaml/badge.svg)](https://github.com/Qiskit-Extensions/quantum-serverless/actions/workflows/repository-verify.yaml) +[![License](https://img.shields.io/github/license/qiskit-community/quantum-prototype-template?label=License)](https://github.com/qiskit-community/quantum-prototype-template/blob/main/LICENSE.txt) +[![Code style: Black](https://img.shields.io/badge/Code%20style-Black-000.svg)](https://github.com/psf/black) +[![Python](https://img.shields.io/badge/3.8%20%7C%203.9%20%7C%203.10-informational)](https://www.python.org/) + +# Quantum Serverless repository + +Repository API for the quantum serverless project. +It manages the access to resources like: programs, users and authentication flow. + +### Table of Contents + +1. [Installation](#installation) +2. [Usage](#usage) + +---------------------------------------------------------------------------------------------------- + +### Installation + +```shell +pip install -U -r requirements.txt -r requirements-dev.txt +``` + +---------------------------------------------------------------------------------------------------- + +### Usage + +To run the API you just need to execute: + +```shell +python manage.py runserver +``` + +This command will run the API in port `8000`. +If this is your first run you will need to apply the database changes first: + +```shell +python manage.py migrate +``` diff --git a/repository/api/__init__.py b/repository/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repository/api/admin.py b/repository/api/admin.py new file mode 100644 index 00000000..3285f4a7 --- /dev/null +++ b/repository/api/admin.py @@ -0,0 +1,8 @@ +""" +Register the models for the admin panel. +""" + +from django.contrib import admin +from .models import NestedProgram + +admin.site.register(NestedProgram) diff --git a/repository/api/apps.py b/repository/api/apps.py new file mode 100644 index 00000000..f708d366 --- /dev/null +++ b/repository/api/apps.py @@ -0,0 +1,14 @@ +""" +Configuration for the api application. +""" + +from django.apps import AppConfig + + +class ApiConfig(AppConfig): + """ + Application api configuration class. + """ + + default_auto_field = "django.db.models.BigAutoField" + name = "api" diff --git a/repository/api/migrations/0001_initial.py b/repository/api/migrations/0001_initial.py new file mode 100644 index 00000000..6bcf2bbe --- /dev/null +++ b/repository/api/migrations/0001_initial.py @@ -0,0 +1,51 @@ +# Generated by Django 4.1.6 on 2023-03-15 11:06 + +import api.models +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="NestedProgram", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ("title", models.CharField(max_length=255)), + ("description", models.TextField(blank=True, default="")), + ("entrypoint", models.CharField(max_length=255)), + ("working_dir", models.CharField(default="./", max_length=255)), + ("version", models.CharField(default="0.0.0", max_length=100)), + ( + "dependencies", + models.JSONField(default=api.models.empty_list, null=True), + ), + ( + "env_vars", + models.JSONField(default=api.models.empty_dict, null=True), + ), + ( + "arguments", + models.JSONField(default=api.models.empty_dict, null=True), + ), + ("tags", models.JSONField(default=api.models.empty_list, null=True)), + ("public", models.BooleanField(default=True)), + ("artifact", models.FileField(upload_to="artifacts_%Y_%m_%d")), + ], + ), + ] diff --git a/repository/api/migrations/__init__.py b/repository/api/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repository/api/models.py b/repository/api/models.py new file mode 100644 index 00000000..f322617f --- /dev/null +++ b/repository/api/models.py @@ -0,0 +1,45 @@ +""" +Django Rest framework models for api application: + - NestedProgram +""" + +import uuid +from django.db import models + + +def empty_list(): + """ + Returns an empty list. + """ + return [] + + +def empty_dict(): + """ + Returns an empty dict. + """ + return {} + + +class NestedProgram(models.Model): + """ + Nested Program database model. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + # author = TODO: relationship with user, pending review integration with keycloack + title = models.CharField(max_length=255, blank=False, null=False) + description = models.TextField(blank=True, default="") + entrypoint = models.CharField(max_length=255, blank=False, null=False) + working_dir = models.CharField( + max_length=255, blank=False, null=False, default="./" + ) + version = models.CharField(max_length=100, blank=False, null=False, default="0.0.0") + dependencies = models.JSONField(null=True, default=empty_list) + env_vars = models.JSONField(null=True, default=empty_dict) + arguments = models.JSONField(null=True, default=empty_dict) + tags = models.JSONField(null=True, default=empty_list) + public = models.BooleanField(default=True) + artifact = models.FileField(upload_to="artifacts_%Y_%m_%d", null=False, blank=False) diff --git a/repository/api/serializers.py b/repository/api/serializers.py new file mode 100644 index 00000000..37ef212c --- /dev/null +++ b/repository/api/serializers.py @@ -0,0 +1,27 @@ +""" +Django Rest framework serializers for api application: + - NestedProgramSerializer + +Version serializers inherit from the different serializers. +""" + +from rest_framework import serializers +from .models import NestedProgram +from .validators import list_validator, dict_validator + + +class NestedProgramSerializer(serializers.ModelSerializer): + """ + Serializer for the nested program model. + """ + + class Meta: + model = NestedProgram + validators = [ + list_validator.ListValidator( + fields=["dependencies", "tags"], nullable=True + ), + dict_validator.DictValidator( + fields=["env_vars", "arguments"], nullable=True + ), + ] diff --git a/repository/api/v1/__init__.py b/repository/api/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repository/api/v1/serializers.py b/repository/api/v1/serializers.py new file mode 100644 index 00000000..65f0024d --- /dev/null +++ b/repository/api/v1/serializers.py @@ -0,0 +1,29 @@ +""" +Serializers api for V1. +""" + +from api import serializers + + +class NestedProgramSerializer(serializers.NestedProgramSerializer): + """ + Nested program serializer first version. Include basic fields from the initial model. + """ + + class Meta(serializers.NestedProgramSerializer.Meta): + fields = ( + "id", + "created", + "updated", + "title", + "description", + "entrypoint", + "working_dir", + "version", + "dependencies", + "env_vars", + "arguments", + "tags", + "public", + "artifact", + ) diff --git a/repository/api/v1/urls.py b/repository/api/v1/urls.py new file mode 100644 index 00000000..b27a799a --- /dev/null +++ b/repository/api/v1/urls.py @@ -0,0 +1,15 @@ +""" +URL Patterns for V1 api application. +""" + +from rest_framework import routers +from . import views as v1_views + +router = routers.DefaultRouter() +router.register( + r"nested-programs", + v1_views.NestedProgramViewSet, + basename=v1_views.NestedProgramViewSet.BASE_NAME, +) + +urlpatterns = router.urls diff --git a/repository/api/v1/views.py b/repository/api/v1/views.py new file mode 100644 index 00000000..821aed72 --- /dev/null +++ b/repository/api/v1/views.py @@ -0,0 +1,16 @@ +""" +Views api for V1. +""" + +from api import views +from . import serializers as v1_serializers + + +class NestedProgramViewSet( + views.NestedProgramViewSet +): # pylint: disable=too-many-ancestors + """ + Nested program view set first version. Use NestedProgramSerializer V1. + """ + + serializer_class = v1_serializers.NestedProgramSerializer diff --git a/repository/api/validators/__init__.py b/repository/api/validators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repository/api/validators/dict_validator.py b/repository/api/validators/dict_validator.py new file mode 100644 index 00000000..4870e9a6 --- /dev/null +++ b/repository/api/validators/dict_validator.py @@ -0,0 +1,55 @@ +""" +Dict validator to be used in the serializers. +""" + +from typing import Any, Dict, List, OrderedDict, Union +from rest_framework import serializers + + +# pylint: disable=duplicate-code +class DictValidator: + """ + This validator checks if a specific set of fields contains: + - A value of type `dict` + - If the value can be `null` or not + + :param fields: set of fields that will be checked by this validator + :param nullable: specifies if the validator may accept `null` values + """ + + def __init__(self, fields: List[str], nullable=True): + self.fields = fields + self.nullable = nullable + + def __call__(self, attrs: OrderedDict[str, Any]): + error_messages = {} + + for field in self.fields: + error_message = self.validate(field, attrs[field]) + if error_message is not None: + error_messages.update(error_message) + + if error_messages: + raise serializers.ValidationError(error_messages) + + def validate(self, field: str, value: Any) -> Union[Dict[str, str], None]: + """ + Method that checks if the value of a specific field is a valid `dict`. + + :param field: name of the model attribute + :param value: field content to be checked + :return: + an error message of type: + `{ f"{field}": f"{error_message}" }` + if the value is not valid + """ + + if value is None: + if not self.nullable: + return {f"{field}": "This field may not be null."} + else: + # Using `type` instead of `isinstance` to validate that it is a dict and no a subtype + value_type = type(value) + if value_type is not dict: + return {f"{field}": "This field must be a valid dict."} + return None diff --git a/repository/api/validators/list_validator.py b/repository/api/validators/list_validator.py new file mode 100644 index 00000000..538c69aa --- /dev/null +++ b/repository/api/validators/list_validator.py @@ -0,0 +1,55 @@ +""" +List validator to be used in the serializers. +""" + +from typing import Any, Dict, List, OrderedDict, Union +from rest_framework import serializers + + +# pylint: disable=duplicate-code +class ListValidator: + """ + This validator checks if a specific set of fields contains: + - A value of type `list` + - If the value can be `null` or not + + :param fields: set of fields that will be checked by this validator + :param nullable: specifies if the validator may accept `null` values + """ + + def __init__(self, fields: List[str], nullable=True): + self.fields = fields + self.nullable = nullable + + def __call__(self, attrs: OrderedDict[str, Any]): + error_messages = {} + + for field in self.fields: + error_message = self.validate(field, attrs[field]) + if error_message is not None: + error_messages.update(error_message) + + if error_messages: + raise serializers.ValidationError(error_messages) + + def validate(self, field: str, value: Any) -> Union[Dict[str, str], None]: + """ + Method that checks if the value of a specific field is a valid `list`. + + :param field: name of the model attribute + :param value: field content to be checked + :return: + an error message of type: + `{ f"{field}": f"{error_message}" }` + if the value is not valid + """ + + if value is None: + if not self.nullable: + return {f"{field}": "This field may not be null."} + else: + # Using `type` instead of `isinstance` to validate that it is a list and no a subtype + value_type = type(value) + if value_type is not list: + return {f"{field}": "This field must be a valid list."} + return None diff --git a/repository/api/views.py b/repository/api/views.py new file mode 100644 index 00000000..5d62fccb --- /dev/null +++ b/repository/api/views.py @@ -0,0 +1,32 @@ +""" +Django Rest framework views for api application: + - Nested Program ViewSet + +Version views inherit from the different views. +""" + +from rest_framework import permissions +from rest_framework import viewsets + +from .models import NestedProgram + + +class NestedProgramViewSet(viewsets.ModelViewSet): # pylint: disable=too-many-ancestors + """ + Nested Program ViewSet configuration using ModelViewSet. + """ + + BASE_NAME = "nested-programs" + + queryset = NestedProgram.objects.all().order_by("created") + permission_classes = [permissions.IsAuthenticatedOrReadOnly] + + def get_queryset(self): + query_params = self.request.query_params + queryset = NestedProgram.objects.all().order_by("created") + + # if name is specified in query parameters + title = query_params.get("title", None) + if title: + queryset = queryset.filter(title__exact=title) + return queryset diff --git a/repository/main/__init__.py b/repository/main/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repository/main/asgi.py b/repository/main/asgi.py new file mode 100644 index 00000000..3f8777f3 --- /dev/null +++ b/repository/main/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for main project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "main.settings") + +application = get_asgi_application() diff --git a/repository/main/settings.py b/repository/main/settings.py new file mode 100644 index 00000000..dc21372e --- /dev/null +++ b/repository/main/settings.py @@ -0,0 +1,140 @@ +""" +Django settings for main project. + +Generated by 'django-admin startproject' using Django 4.1.6. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.1/ref/settings/ +""" +import os +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "django-insecure-o%6p_(-fq39^ihz_ca3@mp14%+nks%!a-i9gma@*1qp9g9-(p2" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "rest_framework", + "drf_yasg", + "api", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "main.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "main.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/4.1/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.1/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.1/howto/static-files/ + +STATIC_URL = "static/" + +MEDIA_ROOT = os.path.join(BASE_DIR, "media") +MEDIA_URL = "/media/" + +# Default primary key field type +# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +# Django Rest configuration +REST_FRAMEWORK = { + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", + "PAGE_SIZE": 10, + "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.NamespaceVersioning", + "DEFAULT_VERSION": "v1", + "VERSION_PARAM": "version", +} + +SWAGGER_SETTINGS = {"LOGIN_URL": "/api-auth/login/", "LOGOUT_URL": "/api-auth/logout/"} diff --git a/repository/main/urls.py b/repository/main/urls.py new file mode 100644 index 00000000..0edd550f --- /dev/null +++ b/repository/main/urls.py @@ -0,0 +1,53 @@ +"""main URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.1/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.conf import settings +from django.conf.urls.static import static +from django.contrib import admin +from django.urls import include, path, re_path +from rest_framework import permissions +from drf_yasg.views import get_schema_view +from drf_yasg import openapi + +schema = get_schema_view( # pylint: disable=invalid-name + openapi.Info( + title="Program repository API", + default_version="v1", + description="List of available API endpoint for Program repository.", + ), + public=False, + permission_classes=[permissions.AllowAny], +) + +urlpatterns = [ + path("admin/", admin.site.urls), + re_path(r"^v1/api/", include(("api.v1.urls", "api"), namespace="v1")), + path("api-auth/", include("rest_framework.urls", namespace="rest_framework")), + # docs + re_path( + r"^swagger(?P\.json|\.yaml)$", + schema.without_ui(cache_timeout=0), + name="schema-json", + ), + re_path( + r"^swagger/$", + schema.with_ui("swagger", cache_timeout=0), + name="schema-swagger-ui", + ), + re_path(r"^redoc/$", schema.with_ui("redoc", cache_timeout=0), name="schema-redoc"), +] + +if settings.DEBUG: + urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/repository/main/wsgi.py b/repository/main/wsgi.py new file mode 100644 index 00000000..c9abe16b --- /dev/null +++ b/repository/main/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for main project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "main.settings") + +application = get_wsgi_application() diff --git a/repository/manage.py b/repository/manage.py new file mode 100755 index 00000000..63d3ae89 --- /dev/null +++ b/repository/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "main.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/repository/requirements-dev.txt b/repository/requirements-dev.txt new file mode 100644 index 00000000..3df2bfc9 --- /dev/null +++ b/repository/requirements-dev.txt @@ -0,0 +1,6 @@ +pylint>=2.9.5 +pytest>=6.2.5 +pylint-django>=2.5.3 +# Black's formatting rules can change between major versions, so we use +# the ~= specifier for it. +black~=22.1 \ No newline at end of file diff --git a/repository/requirements.txt b/repository/requirements.txt new file mode 100644 index 00000000..da2e3630 --- /dev/null +++ b/repository/requirements.txt @@ -0,0 +1,10 @@ +asgiref>=3.6.0 +Django>=4.1.6 +django-filter>=22.1 +djangorestframework>=3.14.0 +importlib-metadata>=6.0.0 +Markdown>=3.4.1 +pytz>=2022.7.1 +sqlparse>=0.4.3 +zipp>=3.12.0 +drf-yasg>=1.21.5 diff --git a/repository/tests/__init__.py b/repository/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repository/tests/core/__init__.py b/repository/tests/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repository/tests/core/test_v1_nested_program.py b/repository/tests/core/test_v1_nested_program.py new file mode 100644 index 00000000..f7f85e5a --- /dev/null +++ b/repository/tests/core/test_v1_nested_program.py @@ -0,0 +1,247 @@ +import json +import os.path +from pathlib import Path + +from django.contrib.auth.models import User +from django.core.files import File +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +from api.models import NestedProgram + + +class NestedProgramTests(APITestCase): + fixtures = ["tests/fixtures/initial_data.json"] + + def test_get_nested_program_returns_200(self): + """ + Retrieve information about a specific nested program + """ + nested_program_id = "1a7947f9-6ae8-4e3d-ac1e-e7d608deec82" + url = reverse("v1:nested-programs-detail", args=[nested_program_id]) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_get_non_created_nested_program_returns_404(self): + """ + Retrieve information about a specific nested program that doesn't exist returns a 404 + """ + url = reverse("v1:nested-programs-detail", args=[2]) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_create_nested_program_unauthenticated_returns_403(self): + """ + Create a nested program without being authenticated returns a 403 + """ + nested_program_input = {} + + url = reverse("v1:nested-programs-list") + response = self.client.post(url, data=nested_program_input, format="json") + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_create_nested_program_with_empty_object_validation(self): + """ + Create a nested program with an empty object as input should return a validation error + """ + nested_program_input = {} + fields_to_check = ["title", "entrypoint", "artifact"] + test_user = User.objects.get(username="test_user") + + self.client.force_login(test_user) + + url = reverse("v1:nested-programs-list") + response = self.client.post(url, data=nested_program_input, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + failed_validation_fields_list = list(response.json().keys()) + self.assertListEqual(failed_validation_fields_list, fields_to_check) + + def test_create_nested_program_with_blank_values_validation(self): + """ + Create a nested program with an object with blank values should return a validation error + """ + nested_program_input = { + "title": "", + "description": "", + "entrypoint": "", + "working_dir": "", + "version": "", + "dependencies": None, + "env_vars": None, + "arguments": None, + "tags": None, + "public": True, + } + fields_to_check = ["title", "entrypoint", "working_dir", "version", "artifact"] + test_user = User.objects.get(username="test_user") + + self.client.force_login(test_user) + + url = reverse("v1:nested-programs-list") + response = self.client.post(url, data=nested_program_input, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + failed_validation_fields_list = list(response.json().keys()) + self.assertListEqual(failed_validation_fields_list, fields_to_check) + + def test_create_nested_program_returns_201(self): + """ + Create a nested program + """ + nested_program_input = { + "title": "Awesome nested program", + "description": "Awesome nested program description", + "entrypoint": "nested_program.py", + "working_dir": "./", + "version": "0.0.1", + "env_vars": json.dumps({"DEBUG": True}), + "arguments": json.dumps({}), + "tags": json.dumps(["dev"]), + "dependencies": json.dumps([]), + "public": True, + } + test_user = User.objects.get(username="test_user") + + self.client.force_login(test_user) + + url = reverse("v1:nested-programs-list") + with open( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "..", + "fixtures", + "initial_data.json", + ) + ) as file: + nested_program_input["artifact"] = File(file) + response = self.client.post( + url, data=nested_program_input, format="multipart" + ) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(NestedProgram.objects.count(), 2) + + def test_count_of_all_nested_programs_created_must_be_one(self): + """ + List all the nested programs created and check that there is one + """ + url = reverse("v1:nested-programs-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json()["count"], 1) + + def test_delete_nested_program_unauthenticated_returns_403(self): + """ + Delete a nested program with an unauthenticated user + """ + nested_program_id = "1a7947f9-6ae8-4e3d-ac1e-e7d608deec82" + + url = reverse("v1:nested-programs-detail", args=[nested_program_id]) + response = self.client.delete(url, format="json") + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_delete_a_non_created_nested_program_returns_404(self): + """ + Delete a nested program that doesn't exist returns a 404 + """ + test_user = User.objects.get(username="test_user") + + self.client.force_login(test_user) + + url = reverse("v1:nested-programs-detail", args=[2]) + response = self.client.delete(url, format="json") + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_delete_nested_program_returns_204(self): + """ + Delete a nested program that exists 204 + """ + nested_program_id = "1a7947f9-6ae8-4e3d-ac1e-e7d608deec82" + test_user = User.objects.get(username="test_user") + + self.client.force_login(test_user) + + url = reverse("v1:nested-programs-detail", args=[nested_program_id]) + response = self.client.delete(url, format="json") + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertEqual(NestedProgram.objects.count(), 0) + + def test_nested_program_list_validation_returns_400(self): + """ + The value for dependencies and tags is a dict, a non-allowed value for these fields and returns a 400 + """ + fields_to_check = ["dependencies", "tags"] + nested_program_input = { + "title": "Awesome nested program", + "description": "Awesome nested program description", + "entrypoint": "nested_program.py", + "working_dir": "./", + "version": "0.0.1", + "dependencies": json.dumps({}), + "env_vars": json.dumps({"DEBUG": True}), + "arguments": json.dumps(None), + "tags": json.dumps({}), + "public": True, + } + test_user = User.objects.get(username="test_user") + + self.client.force_login(test_user) + + url = reverse("v1:nested-programs-list") + with open( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "..", + "fixtures", + "initial_data.json", + ) + ) as file: + nested_program_input["artifact"] = File(file) + response = self.client.post( + url, data=nested_program_input, format="multipart" + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + failed_validation_fields_list = list(response.json().keys()) + self.assertListEqual(failed_validation_fields_list, fields_to_check) + + def test_nested_program_dict_validation_returns_400(self): + """ + The value for env_vars and arguments is a list, a non-allowed value for these fields and returns a 400 + """ + fields_to_check = ["env_vars", "arguments"] + nested_program_input = { + "title": "Awesome nested program", + "description": "Awesome nested program description", + "entrypoint": "nested_program.py", + "working_dir": "./", + "version": "0.0.1", + "dependencies": json.dumps([]), + "env_vars": json.dumps([]), + "arguments": json.dumps([]), + "tags": json.dumps(["dev"]), + "public": True, + } + test_user = User.objects.get(username="test_user") + + self.client.force_login(test_user) + + url = reverse("v1:nested-programs-list") + with open( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "..", + "fixtures", + "initial_data.json", + ) + ) as file: + nested_program_input["artifact"] = File(file) + response = self.client.post( + url, data=nested_program_input, format="multipart" + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + failed_validation_fields_list = list(response.json().keys()) + self.assertListEqual(failed_validation_fields_list, fields_to_check) diff --git a/repository/tests/fixtures/initial_data.json b/repository/tests/fixtures/initial_data.json new file mode 100644 index 00000000..9bb256b4 --- /dev/null +++ b/repository/tests/fixtures/initial_data.json @@ -0,0 +1,33 @@ +[ + { + "model": "auth.user", + "pk": 1, + "fields": { + "username": "test_user", + "password": "password_for_test_user" + } + }, + { + "model": "api.nestedprogram", + "pk": "1a7947f9-6ae8-4e3d-ac1e-e7d608deec82", + "fields": { + "created": "2023-02-01T15:30:43.281796Z", + "updated": "2023-02-01T15:30:43.281830Z", + "title": "Nested program", + "description": "Best nested program in the world", + "entrypoint": "nested_program.py", + "working_dir": "./", + "version": "0.0.0", + "dependencies": null, + "env_vars": { + "DEBUG": "true" + }, + "arguments": null, + "tags": [ + "dev" + ], + "public": true, + "artifact": "path" + } + } +] diff --git a/repository/tests/resources/__init__.py b/repository/tests/resources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repository/tox.ini b/repository/tox.ini new file mode 100644 index 00000000..031c9a56 --- /dev/null +++ b/repository/tox.ini @@ -0,0 +1,35 @@ +[tox] +minversion = 2.1 +envlist = py38, py39, py310, lint, coverage +# CI: skip-next-line +skipsdist = true +# CI: skip-next-line +skip_missing_interpreters = true + +[testenv] +# CI: skip-next-line +usedevelop = true +install_command = + pip install -U {opts} {packages} +setenv = + VIRTUAL_ENV={envdir} + LANGUAGE=en_US + LC_ALL=en_US.utf-8 + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python +deps = -rrequirements.txt + -rrequirements-dev.txt +commands = + pip check + python manage.py test + +[testenv:lint] +envdir = .tox/lint +skip_install = true +commands = + black --check . + pylint --load-plugins pylint_django --load-plugins pylint_django.checkers.migrations --django-settings-module=main.settings --ignore api.migrations -rn api main + +[testenv:black] +envdir = .tox/lint +skip_install = true +commands = black .