From ed1fb1279fed0c163b0fbf50c393ff70c5e00556 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Thu, 15 Jun 2023 17:17:56 +0200 Subject: [PATCH] ci: add docker building steps --- .github/workflows/force-docker-build.yml | 135 ++---------------- .github/workflows/force-docs-build.yml | 113 --------------- .../{Dockerfile => vectordb.Dockerfile} | 0 3 files changed, 15 insertions(+), 233 deletions(-) delete mode 100644 .github/workflows/force-docs-build.yml rename Dockerfiles/{Dockerfile => vectordb.Dockerfile} (100%) diff --git a/.github/workflows/force-docker-build.yml b/.github/workflows/force-docker-build.yml index 39e1bf1..b2bedf6 100644 --- a/.github/workflows/force-docker-build.yml +++ b/.github/workflows/force-docker-build.yml @@ -21,118 +21,28 @@ jobs: touch SUCCESS if: inputs.release_token == env.release_token env: - release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }} + release_token: ${{ secrets.VECTORDB_RELEASE_TOKEN }} - name: Fail release token run: | [[ -f SUCCESS ]] - regular-release: needs: token-check runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - pip_tag: [ "", "perf", "standard", "devel"] # default: "" = core - py_version: [ "3.7", "3.8", "3.9" , "3.10", "3.11"] # default "" = 3.7 steps: - uses: actions/checkout@v2.5.0 with: fetch-depth: 100 - - name: Set envs and versions + - name: Set up Python 3.9 + uses: actions/setup-python@v1 + with: + python-version: 3.9 + + - name: Get vectordb version run: | - DEFAULT_PY_VERSION="3.8" - VCS_REF=${{ github.ref }} - echo "VCS_REF=$VCS_REF" >> $GITHUB_ENV - echo "Will build $VCS_REF" - echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV - echo "BUILD_TARGET=jina" >> $GITHUB_ENV - - if [[ "${{ matrix.pip_tag }}" == "perf" ]]; then - echo "JINA_PIP_INSTALL_PERF=1" >> $GITHUB_ENV - fi - - if [[ "${{ matrix.pip_tag }}" == "" ]]; then - echo "JINA_PIP_INSTALL_CORE=1" >> $GITHUB_ENV - fi - - JINA_VERSION=$(sed -n '/^__version__/p' ./jina/__init__.py | cut -d \' -f2) - V_JINA_VERSION=v${JINA_VERSION} - JINA_MINOR_VERSION=${JINA_VERSION%.*} - JINA_MAJOR_VERSION=${JINA_MINOR_VERSION%.*} - - PY_TAG=${{matrix.py_version}} - if [ -n "${PY_TAG}" ]; then - PY_TAG=-py${PY_TAG//./} - fi - - PIP_TAG=${{ matrix.pip_tag }} - if [ -n "${PIP_TAG}" ]; then - PIP_TAG=-${PIP_TAG} - fi - - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - LAST_VER_TAG=$(git tag -l | sort -V | tail -n1) - PRE_VERSION=-dev$(git rev-list $LAST_VER_TAG..HEAD --count) - - if [[ "${{ github.event.inputs.triggered_by }}" == "CD" ]]; then - - if [[ "${{ matrix.py_version }}" == "$DEFAULT_PY_VERSION" ]]; then - echo "TAG_ALIAS=\ - jinaai/jina:master${PY_TAG}${PIP_TAG}, \ - jinaai/jina:master${PIP_TAG}, \ - jinaai/jina:${JINA_VERSION}${PRE_VERSION}${PIP_TAG}, \ - jinaai/jina:${JINA_VERSION}${PRE_VERSION}${PY_TAG}${PIP_TAG}" \ - >> $GITHUB_ENV - else - # on every CD - echo "TAG_ALIAS=\ - jinaai/jina:master${PY_TAG}${PIP_TAG}, \ - jinaai/jina:${JINA_VERSION}${PRE_VERSION}${PY_TAG}${PIP_TAG}" \ - >> $GITHUB_ENV - fi - - elif [[ "${{ github.event.inputs.triggered_by }}" == "TAG" ]]; then - # on every tag release - - if [[ "${{ matrix.py_version }}" == "$DEFAULT_PY_VERSION" ]]; then - echo "TAG_ALIAS=\ - jinaai/jina:latest${PY_TAG}${PIP_TAG}, \ - jinaai/jina:${JINA_VERSION}${PY_TAG}${PIP_TAG}, \ - jinaai/jina:${JINA_MINOR_VERSION}${PY_TAG}${PIP_TAG}, \ - jinaai/jina:${JINA_MAJOR_VERSION}${PY_TAG}${PIP_TAG}, \ - jinaai/jina:latest${PIP_TAG}, \ - jinaai/jina:${JINA_VERSION}${PIP_TAG}, \ - jinaai/jina:${JINA_MINOR_VERSION}${PIP_TAG}, \ - jinaai/jina:${JINA_MAJOR_VERSION}${PIP_TAG} \ - " >> $GITHUB_ENV - else - echo "TAG_ALIAS=\ - jinaai/jina:latest${PY_TAG}${PIP_TAG}, \ - jinaai/jina:${JINA_VERSION}${PY_TAG}${PIP_TAG}, \ - jinaai/jina:${JINA_MINOR_VERSION}${PY_TAG}${PIP_TAG}, \ - jinaai/jina:${JINA_MAJOR_VERSION}${PY_TAG}${PIP_TAG} \ - " >> $GITHUB_ENV - fi - elif [[ "${{ github.event.inputs.triggered_by }}" == "MANUAL" ]]; then - # on every manual release - if [[ "${{ matrix.py_version }}" == "$DEFAULT_PY_VERSION" ]]; then - echo "TAG_ALIAS=\ - jinaai/jina:${JINA_VERSION}${PIP_TAG}, \ - jinaai/jina:${JINA_VERSION}${PY_TAG}${PIP_TAG} \ - " >> $GITHUB_ENV - else - echo "TAG_ALIAS=\ - jinaai/jina:${JINA_VERSION}${PY_TAG}${PIP_TAG} \ - " >> $GITHUB_ENV - fi - else - echo "Bad triggered_by: ${{ github.event.inputs.triggered_by }}!" - exit 1 - fi - - echo "JINA_VERSION=${JINA_VERSION}" >> $GITHUB_ENV - + pip install -e . + echo "VECTORDB_VERSION=$(python -c 'import vectordb; print(vectordb.__version__)')" >> $GITHUB_ENV + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v1 @@ -141,27 +51,12 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKERHUB_DEVBOT_USER }} - password: ${{ secrets.DOCKERHUB_DEVBOT_TOKEN }} - - run: | - # https://github.com/docker/buildx/issues/464#issuecomment-741507760 - # https://github.com/kubernetes-sigs/azuredisk-csi-driver/pull/808/files - docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64 - docker run --rm --privileged tonistiigi/binfmt --install all + username: ${{ secrets.DOCKERHUB_JINAVECTORDB_USER }} + password: ${{ secrets.DOCKERHUB_JINAVECTORDB_TOKEN }} - name: Build and push uses: docker/build-push-action@v2 with: - context: . - file: Dockerfiles/debianx.Dockerfile - platforms: linux/amd64,linux/arm64 push: true - tags: ${{env.TAG_ALIAS}} - build-args: | - BUILD_DATE=${{env.BUILD_DATE}} - JINA_VERSION=${{env.JINA_VERSION}} - VCS_REF=${{env.VCS_REF}} - PIP_INSTALL_CORE=${{env.JINA_PIP_INSTALL_CORE}} - PIP_INSTALL_PERF=${{env.JINA_PIP_INSTALL_PERF}} - PY_VERSION=${{matrix.py_version}} - PIP_TAG=${{matrix.pip_tag}} - target: ${{env.BUILD_TARGET}} + context: . + file: Dockerfiles/vectordb.Dockerfile + tags: jinaai/vectordb:latest, jinaai/vectordb:${{ env.VECTORDB_VERSION }} diff --git a/.github/workflows/force-docs-build.yml b/.github/workflows/force-docs-build.yml deleted file mode 100644 index a7ad000..0000000 --- a/.github/workflows/force-docs-build.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: Manual Docs Build - -on: - workflow_dispatch: - inputs: - release_token: - description: 'Your release token' - required: true - triggered_by: - description: 'CD | TAG | MANUAL' - required: false - default: MANUAL - build_old_docs: - description: 'Whether to build old docs (TRUE | FALSE)' - type: string - default: 'FALSE' - package: - description: The name of the repo to build documentation for. - type: string - default: jina - repo_owner: - description: The owner of the repo to build documentation for. Defaults to 'jina-ai'. - type: string - default: jina-ai - pages_branch: - description: Branch that Github Pages observes - type: string - default: gh-pages - git_config_name: - type: string - default: Jina Dev Bot - git_config_email: - type: string - default: dev-bot@jina.ai - -jobs: - token-check: - runs-on: ubuntu-latest - steps: - - name: Check release token - id: token-check - run: | - touch SUCCESS - if: inputs.release_token == env.release_token - env: - release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }} - - name: Fail release token - run: | - [[ -f SUCCESS ]] - - build-and-push-latest-docs: - needs: token-check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - uses: actions/setup-python@v4 - with: - python-version: '3.7' - - name: Install Dependencies - run: | - pip install .[devel] - cd docs - pip install -r requirements.txt - pip install --pre -U furo - pip install sphinx-markdown-tables==0.0.17 - - name: Sphinx Build - run: | - cd docs - bash makedoc.sh local-only - mv ./_build/dirhtml /tmp/gen-html - cd .. - - name: Checkout to GH pages branch (${{ inputs.pages_branch }}) - run: | - git fetch origin ${{ inputs.pages_branch }}:${{ inputs.pages_branch }} --depth 1 - git checkout -f ${{ inputs.pages_branch }} - git reset --hard HEAD - - name: Small config stuff - run: | - touch /tmp/gen-html/.nojekyll - cp ./docs/_versions.json /tmp/gen-html/_versions.json - cp ./docs/CNAME /tmp/gen-html/CNAME - cp /tmp/gen-html/404/index.html /tmp/gen-html/404.html - sed -i 's/href="\.\./href="/' /tmp/gen-html/404.html # fix asset urls that needs to be updated in 404.html - - name: Moving old doc versions - run: | - cd docs - for i in $(cat _versions.json | jq '.[].version' | tr -d '"'); do if [ -d "$i" ]; then mv "$i" /tmp/gen-html; fi; done - - name: Swap in new docs - run: | - rm -rf ./docs - mv /tmp/gen-html ./docs - - name: Push it up! - run: | - git config --local user.email "${{ inputs.git_config_email }}" - git config --local user.name "${{ inputs.git_config_name }}" - git show --summary - git add ./docs && git commit -m "chore(docs): update docs due to ${{github.event_name}} on ${{github.repository}}" - git push origin ${{ inputs.pages_branch }} - - build-old-docs: - needs: build-and-push-latest-docs - runs-on: ubuntu-latest - if: inputs.build_old_docs == 'TRUE' - steps: - - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: Build old docs - token: ${{ secrets.JINA_DEV_BOT }} - inputs: '{ "release_token": "${{ env.release_token }}", "triggered_by": "TAG"}' - env: - release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }} diff --git a/Dockerfiles/Dockerfile b/Dockerfiles/vectordb.Dockerfile similarity index 100% rename from Dockerfiles/Dockerfile rename to Dockerfiles/vectordb.Dockerfile