ci: add docker building steps
This commit is contained in:
parent
795178d3f6
commit
ed1fb1279f
|
@ -21,117 +21,27 @@ jobs:
|
||||||
touch SUCCESS
|
touch SUCCESS
|
||||||
if: inputs.release_token == env.release_token
|
if: inputs.release_token == env.release_token
|
||||||
env:
|
env:
|
||||||
release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }}
|
release_token: ${{ secrets.VECTORDB_RELEASE_TOKEN }}
|
||||||
- name: Fail release token
|
- name: Fail release token
|
||||||
run: |
|
run: |
|
||||||
[[ -f SUCCESS ]]
|
[[ -f SUCCESS ]]
|
||||||
|
|
||||||
|
|
||||||
regular-release:
|
regular-release:
|
||||||
needs: token-check
|
needs: token-check
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
pip_tag: [ "", "perf", "standard", "devel"] # default: "" = core
|
|
||||||
py_version: [ "3.7", "3.8", "3.9" , "3.10", "3.11"] # default "" = 3.7
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2.5.0
|
- uses: actions/checkout@v2.5.0
|
||||||
with:
|
with:
|
||||||
fetch-depth: 100
|
fetch-depth: 100
|
||||||
- name: Set envs and versions
|
- name: Set up Python 3.9
|
||||||
|
uses: actions/setup-python@v1
|
||||||
|
with:
|
||||||
|
python-version: 3.9
|
||||||
|
|
||||||
|
- name: Get vectordb version
|
||||||
run: |
|
run: |
|
||||||
DEFAULT_PY_VERSION="3.8"
|
pip install -e .
|
||||||
VCS_REF=${{ github.ref }}
|
echo "VECTORDB_VERSION=$(python -c 'import vectordb; print(vectordb.__version__)')" >> $GITHUB_ENV
|
||||||
echo "VCS_REF=$VCS_REF" >> $GITHUB_ENV
|
|
||||||
echo "Will build $VCS_REF"
|
|
||||||
echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV
|
|
||||||
echo "BUILD_TARGET=jina" >> $GITHUB_ENV
|
|
||||||
|
|
||||||
if [[ "${{ matrix.pip_tag }}" == "perf" ]]; then
|
|
||||||
echo "JINA_PIP_INSTALL_PERF=1" >> $GITHUB_ENV
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "${{ matrix.pip_tag }}" == "" ]]; then
|
|
||||||
echo "JINA_PIP_INSTALL_CORE=1" >> $GITHUB_ENV
|
|
||||||
fi
|
|
||||||
|
|
||||||
JINA_VERSION=$(sed -n '/^__version__/p' ./jina/__init__.py | cut -d \' -f2)
|
|
||||||
V_JINA_VERSION=v${JINA_VERSION}
|
|
||||||
JINA_MINOR_VERSION=${JINA_VERSION%.*}
|
|
||||||
JINA_MAJOR_VERSION=${JINA_MINOR_VERSION%.*}
|
|
||||||
|
|
||||||
PY_TAG=${{matrix.py_version}}
|
|
||||||
if [ -n "${PY_TAG}" ]; then
|
|
||||||
PY_TAG=-py${PY_TAG//./}
|
|
||||||
fi
|
|
||||||
|
|
||||||
PIP_TAG=${{ matrix.pip_tag }}
|
|
||||||
if [ -n "${PIP_TAG}" ]; then
|
|
||||||
PIP_TAG=-${PIP_TAG}
|
|
||||||
fi
|
|
||||||
|
|
||||||
git fetch --depth=1 origin +refs/tags/*:refs/tags/*
|
|
||||||
LAST_VER_TAG=$(git tag -l | sort -V | tail -n1)
|
|
||||||
PRE_VERSION=-dev$(git rev-list $LAST_VER_TAG..HEAD --count)
|
|
||||||
|
|
||||||
if [[ "${{ github.event.inputs.triggered_by }}" == "CD" ]]; then
|
|
||||||
|
|
||||||
if [[ "${{ matrix.py_version }}" == "$DEFAULT_PY_VERSION" ]]; then
|
|
||||||
echo "TAG_ALIAS=\
|
|
||||||
jinaai/jina:master${PY_TAG}${PIP_TAG}, \
|
|
||||||
jinaai/jina:master${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_VERSION}${PRE_VERSION}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_VERSION}${PRE_VERSION}${PY_TAG}${PIP_TAG}" \
|
|
||||||
>> $GITHUB_ENV
|
|
||||||
else
|
|
||||||
# on every CD
|
|
||||||
echo "TAG_ALIAS=\
|
|
||||||
jinaai/jina:master${PY_TAG}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_VERSION}${PRE_VERSION}${PY_TAG}${PIP_TAG}" \
|
|
||||||
>> $GITHUB_ENV
|
|
||||||
fi
|
|
||||||
|
|
||||||
elif [[ "${{ github.event.inputs.triggered_by }}" == "TAG" ]]; then
|
|
||||||
# on every tag release
|
|
||||||
|
|
||||||
if [[ "${{ matrix.py_version }}" == "$DEFAULT_PY_VERSION" ]]; then
|
|
||||||
echo "TAG_ALIAS=\
|
|
||||||
jinaai/jina:latest${PY_TAG}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_VERSION}${PY_TAG}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_MINOR_VERSION}${PY_TAG}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_MAJOR_VERSION}${PY_TAG}${PIP_TAG}, \
|
|
||||||
jinaai/jina:latest${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_VERSION}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_MINOR_VERSION}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_MAJOR_VERSION}${PIP_TAG} \
|
|
||||||
" >> $GITHUB_ENV
|
|
||||||
else
|
|
||||||
echo "TAG_ALIAS=\
|
|
||||||
jinaai/jina:latest${PY_TAG}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_VERSION}${PY_TAG}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_MINOR_VERSION}${PY_TAG}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_MAJOR_VERSION}${PY_TAG}${PIP_TAG} \
|
|
||||||
" >> $GITHUB_ENV
|
|
||||||
fi
|
|
||||||
elif [[ "${{ github.event.inputs.triggered_by }}" == "MANUAL" ]]; then
|
|
||||||
# on every manual release
|
|
||||||
if [[ "${{ matrix.py_version }}" == "$DEFAULT_PY_VERSION" ]]; then
|
|
||||||
echo "TAG_ALIAS=\
|
|
||||||
jinaai/jina:${JINA_VERSION}${PIP_TAG}, \
|
|
||||||
jinaai/jina:${JINA_VERSION}${PY_TAG}${PIP_TAG} \
|
|
||||||
" >> $GITHUB_ENV
|
|
||||||
else
|
|
||||||
echo "TAG_ALIAS=\
|
|
||||||
jinaai/jina:${JINA_VERSION}${PY_TAG}${PIP_TAG} \
|
|
||||||
" >> $GITHUB_ENV
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "Bad triggered_by: ${{ github.event.inputs.triggered_by }}!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "JINA_VERSION=${JINA_VERSION}" >> $GITHUB_ENV
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
id: buildx
|
id: buildx
|
||||||
|
@ -141,27 +51,12 @@ jobs:
|
||||||
- name: Login to DockerHub
|
- name: Login to DockerHub
|
||||||
uses: docker/login-action@v1
|
uses: docker/login-action@v1
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKERHUB_DEVBOT_USER }}
|
username: ${{ secrets.DOCKERHUB_JINAVECTORDB_USER }}
|
||||||
password: ${{ secrets.DOCKERHUB_DEVBOT_TOKEN }}
|
password: ${{ secrets.DOCKERHUB_JINAVECTORDB_TOKEN }}
|
||||||
- run: |
|
|
||||||
# https://github.com/docker/buildx/issues/464#issuecomment-741507760
|
|
||||||
# https://github.com/kubernetes-sigs/azuredisk-csi-driver/pull/808/files
|
|
||||||
docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64
|
|
||||||
docker run --rm --privileged tonistiigi/binfmt --install all
|
|
||||||
- name: Build and push
|
- name: Build and push
|
||||||
uses: docker/build-push-action@v2
|
uses: docker/build-push-action@v2
|
||||||
with:
|
with:
|
||||||
context: .
|
|
||||||
file: Dockerfiles/debianx.Dockerfile
|
|
||||||
platforms: linux/amd64,linux/arm64
|
|
||||||
push: true
|
push: true
|
||||||
tags: ${{env.TAG_ALIAS}}
|
context: .
|
||||||
build-args: |
|
file: Dockerfiles/vectordb.Dockerfile
|
||||||
BUILD_DATE=${{env.BUILD_DATE}}
|
tags: jinaai/vectordb:latest, jinaai/vectordb:${{ env.VECTORDB_VERSION }}
|
||||||
JINA_VERSION=${{env.JINA_VERSION}}
|
|
||||||
VCS_REF=${{env.VCS_REF}}
|
|
||||||
PIP_INSTALL_CORE=${{env.JINA_PIP_INSTALL_CORE}}
|
|
||||||
PIP_INSTALL_PERF=${{env.JINA_PIP_INSTALL_PERF}}
|
|
||||||
PY_VERSION=${{matrix.py_version}}
|
|
||||||
PIP_TAG=${{matrix.pip_tag}}
|
|
||||||
target: ${{env.BUILD_TARGET}}
|
|
||||||
|
|
|
@ -1,113 +0,0 @@
|
||||||
name: Manual Docs Build
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
inputs:
|
|
||||||
release_token:
|
|
||||||
description: 'Your release token'
|
|
||||||
required: true
|
|
||||||
triggered_by:
|
|
||||||
description: 'CD | TAG | MANUAL'
|
|
||||||
required: false
|
|
||||||
default: MANUAL
|
|
||||||
build_old_docs:
|
|
||||||
description: 'Whether to build old docs (TRUE | FALSE)'
|
|
||||||
type: string
|
|
||||||
default: 'FALSE'
|
|
||||||
package:
|
|
||||||
description: The name of the repo to build documentation for.
|
|
||||||
type: string
|
|
||||||
default: jina
|
|
||||||
repo_owner:
|
|
||||||
description: The owner of the repo to build documentation for. Defaults to 'jina-ai'.
|
|
||||||
type: string
|
|
||||||
default: jina-ai
|
|
||||||
pages_branch:
|
|
||||||
description: Branch that Github Pages observes
|
|
||||||
type: string
|
|
||||||
default: gh-pages
|
|
||||||
git_config_name:
|
|
||||||
type: string
|
|
||||||
default: Jina Dev Bot
|
|
||||||
git_config_email:
|
|
||||||
type: string
|
|
||||||
default: dev-bot@jina.ai
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
token-check:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Check release token
|
|
||||||
id: token-check
|
|
||||||
run: |
|
|
||||||
touch SUCCESS
|
|
||||||
if: inputs.release_token == env.release_token
|
|
||||||
env:
|
|
||||||
release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }}
|
|
||||||
- name: Fail release token
|
|
||||||
run: |
|
|
||||||
[[ -f SUCCESS ]]
|
|
||||||
|
|
||||||
build-and-push-latest-docs:
|
|
||||||
needs: token-check
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
with:
|
|
||||||
fetch-depth: 1
|
|
||||||
- uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: '3.7'
|
|
||||||
- name: Install Dependencies
|
|
||||||
run: |
|
|
||||||
pip install .[devel]
|
|
||||||
cd docs
|
|
||||||
pip install -r requirements.txt
|
|
||||||
pip install --pre -U furo
|
|
||||||
pip install sphinx-markdown-tables==0.0.17
|
|
||||||
- name: Sphinx Build
|
|
||||||
run: |
|
|
||||||
cd docs
|
|
||||||
bash makedoc.sh local-only
|
|
||||||
mv ./_build/dirhtml /tmp/gen-html
|
|
||||||
cd ..
|
|
||||||
- name: Checkout to GH pages branch (${{ inputs.pages_branch }})
|
|
||||||
run: |
|
|
||||||
git fetch origin ${{ inputs.pages_branch }}:${{ inputs.pages_branch }} --depth 1
|
|
||||||
git checkout -f ${{ inputs.pages_branch }}
|
|
||||||
git reset --hard HEAD
|
|
||||||
- name: Small config stuff
|
|
||||||
run: |
|
|
||||||
touch /tmp/gen-html/.nojekyll
|
|
||||||
cp ./docs/_versions.json /tmp/gen-html/_versions.json
|
|
||||||
cp ./docs/CNAME /tmp/gen-html/CNAME
|
|
||||||
cp /tmp/gen-html/404/index.html /tmp/gen-html/404.html
|
|
||||||
sed -i 's/href="\.\./href="/' /tmp/gen-html/404.html # fix asset urls that needs to be updated in 404.html
|
|
||||||
- name: Moving old doc versions
|
|
||||||
run: |
|
|
||||||
cd docs
|
|
||||||
for i in $(cat _versions.json | jq '.[].version' | tr -d '"'); do if [ -d "$i" ]; then mv "$i" /tmp/gen-html; fi; done
|
|
||||||
- name: Swap in new docs
|
|
||||||
run: |
|
|
||||||
rm -rf ./docs
|
|
||||||
mv /tmp/gen-html ./docs
|
|
||||||
- name: Push it up!
|
|
||||||
run: |
|
|
||||||
git config --local user.email "${{ inputs.git_config_email }}"
|
|
||||||
git config --local user.name "${{ inputs.git_config_name }}"
|
|
||||||
git show --summary
|
|
||||||
git add ./docs && git commit -m "chore(docs): update docs due to ${{github.event_name}} on ${{github.repository}}"
|
|
||||||
git push origin ${{ inputs.pages_branch }}
|
|
||||||
|
|
||||||
build-old-docs:
|
|
||||||
needs: build-and-push-latest-docs
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
if: inputs.build_old_docs == 'TRUE'
|
|
||||||
steps:
|
|
||||||
- uses: benc-uk/workflow-dispatch@v1
|
|
||||||
with:
|
|
||||||
workflow: Build old docs
|
|
||||||
token: ${{ secrets.JINA_DEV_BOT }}
|
|
||||||
inputs: '{ "release_token": "${{ env.release_token }}", "triggered_by": "TAG"}'
|
|
||||||
env:
|
|
||||||
release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }}
|
|
Loading…
Reference in New Issue