Merge branch 'citemplate' into 'dev'
Switch to ci template See merge request epi2melabs/workflow-containers/wf-template!7
This commit is contained in:
Chris Wright
commit
ffb895ef7e
196
.gitlab-ci.yml
196
.gitlab-ci.yml
@ -1,197 +1,17 @@
|
||||
stages:
|
||||
- pretest
|
||||
- test
|
||||
- pre-release
|
||||
- release-checks
|
||||
- release
|
||||
include:
|
||||
- project: "epi2melabs/ci-templates"
|
||||
file: "wf-containers.yaml"
|
||||
|
||||
image: ${IMAGE}
|
||||
variables:
|
||||
BASEIMAGE: ${CI_REGISTRY}/${CURRENT_TEMPLATE_IMAGE}
|
||||
|
||||
.install-nextflow: &install-nextflow |
|
||||
wget -qO- https://get.nextflow.io | bash
|
||||
|
||||
.install-conda: &install-conda |
|
||||
wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
|
||||
bash Miniconda3-latest-Linux-x86_64.sh -b -p ./miniconda
|
||||
source ./miniconda/bin/activate
|
||||
conda init
|
||||
|
||||
|
||||
# insist that the wfversion in nextflow.config matches the git tag
|
||||
.check-versions: &check-versions |
|
||||
WFVER=$(grep "wfversion = " nextflow.config | awk '{gsub("\"","",$3); print $3}')
|
||||
TAGVER=${CI_COMMIT_TAG}
|
||||
if [[ "${TAGVER}" != "${WFVER}" ]]; then
|
||||
echo "Mismatching TAG and WORKFLOW versions:"
|
||||
echo " - TAG:$TAGVER"
|
||||
echo " - WF:$WFVER"
|
||||
exit 1
|
||||
else
|
||||
echo "TAG and WORKFLOW versions agree: ${WFVER}"
|
||||
fi
|
||||
|
||||
# Insist a CHANGELOG entry has been made for tags
|
||||
.check-changelog: &check-changelog |
|
||||
TAGVER=${CI_COMMIT_TAG}
|
||||
MATCHES=$(grep -c "## \[${TAGVER}\]" CHANGELOG.md || exit 0)
|
||||
if [[ "${MATCHES}" != "1" ]]; then
|
||||
echo "Expected one match to '${CI_COMMIT_TAG}' in CHANGELOG, found ${MATCHES}"
|
||||
exit 1
|
||||
else
|
||||
echo "Found CHANGELOG.md entry for tag"
|
||||
fi
|
||||
|
||||
|
||||
# setup image for pushing to github
|
||||
.setup-gh-push: &setup-gh-push |
|
||||
apt-get update -qq && apt-get install -y -qq git python3-all-dev git-lfs python3-venv
|
||||
mkdir ~/.ssh/
|
||||
cp $LABS_BOT_GH_KEY ~/.ssh/id_rsa && chmod 600 ~/.ssh/id_rsa
|
||||
echo -e "Host github.com\n\tStrictHostKeyChecking no\n\tHostname ssh.github.com\n\tPort 443\n\n" > ~/.ssh/config
|
||||
git config --global user.email "epi2melabs@nanoporetech.com"
|
||||
git config --global user.name "epi2melabs-bot"
|
||||
|
||||
RUN: "./nextflow run main.nf -w ${OUTPUT}/workspace --out_dir ${OUTPUT} --fastq test_data/reads.fq.gz"
|
||||
|
||||
conda-run:
|
||||
image: ubuntu:20.04
|
||||
stage: test
|
||||
before_script:
|
||||
- apt update && apt install -y wget default-jre
|
||||
- *install-nextflow
|
||||
- *install-conda
|
||||
extends: .conda-run
|
||||
script:
|
||||
- OUTPUT=${CI_PROJECT_NAME}
|
||||
- ./nextflow run main.nf
|
||||
-w ${OUTPUT}/workspace
|
||||
-profile conda
|
||||
--fastq test_data/reads.fq.gz
|
||||
--out_dir ${OUTPUT}
|
||||
only:
|
||||
- branches
|
||||
|
||||
- "${RUN} -profile conda"
|
||||
|
||||
docker-run:
|
||||
stage: test
|
||||
before_script:
|
||||
- apk add wget openjdk11 bash
|
||||
- *install-nextflow
|
||||
extends: .docker-run
|
||||
script:
|
||||
# build image
|
||||
- TAG="${DOCKERHUB_NAMESPACE}/${CI_PROJECT_NAME}:latest"
|
||||
- docker build --no-cache -t "${TAG}" -f Dockerfile . --build-arg BASEIMAGE=${BASEIMAGE}
|
||||
# run letting nextflow orchestrate the containers
|
||||
- OUTPUT=${CI_PROJECT_NAME}
|
||||
- ./nextflow run main.nf
|
||||
-w ${OUTPUT}/workspace
|
||||
-profile standard
|
||||
--wfversion latest
|
||||
--fastq test_data/reads.fq.gz
|
||||
--out_dir ${OUTPUT}
|
||||
# push, to ont for later tagging, to dockerhub for release testing
|
||||
- if [[ ${CI_COMMIT_BRANCH} == 'dev' ]]; then
|
||||
echo "Pushing to ONT registry";
|
||||
echo ${CI_BUILD_TOKEN} | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY};
|
||||
SHATAG=${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA};
|
||||
echo " - tag is:${SHATAG}";
|
||||
docker tag ${TAG} ${SHATAG};
|
||||
docker push ${SHATAG};
|
||||
echo "Pushing to Dockerhub";
|
||||
echo ${DOCKERHUB_TOKEN} | docker login --username epi2melabs --password-stdin;
|
||||
HUBTAG="${DOCKERHUB_NAMESPACE}/${CI_PROJECT_NAME}:${CI_COMMIT_SHORT_SHA}";
|
||||
echo " - tag is:${HUBTAG}";
|
||||
docker tag ${TAG} ${HUBTAG};
|
||||
docker push ${HUBTAG};
|
||||
fi;
|
||||
only:
|
||||
- branches
|
||||
artifacts:
|
||||
paths:
|
||||
# Add the output directory for the test
|
||||
- ${CI_PROJECT_NAME}
|
||||
expire_in: 1 day
|
||||
|
||||
###
|
||||
# Push dev branch to github for release testers
|
||||
release-dev:
|
||||
stage: pre-release
|
||||
image: $GH_PUSH_IMAGE
|
||||
before_script:
|
||||
- *setup-gh-push
|
||||
script:
|
||||
# Merge ont/dev into prerelease
|
||||
- git remote add ont ${CI_REPOSITORY_URL} || true
|
||||
- git remote add github git@github.com:epi2me-labs/${CI_PROJECT_NAME}.git || true
|
||||
- git fetch ont dev
|
||||
- git checkout ont/dev
|
||||
- git branch -D prerelease || echo "branch 'prerelease' not present"
|
||||
- git checkout -b prerelease
|
||||
# Edit nextflow config to set wfversion
|
||||
- grep 'wfversion = ".*"' nextflow.config || { echo "Failed to replace wfversion"; exit 1; }
|
||||
- sed -i 's/wfversion = ".*"/wfversion = "'${CI_COMMIT_SHORT_SHA}'"/' nextflow.config
|
||||
- git add nextflow.config
|
||||
- git commit -m "Setting wfversion to:${CI_COMMIT_SHORT_SHA}"
|
||||
- git push -f github prerelease
|
||||
only:
|
||||
- '/^dev$/'
|
||||
except:
|
||||
- tags
|
||||
|
||||
# Check things before release
|
||||
release-checks:
|
||||
stage: release-checks
|
||||
script:
|
||||
- *check-versions
|
||||
- *check-changelog
|
||||
only:
|
||||
- tags
|
||||
|
||||
|
||||
# To dockerhub
|
||||
release-hub:
|
||||
stage: release
|
||||
before_script:
|
||||
- echo ${CI_BUILD_TOKEN} | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY}
|
||||
- SHATAG=${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}
|
||||
- echo "Pulling ${SHATAG}"
|
||||
- docker pull ${SHATAG}
|
||||
script:
|
||||
- echo ${DOCKERHUB_TOKEN} | docker login --username epi2melabs --password-stdin
|
||||
- RELTAG="${DOCKERHUB_NAMESPACE}/${CI_PROJECT_NAME}:${CI_COMMIT_TAG}"
|
||||
- LATEST="${DOCKERHUB_NAMESPACE}/${CI_PROJECT_NAME}:latest"
|
||||
- echo "Pushing ${RELTAG}"
|
||||
- docker tag ${SHATAG} ${RELTAG}
|
||||
- docker push ${RELTAG}
|
||||
- docker tag ${SHATAG} ${LATEST}
|
||||
- docker push ${LATEST}
|
||||
only:
|
||||
- tags
|
||||
|
||||
# Send all tags matching vX.Y.Z to github (code and release)
|
||||
push-github:
|
||||
stage: release
|
||||
image: $GH_PUSH_IMAGE
|
||||
before_script:
|
||||
- *setup-gh-push
|
||||
script:
|
||||
# Push master and tag to github
|
||||
- git remote add ont ${CI_REPOSITORY_URL} || true
|
||||
- git remote add github git@github.com:epi2me-labs/${CI_PROJECT_NAME}.git || true
|
||||
- git fetch ont dev --tags
|
||||
- git fetch ont master
|
||||
- git fetch github master || echo "No 'master' branch present on github"
|
||||
- git checkout --track github/master || git checkout master
|
||||
- git merge ${CI_COMMIT_TAG}
|
||||
- git push github master
|
||||
- git push github ${CI_COMMIT_TAG}
|
||||
# Make a github release page
|
||||
- python3 -m venv release_env --prompt "(release) "
|
||||
- source release_env/bin/activate
|
||||
- pip install pip --upgrade
|
||||
- pip install git+https://github.com/epi2me-labs/github_release.git
|
||||
- github_release ${CI_PROJECT_NAME} ${CI_COMMIT_TAG} CHANGELOG.md ${LABS_BOT_GH_TOKEN}
|
||||
only:
|
||||
- /^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$/
|
||||
except:
|
||||
- branches
|
||||
- "${RUN} -profile standard --wfversion latest"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user