-
Notifications
You must be signed in to change notification settings - Fork 770
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Unify CI from Travis and AppVeyor to GitHub Actions (#942)
- Loading branch information
1 parent
166794f
commit 8ad8a4d
Showing
12 changed files
with
399 additions
and
378 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,350 @@ | ||
| name: Build | ||
| on: [push, pull_request] | ||
| env: | ||
| GCS_BUCKET: autobuilds.grr-response.com | ||
| GCS_BUCKET_OPENAPI: autobuilds-grr-openapi | ||
| GCS_LATEST_PATH: _latest_server_deb | ||
| DOCKER_REPOSITORY: grrdocker/grr | ||
| jobs: | ||
|
|
||
| test-ubuntu: | ||
| runs-on: ubuntu-18.04 | ||
| env: | ||
| CHROME_DEB: google-chrome-stable_current_amd64.deb | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: '3.7' | ||
| - name: Install | ||
| run: | | ||
| free -hmw | ||
| lscpu | ||
| sudo apt install -y libmysqlclient-dev | ||
| if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then | ||
| wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}"; | ||
| fi | ||
| python -m venv "${HOME}/INSTALL" | ||
| travis/install.sh | ||
| - name: Test | ||
| run: | | ||
| source "${HOME}/INSTALL/bin/activate" | ||
| pip install pytest-xdist==2.2.1 | ||
| # We have 4 vCPUs available, but only use 3 here to avoid timeouts like | ||
| # https://ci.appveyor.com/project/grr/grr-ia94e/builds/20483467/messages , | ||
| # which happen when tests stall. | ||
| pytest --verbose -n 3 grr/ --ignore grr/server/grr_response_server/gui/selenium_tests/ --ignore grr/client/grr_response_client/client_actions/windows/ | ||
| # jsTree tests seem to fail on Chrome 71 headless due to https://github.com/GoogleChrome/puppeteer/issues/3463 | ||
| if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi | ||
| build-openapi: | ||
| runs-on: ubuntu-18.04 | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: '3.7' | ||
| - name: Install | ||
| run: | | ||
| sudo apt install -y libmysqlclient-dev | ||
| python -m venv "${HOME}/INSTALL" | ||
| travis/install.sh | ||
| - name: Build | ||
| run: | | ||
| source "${HOME}/INSTALL/bin/activate" | ||
| mkdir -p _openapi_artifacts/openapi_description | ||
| mkdir -p _openapi_artifacts/documentation | ||
| travis/build_api_documentation.sh "_openapi_artifacts/openapi_description/openapi_description.json" "_openapi_artifacts/documentation/openapi_documentation.html" | ||
| ls -la _openapi_artifacts/* | ||
| - name: Upload OpenAPI to GitHub artifacts | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: openapi | ||
| path: _openapi_artifacts/ | ||
| retention-days: 1 | ||
|
|
||
| build-linux: | ||
| runs-on: ubuntu-18.04 | ||
| env: | ||
| GCS_TAG: ubuntu_64bit | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: '3.6' | ||
| - name: Set up | ||
| run: | | ||
| sudo apt install fakeroot debhelper libffi-dev libssl-dev | ||
| pip install virtualenv | ||
| virtualenv "${HOME}/INSTALL" | ||
| - name: Build | ||
| run: | | ||
| travis/install_client_builder.sh | ||
| travis/build_templates.sh | ||
| ls -la gcs_upload_dir | ||
| - name: Upload installers to GitHub artifacts | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: ubuntu-installers | ||
| path: gcs_upload_dir/ | ||
| retention-days: 1 | ||
|
|
||
| build-osx: | ||
| runs-on: macos-latest | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: '3.6' | ||
| - name: Set up | ||
| run: | | ||
| pip install --upgrade setuptools virtualenv | ||
| virtualenv "${HOME}/INSTALL" | ||
| - name: Build installers | ||
| run: | | ||
| travis/install_client_builder.sh | ||
| travis/build_templates.sh | ||
| ls -la gcs_upload_dir | ||
| - name: Upload installers to GitHub artifacts | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: osx-installers | ||
| path: gcs_upload_dir/ | ||
| retention-days: 1 | ||
|
|
||
| build-centos: | ||
| runs-on: ubuntu-18.04 | ||
| env: | ||
| GCS_TAG: centos_64bit | ||
| DOCKER_IMG: grrdocker/centos7 | ||
| DOCKER_CONTAINER: centos_64bit_container | ||
| DOCKER_USER: grrbot | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: '3.6' | ||
| - name: Build installers | ||
| run: | | ||
| sudo docker run -dit \ | ||
| --volume "${PWD}:/mnt/grr" \ | ||
| --workdir /mnt/grr \ | ||
| --env DOCKER_USER="${DOCKER_USER}" \ | ||
| --env TRAVIS_OS_NAME="linux" \ | ||
| --name "${DOCKER_CONTAINER}" \ | ||
| "${DOCKER_IMG}" | ||
| sudo docker exec "${DOCKER_CONTAINER}" travis/set_up_test_user.sh | ||
| sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" /usr/local/bin/python3.6 -m venv "/home/${DOCKER_USER}/INSTALL" | ||
| sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/install_client_builder.sh | ||
| sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" travis/build_templates.sh | ||
| sudo docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm | ||
| ls -la gcs_upload_dir | ||
| - name: Upload installers to GitHub artifacts | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: centos-installers | ||
| path: gcs_upload_dir/ | ||
| retention-days: 1 | ||
|
|
||
| build-windows: | ||
| runs-on: windows-2016 | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: '3.7' | ||
| - name: Build installers | ||
| shell: bash | ||
| run: | | ||
| set -ex | ||
| pip install virtualenv wheel | ||
| python -u appveyor/windows_templates/build_windows_templates.py --grr_src=$GITHUB_WORKSPACE --output_dir=$GITHUB_WORKSPACE/output --test_repack_install | ||
| python -u appveyor/windows_templates/build_windows_templates.py --grr_src=$GITHUB_WORKSPACE --output_dir=$GITHUB_WORKSPACE/output_msi --test_repack_install --build_msi | ||
| mkdir -p gcs_upload_dir | ||
| mv -v output*/* gcs_upload_dir | ||
| ls -la gcs_upload_dir | ||
| - name: Upload installers to GitHub artifacts | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: windows-installers | ||
| path: gcs_upload_dir/ | ||
| retention-days: 1 | ||
|
|
||
| build-server-deb: | ||
| runs-on: ubuntu-18.04 | ||
| env: | ||
| GCS_TAG: server_deb | ||
| needs: | ||
| - build-centos | ||
| - build-linux | ||
| - build-osx | ||
| - build-windows | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - name: Download installers from GitHub artifacts | ||
| id: download | ||
| uses: actions/download-artifact@v2 | ||
| with: | ||
| path: ~/_artifacts | ||
| - name: Set up | ||
| run: | | ||
| sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv wget openjdk-8-jdk zip git devscripts dh-systemd libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1 | ||
| pip3 install --upgrade setuptools virtualenv | ||
| virtualenv "${HOME}/INSTALL" | ||
| - name: Build | ||
| run: | | ||
| travis/install.sh | ||
| mkdir -p grr/config/grr_response_templates/templates | ||
| mv -v ~/_artifacts/windows-installers/GRR_*_amd64.exe.zip grr/config/grr_response_templates/templates | ||
| mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates | ||
| mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates | ||
| mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates | ||
| mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates | ||
| travis/build_local_pyindex.sh | ||
| travis/build_server_deb.sh | ||
| ls -la gcs_upload_dir | ||
| - name: Upload installers to GitHub artifacts | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: server-deb | ||
| path: gcs_upload_dir/ | ||
| retention-days: 1 | ||
|
|
||
| test-ubuntu-e2e: | ||
| continue-on-error: true # Debug follow up step. | ||
| runs-on: ubuntu-18.04 | ||
| env: | ||
| GRR_ADMIN_PASS: 'e2e_tests' | ||
| APPVEYOR_MYSQL_PASS: 'root' | ||
| needs: | ||
| - build-server-deb | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: '3.6' | ||
| - name: Set up MySQL | ||
| run: | | ||
| sudo /etc/init.d/mysql start | ||
| - name: Download installers from GitHub artifacts | ||
| id: download | ||
| uses: actions/download-artifact@v2 | ||
| with: | ||
| name: server-deb | ||
| path: _artifacts | ||
| - name: Install | ||
| run: | | ||
| free -hmw | ||
| lscpu | ||
| sudo apt install -y libmysqlclient-dev | ||
| if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then | ||
| wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}"; | ||
| fi | ||
| python -m venv "${HOME}/INSTALL" | ||
| travis/install.sh | ||
| sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh | ||
| sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh | ||
| - name: Test | ||
| run: | | ||
| sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh | ||
| sudo -EH ./appveyor/e2e_tests/test_repack.sh | ||
| ./appveyor/e2e_tests/upload_e2e_artifacts.sh | ||
| - name: Upload logs and configs to GitHub artifacts | ||
| uses: actions/upload-artifact@v2 | ||
| with: | ||
| name: e2e-test | ||
| path: appveyor_e2e_artifacts/ | ||
| retention-days: 1 | ||
|
|
||
|
|
||
| build-push-docker: | ||
| runs-on: ubuntu-18.04 | ||
| needs: | ||
| - build-server-deb | ||
| # - test-ubuntu-e2e # TODO: Comment back in after debugging is finished. | ||
| - test-ubuntu | ||
| - build-openapi | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - name: Download installers from GitHub artifacts | ||
| id: download | ||
| uses: actions/download-artifact@v2 | ||
| with: | ||
| name: server-deb | ||
| path: _artifacts | ||
| - name: Build Docker image | ||
| run: | | ||
| export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3) | ||
| ./appveyor/docker_build/build_docker_image.sh | ||
| - if: ${{ github.event_name == 'push' }} | ||
| name: Log in to Docker Hub | ||
| uses: docker/login-action@v1 | ||
| with: | ||
| username: ${{ secrets.DOCKER_USERNAME }} | ||
| password: ${{ secrets.DOCKER_PASSWORD }} | ||
| - if: ${{ github.event_name == 'push' }} | ||
| name: Push to ${{ env.DOCKER_REPOSITORY }} | ||
| run: | | ||
| docker push ${{ env.DOCKER_REPOSITORY }} | ||
| upload: | ||
| if: ${{ github.event_name == 'push' }} | ||
| runs-on: ubuntu-18.04 | ||
| needs: | ||
| - build-push-docker | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - name: Download installers from GitHub artifacts | ||
| id: download | ||
| uses: actions/download-artifact@v2 | ||
| with: | ||
| path: _artifacts | ||
| - run: | | ||
| ls -la _artifacts/*/ | ||
| COMMIT_TIME=$(git show -s --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" $GITHUB_SHA) | ||
| OUTPUT_DIR=gcs_upload_dir/${COMMIT_TIME}_${GITHUB_SHA}/ | ||
| echo "OUTPUT_DIR=$OUTPUT_DIR" >> $GITHUB_ENV | ||
| mkdir -p $OUTPUT_DIR/centos/ | ||
| mv -v _artifacts/centos-installers/* $OUTPUT_DIR/centos | ||
| mkdir -p $OUTPUT_DIR/ubuntu/ | ||
| mv -v _artifacts/ubuntu-installers/* $OUTPUT_DIR/ubuntu | ||
| mkdir -p $OUTPUT_DIR/osx/ | ||
| mv -v _artifacts/osx-installers/* $OUTPUT_DIR/osx | ||
| mkdir -p $OUTPUT_DIR/windows/ | ||
| mv -v _artifacts/windows-installers/* $OUTPUT_DIR/windows | ||
| mkdir -p $OUTPUT_DIR/server_deb/ | ||
| mv -v _artifacts/server-deb/* $OUTPUT_DIR/server_deb | ||
| - name: Set up Cloud SDK | ||
| uses: google-github-actions/[email protected] | ||
| with: | ||
| project_id: ${{ secrets.GCP_PROJECT_ID }} | ||
| service_account_key: ${{ secrets.GCP_SA_KEY }} | ||
| export_default_credentials: true | ||
| - name: Upload installers to GCS | ||
| uses: google-github-actions/upload-cloud-storage@main | ||
| with: | ||
| path: gcs_upload_dir/ | ||
| destination: ${{ env.GCS_BUCKET }} | ||
| # Omit `path` (e.g. /home/runner/deploy/) in final GCS path. | ||
| parent: false | ||
| - name: Replace ${{ env.GCS_LATEST_PATH }} folder in GCS | ||
| run: | | ||
| gsutil rm gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/** || true | ||
| gsutil cp -r $OUTPUT_DIR/server_deb/* gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/ | ||
| - name: Upload OpenAPI to GCS | ||
| uses: google-github-actions/upload-cloud-storage@main | ||
| with: | ||
| path: _artifacts/openapi/ | ||
| destination: ${{ env.GCS_BUCKET_OPENAPI }} | ||
| # Omit `path` (e.g. /home/runner/deploy/) in final GCS path. | ||
| parent: false |
Oops, something went wrong.