API-284: Trying to bq load #45
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Entur/Api/Lint | |
| on: | |
| push: #Remove later. Schedule only runs on default branch | |
| schedule: | |
| - cron: "*/1 * * * *" | |
| jobs: | |
| lint-and-store-result: | |
| name: OpenAPI Lint | |
| runs-on: ubuntu-24.04 | |
| environment: dev | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout guidelines | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: entur/api-guidelines | |
| path: rulesets | |
| sparse-checkout: | | |
| .spectral.yml | |
| - name: Authenticate with Google Cloud | |
| id: login-gcp | |
| uses: google-github-actions/[email protected] | |
| with: | |
| workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ vars.SERVICE_ACCOUNT }} | |
| - name: Check Spec Hashes and Identify Changed Specs | |
| id: check-hashes | |
| shell: bash | |
| run: | | |
| # Initialize an empty array to hold changed spec file paths. | |
| changed_specs=() | |
| #Calulate hash of the linting rules. If the rules change, all specs should be linted. | |
| computed_rules_hash=$(sha256sum "rulesets/.spectral.yml" | awk '{print $1}') | |
| # Check if the /specs directory is empty | |
| if [ -z "$(ls -A ./specs)" ]; then | |
| echo "No spec files found in ./specs. Aborting job." | |
| exit 0 #Variable "changes" is not set to true, so steps below will be ignored. | |
| fi | |
| # Loop over each file in the /specs directory. | |
| for spec in ./specs/*; do | |
| echo "Processing spec file: $spec" | |
| # Compute a SHA256 hash for the spec file. | |
| computed_spec_hash=$(sha256sum "$spec" | awk '{print $1}') | |
| # Extract the service name from the spec. | |
| if [[ "$spec" == *.json ]]; then | |
| service=$(jq -r '.info.title' "$spec") | |
| else | |
| service=$(yq e '.info.title' "$spec") | |
| fi | |
| # Query BigQuery for the latest stored hashes for this service. | |
| query="SELECT spec_hash, rules_hash FROM \`ent-apidata-dev.api.rest_api_lint\` WHERE service_name = '$service' ORDER BY created DESC LIMIT 1" | |
| # Run the query using the bq CLI, outputting results in JSON format. | |
| response=$(bq query --project_id=ent-apidata-dev --format=json --nouse_legacy_sql "$query") | |
| # Extract the spec_hash and rules_hash from the first row. | |
| stored_spec_hash=$(echo "$response" | jq -r '.[0].spec_hash') | |
| stored_rules_hash=$(echo "$response" | jq -r '.[0].rules_hash') | |
| # Compare computed hash with the stored hash. | |
| if [ "$computed_spec_hash" != "$stored_spec_hash" ]; then | |
| echo "Spec $spec has changed." | |
| changed_specs+=("$spec") | |
| elif [ "$computed_rules_hash" != "$stored_rules_hash" ]; then | |
| echo "Linting rules have changed. Linting $spec." | |
| changed_specs+=("$spec") | |
| else | |
| echo "Spec $spec is unchanged and linting rules have not changed. Not linting spec." | |
| fi | |
| done | |
| # If no spec file has changed, exit the job. | |
| if [ ${#changed_specs[@]} -eq 0 ]; then | |
| echo "No spec changes detected. Aborting job." | |
| exit 0 #Variable "changes" is not set to true, so steps below will be ignored. | |
| fi | |
| echo "changes=true" >> $GITHUB_OUTPUT | |
| # Convert the array of changed spec files into a space-separated string. | |
| changed_specs_list=$(printf "%s " "${changed_specs[@]}") | |
| # Set an output variable so later steps know which specs to lint. | |
| echo "::set-output name=changed_specs::$changed_specs_list" | |
| - name: Setup Node.js | |
| if: steps.check-hashes.outputs.changes == 'true' | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: "16" | |
| - name: Install Spectral CLI | |
| if: steps.check-hashes.outputs.changes == 'true' | |
| run: npm install -g @stoplight/spectral-cli | |
| - name: Run Spectral Linting on Changed Specs | |
| if: steps.check-hashes.outputs.changes == 'true' | |
| id: lint | |
| run: | | |
| # Retrieve the list of changed specs from the previous step. | |
| specs=$(echo "${{ steps.check-hashes.outputs.changed_specs }}") | |
| echo "Linting changed specs: $specs" | |
| spectral lint $specs -o lint-result.json --ruleset rulesets/.spectral.yml -f json --quiet || true | |
| - name: Transform lint results into grouped JSON | |
| if: steps.check-hashes.outputs.changes == 'true' | |
| run: | | |
| # Get the spec source file paths from lint-result.json | |
| sources=$(jq -r '.[].source' lint-result.json | sort -u) | |
| # Initialize an empty array for JSON rows. | |
| rows_array=() | |
| # Iterate over each unique source | |
| while IFS= read -r src; do | |
| if [[ "$src" == *.json ]]; then | |
| # If the file is JSON, use jq to extract the service name. | |
| service=$(jq -r '.info.title' "$src") | |
| else | |
| # Otherwise, assume it's YAML and use yq. | |
| service=$(yq e '.info.title' "$src") | |
| fi | |
| # Re-compute the SHA256 hash for the spec file | |
| computed_spec_hash=$(sha256sum "$src" | awk '{print $1}') | |
| # Re-compute the SHA256 hash for the rules file | |
| computed_rules_hash=$(sha256sum "rulesets/.spectral.yml" | awk '{print $1}') | |
| # Get rule violations from lint-result.json for entries with this source and map each to {rule, severity} | |
| results=$(jq --arg src "$src" '[.[] | select(.source == $src) | {rule: .code, severity: .severity}]' lint-result.json) | |
| # Build a row JSON object with the extracted service name and the result array | |
| row=$(jq -n \ | |
| --arg service "$service" \ | |
| --argjson results "$results" \ | |
| --arg spec_hash "$computed_spec_hash" \ | |
| --arg rules_hash "$computed_rules_hash" \ | |
| '{service_name: $service, lint_result: $results, spec_hash: $spec_hash, rules_hash: $rules_hash}') | |
| # Append the row to our rows_array | |
| rows_array+=("$row") | |
| done <<< "$sources" | |
| printf '%s\n' "${rows_array[@]}" > transformed.json | |
| - name: Write to BigQuery | |
| if: steps.check-hashes.outputs.changes == 'true' | |
| id: write-to-bigquery | |
| shell: bash | |
| run: | | |
| bq load \ | |
| --source_format=NEWLINE_DELIMITED_JSON \ | |
| --autodetect \ | |
| --project_id=ent-apidata-dev \ | |
| ent-apidata-dev:api.rest_api_lint \ | |
| transformed.json |