Skip to content

Commit 4a8658e

Browse files
rikard-swahnrikard-swahn
authored
Merge pull request #17 from entur/API-281-lint-v2
Store api spec lint results
2 parents 33f6be2 + 2f6fb5f commit 4a8658e

File tree

3 files changed

+14902
-0
lines changed

3 files changed

+14902
-0
lines changed

‎.github/workflows/store-api-spec-lint-results.yml

+173
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
name: Entur/Api/Lint
2+
3+
on:
4+
push: #Remove later. Schedule only runs on default branch
5+
schedule:
6+
- cron: "*/1 * * * *"
7+
8+
jobs:
9+
lint-and-store-result:
10+
name: OpenAPI Lint
11+
runs-on: ubuntu-24.04
12+
environment: dev
13+
permissions:
14+
contents: read
15+
id-token: write
16+
steps:
17+
- name: Checkout Repository
18+
uses: actions/checkout@v4
19+
- name: Checkout guidelines
20+
uses: actions/checkout@v4
21+
with:
22+
repository: entur/api-guidelines
23+
path: rulesets
24+
sparse-checkout: |
25+
.spectral.yml
26+
27+
- name: Authenticate with Google Cloud
28+
id: login-gcp
29+
uses: google-github-actions/[email protected]
30+
with:
31+
workload_identity_provider: ${{ vars.WORKLOAD_IDENTITY_PROVIDER }}
32+
service_account: ${{ vars.SERVICE_ACCOUNT }}
33+
token_format: "access_token"
34+
35+
- name: Check Spec Hashes and Identify Changed Specs
36+
id: check-hashes
37+
shell: bash
38+
run: |
39+
# Initialize an empty array to hold changed spec file paths.
40+
changed_specs=()
41+
42+
#Calulate hash of the linting rules. If the rules change, all specs should be linted.
43+
computed_rules_hash=$(sha256sum "rulesets/.spectral.yml" | awk '{print $1}')
44+
45+
# Check if the /specs directory is empty
46+
if [ -z "$(ls -A ./specs)" ]; then
47+
echo "No spec files found in ./specs. Aborting job."
48+
exit 0 #Variable "changes" is not set to true, so steps below will be ignored.
49+
fi
50+
51+
# Loop over each file in the /specs directory.
52+
for spec in ./specs/*; do
53+
echo "Processing spec file: $spec"
54+
55+
# Compute a SHA256 hash for the spec file.
56+
computed_spec_hash=$(sha256sum "$spec" | awk '{print $1}')
57+
58+
# Extract the service name from the spec.
59+
if [[ "$spec" == *.json ]]; then
60+
service=$(jq -r '.info.title' "$spec")
61+
else
62+
service=$(yq e '.info.title' "$spec")
63+
fi
64+
65+
# Query BigQuery for the latest stored hash for this service.
66+
query="SELECT spec_hash, rules_hash FROM \`ent-apidata-dev.api.rest_api_lint\` WHERE service_name = '$service' ORDER BY created DESC LIMIT 1"
67+
response=$(curl -s -X POST "https://bigquery.googleapis.com/bigquery/v2/projects/ent-apidata-dev/queries" \
68+
--header "Authorization: Bearer ${{ steps.login-gcp.outputs.access_token }}" \
69+
--header "Content-Type: application/json" \
70+
--data "{\"query\": \"$query\", \"useLegacySql\": false}")
71+
72+
stored_spec_hash=$(echo "$response" | jq -r '.rows[0].f[0].v')
73+
stored_rules_hash=$(echo "$response" | jq -r '.rows[0].f[1].v')
74+
75+
# Compare computed hash with the stored hash.
76+
if [ "$computed_spec_hash" != "$stored_spec_hash" ]; then
77+
echo "Spec $spec has changed."
78+
changed_specs+=("$spec")
79+
elif [ "$computed_rules_hash" != "$stored_rules_hash" ]; then
80+
echo "Linting rules have changed. Linting $spec."
81+
changed_specs+=("$spec")
82+
else
83+
echo "Spec $spec is unchanged and linting rules have not changed. Not linting spec."
84+
fi
85+
done
86+
87+
# If no spec file has changed, exit the job.
88+
if [ ${#changed_specs[@]} -eq 0 ]; then
89+
echo "No spec changes detected. Aborting job."
90+
exit 0 #Variable "changes" is not set to true, so steps below will be ignored.
91+
fi
92+
93+
echo "changes=true" >> $GITHUB_OUTPUT
94+
95+
# Convert the array of changed spec files into a space-separated string.
96+
changed_specs_list=$(printf "%s " "${changed_specs[@]}")
97+
# Set an output variable so later steps know which specs to lint.
98+
echo "::set-output name=changed_specs::$changed_specs_list"
99+
- name: Setup Node.js
100+
if: steps.check-hashes.outputs.changes == 'true'
101+
uses: actions/setup-node@v3
102+
with:
103+
node-version: "16"
104+
- name: Install Spectral CLI
105+
if: steps.check-hashes.outputs.changes == 'true'
106+
run: npm install -g @stoplight/spectral-cli
107+
- name: Run Spectral Linting on Changed Specs
108+
if: steps.check-hashes.outputs.changes == 'true'
109+
id: lint
110+
run: |
111+
# Retrieve the list of changed specs from the previous step.
112+
specs=$(echo "${{ steps.check-hashes.outputs.changed_specs }}")
113+
echo "Linting changed specs: $specs"
114+
spectral lint $specs -o lint-result.json --ruleset rulesets/.spectral.yml -f json --quiet || true
115+
116+
- name: Transform lint results into grouped JSON
117+
if: steps.check-hashes.outputs.changes == 'true'
118+
run: |
119+
# Get the spec source file paths from lint-result.json
120+
sources=$(jq -r '.[].source' lint-result.json | sort -u)
121+
122+
# Initialize an empty array for JSON rows.
123+
rows_array=()
124+
125+
# Iterate over each unique source
126+
while IFS= read -r src; do
127+
if [[ "$src" == *.json ]]; then
128+
# If the file is JSON, use jq to extract the service name.
129+
service=$(jq -r '.info.title' "$src")
130+
else
131+
# Otherwise, assume it's YAML and use yq.
132+
service=$(yq e '.info.title' "$src")
133+
fi
134+
135+
# Re-compute the SHA256 hash for the spec file
136+
computed_spec_hash=$(sha256sum "$src" | awk '{print $1}')
137+
138+
# Re-compute the SHA256 hash for the rules file
139+
computed_rules_hash=$(sha256sum "rulesets/.spectral.yml" | awk '{print $1}')
140+
141+
# Get rule violations from lint-result.json for entries with this source and map each to {rule, severity}
142+
results=$(jq --arg src "$src" '[.[] | select(.source == $src) | {rule: .code, severity: .severity}]' lint-result.json)
143+
144+
# Build a row JSON object with the extracted service name and the result array
145+
row=$(jq -n \
146+
--arg service "$service" \
147+
--argjson results "$results" \
148+
--arg spec_hash "$computed_spec_hash" \
149+
--arg rules_hash "$computed_rules_hash" \
150+
'{json: {service_name: $service, lint_result: $results, spec_hash: $spec_hash, rules_hash: $rules_hash}}')
151+
152+
# Append the row to our rows_array
153+
rows_array+=("$row")
154+
done <<< "$sources"
155+
156+
# Combine all rows into the final JSON structure
157+
combined=$(printf '%s\n' "${rows_array[@]}" | jq -s '{rows: .}')
158+
159+
# Write the output to a file
160+
echo "$combined" > transformed.json
161+
162+
- name: Write to BigQuery
163+
if: steps.check-hashes.outputs.changes == 'true'
164+
id: write-to-bigquery
165+
shell: bash
166+
run: |
167+
curl --request POST \
168+
'https://bigquery.googleapis.com/bigquery/v2/projects/ent-apidata-dev/datasets/api/tables/rest_api_lint/insertAll' \
169+
--header "Authorization: Bearer ${{ steps.login-gcp.outputs.access_token }}" \
170+
--header "Accept: application/json" \
171+
--header "Content-Type: application/json" \
172+
--data @transformed.json \
173+
--compressed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
.idea

0 commit comments

Comments
 (0)