Skip to content

Commit 0f27e49

Browse files
disqdisq
authored
feat: Initial implementation (#1)
Co-authored-by: Kemal Hadimli <[email protected]>
1 parent 591a0ac commit 0f27e49

File tree

192 files changed

+8194
-1
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

192 files changed

+8194
-1
lines changed

‎.gitignore

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
dist/
2+
vendor/
3+
bin/
4+
cq-source-scaleway
5+
.DS_Store
6+
.idea

‎.golangci.yml

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
run:
2+
tests: true
3+
skip-dirs:
4+
- bin
5+
- docs
6+
- client/mocks
7+
- resources/forks
8+
- codegen
9+
timeout: 10m
10+
build-tags:
11+
- all
12+
13+
linters-settings:
14+
errcheck:
15+
check-blank: false
16+
ignore: fmt:.*,[rR]ead|[wW]rite|[cC]lose,io:Copy
17+
gocritic:
18+
disabled-checks:
19+
- commentFormatting
20+
dupl:
21+
# tokens count to trigger issue, 150 by default
22+
threshold: 500
23+
24+
misspell:
25+
ignore-words:
26+
- hdinsight
27+
28+
revive:
29+
enable-all-rules: true
30+
rules:
31+
- name: cyclomatic
32+
disabled: true
33+
- name: argument-limit
34+
disabled: true
35+
- name: function-length
36+
disabled: true
37+
- name: function-result-limit
38+
disabled: true
39+
- name: line-length-limit
40+
disabled: true
41+
- name: file-header
42+
disabled: true
43+
- name: cognitive-complexity
44+
disabled: true
45+
- name: banned-characters
46+
disabled: true
47+
- name: max-public-structs
48+
disabled: true
49+
- name: add-constant
50+
disabled: true
51+
- name: unhandled-error
52+
disabled: true
53+
- name: var-naming
54+
disabled: true
55+
- name: deep-exit
56+
disabled: true
57+
- name: exported
58+
disabled: false
59+
arguments:
60+
- 'disableStutteringCheck'
61+
- name: unused-parameter
62+
disabled: true
63+
- name: confusing-naming
64+
disabled: true
65+
- name: confusing-results
66+
disabled: true
67+
- name: flag-parameter
68+
disabled: true
69+
- name: nested-structs
70+
disabled: true
71+
72+
gofmt:
73+
rewrite-rules:
74+
- pattern: 'interface{}'
75+
replacement: 'any'
76+
- pattern: 'a[b:len(a)]'
77+
replacement: 'a[b:]'
78+
79+
linters:
80+
enable:
81+
- asciicheck
82+
- bodyclose
83+
- depguard
84+
- dupl
85+
- errcheck
86+
- gocritic
87+
- gofmt
88+
- gosimple
89+
- govet
90+
- ineffassign
91+
- importas
92+
- misspell
93+
- nakedret
94+
- prealloc
95+
- revive
96+
- staticcheck
97+
- unconvert
98+
- unparam
99+
- unused
100+
101+
issues:
102+
exclude-rules:
103+
# Exclude some linters from running on tests files.
104+
- path: _test\.go
105+
linters:
106+
- dupl
107+
- gocritic
108+
# Exclude some linters from running on resource files.
109+
- path: resources(\\|\/).*\.go
110+
linters:
111+
- dupl
112+
# Exclude some linters from running on services files.
113+
- path: services\.go
114+
linters:
115+
- dupl

‎.goreleaser.yaml

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
before:
2+
hooks:
3+
- go mod download
4+
builds:
5+
- flags:
6+
- -buildmode=exe
7+
env:
8+
- CGO_ENABLED=0
9+
- GO111MODULE=on
10+
ldflags:
11+
- -s -w -X github.com/scaleway/cq-source-scaleway/plugin.Version={{.Version}}
12+
goos:
13+
- windows
14+
- linux
15+
- darwin
16+
goarch:
17+
- amd64
18+
- arm64
19+
ignore:
20+
- goos: windows
21+
goarch: arm64
22+
archives:
23+
- name_template: "{{ .Binary }}_{{ .Os }}_{{ .Arch }}"
24+
format: zip
25+
checksum:
26+
name_template: "checksums.txt"
27+
changelog:
28+
sort: asc
29+
filters:
30+
exclude:
31+
- "^docs:"
32+
- "^test:"
33+
34+
release:
35+
prerelease: auto

‎Makefile

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
.PHONY: test
2+
test:
3+
go test -timeout 3m ./...
4+
5+
.PHONY: lint
6+
lint:
7+
@golangci-lint run --timeout 10m --verbose
8+
9+
.PHONY: gen-docs
10+
gen-docs:
11+
rm -rf ./docs/tables/*
12+
go run main.go doc ./docs/tables
13+
14+
# All gen targets
15+
.PHONY: gen
16+
gen: gen-docs

‎README.md

+63-1
Original file line numberDiff line numberDiff line change
@@ -1 +1,63 @@
1-
# cq-source-scaleway
1+
# CloudQuery Scaleway Source Plugin
2+
3+
[![test](https://github.com/scaleway/cq-source-scaleway/actions/workflows/test.yaml/badge.svg)](https://github.com/scaleway/cq-source-scaleway/actions/workflows/test.yaml)
4+
[![Go Report Card](https://goreportcard.com/badge/github.com/scaleway/cq-source-scaleway)](https://goreportcard.com/report/github.com/scaleway/cq-source-scaleway)
5+
6+
A [Scaleway](https://scaleway.com/) source plugin for CloudQuery that loads data from Scaleway to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more.
7+
8+
## Links
9+
10+
- [CloudQuery Quickstart Guide](https://www.cloudquery.io/docs/quickstart)
11+
- [Supported Tables](docs/tables/README.md)
12+
13+
## Authentication
14+
15+
Credentials are used from the default config file and the environment. Get your credentials from the [IAM dashboard](https://console.scaleway.com/iam/api-keys)
16+
17+
Set credentials in environment variables or use the default config file at `~/.config/scw/config.yaml`. Config format and locations for other platforms are documented in the [SDK Docs](https://github.com/scaleway/scaleway-sdk-go/tree/master/scw#scaleway-config)
18+
19+
The environment variables to set are:
20+
- `SCW_ACCESS_KEY`
21+
- `SCW_SECRET_KEY`
22+
- `SCW_DEFAULT_ORGANIZATION_ID`
23+
24+
Env vars override config values if both are set. By default all regions and zones are queried.
25+
26+
## Incremental Syncing
27+
28+
The Scaleway plugin supports incremental syncing. This means that only new data will be fetched from Scaleway and loaded into your destination for supported tables (support depending on API endpoint). This is done by keeping track of the last item fetched and only fetching data that has been created since then.
29+
To enable this, `backend` option must be set in the spec (as shown below). This is documented in the [Managing Incremental Tables](https://www.cloudquery.io/docs/advanced-topics/managing-incremental-tables) section.
30+
31+
## Configuration
32+
33+
The following source configuration file will sync all data from Scaleway to a PostgreSQL database. See [the CloudQuery Quickstart](https://www.cloudquery.io/docs/quickstart) for more information on how to configure the source and destination.
34+
35+
```yaml
36+
kind: source
37+
spec:
38+
name: "scaleway"
39+
path: "scaleway/scaleway"
40+
version: "${VERSION}"
41+
# backend: "local" # use this to enable incremental syncing
42+
tables:
43+
- "*"
44+
skip_tables:
45+
- "scaleway_ipfs_volumes"
46+
- "scaleway_marketplace_image_versions"
47+
destinations:
48+
- "postgresql"
49+
backend: local
50+
spec:
51+
# plugin spec section
52+
```
53+
54+
### Plugin Spec
55+
56+
- `regions` (list of string, optional. Defaults to all regions):
57+
List of regions to query.
58+
59+
- `zones` (list of string, optional. Defaults to all zones):
60+
List of zones to query.
61+
62+
- `timeout_secs` (integer in seconds, optional. Default: 10):
63+
Timeout for requests against the Scaleway API endpoint.

‎client/client.go

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package client
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"net/http"
8+
"time"
9+
10+
"github.com/cloudquery/plugin-sdk/backend"
11+
"github.com/cloudquery/plugin-sdk/plugins/source"
12+
"github.com/cloudquery/plugin-sdk/schema"
13+
"github.com/cloudquery/plugin-sdk/specs"
14+
"github.com/rs/zerolog"
15+
"github.com/scaleway/scaleway-sdk-go/scw"
16+
)
17+
18+
type Client struct {
19+
Logger zerolog.Logger
20+
SCWClient *scw.Client
21+
Backend backend.Backend
22+
OrgID string
23+
Region scw.Region
24+
Zone scw.Zone
25+
26+
Spec Spec
27+
sourceSpec specs.Source
28+
}
29+
30+
func (c *Client) ID() string {
31+
return c.sourceSpec.Name + ":" + string(c.Region) + ":" + string(c.Zone)
32+
}
33+
34+
func (c *Client) WithRegion(r scw.Region) *Client {
35+
return &Client{
36+
Logger: c.Logger.With().Str("region", string(r)).Logger(),
37+
SCWClient: c.SCWClient,
38+
Backend: c.Backend,
39+
OrgID: c.OrgID,
40+
Region: r,
41+
Zone: c.Zone,
42+
Spec: c.Spec,
43+
sourceSpec: c.sourceSpec,
44+
}
45+
}
46+
47+
func (c *Client) WithZone(z scw.Zone) *Client {
48+
return &Client{
49+
Logger: c.Logger.With().Str("zone", string(z)).Logger(),
50+
SCWClient: c.SCWClient,
51+
Backend: c.Backend,
52+
OrgID: c.OrgID,
53+
Region: c.Region,
54+
Zone: z,
55+
Spec: c.Spec,
56+
sourceSpec: c.sourceSpec,
57+
}
58+
}
59+
60+
func New(_ context.Context, logger zerolog.Logger, s specs.Source, opts source.Options) (schema.ClientMeta, error) {
61+
var pluginSpec Spec
62+
if err := s.UnmarshalSpec(&pluginSpec); err != nil {
63+
return nil, fmt.Errorf("failed to unmarshal plugin spec: %w", err)
64+
}
65+
err := pluginSpec.Validate()
66+
if err != nil {
67+
return nil, fmt.Errorf("failed to validate plugin spec: %w", err)
68+
}
69+
pluginSpec.SetDefaults()
70+
71+
scwOpts := []scw.ClientOption{
72+
scw.WithEnv(), // existing env variables may overwrite active profile
73+
74+
scw.WithHTTPClient(&http.Client{
75+
Timeout: time.Duration(pluginSpec.Timeout) * time.Second,
76+
}),
77+
scw.WithUserAgent("cq-plugin-scaleway/" + s.Version),
78+
}
79+
80+
cf, err := scw.LoadConfig()
81+
if err != nil {
82+
var configNotFoundError *scw.ConfigFileNotFoundError
83+
if !errors.As(err, &configNotFoundError) {
84+
return nil, err
85+
}
86+
}
87+
if cf != nil {
88+
p, err := cf.GetActiveProfile()
89+
if err != nil {
90+
return nil, err
91+
}
92+
scwOpts = append([]scw.ClientOption{
93+
scw.WithProfile(p), // active profile applies first
94+
}, scwOpts...)
95+
}
96+
97+
// Create a Scaleway client
98+
scwClient, err := scw.NewClient(scwOpts...)
99+
if err != nil {
100+
return nil, err
101+
}
102+
103+
orgID, ok := scwClient.GetDefaultOrganizationID()
104+
if !ok {
105+
return nil, fmt.Errorf("SCW_DEFAULT_ORGANIZATION_ID or default_organization_id not set, get yours from https://console.scaleway.com/organization/settings")
106+
}
107+
108+
return &Client{
109+
Logger: logger,
110+
Backend: opts.Backend,
111+
OrgID: orgID,
112+
Spec: pluginSpec,
113+
SCWClient: scwClient,
114+
sourceSpec: s,
115+
}, nil
116+
}

0 commit comments

Comments
 (0)