Feature multi select (#176)

* Implement multi-pattern support for groups and users

* adjust regex for multi-pattern

* Allow - and ' ' in names

* UPDATE envVar creation logic for Match

* Update template.yaml

* Update template.yaml

* Adding support for '*' to sync all and empty to sync nothing.

* Improvements to Filtering

UserMatch now considered in addition to GroupMatch.

Improved filtering for external users
proper handling of nested groups.

* Improve logging

Added dump of envVars
Corrected copy&paste error in log message.

* Adding user detail caching

To reduce repeated calls to the directory api, prefetch all users and use when processing groups.

* Update README.md

Author: ChrisPates

README.md

@@ -11,7 +11,7 @@
 SSO Sync will run on any platform that Go can build for. It is available in the [AWS Serverless Application Repository](https://console.aws.amazon.com/lambda/home#/create/app?applicationId=arn:aws:serverlessrepo:us-east-2:004480582608:applications/SSOSync)
 
 > [!CAUTION]
-> When using ssosync with an instance or IAM Identity Center integrated with AWS Control Tower. AWS Control Tower creates a number of groups and users (directly via the Identity Store API), when an external identity provider is configured these users and groups are can not be used to log in. However it is important to remember that because ssosync implemements a uni-directional sync it will make the IAM Identity Store match the subset of your Google Workspaces directory you specify, including removing these groups and users created by AWS Control Tower. There is a PFR [#88 - ssosync deletes Control Tower groups](https://github.com/awslabs/ssosync/issues/88) to implement an option to ignore these users and groups, hopefully this will be implemented in version 3.x.
+> When using ssosync with an instance of IAM Identity Center integrated with AWS Control Tower. AWS Control Tower creates a number of groups and users (directly via the Identity Store API), when an external identity provider is configured these users and groups are can not be used to log in. However it is important to remember that because ssosync implemements a uni-directional sync it will make the IAM Identity Store match the subset of your Google Workspaces directory you specify, including removing these groups and users created by AWS Control Tower. There is a PFR [#88 - ssosync deletes Control Tower groups](https://github.com/awslabs/ssosync/issues/88) to implement an option to ignore these users and groups, hopefully this will be implemented in version 3.x.
 
 > [!WARNING]
 > There are breaking changes for versions `>= 0.02`
@@ -30,6 +30,13 @@ SSO Sync will run on any platform that Go can build for. It is available in the
 > [!IMPORTANT]
 > `>= 2.1.0` switched to using `provided.al2` powered by ARM64 instances.
 
+> [!Info]
+> As of `v2.2.0` multiple query patterns are supported for both Group and User matching, simply separate each query with a `,`. For full sync of groups and/or users specify '*' in the relevant match field. 
+> User match and group match can now be used in combination with the sync method of groups.
+> Nested groups will now be flattened into the top level groups.
+> external users are ignored.
+> User details are now cached to reduce the number of api calls and improve execution times on large directories.
+
 ## Why?
 
 As per the [AWS SSO](https://aws.amazon.com/single-sign-on/) Homepage:
@@ -146,15 +153,15 @@ Flags:
   -e, --endpoint string             AWS SSO SCIM API Endpoint
   -u, --google-admin string         Google Workspace admin user email
   -c, --google-credentials string   path to Google Workspace credentials file (default "credentials.json")
-  -g, --group-match string          Google Workspace Groups filter query parameter, example: 'name:Admin* email:aws-*', see: https://developers.google.com/admin-sdk/directory/v1/guides/search-groups
+  -g, --group-match string          Google Workspace Groups filter query parameter, a simple '*' denotes sync all groups (and any users that are members of those groups). example: 'name:Admin*,email:aws-*', 'name=Admins' or '*' see: https://developers.google.com/admin-sdk/directory/v1/guides/search-groups
   -h, --help                        help for ssosync
       --ignore-groups strings       ignores these Google Workspace groups
       --ignore-users strings        ignores these Google Workspace users
       --include-groups strings      include only these Google Workspace groups, NOTE: only works when --sync-method 'users_groups'
       --log-format string           log format (default "text")
       --log-level string            log level (default "info")
   -s, --sync-method string          Sync method to use (users_groups|groups) (default "groups")
-  -m, --user-match string           Google Workspace Users filter query parameter, example: 'name:John* email:admin*', see: https://developers.google.com/admin-sdk/directory/v1/guides/search-users
+  -m, --user-match string           Google Workspace Users filter query parameter, a simple '*' denotes sync all users in the directory. example: 'name:John*,email:admin*', '*' or name=John Doe,email:admin*' see: https://developers.google.com/admin-sdk/directory/v1/guides/search-users
   -v, --version                     version for ssosync
   -r, --region                      AWS region where identity store exists
   -i, --identity-store-id           AWS Identity Store ID

cmd/root.go

@@ -199,78 +199,86 @@ func configLambda() {
 
 	unwrap, err := secrets.GoogleAdminEmail(os.Getenv("GOOGLE_ADMIN"))
 	if err != nil {
-		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
+		log.Fatalf(errors.Wrap(err, "cannot read config: GOOGLE_ADMIN").Error())
 	}
 	cfg.GoogleAdmin = unwrap
 
 	unwrap, err = secrets.GoogleCredentials(os.Getenv("GOOGLE_CREDENTIALS"))
 	if err != nil {
-		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
+		log.Fatalf(errors.Wrap(err, "cannot read config: GOOGLE_CREDENTIALS").Error())
 	}
 	cfg.GoogleCredentials = unwrap
 
 	unwrap, err = secrets.SCIMAccessToken(os.Getenv("SCIM_ACCESS_TOKEN"))
 	if err != nil {
-		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
+		log.Fatalf(errors.Wrap(err, "cannot read config: SCIM_ACCESS_TOKEN").Error())
 	}
 	cfg.SCIMAccessToken = unwrap
 
 	unwrap, err = secrets.SCIMEndpointUrl(os.Getenv("SCIM_ENDPOINT"))
 	if err != nil {
-		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
+		log.Fatalf(errors.Wrap(err, "cannot read config: SCIM_ENDPOINT").Error())
 	}
 	cfg.SCIMEndpoint = unwrap
 
 	unwrap, err = secrets.Region(os.Getenv("REGION"))
 	if err != nil {
-		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
+		log.Fatalf(errors.Wrap(err, "cannot read config: REGION").Error())
 	}
 	cfg.Region = unwrap
 
 	unwrap, err = secrets.IdentityStoreID(os.Getenv("IDENTITY_STORE_ID"))
 	if err != nil {
-		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
+		log.Fatalf(errors.Wrap(err, "cannot read config: IDENTITY_STORE_ID").Error())
 	}
 	cfg.IdentityStoreID = unwrap
 
         unwrap = os.Getenv("LOG_LEVEL")
         if len([]rune(unwrap)) != 0 {
            cfg.LogLevel = unwrap
+	   log.WithField("LogLevel", unwrap).Debug("from EnvVar")
         }
 
         unwrap = os.Getenv("LOG_FORMAT")
         if len([]rune(unwrap)) != 0 {
            cfg.LogFormat = unwrap
+	   log.WithField("LogFormay", unwrap).Debug("from EnvVar")
         }
 
 	unwrap = os.Getenv("SYNC_METHOD")
         if len([]rune(unwrap)) != 0 {
            cfg.SyncMethod = unwrap
+	   log.WithField("SyncMethod", unwrap).Debug("from EnvVar")
         }
 
 	unwrap = os.Getenv("USER_MATCH")
         if len([]rune(unwrap)) != 0 {
 	   cfg.UserMatch = unwrap
+	   log.WithField("UserMatch", unwrap).Debug("from EnvVar")
         }
 
 	unwrap = os.Getenv("GROUP_MATCH")
         if len([]rune(unwrap)) != 0 {
            cfg.GroupMatch = unwrap
+	   log.WithField("GroupMatch", unwrap).Debug("from EnvVar")
         }
 
         unwrap = os.Getenv("IGNORE_GROUPS")
         if len([]rune(unwrap)) != 0 {
            cfg.IgnoreGroups = strings.Split(unwrap, ",")
+	   log.WithField("IgnoreGroups", unwrap).Debug("from EnvVar")
         }
 
         unwrap = os.Getenv("IGNORE_USERS")
         if len([]rune(unwrap)) != 0 {
            cfg.IgnoreUsers = strings.Split(unwrap, ",")
+	   log.WithField("IgnoreUsers", unwrap).Debug("from EnvVar")
         }
 
         unwrap = os.Getenv("INCLUDE_GROUPS")
         if len([]rune(unwrap)) != 0 {
            cfg.IncludeGroups = strings.Split(unwrap, ",")
+	   log.WithField("IncludeGroups", unwrap).Debug("from EnvVar")
         }
 
 }
@@ -287,8 +295,8 @@ func addFlags(cmd *cobra.Command, cfg *config.Config) {
 	rootCmd.Flags().StringSliceVar(&cfg.IgnoreUsers, "ignore-users", []string{}, "ignores these Google Workspace users")
 	rootCmd.Flags().StringSliceVar(&cfg.IgnoreGroups, "ignore-groups", []string{}, "ignores these Google Workspace groups")
 	rootCmd.Flags().StringSliceVar(&cfg.IncludeGroups, "include-groups", []string{}, "include only these Google Workspace groups, NOTE: only works when --sync-method 'users_groups'")
-	rootCmd.Flags().StringVarP(&cfg.UserMatch, "user-match", "m", "", "Google Workspace Users filter query parameter, example: 'name:John* email:admin*', see: https://developers.google.com/admin-sdk/directory/v1/guides/search-users")
-	rootCmd.Flags().StringVarP(&cfg.GroupMatch, "group-match", "g", "", "Google Workspace Groups filter query parameter, example: 'name:Admin* email:aws-*', see: https://developers.google.com/admin-sdk/directory/v1/guides/search-groups")
+	rootCmd.Flags().StringVarP(&cfg.UserMatch, "user-match", "m", "", "Google Workspace Users filter query parameter, example: 'name:John*' 'name=John Doe,email:admin*', to sync all users in the directory specify '*'. For query syntax and more examples see: https://developers.google.com/admin-sdk/directory/v1/guides/search-users")
+	rootCmd.Flags().StringVarP(&cfg.GroupMatch, "group-match", "g", "*", "Google Workspace Groups filter query parameter, example: 'name:Admin*' 'name=Admins,email:aws-*', to sync all groups (and their member users) specify '*'. For query syntax and more examples see: https://developers.google.com/admin-sdk/directory/v1/guides/search-groups")
 	rootCmd.Flags().StringVarP(&cfg.SyncMethod, "sync-method", "s", config.DefaultSyncMethod, "Sync method to use (users_groups|groups)")
 	rootCmd.Flags().StringVarP(&cfg.Region, "region", "r", "", "AWS Region where AWS SSO is enabled")
 	rootCmd.Flags().StringVarP(&cfg.IdentityStoreID, "identity-store-id", "i", "", "Identifier of Identity Store in AWS SSO")

internal/google/client.go

@@ -17,6 +17,7 @@ package google
 
 import (
 	"context"
+	"strings"
 
 	"golang.org/x/oauth2/google"
 	admin "google.golang.org/api/admin/directory/v1"
@@ -100,20 +101,33 @@ func (c *client) GetUsers(query string) ([]*admin.User, error) {
 	u := make([]*admin.User, 0)
 	var err error
 
-	if query != "" {
-		err = c.service.Users.List().Query(query).Customer("my_customer").Pages(c.ctx, func(users *admin.Users) error {
-			u = append(u, users.Users...)
-			return nil
-		})
+	// If we have an empty query, return nothing.
+	if query == "" {
+		return u, err
+	}
 
-	} else {
-		err = c.service.Users.List().Customer("my_customer").Pages(c.ctx, func(users *admin.Users) error {
+	// If we have wildcard then fetch all users
+	if query  == "*" {
+                err = c.service.Users.List().Customer("my_customer").Pages(c.ctx, func(users *admin.Users) error {
+                        u = append(u, users.Users...)
+                        return nil
+                })
+		return u, err
+        }
+
+	// The Google api doesn't support multi-part queries, but we do so we need to split into an array of query strings
+	queries := strings.Split(query, ",")
+
+	// Then call the api one query at a time, appending to our list
+	for _, subQuery := range queries {
+		err = c.service.Users.List().Query(subQuery).Customer("my_customer").Pages(c.ctx, func(users *admin.Users) error {
 			u = append(u, users.Users...)
 			return nil
 		})
 	}
-
 	return u, err
+
+
 }
 
 // GetGroups will get the groups from Google's Admin API
@@ -133,17 +147,29 @@ func (c *client) GetGroups(query string) ([]*admin.Group, error) {
 	g := make([]*admin.Group, 0)
 	var err error
 
-	if query != "" {
-		err = c.service.Groups.List().Customer("my_customer").Query(query).Pages(context.TODO(), func(groups *admin.Groups) error {
-			g = append(g, groups.Groups...)
-			return nil
-		})
-	} else {
+        // If we have an empty query, then we are not looking for groups
+        if query  == "" {
+                return g, err
+        }
+
+        // If we have wildcard then fetch all groups
+        if query  == "*" {
 		err = c.service.Groups.List().Customer("my_customer").Pages(context.TODO(), func(groups *admin.Groups) error {
+                        g = append(g, groups.Groups...)
+                        return nil
+                })
+		return g, err
+	}
+
+        // The Google api doesn't support multi-part queries, but we do so we need to split into an array of query strings
+        queries := strings.Split(query, ",")
+
+        // Then call the api one query at a time, appending to our list
+        for _, subQuery := range queries {
+		err = c.service.Groups.List().Customer("my_customer").Query(subQuery).Pages(context.TODO(), func(groups *admin.Groups) error {
 			g = append(g, groups.Groups...)
 			return nil
 		})
-
 	}
 	return g, err
 }