diff --git a/dev/tools/controllerbuilder/cmd/root.go b/dev/tools/controllerbuilder/cmd/root.go index b8490727cf..5129c01807 100644 --- a/dev/tools/controllerbuilder/cmd/root.go +++ b/dev/tools/controllerbuilder/cmd/root.go @@ -19,6 +19,7 @@ import ( "os" "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/commands/apply" + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/commands/detectnewfields" "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/commands/exportcsv" "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/commands/generatecontroller" "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/commands/generatedirectreconciler" @@ -46,6 +47,7 @@ func Execute() { rootCmd.AddCommand(exportcsv.BuildCommand(&generateOptions)) rootCmd.AddCommand(exportcsv.BuildPromptCommand(&generateOptions)) rootCmd.AddCommand(apply.BuildCommand(&generateOptions)) + rootCmd.AddCommand(detectnewfields.BuildCommand(&generateOptions)) if err := rootCmd.Execute(); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) diff --git a/dev/tools/controllerbuilder/generate-proto.sh b/dev/tools/controllerbuilder/generate-proto.sh index a5a7ee99bd..478d258ee8 100755 --- a/dev/tools/controllerbuilder/generate-proto.sh +++ b/dev/tools/controllerbuilder/generate-proto.sh @@ -21,15 +21,32 @@ set -o pipefail REPO_ROOT="$(git rev-parse --show-toplevel)" cd ${REPO_ROOT}/dev/tools/controllerbuilder +# We share the version with mockgcp, which is maybe a boundary violation, but is convenient. +# (It would be confusing if these were out of sync!) +DEFAULT_GOOGLE_API_VERSION=$(grep https://github.com/googleapis/googleapis ${REPO_ROOT}/mockgcp/git.versions | awk '{print $2}') + +# Take googleapi version as parameter, default to version from git.versions. +# Use "HEAD" to get the latest from remote. +GOOGLEAPI_VERSION=${1:-$DEFAULT_GOOGLE_API_VERSION} + +# Take output path as parameter, default to .build/googleapis.pb +OUTPUT_PATH=${2:-"${REPO_ROOT}/.build/googleapis.pb"} + THIRD_PARTY=${REPO_ROOT}/.build/third_party mkdir -p ${THIRD_PARTY}/ +cd ${THIRD_PARTY} -# We share the version with mockgcp, which is maybe a boundary violation, but is convenient. -# (It would be confusing if these were out of sync!) -GOOGLEAPI_VERSION=$(grep https://github.com/googleapis/googleapis ${REPO_ROOT}/mockgcp/git.versions | awk '{print $2}' ) +if [ ! -d "googleapis" ]; then + git clone https://github.com/googleapis/googleapis.git +fi -cd ${REPO_ROOT}/.build/third_party -git clone https://github.com/googleapis/googleapis.git ${THIRD_PARTY}/googleapis || (cd ${THIRD_PARTY}/googleapis && git reset --hard ${GOOGLEAPI_VERSION}) +cd googleapis +git fetch +if [ "${GOOGLEAPI_VERSION}" = "HEAD" ]; then + git reset --hard origin/master +else + git reset --hard ${GOOGLEAPI_VERSION} +fi if (which protoc); then echo "Found protoc version $(protoc --version)" @@ -63,4 +80,4 @@ protoc --include_imports --include_source_info \ ${THIRD_PARTY}/googleapis/google/monitoring/dashboard/v1/*.proto \ ${THIRD_PARTY}/googleapis/google/devtools/cloudbuild/*/*.proto \ ${THIRD_PARTY}/googleapis/google/spanner/admin/instance/v1/*.proto \ - -o ${REPO_ROOT}/.build/googleapis.pb + -o ${OUTPUT_PATH} 2> >(grep -v "Import .* is unused" >&2) diff --git a/dev/tools/controllerbuilder/pkg/codegen/typegenerator.go b/dev/tools/controllerbuilder/pkg/codegen/typegenerator.go index a5b28cba1e..42d184c08b 100644 --- a/dev/tools/controllerbuilder/pkg/codegen/typegenerator.go +++ b/dev/tools/controllerbuilder/pkg/codegen/typegenerator.go @@ -77,7 +77,7 @@ func (g *TypeGenerator) visitMessage(message protoreflect.MessageDescriptor) err g.visitedMessages = append(g.visitedMessages, message) - msgs, err := FindDependenciesForMessage(message) + msgs, err := FindDependenciesForMessage(message, nil) // TODO: explicitly set ignored fields when generating Go types if err != nil { return err } @@ -445,11 +445,11 @@ func goFieldName(protoField protoreflect.FieldDescriptor) string { } // FindDependenciesForMessage recursively explores the dependent proto messages of the given message. -func FindDependenciesForMessage(message protoreflect.MessageDescriptor) ([]protoreflect.MessageDescriptor, error) { +func FindDependenciesForMessage(message protoreflect.MessageDescriptor, ignoredFields sets.String) ([]protoreflect.MessageDescriptor, error) { msgs := make(map[string]protoreflect.MessageDescriptor) for i := 0; i < message.Fields().Len(); i++ { field := message.Fields().Get(i) - FindDependenciesForField(field, msgs, nil) // TODO: explicitly set ignored fields when generating Go types + FindDependenciesForField(field, msgs, ignoredFields) } RemoveNotMappedToGoStruct(msgs) diff --git a/dev/tools/controllerbuilder/pkg/commands/detectnewfields/README.md b/dev/tools/controllerbuilder/pkg/commands/detectnewfields/README.md new file mode 100644 index 0000000000..030e90ca94 --- /dev/null +++ b/dev/tools/controllerbuilder/pkg/commands/detectnewfields/README.md @@ -0,0 +1,50 @@ +# Field Detection Tool + +This tool identifies changes in GCP API proto definitions by comparing the pinned version specified in the `git.version` file with the latest version at HEAD. + +It can identify: +- New fields **added** to messages +- Fields **removed** from messages +- Fields that **changed type** + +## Usage + +```bash +# Basic usage - checks all proto messages used in "generate.sh" +$ go run . detect-new-fields + +# Check specific messages +$ go run . detect-new-fields \ + --target-messages="google.cloud.bigquery.datatransfer.v1.TransferConfig" + +# Ignore specific fields using a config file +$ go run . detect-new-fields \ + --ignored-fields-file=config/ignored_fields.yaml +``` + +An example ignored_fields.yaml +```yaml +google.cloud.bigquery.connection.v1: + Connection: + - salesforceDataCloud +google.api.apikeys.v2: + Key: + - createTime + - updateTime +``` + +## Example Output + +``` +Changes detected in message: google.cloud.bigquery.datatransfer.v1.TransferConfig + New field: schedule_options_v2 + New field: error +Changes detected in message: google.cloud.discoveryengine.v1.DataStore + New field: billing_estimation + New field: workspace_config +Changes detected in message: google.cloud.discoveryengine.v1.Engine + New field: disable_analytics +Changes detected in message: google.spanner.admin.instance.v1.Instance + New field: default_backup_schedule_type + New field: replica_compute_capacity +``` diff --git a/dev/tools/controllerbuilder/pkg/commands/detectnewfields/detectnewfieldscommand.go b/dev/tools/controllerbuilder/pkg/commands/detectnewfields/detectnewfieldscommand.go new file mode 100644 index 0000000000..7ee571a447 --- /dev/null +++ b/dev/tools/controllerbuilder/pkg/commands/detectnewfields/detectnewfieldscommand.go @@ -0,0 +1,150 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package detectnewfields + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/newfieldsdetector" + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/options" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" +) + +type DetectNewFieldsOptions struct { + *options.GenerateOptions + + targetMessages string // comma-separated list of proto message names + ignoredFieldsFile string // path to ignored fields YAML file + outputFormat string // optional: json, yaml, or text +} + +func (o *DetectNewFieldsOptions) InitDefaults() error { + o.outputFormat = "text" + + // Set default ignored fields file path + _, err := options.RepoRoot() + if err != nil { + return err + } + // TODO: create this file + // o.ignoredFieldsFile = filepath.Join(repoRoot, "dev", "tools", "controllerbuilder", "config", "ignored_fields.yaml") + + return nil +} + +func (o *DetectNewFieldsOptions) BindFlags(cmd *cobra.Command) { + cmd.Flags().StringVar(&o.targetMessages, "target-messages", o.targetMessages, "Comma-separated list of target fully qualified proto message names to check") + cmd.Flags().StringVar(&o.ignoredFieldsFile, "ignored-fields-file", o.ignoredFieldsFile, "Path to YAML file containing ignored fields configuration") + cmd.Flags().StringVar(&o.outputFormat, "output-format", o.outputFormat, "Output format: text, json, or yaml") +} + +func BuildCommand(baseOptions *options.GenerateOptions) *cobra.Command { + opt := &DetectNewFieldsOptions{ + GenerateOptions: baseOptions, + } + + if err := opt.InitDefaults(); err != nil { + fmt.Fprintf(os.Stderr, "Error initializing defaults: %v\n", err) + os.Exit(1) + } + + cmd := &cobra.Command{ + Use: "detect-new-fields", + Short: "Detect new fields between pinned and HEAD versions of proto definitions", + Long: `Detect new fields by comparing the pinned version of proto definitions with the current HEAD version. +The pinned version is determined by the version specified in mockgcp/git.versions.`, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + if err := runNewFieldDetector(ctx, opt); err != nil { + return err + } + return nil + }, + } + + opt.BindFlags(cmd) + + return cmd +} + +func runNewFieldDetector(ctx context.Context, opt *DetectNewFieldsOptions) error { + ignoredFields, err := newfieldsdetector.LoadIgnoredFields(opt.ignoredFieldsFile) + if err != nil { + return fmt.Errorf("loading ignored fields: %w", err) + } + + targetMessages := sets.NewString() + if opt.targetMessages != "" { + targetMessages = sets.NewString(strings.Split(opt.targetMessages, ",")...) + } + newFieldDetector, err := newfieldsdetector.NewFieldDetector(&newfieldsdetector.DetectorOptions{ + TargetMessages: targetMessages, + IgnoredFields: ignoredFields, + }) + if err != nil { + return fmt.Errorf("creating new field detector: %w", err) + } + + diffs, err := newFieldDetector.DetectNewFields() + if err != nil { + return fmt.Errorf("detecting new fields: %w", err) + } + + return outputResults(diffs, opt.outputFormat) +} + +func outputResults(diffs []newfieldsdetector.MessageDiff, format string) error { + if len(diffs) == 0 { + klog.Info("No changes detected in the fields") + return nil + } + + sort.Slice(diffs, func(i, j int) bool { + return diffs[i].MessageName < diffs[j].MessageName + }) + + switch format { + case "text": + for _, diff := range diffs { + fmt.Printf("Changes detected in message: %s\n", diff.MessageName) + for _, field := range diff.NewFields { + fmt.Printf(" New field: %v\n", field) + } + for _, field := range diff.RemovedFields { + fmt.Printf(" Removed field: %v\n", field) + } + for field, change := range diff.ChangedFields { + fmt.Printf(" Changed field %s: %v -> %v (repeated: %v)\n", + field, change.OldType, change.NewType, change.IsRepeated) + } + } + case "json": + // TODO + return fmt.Errorf("JSON output not yet implemented") + case "yaml": + // TODO + return fmt.Errorf("YAML output not yet implemented") + default: + return fmt.Errorf("unsupported output format: %s", format) + } + + return nil +} diff --git a/dev/tools/controllerbuilder/pkg/newfieldsdetector/ignoredfields.go b/dev/tools/controllerbuilder/pkg/newfieldsdetector/ignoredfields.go new file mode 100644 index 0000000000..8293c96fae --- /dev/null +++ b/dev/tools/controllerbuilder/pkg/newfieldsdetector/ignoredfields.go @@ -0,0 +1,79 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package newfieldsdetector + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/util/sets" +) + +// IgnoredFieldsConfig represents the structure of the ignored fields YAML file. +// +// Example YAML: +/* +google.cloud.bigquery.connection.v1: + Connection: + - salesforceDataCloud +google.api.apikeys.v2: + Key: + - createTime + - updateTime +*/ +type IgnoredFieldsConfig struct { + // key is proto package name (e.g., "google.cloud.compute.v1"). + ProtoPackages map[string]MessageFieldIgnores `yaml:",inline"` +} + +type MessageFieldIgnores struct { + // key is proto message name (e.g. "Instance") + // value is list of field names to be ignored in the message. + Messages map[string][]string `yaml:",inline"` +} + +// LoadIgnoredFields loads and parses the ignored fields YAML file +func LoadIgnoredFields(configPath string) (sets.String, error) { + if configPath == "" { + return sets.NewString(), nil + } + + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("reading ignored fields config: %w", err) + } + var config IgnoredFieldsConfig + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("parsing ignored fields config: %w", err) + } + ignoredFields := sets.NewString() + // use fully qualified field names in ignoredFields map. e.g. "google.cloud.compute.v1.Instance.id" + for pkgName, pkgIgnores := range config.ProtoPackages { + for msgName, fields := range pkgIgnores.Messages { + for _, fieldName := range fields { + fullyQualifiedName := fmt.Sprintf("%s.%s.%s", pkgName, msgName, fieldName) + ignoredFields.Insert(fullyQualifiedName) + } + } + } + return ignoredFields, nil +} + +// IsFieldIgnored checks if a field should be ignored based on its fully qualified name +func IsFieldIgnored(ignoredFields sets.String, fullyQualifiedMessageName, fieldName string) bool { + fullyQualifiedFieldName := fmt.Sprintf("%s.%s", fullyQualifiedMessageName, fieldName) + return ignoredFields.Has(fullyQualifiedFieldName) +} diff --git a/dev/tools/controllerbuilder/pkg/newfieldsdetector/newfieldsdetector.go b/dev/tools/controllerbuilder/pkg/newfieldsdetector/newfieldsdetector.go new file mode 100644 index 0000000000..bbfa87b80f --- /dev/null +++ b/dev/tools/controllerbuilder/pkg/newfieldsdetector/newfieldsdetector.go @@ -0,0 +1,221 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package newfieldsdetector + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/options" + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/protoapi" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "k8s.io/apimachinery/pkg/util/sets" +) + +type MessageDiff struct { // This structure tracks the difference in fields for a given proto message + MessageName string + NewFields []string // fields added in the new version + RemovedFields []string // fields removed in the new version + ChangedFields map[string]FieldChange // fields changed in the new version +} + +type FieldChange struct { + OldType protoreflect.Kind + NewType protoreflect.Kind + IsRepeated bool +} + +type DetectorOptions struct { + TargetMessages sets.String + IgnoredFields sets.String +} + +type FieldDetector struct { + opts *DetectorOptions + oldFiles *protoregistry.Files // proto files that are generated with the pinned version + newFiles *protoregistry.Files // proto files that are generated from the remote HEAD +} + +// NewFieldDetector detects any proto field changes between the current pinned and the HEAD. +func NewFieldDetector(opts *DetectorOptions) (*FieldDetector, error) { + repoRoot, err := options.RepoRoot() + if err != nil { + return nil, fmt.Errorf("finding repo root: %w", err) + } + pinnedProtoPath := filepath.Join(repoRoot, ".build", "googleapis.pb") + headProtoPath := filepath.Join(repoRoot, ".build", "googleapis_head.pb") + + // generate pinned version proto. The default version is recorded in "mockgcp/git.versions". + if err := generateProto(repoRoot, pinnedProtoPath, ""); err != nil { + return nil, fmt.Errorf("generating pinned proto: %w", err) + } + old, err := protoapi.LoadProto(pinnedProtoPath) + if err != nil { + return nil, fmt.Errorf("loading old proto: %w", err) + } + // generate HEAD version proto + if err := generateProto(repoRoot, headProtoPath, "HEAD"); err != nil { + return nil, fmt.Errorf("generating HEAD proto: %w", err) + } + new, err := protoapi.LoadProto(headProtoPath) + if err != nil { + return nil, fmt.Errorf("loading new proto: %w", err) + } + + return &FieldDetector{ + opts: opts, + oldFiles: old.Files(), + newFiles: new.Files(), + }, nil +} + +// use the script at dev/tools/controllerbuilder/generate-proto.sh +func generateProto(repoRoot, outputPath, version string) error { + scriptPath := filepath.Join(repoRoot, "dev", "tools", "controllerbuilder", "generate-proto.sh") + + cmd := exec.Command("bash", scriptPath, version, outputPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func (d *FieldDetector) DetectNewFields() ([]MessageDiff, error) { + targets := d.opts.TargetMessages + // auto populate target messages if not specified by user + if len(targets) == 0 { + var err error + targets, err = d.defaultTargetMessages() + if err != nil { + return nil, fmt.Errorf("failed to getdefault target messages: %w", err) + } + } + if len(targets) == 0 { + return nil, fmt.Errorf("no target messages specified") + } + + var diffs []MessageDiff + for fqn := range targets { + diff, err := d.compareMessage(d.oldFiles, d.newFiles, fqn) + if err != nil { + return nil, fmt.Errorf("error when comparing message %s: %w", fqn, err) + } + if hasChanges(diff) { + diffs = append(diffs, diff) + } + } + return diffs, nil +} + +func hasChanges(diff MessageDiff) bool { + return len(diff.NewFields) > 0 || len(diff.RemovedFields) > 0 || len(diff.ChangedFields) > 0 +} + +func (d *FieldDetector) compareMessage(oldFiles, newFiles *protoregistry.Files, messageName string) (MessageDiff, error) { + diff := MessageDiff{ + MessageName: messageName, + ChangedFields: make(map[string]FieldChange), + } + + oldMsg := findMessage(oldFiles, messageName) + newMsg := findMessage(newFiles, messageName) + + if oldMsg == nil && newMsg == nil { + return diff, fmt.Errorf("message %s not found in either file", messageName) + } + + // case 1. new message added + if oldMsg == nil { + newFields := getMessageFields(newMsg) + for fieldName := range newFields { + if !IsFieldIgnored(d.opts.IgnoredFields, messageName, fieldName) { + diff.NewFields = append(diff.NewFields, fieldName) + } + } + return diff, nil + } + + // case 2. message removed + if newMsg == nil { + oldFields := getMessageFields(oldMsg) + for fieldName := range oldFields { + if !IsFieldIgnored(d.opts.IgnoredFields, messageName, fieldName) { + diff.RemovedFields = append(diff.RemovedFields, fieldName) + } + } + return diff, nil + } + + // case 3. message exists in both old and new proto + oldFields := getMessageFields(oldMsg) + newFields := getMessageFields(newMsg) + + // 3.1 Find new and changed fields + for fieldName, newField := range newFields { + if IsFieldIgnored(d.opts.IgnoredFields, messageName, fieldName) { + continue + } + + oldField, exists := oldFields[fieldName] + if !exists { + diff.NewFields = append(diff.NewFields, fieldName) + continue + } + + if oldField.Kind() != newField.Kind() || oldField.IsList() != newField.IsList() { + diff.ChangedFields[fieldName] = FieldChange{ + OldType: oldField.Kind(), + NewType: newField.Kind(), + IsRepeated: newField.IsList(), + } + } + } + // 3.2 Find removed fields + for fieldName := range oldFields { + if IsFieldIgnored(d.opts.IgnoredFields, messageName, fieldName) { + continue + } + + if _, exists := newFields[fieldName]; !exists { + diff.RemovedFields = append(diff.RemovedFields, fieldName) + } + } + + return diff, nil +} + +func findMessage(files *protoregistry.Files, name string) protoreflect.MessageDescriptor { + desc, err := files.FindDescriptorByName(protoreflect.FullName(name)) + if err != nil { + return nil + } + msgDesc, ok := desc.(protoreflect.MessageDescriptor) + if !ok { + return nil + } + return msgDesc +} + +func getMessageFields(msg protoreflect.MessageDescriptor) map[string]protoreflect.FieldDescriptor { + fields := make(map[string]protoreflect.FieldDescriptor) + fieldDescriptors := msg.Fields() + for i := 0; i < fieldDescriptors.Len(); i++ { + field := fieldDescriptors.Get(i) + fields[string(field.Name())] = field + } + return fields +} diff --git a/dev/tools/controllerbuilder/pkg/newfieldsdetector/targetfields.go b/dev/tools/controllerbuilder/pkg/newfieldsdetector/targetfields.go new file mode 100644 index 0000000000..7fadb52322 --- /dev/null +++ b/dev/tools/controllerbuilder/pkg/newfieldsdetector/targetfields.go @@ -0,0 +1,174 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package newfieldsdetector + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/codegen" + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/options" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" +) + +// GenerateTypesFlags represents the flags used in "generate-types" command +type GenerateTypesFlags struct { + Service string + Resources []ResourceMapping +} + +type ResourceMapping struct { + KRMType string + ProtoMessage string +} + +// defaultTargetMessages processes the command-line flags used in "generate.sh" to extract fully qualified proto message names. +// These names represent the top-level messages corresponding to Config Connector KRM resources. +// The function also identifies nested messages of these top-level messages to form a comprehensive list of target messages. +// TODO: Create a structured file to store this information for more reliable parsing. The file can be shared between the type generator and the new field detector. +func (d *FieldDetector) defaultTargetMessages() (sets.String, error) { + topLevelMessages, err := extractMessagesFromGenerateTypesScript() + if err != nil { + return nil, fmt.Errorf("failed to extract messages from generate.sh script: %w", err) + } + + expandedMessages, err := d.expandToIncludeNestedMessages(topLevelMessages, d.newFiles) // note: using new proto files here + if err != nil { + return nil, fmt.Errorf("") + } + return expandedMessages, nil +} + +func extractMessagesFromGenerateTypesScript() (sets.String, error) { + repoRoot, err := options.RepoRoot() + if err != nil { + return nil, fmt.Errorf("finding repo root: %w", err) + } + scriptPath := filepath.Join(repoRoot, "dev", "tools", "controllerbuilder", "generate.sh") + + flags, err := parseGenerateTypesScript(scriptPath) + if err != nil { + return nil, fmt.Errorf("parsing generate types script: %w", err) + } + + targetMessages := sets.NewString() + for _, flag := range flags { + for _, resource := range flag.Resources { + targetMessages.Insert(fmt.Sprintf("%s.%s", flag.Service, resource.ProtoMessage)) + } + } + + return targetMessages, nil +} + +// expandToIncludeNestedMessages expands initial target messages to include all nested messages +func (d *FieldDetector) expandToIncludeNestedMessages(initialTargets sets.String, files *protoregistry.Files) (sets.String, error) { + allTargets := sets.NewString() + allTargets.Insert(initialTargets.List()...) // make a copy + + for name := range initialTargets { + desc, err := files.FindDescriptorByName(protoreflect.FullName(name)) + if err != nil { + klog.Warningf("Proto message not found %s: %v", name, err) + continue + } + + message, ok := desc.(protoreflect.MessageDescriptor) + if !ok { + return nil, fmt.Errorf("unexpected descriptor type: %T", desc) + } + + deps, err := codegen.FindDependenciesForMessage(message, d.opts.IgnoredFields) + if err != nil { + return nil, fmt.Errorf("failed to find dependencies for message %s: %w", name, err) + } + for _, dep := range deps { + allTargets.Insert(string(dep.FullName())) + } + } + + return allTargets, nil +} + +func parseGenerateTypesScript(scriptPath string) ([]GenerateTypesFlags, error) { + file, err := os.Open(scriptPath) + if err != nil { + return nil, fmt.Errorf("opening script file: %w", err) + } + defer file.Close() + + var flags []GenerateTypesFlags + var currentFlags *GenerateTypesFlags + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + // Skip empty lines, comments, and non-flag lines + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + // Start of new "generate-types" command + if strings.Contains(line, "generate-types") { + if currentFlags != nil { + flags = append(flags, *currentFlags) + } + currentFlags = &GenerateTypesFlags{} + continue + } + + if currentFlags == nil { + continue + } + + // Parse flags to get proto package and proto message name + if strings.Contains(line, "--service") { + parts := strings.Fields(line) + if len(parts) >= 2 { + currentFlags.Service = parts[1] + } + } else if strings.Contains(line, "--resource") { + parts := strings.Fields(line) + if len(parts) >= 2 { + resourceParts := strings.Split(parts[1], ":") + if len(resourceParts) == 2 { + currentFlags.Resources = append(currentFlags.Resources, ResourceMapping{ + KRMType: resourceParts[0], + ProtoMessage: resourceParts[1], + }) + } + } + } + + // End of a "generate-types" command + if !strings.HasSuffix(line, "\\") && currentFlags != nil { + flags = append(flags, *currentFlags) + currentFlags = nil + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("scanning script file: %w", err) + } + + return flags, nil +}