From 20efed171cc973f6451e81509a34050d86f97521 Mon Sep 17 00:00:00 2001 From: Emily McMullan Date: Tue, 3 Sep 2024 09:42:15 -0400 Subject: [PATCH] containerless support for java Signed-off-by: Emily McMullan --- cmd/analyze-bin.go | 831 +++++++++++++++++ cmd/root.go | 1 + cmd/version.go | 2 + docs/containerless.md | 29 + go.mod | 23 +- go.sum | 65 +- pkg/java_external_provider/dependency.go | 923 +++++++++++++++++++ pkg/java_external_provider/filter.go | 276 ++++++ pkg/java_external_provider/provider.go | 882 ++++++++++++++++++ pkg/java_external_provider/provider_test.go | 53 ++ pkg/java_external_provider/service_client.go | 282 ++++++ pkg/java_external_provider/snipper.go | 54 ++ pkg/java_external_provider/util.go | 610 ++++++++++++ pkg/java_external_provider/util_test.go | 99 ++ static-report.sh | 19 + 15 files changed, 4139 insertions(+), 10 deletions(-) create mode 100644 cmd/analyze-bin.go create mode 100644 docs/containerless.md create mode 100644 pkg/java_external_provider/dependency.go create mode 100644 pkg/java_external_provider/filter.go create mode 100644 pkg/java_external_provider/provider.go create mode 100644 pkg/java_external_provider/provider_test.go create mode 100644 pkg/java_external_provider/service_client.go create mode 100644 pkg/java_external_provider/snipper.go create mode 100644 pkg/java_external_provider/util.go create mode 100644 pkg/java_external_provider/util_test.go create mode 100755 static-report.sh diff --git a/cmd/analyze-bin.go b/cmd/analyze-bin.go new file mode 100644 index 0000000..40421de --- /dev/null +++ b/cmd/analyze-bin.go @@ -0,0 +1,831 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + + "github.com/bombsimon/logrusr/v3" + "github.com/go-logr/logr" + java "github.com/konveyor-ecosystem/kantra/pkg/java_external_provider" + "github.com/konveyor/analyzer-lsp/engine" + "github.com/konveyor/analyzer-lsp/engine/labels" + "github.com/konveyor/analyzer-lsp/output/v1/konveyor" + outputv1 "github.com/konveyor/analyzer-lsp/output/v1/konveyor" + "github.com/konveyor/analyzer-lsp/parser" + "github.com/konveyor/analyzer-lsp/provider" + "github.com/konveyor/analyzer-lsp/provider/lib" + "github.com/konveyor/analyzer-lsp/tracing" + "github.com/sirupsen/logrus" + "go.lsp.dev/uri" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "gopkg.in/yaml.v2" + + "github.com/spf13/cobra" +) + +type analyzeBinCommand struct { + listSources bool + listTargets bool + skipStaticReport bool + analyzeKnownLibraries bool + jsonOutput bool + overwrite bool + mavenSettingsFile string + sources []string + targets []string + labelSelector string + input string + output string + mode string + httpProxy string + httpsProxy string + noProxy string + rules []string + jaegerEndpoint string + enableDefaultRulesets bool + contextLines int + incidentSelector string + + //for cleaning + binMap map[string]string + homeKantraDir string + log logr.Logger + // isFileInput is set when input points to a file and not a dir + isFileInput bool + logLevel *uint32 + cleanup bool +} + +// analyzeCmd represents the analyze command +func NewAnalyzeBinCmd(log logr.Logger) *cobra.Command { + analyzeBinCmd := &analyzeBinCommand{ + log: log, + cleanup: true, + } + analyzeBinCommand := &cobra.Command{ + Use: "analyze-bin", + Short: "Analyze Java application source code", + PreRunE: func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Lookup("list-sources").Changed && + !cmd.Flags().Lookup("list-targets").Changed { + //cmd.MarkFlagRequired("input") + cmd.MarkFlagRequired("output") + if err := cmd.ValidateRequiredFlags(); err != nil { + return err + } + } + err := analyzeBinCmd.Validate(cmd.Context()) + if err != nil { + log.Error(err, "failed to validate flags") + return err + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + if val, err := cmd.Flags().GetUint32(logLevelFlag); err == nil { + analyzeBinCmd.logLevel = &val + } + if val, err := cmd.Flags().GetBool(noCleanupFlag); err == nil { + analyzeBinCmd.cleanup = !val + } + if analyzeBinCmd.listSources || analyzeBinCmd.listTargets { + err := analyzeBinCmd.ListLabels(cmd.Context()) + if err != nil { + log.Error(err, "failed to list rule labels") + return err + } + return nil + } + + if analyzeBinCmd.binMap == nil { + analyzeBinCmd.binMap = make(map[string]string) + } + + defer os.Remove(filepath.Join(analyzeBinCmd.output, "settings.json")) + + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + analysisLogFilePath := filepath.Join(analyzeBinCmd.output, "analysis.log") + analysisLog, err := os.Create(analysisLogFilePath) + if err != nil { + return fmt.Errorf("failed creating provider log file at %s", analysisLogFilePath) + } + defer analysisLog.Close() + + logrusLog := logrus.New() + logrusLog.SetOutput(analysisLog) + logrusLog.SetFormatter(&logrus.TextFormatter{}) + // need to do research on mapping in logrusr to level here TODO + logrusLog.SetLevel(logrus.Level(logLevel)) + log := logrusr.New(logrusLog) + + logrusErrLog := logrus.New() + logrusErrLog.SetOutput(analysisLog) + errLog := logrusr.New(logrusErrLog) + + fmt.Println("running analysis") + selectors := []engine.RuleSelector{} + if analyzeBinCmd.labelSelector != "" { + selector, err := labels.NewLabelSelector[*engine.RuleMeta](analyzeBinCmd.labelSelector, nil) + if err != nil { + log.Error(err, "failed to create label selector from expression", "selector", analyzeBinCmd.labelSelector) + os.Exit(1) + } + selectors = append(selectors, selector) + } + + err = analyzeBinCmd.setBins() + if err != nil { + log.Error(err, "unable to get binaries") + os.Exit(1) + } + + // Get the configs + configs, err := analyzeBinCmd.createJavaProviderConfig() + if err != nil { + log.Error(err, "unable to get Java configuration") + os.Exit(1) + } + // we add builtin configs by default for all locations + defaultBuiltinConfigs := []provider.InitConfig{} + seenBuiltinConfigs := map[string]bool{} + finalConfigs := []provider.Config{} + for _, config := range configs { + if config.Name != "builtin" { + finalConfigs = append(finalConfigs, config) + } + for _, initConf := range config.InitConfig { + if _, ok := seenBuiltinConfigs[initConf.Location]; !ok { + if initConf.Location != "" { + if stat, err := os.Stat(initConf.Location); err == nil && stat.IsDir() { + builtinLocation, err := filepath.Abs(initConf.Location) + if err != nil { + builtinLocation = initConf.Location + } + seenBuiltinConfigs[builtinLocation] = true + builtinConf := provider.InitConfig{Location: builtinLocation} + if config.Name == "builtin" { + builtinConf.ProviderSpecificConfig = initConf.ProviderSpecificConfig + } + defaultBuiltinConfigs = append(defaultBuiltinConfigs, builtinConf) + } + } + } + } + } + + finalConfigs = append(finalConfigs, provider.Config{ + Name: "builtin", + InitConfig: defaultBuiltinConfigs, + }) + + providers := map[string]provider.InternalProviderClient{} + providerLocations := []string{} + for _, config := range finalConfigs { + config.ContextLines = analyzeBinCmd.contextLines + for _, ind := range config.InitConfig { + providerLocations = append(providerLocations, ind.Location) + } + // IF analsyis mode is set from the CLI, then we will override this for each init config + if analyzeBinCmd.mode != "" { + inits := []provider.InitConfig{} + for _, i := range config.InitConfig { + i.AnalysisMode = provider.AnalysisMode(analyzeBinCmd.mode) + inits = append(inits, i) + } + config.InitConfig = inits + } + var prov provider.InternalProviderClient + // only create java and builtin providers + if config.Name == "java" { + prov = java.NewJavaProvider(log, config, analyzeBinCmd.contextLines) + + } else if config.Name == "builtin" { + prov, err = lib.GetProviderClient(config, log) + if err != nil { + log.Error(err, "failed to create builtin provider") + os.Exit(1) + } + } + providers[config.Name] = prov + } + + engineCtx, engineSpan := tracing.StartNewSpan(ctx, "rule-engine") + //start up the rule eng + eng := engine.CreateRuleEngine(engineCtx, + 10, + log, + engine.WithContextLines(analyzeBinCmd.contextLines), + engine.WithIncidentSelector(analyzeBinCmd.incidentSelector), + engine.WithLocationPrefixes(providerLocations), + ) + + parser := parser.RuleParser{ + ProviderNameToClient: providers, + Log: log.WithName("parser"), + } + + ruleSets := []engine.RuleSet{} + needProviders := map[string]provider.InternalProviderClient{} + + if analyzeBinCmd.enableDefaultRulesets { + analyzeBinCmd.rules = append(analyzeBinCmd.rules, filepath.Join(analyzeBinCmd.homeKantraDir, RulesetsLocation)) + } + for _, f := range analyzeBinCmd.rules { + internRuleSet, internNeedProviders, err := parser.LoadRules(f) + if err != nil { + log.Error(err, "unable to parse all the rules for ruleset", "file", f) + } + ruleSets = append(ruleSets, internRuleSet...) + for k, v := range internNeedProviders { + needProviders[k] = v + } + } + + // Now that we have all the providers, we need to start them. + additionalBuiltinConfigs := []provider.InitConfig{} + for name, provider := range needProviders { + switch name { + // other providers can return additional configs for the builtin provider + // therefore, we initiate builtin provider separately at the end + case "builtin": + continue + default: + initCtx, initSpan := tracing.StartNewSpan(ctx, "init", + attribute.Key("provider").String(name)) + additionalBuiltinConfs, err := provider.ProviderInit(initCtx, nil) + if err != nil { + log.Error(err, "unable to init the providers", "provider", name) + os.Exit(1) + } + if additionalBuiltinConfs != nil { + additionalBuiltinConfigs = append(additionalBuiltinConfigs, additionalBuiltinConfs...) + } + initSpan.End() + } + } + + if builtinClient, ok := needProviders["builtin"]; ok { + if _, err = builtinClient.ProviderInit(ctx, additionalBuiltinConfigs); err != nil { + os.Exit(1) + } + } + + wg := &sync.WaitGroup{} + var depSpan trace.Span + + if analyzeBinCmd.mode == string(provider.FullAnalysisMode) { + var depCtx context.Context + depCtx, depSpan = tracing.StartNewSpan(ctx, "dep") + wg.Add(1) + + fmt.Println("running dependency analysis") + go analyzeBinCmd.DependencyOutput(depCtx, providers, log, errLog, "dependencies.yaml", wg) + } + + // This will already wait + rulesets := eng.RunRules(ctx, ruleSets, selectors...) + engineSpan.End() + wg.Wait() + if depSpan != nil { + depSpan.End() + } + eng.Stop() + + for _, provider := range needProviders { + provider.Stop() + } + + sort.SliceStable(rulesets, func(i, j int) bool { + return rulesets[i].Name < rulesets[j].Name + }) + + // Write results out to CLI + b, err := yaml.Marshal(rulesets) + if err != nil { + return err + } + + fmt.Println("writing analysis results to output", "output", analyzeBinCmd.output) + err = os.WriteFile(filepath.Join(analyzeBinCmd.output, "output.yaml"), b, 0644) + if err != nil { + os.Exit(1) // Treat the error as a fatal error + } + + err = analyzeBinCmd.createJSONOutput() + if err != nil { + log.Error(err, "failed to create json output file") + return err + } + + err = analyzeBinCmd.GenerateStaticReport(cmd.Context()) + if err != nil { + log.Error(err, "failed to generate static report") + return err + } + + return nil + }, + } + analyzeBinCommand.Flags().BoolVar(&analyzeBinCmd.listSources, "list-sources", false, "list rules for available migration sources") + analyzeBinCommand.Flags().BoolVar(&analyzeBinCmd.listTargets, "list-targets", false, "list rules for available migration targets") + analyzeBinCommand.Flags().StringArrayVarP(&analyzeBinCmd.sources, "source", "s", []string{}, "source technology to consider for analysis. Use multiple times for additional sources: --source --source ...") + analyzeBinCommand.Flags().StringArrayVarP(&analyzeBinCmd.targets, "target", "t", []string{}, "target technology to consider for analysis. Use multiple times for additional targets: --target --target ...") + analyzeBinCommand.Flags().StringVarP(&analyzeBinCmd.labelSelector, "label-selector", "l", "", "run rules based on specified label selector expression") + analyzeBinCommand.Flags().StringArrayVar(&analyzeBinCmd.rules, "rules", []string{}, "filename or directory containing rule files. Use multiple times for additional rules: --rules --rules ...") + analyzeBinCommand.Flags().StringVarP(&analyzeBinCmd.input, "input", "i", "", "path to application source code or a binary") + analyzeBinCommand.Flags().StringVarP(&analyzeBinCmd.output, "output", "o", "", "path to the directory for analysis output") + analyzeBinCommand.Flags().BoolVar(&analyzeBinCmd.skipStaticReport, "skip-static-report", false, "do not generate static report") + analyzeBinCommand.Flags().BoolVar(&analyzeBinCmd.analyzeKnownLibraries, "analyze-known-libraries", false, "analyze known open-source libraries") + analyzeBinCommand.Flags().StringVar(&analyzeBinCmd.mavenSettingsFile, "maven-settings", "", "path to a custom maven settings file to use") + analyzeBinCommand.Flags().StringVarP(&analyzeBinCmd.mode, "mode", "m", string(provider.FullAnalysisMode), "analysis mode. Must be one of 'full' (source + dependencies) or 'source-only'") + analyzeBinCommand.Flags().BoolVar(&analyzeBinCmd.jsonOutput, "json-output", false, "create analysis and dependency output as json") + analyzeBinCommand.Flags().BoolVar(&analyzeBinCmd.overwrite, "overwrite", false, "overwrite output directory") + analyzeBinCommand.Flags().StringVar(&analyzeBinCmd.httpProxy, "http-proxy", loadEnvInsensitive("http_proxy"), "HTTP proxy string URL") + analyzeBinCommand.Flags().StringVar(&analyzeBinCmd.httpsProxy, "https-proxy", loadEnvInsensitive("https_proxy"), "HTTPS proxy string URL") + analyzeBinCommand.Flags().StringVar(&analyzeBinCmd.noProxy, "no-proxy", loadEnvInsensitive("no_proxy"), "proxy excluded URLs (relevant only with proxy)") + analyzeBinCommand.Flags().StringVar(&analyzeBinCmd.jaegerEndpoint, "jaeger-endpoint", "", "jaeger endpoint to collect traces") + analyzeBinCommand.Flags().BoolVar(&analyzeBinCmd.enableDefaultRulesets, "enable-default-rulesets", true, "run default rulesets with analysis") + analyzeBinCommand.Flags().IntVar(&analyzeBinCmd.contextLines, "context-lines", 10, "number of lines of source code to include in the output for each incident") + analyzeBinCommand.Flags().StringVar(&analyzeBinCmd.incidentSelector, "incident-selector", "", "an expression to select incidents based on custom variables. ex: (!package=io.konveyor.demo.config-utils)") + + return analyzeBinCommand +} + +func (b *analyzeBinCommand) Validate(ctx context.Context) error { + if b.listSources || b.listTargets { + return nil + } + // if b.labelSelector != "" && (len(b.sources) > 0 || len(b.targets) > 0) { + // return fmt.Errorf("must not specify label-selector and sources or targets") + // } + // Validate source labels + // if len(b.sources) > 0 { + // var sourcesRaw bytes.Buffer + // b.fetchLabels(ctx, true, false, &sourcesRaw) + // knownSources := strings.Split(sourcesRaw.String(), "\n") + // for _, source := range b.sources { + // found := false + // for _, knownSource := range knownSources { + // if source == knownSource { + // found = true + // } + // } + // if !found { + // return fmt.Errorf("unknown source: \"%s\"", source) + // } + // } + // } + // // Validate target labels + // if len(b.targets) > 0 { + // var targetRaw bytes.Buffer + // b.fetchLabels(ctx, false, true, &targetRaw) + // knownTargets := strings.Split(targetRaw.String(), "\n") + // for _, source := range b.targets { + // found := false + // for _, knownTarget := range knownTargets { + // if source == knownTarget { + // found = true + // } + // } + // if !found { + // return fmt.Errorf("unknown target: \"%s\"", source) + // } + // } + // } + if b.input != "" { + // do not allow multiple input applications + inputNum := 0 + for _, arg := range os.Args { + if arg == "-i" || strings.Contains(arg, "--input") { + inputNum += 1 + if inputNum > 1 { + return fmt.Errorf("must specify only one input source") + } + } + } + stat, err := os.Stat(b.input) + if err != nil { + return fmt.Errorf("%w failed to stat input path %s", err, b.input) + } + // when input isn't a dir, it's pointing to a binary + // we need abs path to mount the file correctly + if !stat.Mode().IsDir() { + // validate file types + fileExt := filepath.Ext(b.input) + switch fileExt { + case JavaArchive, WebArchive, EnterpriseArchive, ClassFile: + b.log.V(5).Info("valid java file found") + default: + return fmt.Errorf("invalid file type %v", fileExt) + } + b.input, err = filepath.Abs(b.input) + if err != nil { + return fmt.Errorf("%w failed to get absolute path for input file %s", err, b.input) + } + // make sure we mount a file and not a dir + SourceMountPath = path.Join(SourceMountPath, filepath.Base(b.input)) + b.isFileInput = true + } + } + err := b.CheckOverwriteOutput() + if err != nil { + return err + } + stat, err := os.Stat(b.output) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + err = os.MkdirAll(b.output, os.ModePerm) + if err != nil { + return fmt.Errorf("%w failed to create output dir %s", err, b.output) + } + } else { + return fmt.Errorf("failed to stat output directory %s", b.output) + } + } + if stat != nil && !stat.IsDir() { + return fmt.Errorf("output path %s is not a directory", b.output) + } + + if b.mode != string(provider.FullAnalysisMode) && + b.mode != string(provider.SourceOnlyAnalysisMode) { + return fmt.Errorf("mode must be one of 'full' or 'source-only'") + } + if _, err := os.Stat(b.mavenSettingsFile); b.mavenSettingsFile != "" && err != nil { + return fmt.Errorf("%w failed to stat maven settings file at path %s", err, b.mavenSettingsFile) + } + // try to get abs path, if not, continue with relative path + if absPath, err := filepath.Abs(b.output); err == nil { + b.output = absPath + } + if absPath, err := filepath.Abs(b.input); err == nil { + b.input = absPath + } + if absPath, err := filepath.Abs(b.mavenSettingsFile); b.mavenSettingsFile != "" && err == nil { + b.mavenSettingsFile = absPath + } + if !b.enableDefaultRulesets && len(b.rules) == 0 { + return fmt.Errorf("must specify rules if default rulesets are not enabled") + } + return nil +} + +func (b *analyzeBinCommand) ListLabels(ctx context.Context) error { + return b.fetchLabels(ctx, b.listSources, b.listTargets, os.Stdout) +} + +func (b *analyzeBinCommand) fetchLabels(ctx context.Context, listSources, listTargets bool, out io.Writer) error { + // reserved labels + sourceLabel := outputv1.SourceTechnologyLabel + targetLabel := outputv1.TargetTechnologyLabel + + if listSources { + sourceSlice, err := readRuleFilesForLabels(sourceLabel) + if err != nil { + b.log.Error(err, "failed to read rule labels") + return err + } + listOptionsFromLabels(sourceSlice, sourceLabel, out) + return nil + } + if listTargets { + targetsSlice, err := readRuleFilesForLabels(targetLabel) + if err != nil { + b.log.Error(err, "failed to read rule labels") + return err + } + listOptionsFromLabels(targetsSlice, targetLabel, out) + return nil + } + + args := []string{"analyze"} + if listSources { + args = append(args, "--list-sources") + } else { + args = append(args, "--list-targets") + } + + return nil +} + +func (a *analyzeBinCommand) CheckOverwriteOutput() error { + // default overwrite to false so check for already existing output dir + stat, err := os.Stat(a.output) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + } + if a.overwrite && stat != nil { + err := os.RemoveAll(a.output) + if err != nil { + return err + } + } + return nil +} + +func (b *analyzeBinCommand) setBins() error { + var homeDir string + var set bool + ops := runtime.GOOS + if ops == "linux" { + homeDir, set = os.LookupEnv("XDG_CONFIG_HOME") + } + if ops != "linux" || homeDir == "" || !set { + // on Unix, including macOS, this returns the $HOME environment variable. On Windows, it returns %USERPROFILE% + var err error + homeDir, err = os.UserHomeDir() + if err != nil { + return err + } + } + binDir := filepath.Join(homeDir, ".kantra") + b.homeKantraDir = binDir + + b.binMap["bundle"] = filepath.Join(binDir, JavaBundlesLocation) + b.binMap["jdtls"] = filepath.Join(binDir, JDTLSBinLocation) + + // validate + for _, v := range b.binMap { + stat, err := os.Stat(v) + if err != nil { + return fmt.Errorf("%w failed to stat bin %s", err, v) + } + if stat.Mode().IsDir() { + return fmt.Errorf("unable to find expected file at %s", v) + } + } + return nil +} + +func (b *analyzeBinCommand) createJavaProviderConfig() ([]provider.Config, error) { + javaConfig := provider.Config{ + Name: javaProvider, + BinaryPath: b.binMap["jdtls"], + InitConfig: []provider.InitConfig{ + { + Location: b.input, + AnalysisMode: provider.AnalysisMode(b.mode), + ProviderSpecificConfig: map[string]interface{}{ + "lspServerName": javaProvider, + "bundles": b.binMap["bundle"], + provider.LspServerPathConfigKey: b.binMap["jdtls"], + }, + }, + }, + } + if b.mavenSettingsFile != "" { + javaConfig.InitConfig[0].ProviderSpecificConfig["mavenSettingsFile"] = b.mavenSettingsFile + } + + provConfig := []provider.Config{ + { + Name: "builtin", + InitConfig: []provider.InitConfig{ + { + Location: b.input, + AnalysisMode: provider.AnalysisMode(b.mode), + }, + }, + }, + } + provConfig = append(provConfig, javaConfig) + + for i := range provConfig { + // Set proxy to providers + if b.httpProxy != "" || b.httpsProxy != "" { + proxy := provider.Proxy{ + HTTPProxy: b.httpProxy, + HTTPSProxy: b.httpsProxy, + NoProxy: b.noProxy, + } + + provConfig[i].Proxy = &proxy + } + provConfig[i].ContextLines = b.contextLines + } + + jsonData, err := json.MarshalIndent(&provConfig, "", " ") + if err != nil { + b.log.V(1).Error(err, "failed to marshal provider config") + return nil, err + } + err = os.WriteFile(filepath.Join(b.output, "settings.json"), jsonData, os.ModePerm) + if err != nil { + b.log.V(1).Error(err, + "failed to write provider config", "dir", b.output, "file", "settings.json") + return nil, err + } + return provConfig, nil +} + +func (b *analyzeBinCommand) DependencyOutput(ctx context.Context, providers map[string]provider.InternalProviderClient, log logr.Logger, errLog logr.Logger, depOutputFile string, wg *sync.WaitGroup) { + defer wg.Done() + var depsFlat []konveyor.DepsFlatItem + var depsTree []konveyor.DepsTreeItem + var err error + + for _, prov := range providers { + deps, err := prov.GetDependencies(ctx) + if err != nil { + errLog.Error(err, "failed to get list of dependencies for provider", "provider", "java") + } + for u, ds := range deps { + newDeps := ds + depsFlat = append(depsFlat, konveyor.DepsFlatItem{ + Provider: "java", + FileURI: string(u), + Dependencies: newDeps, + }) + } + + if depsFlat == nil && depsTree == nil { + errLog.Info("failed to get dependencies from all given providers") + return + } + } + + var by []byte + // Sort depsFlat + sort.SliceStable(depsFlat, func(i, j int) bool { + if depsFlat[i].Provider == depsFlat[j].Provider { + return depsFlat[i].FileURI < depsFlat[j].FileURI + } else { + return depsFlat[i].Provider < depsFlat[j].Provider + } + }) + + by, err = yaml.Marshal(depsFlat) + if err != nil { + errLog.Error(err, "failed to marshal dependency data as yaml") + return + } + + err = os.WriteFile(filepath.Join(b.output, depOutputFile), by, 0644) + if err != nil { + errLog.Error(err, "failed to write dependencies to output file", "file", depOutputFile) + return + } + +} + +func (b *analyzeBinCommand) createJSONOutput() error { + if !b.jsonOutput { + return nil + } + outputPath := filepath.Join(b.output, "output.yaml") + depPath := filepath.Join(b.output, "dependencies.yaml") + + data, err := os.ReadFile(outputPath) + if err != nil { + return err + } + ruleOutput := &[]outputv1.RuleSet{} + err = yaml.Unmarshal(data, ruleOutput) + if err != nil { + b.log.V(1).Error(err, "failed to unmarshal output yaml") + return err + } + + jsonData, err := json.MarshalIndent(ruleOutput, "", " ") + if err != nil { + b.log.V(1).Error(err, "failed to marshal output file to json") + return err + } + err = os.WriteFile(filepath.Join(b.output, "output.json"), jsonData, os.ModePerm) + if err != nil { + b.log.V(1).Error(err, "failed to write json output", "dir", b.output, "file", "output.json") + return err + } + + // in case of no dep output + _, noDepFileErr := os.Stat(filepath.Join(b.output, "dependencies.yaml")) + if errors.Is(noDepFileErr, os.ErrNotExist) || b.mode == string(provider.SourceOnlyAnalysisMode) { + b.log.Info("skipping dependency output for json output") + return nil + } + depData, err := os.ReadFile(depPath) + if err != nil { + return err + } + depOutput := &[]outputv1.DepsFlatItem{} + err = yaml.Unmarshal(depData, depOutput) + if err != nil { + b.log.V(1).Error(err, "failed to unmarshal dependencies yaml") + return err + } + + jsonDataDep, err := json.MarshalIndent(depOutput, "", " ") + if err != nil { + b.log.V(1).Error(err, "failed to marshal dependencies file to json") + return err + } + err = os.WriteFile(filepath.Join(b.output, "dependencies.json"), jsonDataDep, os.ModePerm) + if err != nil { + b.log.V(1).Error(err, "failed to write json dependencies output", "dir", b.output, "file", "dependencies.json") + return err + } + + return nil +} + +func (b *analyzeBinCommand) buildStaticReportFile(ctx context.Context, staticReportPath string, hasDeps bool) error { + args := []string{} + // Prepare report args list with single input analysis + applicationName := filepath.Base(b.input) + outputAnalysis := filepath.Join(b.output, "output.yaml") + outputDeps := filepath.Join(b.output, "dependencies.yaml") + outputJSPath := filepath.Join(staticReportPath, "public", "output.js") + + args = append(args, + fmt.Sprintf("--analysis-output-list=%s", outputAnalysis), + fmt.Sprintf("--application-name-list=%s", applicationName), + fmt.Sprintf("--output-path=%s", outputJSPath), + ) + + if b.mode == string(provider.FullAnalysisMode) && !hasDeps { + args = append(args, + fmt.Sprintf("--deps-output-list=%s", outputDeps)) + } + + // create output.js file from analysis output.yaml + cmd := exec.Command(filepath.Join(staticReportPath, "analyzer-output-parser", "analyzer-output-parser"), args...) + var outb bytes.Buffer + cmd.Stdout = &outb + err := cmd.Run() + if err != nil { + return err + } + + return nil +} + +func (b *analyzeBinCommand) buildStaticReportOutput(ctx context.Context, log *os.File) error { + // build static report + cmd := exec.Command("./static-report.sh") + cmd.Stdout = log + err := cmd.Run() + if err != nil { + return err + } + + // move build dir to analysis output + cmd = exec.Command("cp", "-r", filepath.Join(b.homeKantraDir, "static-report", "static-report"), b.output) + cmd.Stdout = log + err = cmd.Run() + if err != nil { + return err + } + + return nil +} + +func (b *analyzeBinCommand) GenerateStaticReport(ctx context.Context) error { + if b.skipStaticReport { + return nil + } + staticReportLogFilePath := filepath.Join(b.output, "static-report.log") + staticReportLog, err := os.Create(staticReportLogFilePath) + if err != nil { + return fmt.Errorf("failed creating provider log file at %s", staticReportLogFilePath) + } + defer staticReportLog.Close() + + // it's possible for dependency analysis to fail + // in this case we still want to generate a static report for successful source analysis + _, noDepFileErr := os.Stat(filepath.Join(b.output, "dependencies.yaml")) + if errors.Is(noDepFileErr, os.ErrNotExist) { + b.log.Info("unable to get dependency output in static report. generating static report from source analysis only") + } else if noDepFileErr != nil && !errors.Is(noDepFileErr, os.ErrNotExist) { + return noDepFileErr + } + + staticReportAanlyzePath := filepath.Join(b.homeKantraDir, "static-report") + err = b.buildStaticReportFile(ctx, staticReportAanlyzePath, errors.Is(noDepFileErr, os.ErrNotExist)) + if err != nil { + return err + } + err = b.buildStaticReportOutput(ctx, staticReportLog) + if err != nil { + return err + } + uri := uri.File(filepath.Join(b.output, "static-report", "index.html")) + b.log.Info("Static report created. Access it at this URL:", "URL", string(uri)) + + return nil +} diff --git a/cmd/root.go b/cmd/root.go index ec1927c..51ce5ac 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -48,6 +48,7 @@ func init() { rootCmd.AddCommand(NewAnalyzeCmd(logger)) rootCmd.AddCommand(NewTestCommand(logger)) rootCmd.AddCommand(NewVersionCommand()) + rootCmd.AddCommand(NewAnalyzeBinCmd(logger)) } // Execute adds all child commands to the root command and sets flags appropriately. diff --git a/cmd/version.go b/cmd/version.go index d485055..8f4cc9d 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -12,6 +12,8 @@ var ( RunnerImage = "quay.io/konveyor/kantra" RootCommandName = "kantra" JavaBundlesLocation = "/jdtls/java-analyzer-bundle/java-analyzer-bundle.core/target/java-analyzer-bundle.core-1.0.0-SNAPSHOT.jar" + JDTLSBinLocation = "/jdtls/bin/jdtls" + RulesetsLocation = "rulesets/default/generated/00-discovery" JavaProviderImage = "quay.io/konveyor/java-external-provider" GenericProviderImage = "quay.io/konveyor/generic-external-provider" DotnetProviderImage = "quay.io/konveyor/dotnet-external-provider" diff --git a/docs/containerless.md b/docs/containerless.md new file mode 100644 index 0000000..5a544ef --- /dev/null +++ b/docs/containerless.md @@ -0,0 +1,29 @@ +## Test and Run Containerless Kantra + +### Clone the required packages: + +```sh +git clone https://github.com/eemcmullan/containerless-kantra-deps.git +``` + +## Move them to where kantra will look for packged binaries: + +```sh +mv $HOME/containerless-kantra-deps $HOME/.kantra +``` + +## Build the static report output parser: + +```sh +cd $HOME/.kantra/static-report/analyzer-output-parser +``` + +```sh +go build -o analyzer-output-parser main.go +``` + +### From kantra, run: + +```sh +go run main.go analyze-bin --input --output --rules +``` diff --git a/go.mod b/go.mod index 60609db..884633d 100644 --- a/go.mod +++ b/go.mod @@ -6,24 +6,37 @@ require ( github.com/devfile/alizer v1.6.1 github.com/getkin/kin-openapi v0.108.0 github.com/go-logr/logr v1.4.2 + github.com/nxadm/tail v1.4.8 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/spf13/cobra v1.8.1 + github.com/swaggest/openapi-go v0.2.45 + github.com/vifraa/gopom v1.0.0 go.lsp.dev/uri v0.3.0 + go.opentelemetry.io/otel v1.22.0 + go.opentelemetry.io/otel/trace v1.22.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + cloud.google.com/go/compute v1.24.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/antchfx/xpath v1.3.1 // indirect + github.com/bufbuild/protocompile v0.10.0 // indirect github.com/cbroglie/mustache v1.3.0 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/invopop/yaml v0.1.0 // indirect + github.com/jhump/protoreflect v1.16.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -32,26 +45,28 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect github.com/swaggest/jsonschema-go v0.3.64 // indirect - github.com/swaggest/openapi-go v0.2.45 // indirect github.com/swaggest/refl v1.3.0 // indirect - go.opentelemetry.io/otel v1.21.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect - go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.22.0 // indirect go.opentelemetry.io/otel/sdk v1.21.0 // indirect - go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.20.0 // indirect golang.org/x/net v0.24.0 // indirect + golang.org/x/oauth2 v0.17.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.14.0 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect ) require ( github.com/PaesslerAG/gval v1.2.2 // indirect + github.com/antchfx/jsonquery v1.3.5 // indirect + github.com/antchfx/xmlquery v1.4.1 // indirect github.com/bombsimon/logrusr/v3 v3.1.0 github.com/codingconcepts/env v0.0.0-20200821220118-a8fbf8d84482 github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 7f218d1..18a98ef 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,25 @@ +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E= github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI= github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= +github.com/antchfx/jsonquery v1.3.5 h1:243OSaQh02EfmASa3w3weKC9UaiD8RRzJhgfvq3q408= +github.com/antchfx/jsonquery v1.3.5/go.mod h1:qH23yX2Jsj1/k378Yu/EOgPCNgJ35P9tiGOeQdt/GWc= +github.com/antchfx/xmlquery v1.4.1 h1:YgpSwbeWvLp557YFTi8E3z6t6/hYjmFEtiEKbDfEbl0= +github.com/antchfx/xmlquery v1.4.1/go.mod h1:lKezcT8ELGt8kW5L+ckFMTbgdR61/odpPgDv8Gvi1fI= +github.com/antchfx/xpath v1.3.1 h1:PNbFuUqHwWl0xRjvUPjJ95Agbmdj2uzzIwmQKgu4oCk= +github.com/antchfx/xpath v1.3.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/bombsimon/logrusr/v3 v3.1.0 h1:zORbLM943D+hDMGgyjMhSAz/iDz86ZV72qaak/CA0zQ= github.com/bombsimon/logrusr/v3 v3.1.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco= github.com/bool64/dev v0.2.32 h1:DRZtloaoH1Igky3zphaUHV9+SLIV2H3lsf78JsJHFg0= github.com/bool64/dev v0.2.32/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= github.com/bool64/shared v0.1.5/go.mod h1:081yz68YC9jeFB3+Bbmno2RFWvGKv1lPKkMP6MHJlPs= +github.com/bufbuild/protocompile v0.10.0 h1:+jW/wnLMLxaCEG8AX9lD0bQ5v9h1RUiMKOBOT5ll9dM= +github.com/bufbuild/protocompile v0.10.0/go.mod h1:G9qQIQo0xZ6Uyj6CMNz0saGmx2so+KONo8/KrELABiY= github.com/cbroglie/mustache v1.3.0 h1:sj24GVYl8G7MH4b3zaROGsZnF8X79JqtjMx8/6H/nXM= github.com/cbroglie/mustache v1.3.0/go.mod h1:w58RIHjw/L7DPyRX2CcCTduNmcP1dvztaHP72ciSfh0= github.com/codingconcepts/env v0.0.0-20200821220118-a8fbf8d84482 h1:5/aEFreBh9hH/0G+33xtczJCvMaulqsm9nDuu2BZUEo= @@ -21,6 +33,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/devfile/alizer v1.6.1 h1:5G0SHFAcFNTwqsaZu5pir2pBJ04hg/F6d/lfQeAs7YY= github.com/devfile/alizer v1.6.1/go.mod h1:RHcz/nRPIlHNIfG3wTmVLrS8IyM+4CB841S/WKFXDiw= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/getkin/kin-openapi v0.108.0 h1:EYf0GtsKa4hQNIlplGS+Au7NEfGQ1F7MoHD2kcVevPQ= github.com/getkin/kin-openapi v0.108.0/go.mod h1:QtwUNt0PAAgIIBEvFWYfB7dfngxtAaqCX1zYHMZDeK8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -40,7 +54,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -56,6 +73,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/jhump/protoreflect v1.16.0 h1:54fZg+49widqXYQ0b+usAFHbMkBGR4PpXrsHc8+TBDg= +github.com/jhump/protoreflect v1.16.0/go.mod h1:oYPd7nPvcBw/5wlDfm/AVmU9zH9BgqGCI469pGxfj/8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -78,6 +97,8 @@ github.com/moby/buildkit v0.14.1 h1:2epLCZTkn4CikdImtsLtIa++7DzCimrrZCT1sway+oI= github.com/moby/buildkit v0.14.1/go.mod h1:1XssG7cAqv5Bz1xcGMxJL123iCv5TYN4Z/qf647gfuk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -109,6 +130,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/swaggest/assertjson v1.9.0 h1:dKu0BfJkIxv/xe//mkCrK5yZbs79jL7OVf9Ija7o2xQ= @@ -119,24 +141,27 @@ github.com/swaggest/openapi-go v0.2.45 h1:LOMAEleKVLg4E86lSCyioJK7ltjWRx50AaP4LZ github.com/swaggest/openapi-go v0.2.45/go.mod h1:/ykzNtS1ZO7X43OnEtyisMktxCiawQLyGd08rkjV68U= github.com/swaggest/refl v1.3.0 h1:PEUWIku+ZznYfsoyheF97ypSduvMApYyGkYF3nabS0I= github.com/swaggest/refl v1.3.0/go.mod h1:3Ujvbmh1pfSbDYjC6JGG7nMgPvpG0ehQL4iNonnLNbg= +github.com/vifraa/gopom v1.0.0 h1:L9XlKbyvid8PAIK8nr0lihMApJQg/12OBvMA28BcWh0= +github.com/vifraa/gopom v1.0.0/go.mod h1:oPa1dcrGrtlO37WPDBm5SqHAT+wTgF8An1Q71Z6Vv4o= github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.lsp.dev/uri v0.3.0 h1:KcZJmh6nFIBeJzTugn5JTU6OOyG0lDOo3R9KwTxTYbo= go.lsp.dev/uri v0.3.0/go.mod h1:P5sbO1IQR+qySTWOCnhnK7phBx+W3zbLqSMDJNTw88I= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -146,44 +171,70 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= +golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002 h1:V7Da7qt0MkY3noVANIMVBk28nOnijADeOR3i5Hcvpj4= google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= @@ -192,6 +243,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/pkg/java_external_provider/dependency.go b/pkg/java_external_provider/dependency.go new file mode 100644 index 0000000..3569d6d --- /dev/null +++ b/pkg/java_external_provider/dependency.go @@ -0,0 +1,923 @@ +package java + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "maps" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "strings" + + "github.com/konveyor/analyzer-lsp/engine/labels" + "github.com/konveyor/analyzer-lsp/output/v1/konveyor" + "github.com/konveyor/analyzer-lsp/provider" + "github.com/vifraa/gopom" + "go.lsp.dev/uri" +) + +const ( + javaDepSourceInternal = "internal" + javaDepSourceOpenSource = "open-source" + providerSpecificConfigOpenSourceDepListKey = "depOpenSourceLabelsFile" + providerSpecificConfigExcludePackagesKey = "excludePackages" +) + +// keys used in dep.Extras for extra information about a dep +const ( + artifactIdKey = "artifactId" + groupIdKey = "groupId" + pomPathKey = "pomPath" + baseDepKey = "baseDep" +) + +const ( + maven = "maven" + gradle = "gradle" +) + +func (p *javaServiceClient) GetBuildTool() string { + bf := "" + if bf = p.findPom(); bf != "" { + return maven + } else if bf = p.findGradleBuild(); bf != "" { + return gradle + } + return "" +} + +// TODO implement this for real +func (p *javaServiceClient) findPom() string { + var depPath string + if p.config.DependencyPath == "" { + depPath = "pom.xml" + } else { + depPath = p.config.DependencyPath + } + if filepath.IsAbs(depPath) { + return depPath + } + f, err := filepath.Abs(filepath.Join(p.config.Location, depPath)) + if err != nil { + return "" + } + if _, err := os.Stat(f); errors.Is(err, os.ErrNotExist) { + return "" + } + return f +} + +func (p *javaServiceClient) findGradleBuild() string { + if p.config.Location != "" { + path := filepath.Join(p.config.Location, "build.gradle") + _, err := os.Stat(path) + if err != nil { + return "" + } + f, err := filepath.Abs(path) + if err != nil { + return "" + } + return f + } + return "" +} + +func (p *javaServiceClient) GetDependencies(ctx context.Context) (map[uri.URI][]*provider.Dep, error) { + p.log.V(4).Info("running dependency analysis") + + if p.GetBuildTool() == gradle { + p.log.V(2).Info("gradle found - retrieving dependencies") + m := map[uri.URI][]*provider.Dep{} + deps, err := p.getDependenciesForGradle(ctx) + for f, ds := range deps { + deps := []*provider.Dep{} + for _, dep := range ds { + d := dep.Dep + deps = append(deps, &d) + deps = append(deps, provider.ConvertDagItemsToList(dep.AddedDeps)...) + } + m[f] = deps + } + return m, err + } + + p.depsMutex.RLock() + val := p.depsCache + p.depsMutex.RUnlock() + if val != nil { + return val, nil + } + + var err error + var ll map[uri.URI][]konveyor.DepDAGItem + m := map[uri.URI][]*provider.Dep{} + if p.isLocationBinary { + ll = make(map[uri.URI][]konveyor.DepDAGItem, 0) + // for binaries we only find JARs embedded in archive + p.discoverDepsFromJars(p.config.DependencyPath, ll) + if len(ll) == 0 { + p.log.Info("unable to get dependencies from jars, looking for pom") + pomPaths := p.discoverPoms(p.config.DependencyPath, ll) + for _, path := range pomPaths { + dep, err := p.GetDependenciesFallback(ctx, path) + if err != nil { + return m, err + } + maps.Copy(m, dep) + } + return m, nil + } + } else { + ll, err = p.GetDependenciesDAG(ctx) + if err != nil { + p.log.Info("unable to get dependencies, using fallback", "error", err) + return p.GetDependenciesFallback(ctx, "") + } + if len(ll) == 0 { + p.log.Info("unable to get dependencies (none found), using fallback") + return p.GetDependenciesFallback(ctx, "") + } + } + for f, ds := range ll { + deps := []*provider.Dep{} + for _, dep := range ds { + d := dep.Dep + deps = append(deps, &d) + deps = append(deps, provider.ConvertDagItemsToList(dep.AddedDeps)...) + } + m[f] = deps + } + p.depsMutex.Lock() + p.depsCache = m + p.depsMutex.Unlock() + return m, nil +} + +func getMavenLocalRepoPath(mvnSettingsFile string) string { + args := []string{ + "help:evaluate", "-Dexpression=settings.localRepository", "-q", "-DforceStdout", + } + if mvnSettingsFile != "" { + args = append(args, "-s", mvnSettingsFile) + } + cmd := exec.Command("mvn", args...) + var outb bytes.Buffer + cmd.Stdout = &outb + err := cmd.Run() + if err != nil { + return "" + } + + // check errors + return string(outb.String()) +} + +func (p *javaServiceClient) GetDependenciesFallback(ctx context.Context, location string) (map[uri.URI][]*provider.Dep, error) { + deps := []*provider.Dep{} + + m2Repo := getMavenLocalRepoPath(p.mvnSettingsFile) + + path, err := filepath.Abs(p.findPom()) + if err != nil { + return nil, err + } + + if location != "" { + path = location + } + pom, err := gopom.Parse(path) + if err != nil { + p.log.Error(err, "Analyzing POM") + return nil, err + } + p.log.V(10).Info("Analyzing POM", + "POM", fmt.Sprintf("%s:%s:%s", pomCoordinate(pom.GroupID), pomCoordinate(pom.ArtifactID), pomCoordinate(pom.Version)), + "error", err) + + // If the pom object is empty then parse failed silently. + if reflect.DeepEqual(*pom, gopom.Project{}) { + return nil, nil + } + + // have to get both and dependencies (if present) + var pomDeps []gopom.Dependency + if pom.Dependencies != nil && *pom.Dependencies != nil { + pomDeps = append(pomDeps, *pom.Dependencies...) + } + if pom.DependencyManagement != nil { + if pom.DependencyManagement.Dependencies != nil { + pomDeps = append(pomDeps, *pom.DependencyManagement.Dependencies...) + } + } + + // add each dependency found + for _, d := range pomDeps { + if d.GroupID == nil || d.Version == nil || d.ArtifactID == nil { + continue + } + dep := provider.Dep{} + dep.Name = fmt.Sprintf("%s.%s", *d.GroupID, *d.ArtifactID) + dep.Extras = map[string]interface{}{ + groupIdKey: *d.GroupID, + artifactIdKey: *d.ArtifactID, + pomPathKey: path, + } + if d.Version != nil { + if strings.Contains(*d.Version, "$") { + version := strings.TrimSuffix(strings.TrimPrefix(*d.Version, "${"), "}") + p.log.V(10).Info("Searching for property in properties", + "property", version, + "properties", pom.Properties) + if pom.Properties == nil { + p.log.Info("Cannot resolve version property value as POM does not have properties", + "POM", fmt.Sprintf("%s.%s", pomCoordinate(pom.GroupID), pomCoordinate(pom.ArtifactID)), + "property", version, + "dependency", dep.Name) + dep.Version = version + } else { + version = pom.Properties.Entries[version] + if version != "" { + dep.Version = version + } + } + } else { + dep.Version = *d.Version + } + if m2Repo != "" && d.ArtifactID != nil && d.GroupID != nil { + dep.FileURIPrefix = fmt.Sprintf("file://%s", filepath.Join(m2Repo, + strings.Replace(*d.GroupID, ".", "/", -1), *d.ArtifactID, dep.Version)) + } + } + deps = append(deps, &dep) + } + if len(deps) == 0 { + p.log.V(1).Info("unable to get dependencies from pom.xml in fallback", "pom", path) + return nil, nil + } + + m := map[uri.URI][]*provider.Dep{} + m[uri.File(path)] = deps + p.depsMutex.Lock() + p.depsCache = m + p.depsMutex.Unlock() + + // recursively find deps in submodules + if pom.Modules != nil { + for _, mod := range *pom.Modules { + mPath := fmt.Sprintf("%s/%s/pom.xml", filepath.Dir(path), mod) + moreDeps, err := p.GetDependenciesFallback(ctx, mPath) + if err != nil { + return nil, err + } + + // add found dependencies to map + for depPath := range moreDeps { + m[depPath] = moreDeps[depPath] + } + } + } + + return m, nil +} + +func pomCoordinate(value *string) string { + if value != nil { + return *value + } + return "unknown" +} + +func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI][]provider.DepDAGItem, error) { + switch p.GetBuildTool() { + case maven: + return p.getDependenciesForMaven(ctx) + case gradle: + return p.getDependenciesForGradle(ctx) + default: + return nil, fmt.Errorf("no build tool found") + } +} + +func (p *javaServiceClient) getDependenciesForMaven(_ context.Context) (map[uri.URI][]provider.DepDAGItem, error) { + localRepoPath := getMavenLocalRepoPath(p.mvnSettingsFile) + + path := p.findPom() + file := uri.File(path) + + moddir := filepath.Dir(path) + + args := []string{ + "-B", + "dependency:tree", + "-Djava.net.useSystemProxies=true", + } + + if p.mvnSettingsFile != "" { + args = append(args, "-s", p.mvnSettingsFile) + } + + // get the graph output + cmd := exec.Command("mvn", args...) + cmd.Dir = moddir + mvnOutput, err := cmd.CombinedOutput() + if err != nil { + return nil, err + } + + lines := strings.Split(string(mvnOutput), "\n") + submoduleTrees := extractSubmoduleTrees(lines) + + var pomDeps []provider.DepDAGItem + for _, tree := range submoduleTrees { + submoduleDeps, err := p.parseMavenDepLines(tree, localRepoPath, path) + if err != nil { + return nil, err + } + pomDeps = append(pomDeps, submoduleDeps...) + } + + m := map[uri.URI][]provider.DepDAGItem{} + m[file] = pomDeps + + if len(m) == 0 { + // grab the embedded deps + p.discoverDepsFromJars(moddir, m) + } + + return m, nil +} + +// getDependenciesForGradle invokes the Gradle wrapper to get the dependency tree and returns all project dependencies +// TODO: what if no wrapper? +func (p *javaServiceClient) getDependenciesForGradle(_ context.Context) (map[uri.URI][]provider.DepDAGItem, error) { + subprojects, err := p.getGradleSubprojects() + if err != nil { + return nil, err + } + + // command syntax: ./gradlew subproject1:dependencies subproject2:dependencies ... + args := []string{} + if len(subprojects) > 0 { + for _, sp := range subprojects { + args = append(args, fmt.Sprintf("%s:dependencies", sp)) + } + } else { + args = append(args, "dependencies") + } + + // get the graph output + exe, err := filepath.Abs(filepath.Join(p.config.Location, "gradlew")) + if err != nil { + return nil, fmt.Errorf("error calculating gradle wrapper path") + } + if _, err = os.Stat(exe); errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("a gradle wrapper must be present in the project") + } + cmd := exec.Command(exe, args...) + cmd.Dir = p.config.Location + output, err := cmd.CombinedOutput() + if err != nil { + return nil, err + } + + lines := strings.Split(string(output), "\n") + deps := p.parseGradleDependencyOutput(lines) + + // TODO: do we need to separate by submodule somehow? + + path := p.findGradleBuild() + file := uri.File(path) + m := map[uri.URI][]provider.DepDAGItem{} + m[file] = deps + + // TODO: need error? + return m, nil +} + +func (p *javaServiceClient) getGradleSubprojects() ([]string, error) { + args := []string{ + "projects", + } + + // Ideally we'd want to set this in gradle.properties, or as a -Dorg.gradle.java.home arg, + // but it doesn't seem to work in older Gradle versions. This should only affect child processes in any case. + err := os.Setenv("JAVA_HOME", os.Getenv("JAVA8_HOME")) + if err != nil { + return nil, err + } + + exe, err := filepath.Abs(filepath.Join(p.config.Location, "gradlew")) + if err != nil { + return nil, fmt.Errorf("error calculating gradle wrapper path") + } + if _, err = os.Stat(exe); errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("a gradle wrapper must be present in the project") + } + cmd := exec.Command(exe, args...) + cmd.Dir = p.config.Location + output, err := cmd.CombinedOutput() + if err != nil { + return nil, err + } + + beginRegex := regexp.MustCompile(`Root project`) + endRegex := regexp.MustCompile(`To see a list of`) + npRegex := regexp.MustCompile(`No sub-projects`) + pRegex := regexp.MustCompile(`.*- Project '(.*)'`) + + subprojects := []string{} + + gather := false + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if npRegex.Find([]byte(line)) != nil { + return []string{}, nil + } + if beginRegex.Find([]byte(line)) != nil { + gather = true + continue + } + if gather { + if endRegex.Find([]byte(line)) != nil { + return subprojects, nil + } + + if p := pRegex.FindStringSubmatch(line); p != nil { + subprojects = append(subprojects, p[1]) + } + } + } + + return subprojects, fmt.Errorf("error parsing gradle dependency output") +} + +// parseGradleDependencyOutput converts the relevant lines from the dependency output into actual dependencies +// See https://regex101.com/r/9Gp7dW/1 for context +func (p *javaServiceClient) parseGradleDependencyOutput(lines []string) []provider.DepDAGItem { + deps := []provider.DepDAGItem{} + + treeDepRegex := regexp.MustCompile(`^([| ]+)?[+\\]--- (.*)`) + + // map of to + // this is so that children can be added to their respective parents + lastFoundWithDepth := make(map[int]*provider.DepDAGItem) + + for _, line := range lines { + match := treeDepRegex.FindStringSubmatch(line) + if match != nil { + dep := parseGradleDependencyString(match[2]) + if reflect.DeepEqual(dep, provider.DepDAGItem{}) { // ignore empty dependency + continue + } else if match[1] != "" { // transitive dependency + dep.Dep.Indirect = true + depth := len(match[1]) / 5 // get the level of anidation of the dependency within the tree + parent := lastFoundWithDepth[depth-1] // find its parent + parent.AddedDeps = append(parent.AddedDeps, dep) // add child to parent + lastFoundWithDepth[depth] = &parent.AddedDeps[len(parent.AddedDeps)-1] // update last found with given depth + } else { // root level (direct) dependency + deps = append(deps, dep) // add root dependency to result list + lastFoundWithDepth[0] = &deps[len(deps)-1] + continue + } + } + } + + return deps +} + +// parseGradleDependencyString parses the lines of the gradle dependency output, for instance: +// org.codehaus.groovy:groovy:3.0.21 +// org.codehaus.groovy:groovy:3.+ -> 3.0.21 +// com.codevineyard:hello-world:{strictly 1.0.1} -> 1.0.1 +// :simple-jar (n) +func parseGradleDependencyString(s string) provider.DepDAGItem { + // (*) - dependencies omitted (listed previously) + // (n) - Not resolved (configuration is not meant to be resolved) + if strings.HasSuffix(s, "(n)") || strings.HasSuffix(s, "(*)") { + return provider.DepDAGItem{} + } + + depRegex := regexp.MustCompile(`(.+):(.+):((.*) -> )?(.*)`) + libRegex := regexp.MustCompile(`:(.*)`) + + dep := provider.Dep{} + match := depRegex.FindStringSubmatch(s) + if match != nil { + dep.Name = match[1] + "." + match[2] + dep.Version = match[5] + } else if match = libRegex.FindStringSubmatch(s); match != nil { + dep.Name = match[1] + } + + return provider.DepDAGItem{Dep: dep, AddedDeps: []provider.DepDAGItem{}} +} + +// extractSubmoduleTrees creates an array of lines for each submodule tree found in the mvn dependency:tree output +func extractSubmoduleTrees(lines []string) [][]string { + submoduleTrees := [][]string{} + + beginRegex := regexp.MustCompile(`(maven-)*dependency(-plugin)*:[\d\.]+:tree`) + endRegex := regexp.MustCompile(`\[INFO\] -*$`) + + submod := 0 + gather, skipmod := false, true + for _, line := range lines { + if beginRegex.Find([]byte(line)) != nil { + gather = true + submoduleTrees = append(submoduleTrees, []string{}) + continue + } + + if gather { + if endRegex.Find([]byte(line)) != nil { + gather, skipmod = false, true + submod++ + continue + } + if skipmod { // we ignore the first module (base module) + skipmod = false + continue + } + + line = strings.TrimPrefix(line, "[INFO] ") + line = strings.Trim(line, " ") + + // output contains progress report lines that are not deps, skip those + if !(strings.HasPrefix(line, "+") || strings.HasPrefix(line, "|") || strings.HasPrefix(line, "\\")) { + continue + } + + submoduleTrees[submod] = append(submoduleTrees[submod], line) + } + } + + return submoduleTrees +} + +// discoverDepsFromJars walks given path to discover dependencies embedded as JARs +func (p *javaServiceClient) discoverDepsFromJars(path string, ll map[uri.URI][]konveyor.DepDAGItem) { + // for binaries we only find JARs embedded in archive + w := walker{ + deps: ll, + depToLabels: p.depToLabels, + m2RepoPath: getMavenLocalRepoPath(p.mvnSettingsFile), + seen: map[string]bool{}, + initialPath: path, + } + filepath.WalkDir(path, w.walkDirForJar) +} + +type walker struct { + deps map[uri.URI][]provider.DepDAGItem + depToLabels map[string]*depLabelItem + m2RepoPath string + initialPath string + seen map[string]bool + pomPaths []string +} + +func (w *walker) walkDirForJar(path string, info fs.DirEntry, err error) error { + if info == nil { + return nil + } + if info.IsDir() { + return filepath.WalkDir(filepath.Join(path, info.Name()), w.walkDirForJar) + } + if strings.HasSuffix(info.Name(), ".jar") { + seenKey := filepath.Base(info.Name()) + if _, ok := w.seen[seenKey]; ok { + return nil + } + w.seen[seenKey] = true + d := provider.Dep{ + Name: info.Name(), + } + artifact, _ := toDependency(context.TODO(), path) + if (artifact != javaArtifact{}) { + d.Name = fmt.Sprintf("%s.%s", artifact.GroupId, artifact.ArtifactId) + d.Version = artifact.Version + d.Labels = addDepLabels(w.depToLabels, d.Name) + d.ResolvedIdentifier = artifact.sha1 + // when we can successfully get javaArtifact from a jar + // we added it to the pom and it should be in m2Repo path + if w.m2RepoPath != "" { + d.FileURIPrefix = fmt.Sprintf("file://%s", filepath.Join(w.m2RepoPath, + strings.Replace(artifact.GroupId, ".", "/", -1), artifact.ArtifactId, artifact.Version)) + } + } + + w.deps[uri.URI(filepath.Join(path, info.Name()))] = []provider.DepDAGItem{ + { + Dep: d, + }, + } + } + if strings.HasSuffix(info.Name(), ".class") { + // If the class is in WEB-INF we assume this is apart of the application + relPath, _ := filepath.Rel(w.initialPath, path) + relPath = filepath.Dir(relPath) + if strings.Contains(relPath, "WEB-INF") { + return nil + } + if _, ok := w.seen[relPath]; ok { + return nil + } + d := provider.Dep{ + Name: info.Name(), + } + artifact, _ := toFilePathDependency(context.Background(), filepath.Join(relPath, info.Name())) + if (artifact != javaArtifact{}) { + d.Name = fmt.Sprintf("%s.%s", artifact.GroupId, artifact.ArtifactId) + d.Version = artifact.Version + d.Labels = addDepLabels(w.depToLabels, d.Name) + d.ResolvedIdentifier = artifact.sha1 + // when we can successfully get javaArtifact from a jar + // we added it to the pom and it should be in m2Repo path + d.FileURIPrefix = fmt.Sprintf("file://%s", filepath.Join("java-project", "src", "main", + strings.Replace(artifact.GroupId, ".", "/", -1), artifact.ArtifactId)) + } + w.deps[uri.URI(filepath.Join(relPath))] = []provider.DepDAGItem{ + { + Dep: d, + }, + } + w.seen[relPath] = true + } + return nil +} + +func (p *javaServiceClient) discoverPoms(pathStart string, ll map[uri.URI][]konveyor.DepDAGItem) []string { + w := walker{ + deps: ll, + depToLabels: p.depToLabels, + m2RepoPath: "", + seen: map[string]bool{}, + initialPath: pathStart, + pomPaths: []string{}, + } + filepath.WalkDir(pathStart, w.walkDirForPom) + return w.pomPaths +} + +func (w *walker) walkDirForPom(path string, info fs.DirEntry, err error) error { + if info == nil { + return nil + } + if info.IsDir() { + return filepath.WalkDir(filepath.Join(path, info.Name()), w.walkDirForPom) + } + if strings.Contains(info.Name(), "pom.xml") { + w.pomPaths = append(w.pomPaths, path) + } + return nil +} + +// parseDepString parses a java dependency string +func (p *javaServiceClient) parseDepString(dep, localRepoPath, pomPath string) (provider.Dep, error) { + d := provider.Dep{} + // remove all the pretty print characters. + dep = strings.TrimFunc(dep, func(r rune) bool { + if r == '+' || r == '-' || r == '\\' || r == '|' || r == ' ' || r == '"' || r == '\t' { + return true + } + return false + + }) + // Split string on ":" must have 5 parts. + // For now we ignore Type as it appears most everything is a jar + parts := strings.Split(dep, ":") + if len(parts) >= 3 { + // Its always ::: ... then + if len(parts) == 6 { + d.Classifier = parts[3] + d.Version = parts[4] + d.Type = parts[5] + } else if len(parts) == 5 { + d.Version = parts[3] + d.Type = parts[4] + } else { + p.log.Info("Cannot derive version from dependency string", "dependency", dep) + d.Version = "Unknown" + } + } else { + return d, fmt.Errorf("unable to split dependency string %s", dep) + } + + group := parts[0] + artifact := parts[1] + d.Name = fmt.Sprintf("%s.%s", group, artifact) + + fp := resolveDepFilepath(&d, p, group, artifact, localRepoPath) + + d.Labels = addDepLabels(p.depToLabels, d.Name) + d.FileURIPrefix = fmt.Sprintf("file://%v", filepath.Dir(fp)) + + d.Extras = map[string]interface{}{ + groupIdKey: group, + artifactIdKey: artifact, + pomPathKey: pomPath, + } + + return d, nil +} + +// resolveDepFilepath tries to extract a valid filepath for the dependency with either JAR or POM packaging +func resolveDepFilepath(d *provider.Dep, p *javaServiceClient, group string, artifact string, localRepoPath string) string { + groupPath := strings.Replace(group, ".", "/", -1) + + // Try pom packaging (see https://www.baeldung.com/maven-packaging-types#4-pom) + var fp string + if d.Classifier == "" { + fp = filepath.Join(localRepoPath, groupPath, artifact, d.Version, fmt.Sprintf("%v-%v.%v.sha1", artifact, d.Version, "pom")) + } else { + fp = filepath.Join(localRepoPath, groupPath, artifact, d.Version, fmt.Sprintf("%v-%v-%v.%v.sha1", artifact, d.Version, d.Classifier, "pom")) + } + b, err := os.ReadFile(fp) + if err != nil { + // Try jar packaging + if d.Classifier == "" { + fp = filepath.Join(localRepoPath, groupPath, artifact, d.Version, fmt.Sprintf("%v-%v.%v.sha1", artifact, d.Version, "jar")) + } else { + fp = filepath.Join(localRepoPath, groupPath, artifact, d.Version, fmt.Sprintf("%v-%v-%v.%v.sha1", artifact, d.Version, d.Classifier, "jar")) + } + b, err = os.ReadFile(fp) + } + + if err != nil { + // Log the error and continue with the next dependency. + p.log.V(5).Error(err, "error reading SHA hash file for dependency", "d", d.Name) + // Set some default or empty resolved identifier for the dependency. + d.ResolvedIdentifier = "" + } else { + // sometimes sha file contains name of the jar followed by the actual sha + sha, _, _ := strings.Cut(string(b), " ") + d.ResolvedIdentifier = sha + } + + return fp +} + +func addDepLabels(depToLabels map[string]*depLabelItem, depName string) []string { + m := map[string]interface{}{} + for _, d := range depToLabels { + if d.r.Match([]byte(depName)) { + for label := range d.labels { + m[label] = nil + } + } + } + s := []string{} + for k := range m { + s = append(s, k) + } + // if open source label is not found, qualify the dep as being internal by default + if _, openSourceLabelFound := + m[labels.AsString(provider.DepSourceLabel, javaDepSourceOpenSource)]; !openSourceLabelFound { + s = append(s, + labels.AsString(provider.DepSourceLabel, javaDepSourceInternal)) + } + s = append(s, labels.AsString(provider.DepLanguageLabel, "java")) + return s +} + +// parseMavenDepLines recursively parses output lines from maven dependency tree +func (p *javaServiceClient) parseMavenDepLines(lines []string, localRepoPath, pomPath string) ([]provider.DepDAGItem, error) { + if len(lines) > 0 { + baseDepString := lines[0] + baseDep, err := p.parseDepString(baseDepString, localRepoPath, pomPath) + if err != nil { + return nil, err + } + item := provider.DepDAGItem{} + item.Dep = baseDep + item.AddedDeps = []provider.DepDAGItem{} + idx := 1 + // indirect deps are separated by 3 or more spaces after the direct dep + for idx < len(lines) && strings.Count(lines[idx], " ") > 2 { + transitiveDep, err := p.parseDepString(lines[idx], localRepoPath, pomPath) + if err != nil { + return nil, err + } + dm := map[string]interface{}{ + "name": baseDep.Name, + "version": baseDep.Version, + "extras": baseDep.Extras, + } + transitiveDep.Indirect = true + transitiveDep.Extras[baseDepKey] = dm // Minimum needed set of attributes for GetLocation + item.AddedDeps = append(item.AddedDeps, provider.DepDAGItem{Dep: transitiveDep}) + idx += 1 + } + ds, err := p.parseMavenDepLines(lines[idx:], localRepoPath, pomPath) + if err != nil { + return nil, err + } + ds = append(ds, item) + return ds, nil + } + return []provider.DepDAGItem{}, nil +} + +// depInit loads a map of package patterns and their associated labels for easy lookup +func (p *javaServiceClient) depInit() error { + err := p.initOpenSourceDepLabels() + if err != nil { + p.log.V(5).Error(err, "failed to initialize dep labels lookup for open source packages") + return err + } + + err = p.initExcludeDepLabels() + if err != nil { + p.log.V(5).Error(err, "failed to initialize dep labels lookup for excluded packages") + return err + } + + return nil +} + +// initOpenSourceDepLabels reads user provided file that has a list of open source +// packages (supports regex) and loads a map of patterns -> labels for easy lookup +func (p *javaServiceClient) initOpenSourceDepLabels() error { + var ok bool + var v interface{} + if v, ok = p.config.ProviderSpecificConfig[providerSpecificConfigOpenSourceDepListKey]; !ok { + p.log.V(7).Info("Did not find open source dep list.") + return nil + } + + var filePath string + if filePath, ok = v.(string); !ok { + return fmt.Errorf("unable to determine filePath from open source dep list") + } + + fileInfo, err := os.Stat(filePath) + if err != nil { + //TODO(shawn-hurley): consider wrapping error with value + return err + } + + if fileInfo.IsDir() { + return fmt.Errorf("open source dep list must be a file, not a directory") + } + + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + return loadDepLabelItems(file, p.depToLabels, + labels.AsString(provider.DepSourceLabel, javaDepSourceOpenSource)) +} + +// initExcludeDepLabels reads user provided list of excluded packages +// and initiates label lookup for them +func (p *javaServiceClient) initExcludeDepLabels() error { + var ok bool + var v interface{} + if v, ok = p.config.ProviderSpecificConfig[providerSpecificConfigExcludePackagesKey]; !ok { + p.log.V(7).Info("did not find exclude packages list") + return nil + } + var excludePackages []string + if excludePackages, ok = v.([]string); !ok { + return fmt.Errorf("%s config must be a list of packages to exclude", providerSpecificConfigExcludePackagesKey) + } + return loadDepLabelItems(strings.NewReader( + strings.Join(excludePackages, "\n")), p.depToLabels, provider.DepExcludeLabel) +} + +// loadDepLabelItems reads list of patterns from reader and appends given +// label to the list of labels for the associated pattern +func loadDepLabelItems(r io.Reader, depToLabels map[string]*depLabelItem, label string) error { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + pattern := scanner.Text() + r, err := regexp.Compile(pattern) + if err != nil { + return fmt.Errorf("unable to create regexp for string: %v", pattern) + } + //Make sure that we are not adding duplicates + if _, found := depToLabels[pattern]; !found { + depToLabels[pattern] = &depLabelItem{ + r: r, + labels: map[string]interface{}{ + label: nil, + }, + } + } else { + if depToLabels[pattern].labels == nil { + depToLabels[pattern].labels = map[string]interface{}{} + } + depToLabels[pattern].labels[label] = nil + } + } + return nil +} diff --git a/pkg/java_external_provider/filter.go b/pkg/java_external_provider/filter.go new file mode 100644 index 0000000..2dbd7d7 --- /dev/null +++ b/pkg/java_external_provider/filter.go @@ -0,0 +1,276 @@ +package java + +import ( + "bufio" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/konveyor/analyzer-lsp/lsp/protocol" + "github.com/konveyor/analyzer-lsp/provider" + "go.lsp.dev/uri" +) + +const ( + LINE_NUMBER_EXTRA_KEY = "lineNumber" + KIND_EXTRA_KEY = "kind" + SYMBOL_NAME_KEY = "name" + FILE_KEY = "file" +) + +func (p *javaServiceClient) filterVariableDeclaration(symbols []protocol.WorkspaceSymbol) ([]provider.IncidentContext, error) { + incidents := []provider.IncidentContext{} + for _, ref := range symbols { + incident, err := p.convertToIncidentContext(ref) + if err != nil { + return nil, err + } + incidents = append(incidents, incident) + } + return incidents, nil +} + +func (p *javaServiceClient) filterModulesImports(symbols []protocol.WorkspaceSymbol) ([]provider.IncidentContext, error) { + incidents := []provider.IncidentContext{} + for _, symbol := range symbols { + if symbol.Kind != protocol.Module { + continue + } + incident, err := p.convertToIncidentContext(symbol) + if err != nil { + return nil, err + } + incidents = append(incidents, incident) + } + return incidents, nil +} + +func (p *javaServiceClient) filterTypesInheritance(symbols []protocol.WorkspaceSymbol) ([]provider.IncidentContext, error) { + incidents := []provider.IncidentContext{} + for _, symbol := range symbols { + incident, err := p.convertToIncidentContext(symbol) + if err != nil { + return nil, err + } + incidents = append(incidents, incident) + } + + return incidents, nil +} + +func (p *javaServiceClient) filterDefault(symbols []protocol.WorkspaceSymbol) ([]provider.IncidentContext, error) { + incidents := []provider.IncidentContext{} + for _, symbol := range symbols { + incident, err := p.convertToIncidentContext(symbol) + if err != nil { + return nil, err + } + incidents = append(incidents, incident) + } + return incidents, nil +} + +// TODO: we will probably want to filter symbols bassed on if in any way the method is being used in the code directly. +// This will need to be part of a "filtration" concept that windup has. Searching partiular subsets of things (just the application, applicatoin + corp libraries and the everything.) +// Today this is just giving everything. +func (p *javaServiceClient) filterMethodSymbols(symbols []protocol.WorkspaceSymbol) ([]provider.IncidentContext, error) { + incidents := []provider.IncidentContext{} + for _, symbol := range symbols { + incident, err := p.convertToIncidentContext(symbol) + if err != nil { + return nil, err + } + incidents = append(incidents, incident) + + } + return incidents, nil + +} + +func (p *javaServiceClient) convertToIncidentContext(symbol protocol.WorkspaceSymbol) (provider.IncidentContext, error) { + var locationURI protocol.DocumentURI + var locationRange protocol.Range + switch x := symbol.Location.Value.(type) { + case protocol.Location: + locationURI = x.URI + locationRange = x.Range + case protocol.PLocationMsg_workspace_symbol: + locationURI = x.URI + locationRange = protocol.Range{} + default: + locationURI = "" + locationRange = protocol.Range{} + } + + n, u, err := p.getURI(locationURI) + if err != nil { + return provider.IncidentContext{}, err + } + + lineNumber := int(locationRange.Start.Line) + 1 + incident := provider.IncidentContext{ + FileURI: u, + LineNumber: &lineNumber, + Variables: map[string]interface{}{ + KIND_EXTRA_KEY: symbolKindToString(symbol.Kind), + SYMBOL_NAME_KEY: symbol.Name, + FILE_KEY: string(u), + "package": n, + }, + } + + // based on original URI we got, we can tell if this incident appeared in a dep + if locationURI != "" && strings.HasPrefix(locationURI, JDT_CLASS_FILE_URI_PREFIX) { + incident.IsDependencyIncident = true + } + + if locationRange.Start.Line == 0 && locationRange.Start.Character == 0 && locationRange.End.Line == 0 && locationRange.End.Character == 0 { + return incident, nil + } + incident.CodeLocation = &provider.Location{ + StartPosition: provider.Position{ + Line: float64(locationRange.Start.Line), + Character: float64(locationRange.Start.Character), + }, + EndPosition: provider.Position{ + Line: float64(locationRange.End.Line), + Character: float64(locationRange.End.Character), + }, + } + return incident, nil +} + +func (p *javaServiceClient) getURI(refURI string) (string, uri.URI, error) { + if !strings.HasPrefix(refURI, JDT_CLASS_FILE_URI_PREFIX) { + u, err := uri.Parse(refURI) + if err != nil { + return "", uri.URI(""), err + } + file, err := os.Open(u.Filename()) + if err != nil { + p.log.V(4).Info("unable to get package name", "err", err) + return "", u, nil + } + defer file.Close() + scanner := bufio.NewScanner(file) + name := "" + for scanner.Scan() { + if strings.Contains(scanner.Text(), "package") && + // here we have to handle the work package in license's/copyrights. + // Ignoring everyting that looks like a java doc comment. + !strings.Contains(scanner.Text(), "//") && !strings.Contains(scanner.Text(), "/*") && + !strings.HasPrefix(strings.TrimSpace(scanner.Text()), "*") { + + name = strings.ReplaceAll(scanner.Text(), "package ", "") + name = strings.ReplaceAll(name, ";", "") + break + } + } + return name, u, nil + + } + + u, err := url.Parse(refURI) + if err != nil { + return "", uri.URI(""), err + } + // Decompile the jar + sourceRange, err := strconv.ParseBool(u.Query().Get("source-range")) + if err != nil { + // then we got some response that does not make sense or should not be valid + return "", uri.URI(""), fmt.Errorf("unable to get konveyor-jdt source range query parameter") + } + packageName := u.Query().Get("packageName") + + var jarPath string + if sourceRange { + // If there is a source range, we know there is a sources jar + jarName := filepath.Base(u.Path) + s := strings.TrimSuffix(jarName, ".jar") + s = fmt.Sprintf("%v-sources.jar", s) + jarPath = filepath.Join(filepath.Dir(u.Path), s) + } else { + jarName := filepath.Base(u.Path) + jarPath = filepath.Join(filepath.Dir(u.Path), jarName) + } + + path := filepath.Join(strings.Split(strings.TrimSuffix(packageName, ".class"), ".")...) // path: org/apache/logging/log4j/core/appender/FileManager + + javaFileName := fmt.Sprintf("%s.java", filepath.Base(path)) + if i := strings.Index(javaFileName, "$"); i > 0 { + javaFileName = fmt.Sprintf("%v.java", javaFileName[0:i]) + } + + javaFileAbsolutePath := "" + if p.GetBuildTool() == maven { + javaFileAbsolutePath = filepath.Join(filepath.Dir(jarPath), filepath.Dir(path), javaFileName) + + // attempt to decompile when directory for the expected java file doesn't exist + // if directory exists, assume .java file is present within, this avoids decompiling every Jar + if _, err := os.Stat(filepath.Dir(javaFileAbsolutePath)); err != nil { + cmd := exec.Command("jar", "xf", filepath.Base(jarPath)) + cmd.Dir = filepath.Dir(jarPath) + err := cmd.Run() + if err != nil { + fmt.Printf("\n java error%v", err) + return "", "", err + } + } + } else if p.GetBuildTool() == gradle { + sourcesFile := "" + jarFile := filepath.Base(jarPath) + walker := func(path string, d os.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("found error traversing files: %w", err) + } + if !d.IsDir() && d.Name() == jarFile { + sourcesFile = path + return nil + } + return nil + } + root := filepath.Join(jarPath, "..", "..") + err := filepath.WalkDir(root, walker) + if err != nil { + return "", "", err + } + javaFileAbsolutePath = filepath.Join(filepath.Dir(sourcesFile), filepath.Dir(path), javaFileName) + + if _, err := os.Stat(filepath.Dir(javaFileAbsolutePath)); err != nil { + cmd := exec.Command("jar", "xf", filepath.Base(sourcesFile)) + cmd.Dir = filepath.Dir(sourcesFile) + err = cmd.Run() + if err != nil { + fmt.Printf("\n java error%v", err) + return "", "", err + } + } + } + + ui := uri.New(javaFileAbsolutePath) + file, err := os.Open(ui.Filename()) + if err != nil { + p.log.V(4).Info("unable to get package name", "err", err) + return "", ui, nil + } + defer file.Close() + n := "" + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if strings.Contains(scanner.Text(), "package") && + // here we have to handle the work package in license's/copyrights. + // Ignoring everyting that looks like a java doc comment. + !strings.Contains(scanner.Text(), "//") && !strings.Contains(scanner.Text(), "/*") && + !strings.HasPrefix(strings.TrimSpace(scanner.Text()), "*") { + + n = strings.ReplaceAll(scanner.Text(), "package ", "") + n = strings.ReplaceAll(n, ";", "") + break + } + } + return n, ui, nil +} diff --git a/pkg/java_external_provider/provider.go b/pkg/java_external_provider/provider.go new file mode 100644 index 0000000..58cb4cd --- /dev/null +++ b/pkg/java_external_provider/provider.go @@ -0,0 +1,882 @@ +package java + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strings" + "sync" + + "github.com/go-logr/logr" + "github.com/konveyor/analyzer-lsp/engine" + "github.com/konveyor/analyzer-lsp/jsonrpc2" + "github.com/konveyor/analyzer-lsp/lsp/protocol" + "github.com/konveyor/analyzer-lsp/output/v1/konveyor" + "github.com/konveyor/analyzer-lsp/provider" + "github.com/konveyor/analyzer-lsp/tracing" + "github.com/nxadm/tail" + "github.com/swaggest/openapi-go/openapi3" + "go.lsp.dev/uri" +) + +const ( + JavaFile = ".java" + JavaArchive = ".jar" + WebArchive = ".war" + EnterpriseArchive = ".ear" + ClassFile = ".class" + MvnURIPrefix = "mvn://" +) + +// provider specific config keys +const ( + BUNDLES_INIT_OPTION = "bundles" + WORKSPACE_INIT_OPTION = "workspace" + MVN_SETTINGS_FILE_INIT_OPTION = "mavenSettingsFile" + JVM_MAX_MEM_INIT_OPTION = "jvmMaxMem" +) + +// Rule Location to location that the bundle understands +var locationToCode = map[string]int{ + //Type is the default. + "": 0, + "inheritance": 1, + "method_call": 2, + "constructor_call": 3, + "annotation": 4, + "implements_type": 5, + // Not Implemented + "enum_constant": 6, + "return_type": 7, + "import": 8, + "variable_declaration": 9, + "type": 10, + "package": 11, + "field": 12, + "method": 13, +} + +type javaProvider struct { + config provider.Config + Log logr.Logger + contextLines int + + clients []provider.ServiceClient + + lspServerName string + + hasMaven bool + depsMutex sync.RWMutex + depsLocationCache map[string]int + + logFollow sync.Once +} + +//var _ provider.BaseClient = &javaProvider{} + +var _ provider.InternalProviderClient = &javaProvider{} + +var _ provider.DependencyLocationResolver = &javaProvider{} + +type javaCondition struct { + Referenced referenceCondition `yaml:"referenced"` +} + +type referenceCondition struct { + Pattern string `yaml:"pattern"` + Location string `yaml:"location"` + Annotated annotated `yaml:"annotated,omitempty" json:"annotated,omitempty"` +} + +type annotated struct { + Pattern string `yaml:"pattern" json:"pattern"` + Elements []element `yaml:"elements,omitempty" json:"elements,omitempty"` +} + +type element struct { + Name string `yaml:"name" json:"name"` + Value string `yaml:"value" json:"value"` // can be a (java) regex pattern +} + +func NewJavaProvider(log logr.Logger, config provider.Config, contextLines int) *javaProvider { + + _, mvnBinaryError := exec.LookPath("mvn") + + return &javaProvider{ + config: config, + hasMaven: mvnBinaryError == nil, + Log: log, + clients: []provider.ServiceClient{}, + depsLocationCache: make(map[string]int), + contextLines: contextLines, + logFollow: sync.Once{}, + } +} + +func (p *javaProvider) Stop() { + // Ignore the error here, it stopped and we wanted it to. + for _, c := range p.clients { + c.Stop() + } +} + +func (p *javaProvider) Capabilities() []provider.Capability { + r := openapi3.NewReflector() + caps := []provider.Capability{} + refCap, err := provider.ToProviderCap(r, p.Log, javaCondition{}, "referenced") + if err != nil { + p.Log.Error(err, "this is not going to be cool if it fails") + } else { + caps = append(caps, refCap) + } + if p.hasMaven { + depCap, err := provider.ToProviderCap(r, p.Log, provider.DependencyConditionCap{}, "dependency") + if err != nil { + p.Log.Error(err, "this is not goinag to be cool if it fails") + } else { + caps = append(caps, depCap) + } + } + return caps +} + +func (p *javaProvider) Evaluate(ctx context.Context, cap string, conditionInfo []byte) (provider.ProviderEvaluateResponse, error) { + return provider.FullResponseFromServiceClients(ctx, p.clients, cap, conditionInfo) +} + +func (p *javaProvider) ProviderInit(ctx context.Context, additionalConfigs []provider.InitConfig) ([]provider.InitConfig, error) { + builtinConfs := []provider.InitConfig{} + if additionalConfigs != nil { + p.config.InitConfig = append(p.config.InitConfig, additionalConfigs...) + } + for _, c := range p.config.InitConfig { + client, builtinConf, err := p.Init(ctx, p.Log, c) + if err != nil { + return nil, err + } + p.clients = append(p.clients, client) + if builtinConf.Location != "" { + builtinConfs = append(builtinConfs, builtinConf) + } + } + return builtinConfs, nil +} + +func (p *javaProvider) GetDependencies(ctx context.Context) (map[uri.URI][]*provider.Dep, error) { + return provider.FullDepsResponse(ctx, p.clients) +} + +func (p *javaProvider) GetDependenciesDAG(ctx context.Context) (map[uri.URI][]provider.DepDAGItem, error) { + return provider.FullDepDAGResponse(ctx, p.clients) +} + +func symbolKindToString(symbolKind protocol.SymbolKind) string { + switch symbolKind { + case 1: + return "File" + case 2: + return "Module" + case 3: + return "Namespace" + case 4: + return "Package" + case 5: + return "Class" + case 6: + return "Method" + case 7: + return "Property" + case 8: + return "Field" + case 9: + return "Constructor" + case 10: + return "Enum" + case 11: + return "Interface" + case 12: + return "Function" + case 13: + return "Variable" + case 14: + return "Constant" + case 15: + return "String" + case 16: + return "Number" + case 17: + return "Boolean" + case 18: + return "Array" + case 19: + return "Object" + case 20: + return "Key" + case 21: + return "Null" + case 22: + return "EnumMember" + case 23: + return "Struct" + case 24: + return "Event" + case 25: + return "Operator" + case 26: + return "TypeParameter" + } + return "" +} + +func (p *javaProvider) Init(ctx context.Context, log logr.Logger, config provider.InitConfig) (provider.ServiceClient, provider.InitConfig, error) { + // By default, if nothing is set for analysis mode in the config, we should default to full for external providers + var mode provider.AnalysisMode = provider.AnalysisMode(config.AnalysisMode) + // in case of a binary, provider decompiles it and returns + // a builtin config that points to the decompiled archive + additionalBuiltinConfig := provider.InitConfig{} + + if mode == provider.AnalysisMode("") { + mode = provider.FullAnalysisMode + } else if !(mode == provider.FullAnalysisMode || mode == provider.SourceOnlyAnalysisMode) { + return nil, additionalBuiltinConfig, fmt.Errorf("invalid Analysis Mode") + } + log = log.WithValues("provider", "java") + + // read provider settings + bundlesString, ok := config.ProviderSpecificConfig[BUNDLES_INIT_OPTION].(string) + if !ok { + bundlesString = "" + } + bundles := strings.Split(bundlesString, ",") + + workspace, ok := config.ProviderSpecificConfig[WORKSPACE_INIT_OPTION].(string) + if !ok { + workspace = "" + } + + mavenSettingsFile, ok := config.ProviderSpecificConfig[MVN_SETTINGS_FILE_INIT_OPTION].(string) + if !ok { + mavenSettingsFile = "" + } + + lspServerPath, ok := config.ProviderSpecificConfig[provider.LspServerPathConfigKey].(string) + if !ok || lspServerPath == "" { + return nil, additionalBuiltinConfig, fmt.Errorf("invalid lspServerPath provided, unable to init java provider") + } + + isBinary := false + var returnErr error + // each service client should have their own context + ctx, cancelFunc := context.WithCancel(ctx) + // location can be a coordinate to a remote mvn artifact + if strings.HasPrefix(config.Location, MvnURIPrefix) { + mvnUri := strings.Replace(config.Location, MvnURIPrefix, "", 1) + // URI format is :::@ + // is optional & points to a local path where it will be downloaded + mvnCoordinates, destPath, _ := strings.Cut(mvnUri, "@") + mvnCoordinatesParts := strings.Split(mvnCoordinates, ":") + if mvnCoordinates == "" || len(mvnCoordinatesParts) < 3 { + cancelFunc() + return nil, additionalBuiltinConfig, fmt.Errorf("invalid maven coordinates in location %s, must be in format mvn://:::@", config.Location) + } + outputDir := "." + if destPath != "" { + if stat, err := os.Stat(destPath); err != nil || !stat.IsDir() { + cancelFunc() + return nil, additionalBuiltinConfig, fmt.Errorf("output path does not exist or not a directory") + } + outputDir = destPath + } + mvnOptions := []string{ + "dependency:copy", + fmt.Sprintf("-Dartifact=%s", mvnCoordinates), + fmt.Sprintf("-DoutputDirectory=%s", outputDir), + } + if mavenSettingsFile != "" { + mvnOptions = append(mvnOptions, "-s", mavenSettingsFile) + } + log.Info("downloading maven artifact", "artifact", mvnCoordinates, "options", mvnOptions) + cmd := exec.CommandContext(ctx, "mvn", mvnOptions...) + cmd.Dir = outputDir + mvnOutput, err := cmd.CombinedOutput() + if err != nil { + cancelFunc() + return nil, additionalBuiltinConfig, fmt.Errorf("error downloading java artifact %s - %w", mvnUri, err) + } + downloadedPath := filepath.Join(outputDir, + fmt.Sprintf("%s.jar", strings.Join(mvnCoordinatesParts[1:3], "-"))) + if len(mvnCoordinatesParts) == 4 { + downloadedPath = filepath.Join(outputDir, + fmt.Sprintf("%s.%s", strings.Join(mvnCoordinatesParts[1:3], "-"), strings.ToLower(mvnCoordinatesParts[3]))) + } + outputLinePattern := regexp.MustCompile(`.*?Copying.*?to (.*)`) + for _, line := range strings.Split(string(mvnOutput), "\n") { + if outputLinePattern.MatchString(line) { + match := outputLinePattern.FindStringSubmatch(line) + if match != nil { + downloadedPath = match[1] + } + } + } + if _, err := os.Stat(downloadedPath); err != nil { + cancelFunc() + return nil, additionalBuiltinConfig, fmt.Errorf("failed to download maven artifact to path %s - %w", downloadedPath, err) + } + config.Location = downloadedPath + } + + extension := strings.ToLower(path.Ext(config.Location)) + switch extension { + case JavaArchive, WebArchive, EnterpriseArchive: + depLocation, sourceLocation, err := decompileJava(ctx, log, + config.Location, getMavenLocalRepoPath(mavenSettingsFile)) + if err != nil { + cancelFunc() + return nil, additionalBuiltinConfig, err + } + config.Location = sourceLocation + // for binaries, we fallback to looking at .jar files only for deps + config.DependencyPath = depLocation + isBinary = true + } + additionalBuiltinConfig.Location = config.Location + additionalBuiltinConfig.DependencyPath = config.DependencyPath + + // handle proxy settings + // for k, v := range config.Proxy.ToEnvVars() { + // os.Setenv(k, v) + // } + + args := []string{ + "-Djava.net.useSystemProxies=true", + "-configuration", + "./", + //"--jvm-arg=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:1044", + "-data", + workspace, + } + if val, ok := config.ProviderSpecificConfig[JVM_MAX_MEM_INIT_OPTION].(string); ok && val != "" { + args = append(args, fmt.Sprintf("-Xmx%s", val)) + } + cmd := exec.CommandContext(ctx, lspServerPath, args...) + stdin, err := cmd.StdinPipe() + if err != nil { + cancelFunc() + return nil, additionalBuiltinConfig, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + cancelFunc() + return nil, additionalBuiltinConfig, err + } + + go func() { + err := cmd.Start() + if err != nil { + cancelFunc() + returnErr = err + log.Error(err, "unable to start lsp command") + return + } + }() + rpc := jsonrpc2.NewConn(jsonrpc2.NewHeaderStream(stdout, stdin), log) + + rpc.AddHandler(jsonrpc2.NewBackoffHandler(log)) + + go func() { + err := rpc.Run(ctx) + if err != nil { + //TODO: we need to pipe the ctx further into the stream header and run. + // basically it is checking if done, then reading. When it gets EOF it errors. + // We need the read to be at the same level of selection to fully implment graceful shutdown + cancelFunc() + returnErr = err + return + } + }() + + svcClient := javaServiceClient{ + rpc: rpc, + cancelFunc: cancelFunc, + config: config, + cmd: cmd, + bundles: bundles, + workspace: workspace, + log: log, + depToLabels: map[string]*depLabelItem{}, + isLocationBinary: isBinary, + mvnSettingsFile: mavenSettingsFile, + depsLocationCache: make(map[string]int), + includedPaths: provider.GetIncludedPathsFromConfig(config, false), + } + + if mode == provider.FullAnalysisMode { + // we attempt to decompile JARs of dependencies that don't have a sources JAR attached + // we need to do this for jdtls to correctly recognize source attachment for dep + switch svcClient.GetBuildTool() { + case maven: + err := resolveSourcesJarsForMaven(ctx, log, config.Location, mavenSettingsFile) + if err != nil { + // TODO (pgaikwad): should we ignore this failure? + log.Error(err, "failed to resolve maven sources jar for location", "location", config.Location) + } + case gradle: + err = resolveSourcesJarsForGradle(ctx, log, config.Location, mavenSettingsFile, &svcClient) + if err != nil { + log.Error(err, "failed to resolve gradle sources jar for location", "location", config.Location) + } + } + + } + + svcClient.initialization(ctx) + err = svcClient.depInit() + if err != nil { + cancelFunc() + return nil, provider.InitConfig{}, err + } + // Will only set up log follow one time + // Will work in container image and hub, will not work + // When running for long period of time. + p.logFollow.Do(func() { + go func() { + t, err := tail.TailFile(".metadata/.log", tail.Config{ + ReOpen: true, + MustExist: false, + Follow: true, + }) + if err != nil { + log.Error(err, "unable to set up follower") + return + } + + for line := range t.Lines { + if strings.Contains(line.Text, "KONVEYOR_LOG") { + log.Info("language server log", "line", line.Text) + } + } + }() + }) + return &svcClient, additionalBuiltinConfig, returnErr +} + +func resolveSourcesJarsForGradle(ctx context.Context, log logr.Logger, location string, _ string, svc *javaServiceClient) error { + ctx, span := tracing.StartNewSpan(ctx, "resolve-sources") + defer span.End() + + log.V(5).Info("resolving dependency sources for gradle") + + gb := svc.findGradleBuild() + if gb == "" { + return fmt.Errorf("could not find gradle build file for project") + } + + // create a temporary build file to append the task for downloading sources + taskgb := filepath.Join(filepath.Dir(gb), "tmp.gradle") + err := CopyFile(gb, taskgb) + if err != nil { + return fmt.Errorf("error copying file %s to %s", gb, taskgb) + } + defer os.Remove(taskgb) + + // append downloader task + taskfile := "/root/.gradle/task.gradle" + err = AppendToFile(taskfile, taskgb) + if err != nil { + return fmt.Errorf("error appending file %s to %s", taskfile, taskgb) + } + + tmpgbname := filepath.Join(location, "toberenamed.gradle") + err = os.Rename(gb, tmpgbname) + if err != nil { + return fmt.Errorf("error renaming file %s to %s", gb, "toberenamed.gradle") + } + defer os.Rename(tmpgbname, gb) + + err = os.Rename(taskgb, gb) + if err != nil { + return fmt.Errorf("error renaming file %s to %s", gb, "toberenamed.gradle") + } + defer os.Remove(gb) + + // run gradle wrapper with tmp build file + exe, err := filepath.Abs(filepath.Join(svc.config.Location, "gradlew")) + if err != nil { + return fmt.Errorf("error calculating gradle wrapper path") + } + if _, err = os.Stat(exe); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("a gradle wrapper must be present in the project") + } + + // gradle must run with java 8 (see compatibility matrix) + java8home := os.Getenv("JAVA8_HOME") + if java8home == "" { + return fmt.Errorf("") + } + + args := []string{ + "konveyorDownloadSources", + } + cmd := exec.CommandContext(ctx, exe, args...) + cmd.Env = append(cmd.Env, fmt.Sprintf("JAVA_HOME=%s", java8home)) + cmd.Dir = location + output, err := cmd.CombinedOutput() + if err != nil { + return err + } + + log.V(8).WithValues("output", output).Info("got gradle output") + + // TODO: what if all sources available + reader := bytes.NewReader(output) + unresolvedSources, err := parseUnresolvedSourcesForGradle(reader) + if err != nil { + return err + } + + fmt.Printf("%d", len(unresolvedSources)) + + decompileJobs := []decompileJob{} + if len(unresolvedSources) > 1 { + // Gradle cache dir structure changes over time - we need to find where the actual dependencies are stored + cache, err := findGradleCache(unresolvedSources[0].GroupId) + if err != nil { + return err + } + + for _, artifact := range unresolvedSources { + log.V(5).WithValues("artifact", artifact).Info("sources for artifact not found, decompiling...") + + artifactDir := filepath.Join(cache, artifact.GroupId, artifact.ArtifactId) + jarName := fmt.Sprintf("%s-%s.jar", artifact.ArtifactId, artifact.Version) + artifactPath, err := findGradleArtifact(artifactDir, jarName) + if err != nil { + return err + } + decompileJobs = append(decompileJobs, decompileJob{ + artifact: artifact, + inputPath: artifactPath, + outputPath: filepath.Join(filepath.Dir(artifactPath), "decompiled", jarName), + }) + } + err = decompile(ctx, log, alwaysDecompileFilter(true), 10, decompileJobs, "") + if err != nil { + return err + } + // move decompiled files to base location of the jar + for _, decompileJob := range decompileJobs { + jarName := strings.TrimSuffix(filepath.Base(decompileJob.inputPath), ".jar") + err = moveFile(decompileJob.outputPath, + filepath.Join(filepath.Dir(decompileJob.inputPath), + fmt.Sprintf("%s-sources.jar", jarName))) + if err != nil { + log.V(5).Error(err, "failed to move decompiled file", "file", decompileJob.outputPath) + } + } + + } + return nil +} + +// findGradleCache looks for the folder within the Gradle cache where the actual dependencies are stored +// by walking the cache directory looking for a directory equal to the given sample group id +func findGradleCache(sampleGroupId string) (string, error) { + // TODO(jmle): atm taking for granted that the cache is going to be here + root := "/root/.gradle/caches" + cache := "" + walker := func(path string, d os.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("found error looking for cache directory: %w", err) + } + if d.IsDir() && d.Name() == sampleGroupId { + cache = path + return filepath.SkipAll + } + return nil + } + err := filepath.WalkDir(root, walker) + if err != nil { + return "", err + } + cache = filepath.Dir(cache) // return the parent of the found directory + return cache, nil +} + +// findGradleArtifact looks for a given artifact jar within the given root dir +func findGradleArtifact(root string, artifactId string) (string, error) { + artifactPath := "" + walker := func(path string, d os.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("found error looking for artifact: %w", err) + } + if !d.IsDir() && d.Name() == artifactId { + artifactPath = path + return filepath.SkipAll + } + return nil + } + err := filepath.WalkDir(root, walker) + if err != nil { + return "", err + } + return artifactPath, nil +} + +// GetLocation given a dep, attempts to find line number, caches the line number for a given dep +func (j *javaProvider) GetLocation(ctx context.Context, dep konveyor.Dep, file string) (engine.Location, error) { + location := engine.Location{StartPosition: engine.Position{}, EndPosition: engine.Position{}} + + cacheKey := fmt.Sprintf("%s-%s-%s-%v", + dep.Name, dep.Version, dep.ResolvedIdentifier, dep.Indirect) + j.depsMutex.RLock() + val, exists := j.depsLocationCache[cacheKey] + j.depsMutex.RUnlock() + if exists { + if val == -1 { + return location, + fmt.Errorf("unable to get location for dep %s due to a previous error", dep.Name) + } + return engine.Location{ + StartPosition: engine.Position{ + Line: val, + }, + EndPosition: engine.Position{ + Line: val, + }, + }, nil + } + + defer func() { + j.depsMutex.Lock() + j.depsLocationCache[cacheKey] = location.StartPosition.Line + j.depsMutex.Unlock() + }() + + location.StartPosition.Line = -1 + // we know that this provider populates extras with required information + if dep.Extras == nil { + return location, fmt.Errorf("unable to get location for dep %s, dep.Extras not set", dep.Name) + } + extrasKeys := []string{artifactIdKey, groupIdKey, pomPathKey} + for _, key := range extrasKeys { + if val, ok := dep.Extras[key]; !ok { + return location, + fmt.Errorf("unable to get location for dep %s, missing dep.Extras key %s", dep.Name, key) + } else if _, ok := val.(string); !ok { + return location, + fmt.Errorf("unable to get location for dep %s, dep.Extras key %s not a string", dep.Name, key) + } + } + + groupId := dep.Extras[groupIdKey].(string) + artifactId := dep.Extras[artifactIdKey].(string) + path := dep.Extras[pomPathKey].(string) + if path == "" { + path = file + } + if path == "" { + return location, fmt.Errorf("unable to get location for dep %s, empty pom path", dep.Name) + } + lineNumber, err := provider.MultilineGrep(ctx, 2, path, + fmt.Sprintf("(%s|%s).*?(%s|%s).*", + groupId, artifactId, artifactId, groupId)) + if err != nil || lineNumber == -1 { + return location, fmt.Errorf("unable to get location for dep %s, search error - %w", dep.Name, err) + } + location.StartPosition.Line = lineNumber + location.EndPosition.Line = lineNumber + return location, nil +} + +// resolveSourcesJarsForMaven for a given source code location, runs maven to find +// deps that don't have sources attached and decompiles them +func resolveSourcesJarsForMaven(ctx context.Context, log logr.Logger, location, mavenSettings string) error { + // TODO (pgaikwad): when we move to external provider, inherit context from parent + ctx, span := tracing.StartNewSpan(ctx, "resolve-sources") + defer span.End() + + decompileJobs := []decompileJob{} + + log.Info("resolving dependency sources") + + args := []string{ + "-B", + "de.qaware.maven:go-offline-maven-plugin:resolve-dependencies", + "-DdownloadSources", + "-Djava.net.useSystemProxies=true", + } + if mavenSettings != "" { + args = append(args, "-s", mavenSettings) + } + cmd := exec.CommandContext(ctx, "mvn", args...) + cmd.Dir = location + mvnOutput, err := cmd.CombinedOutput() + if err != nil { + return err + } + + reader := bytes.NewReader(mvnOutput) + artifacts, err := parseUnresolvedSources(reader) + if err != nil { + return err + } + + m2Repo := getMavenLocalRepoPath(mavenSettings) + if m2Repo == "" { + return nil + } + for _, artifact := range artifacts { + log.WithValues("artifact", artifact).Info("sources for artifact not found, decompiling...") + + groupDirs := filepath.Join(strings.Split(artifact.GroupId, ".")...) + artifactDirs := filepath.Join(strings.Split(artifact.ArtifactId, ".")...) + jarName := fmt.Sprintf("%s-%s.jar", artifact.ArtifactId, artifact.Version) + decompileJobs = append(decompileJobs, decompileJob{ + artifact: artifact, + inputPath: filepath.Join( + m2Repo, groupDirs, artifactDirs, artifact.Version, jarName), + outputPath: filepath.Join( + m2Repo, groupDirs, artifactDirs, artifact.Version, "decompiled", jarName), + }) + } + err = decompile(ctx, log, alwaysDecompileFilter(true), 10, decompileJobs, "") + if err != nil { + return err + } + // move decompiled files to base location of the jar + for _, decompileJob := range decompileJobs { + jarName := strings.TrimSuffix(filepath.Base(decompileJob.inputPath), ".jar") + err = moveFile(decompileJob.outputPath, + filepath.Join(filepath.Dir(decompileJob.inputPath), + fmt.Sprintf("%s-sources.jar", jarName))) + if err != nil { + log.Error(err, "failed to move decompiled file", "file", decompileJob.outputPath) + } + } + return nil +} + +// parseUnresolvedSources takes the output from the download sources gradle task and returns the artifacts whose sources +// could not be found. Sample gradle output: +// Found 0 sources for :simple-jar: +// Found 1 sources for com.codevineyard:hello-world:1.0.1 +// Found 1 sources for org.codehaus.groovy:groovy:3.0.21 +func parseUnresolvedSourcesForGradle(output io.Reader) ([]javaArtifact, error) { + unresolvedSources := []javaArtifact{} + unresolvedRegex := regexp.MustCompile(`Found 0 sources for (.*)`) + artifactRegex := regexp.MustCompile(`(.+):(.+):(.+)|:(.+):`) + + scanner := bufio.NewScanner(output) + for scanner.Scan() { + line := scanner.Text() + + if match := unresolvedRegex.FindStringSubmatch(line); len(match) != 0 { + gav := artifactRegex.FindStringSubmatch(match[1]) + if gav[4] != "" { // internal library, unknown group/version + artifact := javaArtifact{ + ArtifactId: match[4], + } + unresolvedSources = append(unresolvedSources, artifact) + } else { // external dependency + artifact := javaArtifact{ + GroupId: gav[1], + ArtifactId: gav[2], + Version: gav[3], + } + unresolvedSources = append(unresolvedSources, artifact) + } + } + } + + // dedup artifacts + result := []javaArtifact{} + for _, artifact := range unresolvedSources { + if contains(result, artifact) { + continue + } + result = append(result, artifact) + } + + return result, scanner.Err() +} + +// parseUnresolvedSources takes the output from the go-offline maven plugin and returns the artifacts whose sources +// could not be found. +func parseUnresolvedSources(output io.Reader) ([]javaArtifact, error) { + unresolvedSources := []javaArtifact{} + unresolvedArtifacts := []javaArtifact{} + + scanner := bufio.NewScanner(output) + + unresolvedRegex := regexp.MustCompile(`\[WARNING] The following artifacts could not be resolved`) + artifactRegex := regexp.MustCompile(`([\w\.]+):([\w\-]+):\w+:([\w\.]+):?([\w\.]+)?`) + + for scanner.Scan() { + line := scanner.Text() + + if unresolvedRegex.Find([]byte(line)) != nil { + gavs := artifactRegex.FindAllStringSubmatch(line, -1) + for _, gav := range gavs { + // dependency jar (not sources) also not found + if len(gav) == 5 && gav[3] != "sources" { + artifact := javaArtifact{ + packaging: JavaArchive, + GroupId: gav[1], + ArtifactId: gav[2], + Version: gav[3], + } + unresolvedArtifacts = append(unresolvedArtifacts, artifact) + continue + } + + var v string + if len(gav) == 4 { + v = gav[3] + } else { + v = gav[4] + } + artifact := javaArtifact{ + packaging: JavaArchive, + GroupId: gav[1], + ArtifactId: gav[2], + Version: v, + } + + unresolvedSources = append(unresolvedSources, artifact) + } + } + } + + // if we don't have the dependency itself available, we can't even decompile + result := []javaArtifact{} + for _, artifact := range unresolvedSources { + if contains(unresolvedArtifacts, artifact) || contains(result, artifact) { + continue + } + result = append(result, artifact) + } + + return result, scanner.Err() +} + +func contains(artifacts []javaArtifact, artifactToFind javaArtifact) bool { + if len(artifacts) == 0 { + return false + } + + for _, artifact := range artifacts { + if artifact == artifactToFind { + return true + } + } + + return false +} diff --git a/pkg/java_external_provider/provider_test.go b/pkg/java_external_provider/provider_test.go new file mode 100644 index 0000000..ad92c36 --- /dev/null +++ b/pkg/java_external_provider/provider_test.go @@ -0,0 +1,53 @@ +package java + +import ( + "reflect" + "strings" + "testing" +) + +func Test_parseUnresolvedSources(t *testing.T) { + tests := []struct { + name string + mvnOutput string + wantErr bool + wantList []javaArtifact + }{ + { + name: "valid sources output", + mvnOutput: ` +[INFO] Downloaded from central: https://repo.maven.apache.org/maven2/com/vladsch/flexmark/flexmark-util/0.42.14/flexmark-util-0.42.14.jar (385 kB at 301 kB/s) +[INFO] Downloaded from central: https://repo.maven.apache.org/maven2/javax/enterprise/cdi-api/1.2/cdi-api-1.2.jar (71 kB at 56 kB/s) +[INFO] Downloaded from central: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcore/4.4.14/httpcore-4.4.14.jar (328 kB at 253 kB/s) +[WARNING] The following artifacts could not be resolved: antlr:antlr:jar:sources:2.7.7 (absent), io.konveyor.demo:config-utils:jar:1.0.0 (absent), io.konveyor.demo:config-utils:jar:sources:1.0.0 (absent): Could not find artifact antlr:antlr:jar:sources:2.7.7 in central (https://repo.maven.apache.org/maven2) +[INFO] ------------------------------------------------------------------------ +[INFO] BUILD SUCCESS +[INFO] ------------------------------------------------------------------------ +[INFO] Total time: 16.485 s +[INFO] Finished at: 2023-11-15T12:52:59Z +[INFO] ------------------------------------------------------------------------ +`, + wantErr: false, + wantList: []javaArtifact{ + { + packaging: JavaArchive, + GroupId: "antlr", + ArtifactId: "antlr", + Version: "2.7.7", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + outputReader := strings.NewReader(tt.mvnOutput) + gotList, gotErr := parseUnresolvedSources(outputReader) + if (gotErr != nil) != tt.wantErr { + t.Errorf("parseUnresolvedSources() gotErr = %v, wantErr %v", gotErr, tt.wantErr) + } + if !reflect.DeepEqual(gotList, tt.wantList) { + t.Errorf("parseUnresolvedSources() gotList = %v, wantList %v", gotList, tt.wantList) + } + }) + } +} diff --git a/pkg/java_external_provider/service_client.go b/pkg/java_external_provider/service_client.go new file mode 100644 index 0000000..f3d8d2f --- /dev/null +++ b/pkg/java_external_provider/service_client.go @@ -0,0 +1,282 @@ +package java + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "strings" + "sync" + "time" + + "github.com/go-logr/logr" + "github.com/konveyor/analyzer-lsp/jsonrpc2" + "github.com/konveyor/analyzer-lsp/lsp/protocol" + "github.com/konveyor/analyzer-lsp/provider" + "go.lsp.dev/uri" + "gopkg.in/yaml.v2" +) + +type javaServiceClient struct { + rpc *jsonrpc2.Conn + cancelFunc context.CancelFunc + config provider.InitConfig + log logr.Logger + cmd *exec.Cmd + bundles []string + workspace string + depToLabels map[string]*depLabelItem + isLocationBinary bool + mvnSettingsFile string + depsMutex sync.RWMutex + depsCache map[uri.URI][]*provider.Dep + depsLocationCache map[string]int + includedPaths []string +} + +type depLabelItem struct { + r *regexp.Regexp + labels map[string]interface{} +} + +var _ provider.ServiceClient = &javaServiceClient{} + +func (p *javaServiceClient) Evaluate(ctx context.Context, cap string, conditionInfo []byte) (provider.ProviderEvaluateResponse, error) { + + cond := &javaCondition{} + err := yaml.Unmarshal(conditionInfo, &cond) + if err != nil { + return provider.ProviderEvaluateResponse{}, fmt.Errorf("unable to get query info: %v", err) + } + + if cond.Referenced.Pattern == "" { + return provider.ProviderEvaluateResponse{}, fmt.Errorf("provided query pattern empty") + } + symbols, err := p.GetAllSymbols(ctx, cond.Referenced.Pattern, cond.Referenced.Location, cond.Referenced.Annotated) + if err != nil { + p.log.Error(err, "unable to get symbols", "symbols", symbols, "cap", cap, "conditionInfo", cond) + return provider.ProviderEvaluateResponse{}, err + } + p.log.Info("Symbols retrieved", "symbols", len(symbols), "cap", cap, "conditionInfo", cond) + + incidents := []provider.IncidentContext{} + switch locationToCode[strings.ToLower(cond.Referenced.Location)] { + case 0: + // Filter handle for type, find all the referneces to this type. + incidents, err = p.filterDefault(symbols) + case 1, 5: + incidents, err = p.filterTypesInheritance(symbols) + case 2: + incidents, err = p.filterMethodSymbols(symbols) + case 3: + incidents, err = p.filterDefault(symbols) + case 4: + incidents, err = p.filterDefault(symbols) + case 7: + incidents, err = p.filterMethodSymbols(symbols) + case 8: + incidents, err = p.filterModulesImports(symbols) + case 9: + incidents, err = p.filterVariableDeclaration(symbols) + case 10: + incidents, err = p.filterDefault(symbols) + case 11: + incidents, err = p.filterDefault(symbols) + case 12: + incidents, err = p.filterDefault(symbols) + case 13: + incidents, err = p.filterDefault(symbols) + default: + + } + // push error up for easier printing. + if err != nil { + return provider.ProviderEvaluateResponse{}, err + } + + if len(incidents) == 0 { + return provider.ProviderEvaluateResponse{ + Matched: false, + }, nil + } + return provider.ProviderEvaluateResponse{ + Matched: true, + Incidents: incidents, + }, nil +} + +func (p *javaServiceClient) GetAllSymbols(ctx context.Context, query, location string, annotation annotated) ([]protocol.WorkspaceSymbol, error) { + // This command will run the added bundle to the language server. The command over the wire needs too look like this. + // in this case the project is hardcoded in the init of the Langauge Server above + // workspace/executeCommand '{"command": "io.konveyor.tackle.ruleEntry", "arguments": {"query":"*customresourcedefinition","project": "java"}}' + argumentsMap := map[string]interface{}{ + "query": query, + "project": "java", + "location": fmt.Sprintf("%v", locationToCode[strings.ToLower(location)]), + "analysisMode": string(p.config.AnalysisMode), + } + + if !reflect.DeepEqual(annotation, annotated{}) { + argumentsMap["annotationQuery"] = annotation + } + + if p.includedPaths != nil && len(p.includedPaths) > 0 { + argumentsMap[provider.IncludedPathsConfigKey] = p.includedPaths + p.log.V(5).Info("setting search scope by filepaths", "paths", p.includedPaths) + } + + argumentsBytes, _ := json.Marshal(argumentsMap) + arguments := []json.RawMessage{argumentsBytes} + + wsp := &protocol.ExecuteCommandParams{ + Command: "io.konveyor.tackle.ruleEntry", + Arguments: arguments, + } + + var refs []protocol.WorkspaceSymbol + // If it takes us 5min to complete a request, then we are in trouble + timeOutCtx, _ := context.WithTimeout(ctx, 5*time.Minute) + err := p.rpc.Call(timeOutCtx, "workspace/executeCommand", wsp, &refs) + if err != nil { + if jsonrpc2.IsRPCClosed(err) { + p.log.Error(err, "connection to the language server is closed, language server is not running") + return refs, fmt.Errorf("connection to the language server is closed, language server is not running") + } else { + p.log.Error(err, "unable to ask for Konveyor rule entry") + return refs, fmt.Errorf("unable to ask for Konveyor rule entry") + } + } + + return refs, nil +} + +func (p *javaServiceClient) GetAllReferences(ctx context.Context, symbol protocol.WorkspaceSymbol) []protocol.Location { + var locationURI protocol.DocumentURI + var locationRange protocol.Range + switch x := symbol.Location.Value.(type) { + case protocol.Location: + locationURI = x.URI + locationRange = x.Range + case protocol.PLocationMsg_workspace_symbol: + locationURI = x.URI + locationRange = protocol.Range{} + default: + locationURI = "" + locationRange = protocol.Range{} + } + + if strings.Contains(locationURI, JDT_CLASS_FILE_URI_PREFIX) { + return []protocol.Location{ + { + URI: locationURI, + Range: locationRange, + }, + } + } + params := &protocol.ReferenceParams{ + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ + TextDocument: protocol.TextDocumentIdentifier{ + URI: locationURI, + }, + Position: locationRange.Start, + }, + } + + res := []protocol.Location{} + err := p.rpc.Call(ctx, "textDocument/references", params, &res) + if err != nil { + if jsonrpc2.IsRPCClosed(err) { + p.log.Error(err, "connection to the language server is closed, language server is not running") + } else { + fmt.Printf("Error rpc: %v", err) + } + } + return res +} + +func (p *javaServiceClient) Stop() { + p.cancelFunc() + p.cmd.Wait() +} + +func (p *javaServiceClient) initialization(ctx context.Context) { + absLocation, err := filepath.Abs(p.config.Location) + if err != nil { + p.log.Error(err, "unable to get path to analyize") + panic(1) + } + + var absBundles []string + for _, bundle := range p.bundles { + abs, err := filepath.Abs(bundle) + if err != nil { + p.log.Error(err, "unable to get path to bundles") + panic(1) + } + absBundles = append(absBundles, abs) + + } + downloadSources := true + if p.config.AnalysisMode == provider.SourceOnlyAnalysisMode { + downloadSources = false + } + + //TODO(shawn-hurley): add ability to parse path to URI in a real supported way + params := &protocol.InitializeParams{} + params.RootURI = fmt.Sprintf("file://%v", absLocation) + params.Capabilities = protocol.ClientCapabilities{} + params.ExtendedClientCapilities = map[string]interface{}{ + "classFileContentsSupport": true, + } + // See https://github.com/eclipse-jdtls/eclipse.jdt.ls/blob/1a3dd9323756113bf39cfab82746d57a2fd19474/org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/preferences/Preferences.java + java8home := os.Getenv("JAVA8_HOME") + params.InitializationOptions = map[string]interface{}{ + "bundles": absBundles, + "workspaceFolders": []string{fmt.Sprintf("file://%v", absLocation)}, + "settings": map[string]interface{}{ + "java": map[string]interface{}{ + "configuration": map[string]interface{}{ + "maven": map[string]interface{}{ + "userSettings": p.mvnSettingsFile, + }, + }, + "autobuild": map[string]interface{}{ + "enabled": false, + }, + "maven": map[string]interface{}{ + "downloadSources": downloadSources, + }, + "import": map[string]interface{}{ + "gradle": map[string]interface{}{ + "java": map[string]interface{}{ + "home": java8home, + }, + }, + }, + }, + }, + } + + var result protocol.InitializeResult + for i := 0; i < 10; i++ { + if err := p.rpc.Call(ctx, "initialize", params, &result); err != nil { + if jsonrpc2.IsRPCClosed(err) { + p.log.Error(err, "connection to the language server is closed, language server is not running") + } else { + p.log.Error(err, "initialize failed") + } + continue + } + break + } + if err := p.rpc.Notify(ctx, "initialized", &protocol.InitializedParams{}); err != nil { + fmt.Printf("initialized failed: %v", err) + p.log.Error(err, "initialize failed") + } + p.log.V(2).Info("java connection initialized") + +} diff --git a/pkg/java_external_provider/snipper.go b/pkg/java_external_provider/snipper.go new file mode 100644 index 0000000..62b1c10 --- /dev/null +++ b/pkg/java_external_provider/snipper.go @@ -0,0 +1,54 @@ +package java + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" + + "github.com/konveyor/analyzer-lsp/engine" + "go.lsp.dev/uri" +) + +const ( + JDT_CLASS_FILE_URI_PREFIX = "konveyor-jdt" +) + +var _ engine.CodeSnip = &javaProvider{} + +func (p *javaProvider) GetCodeSnip(u uri.URI, loc engine.Location) (string, error) { + if !strings.Contains(string(u), uri.FileScheme) { + return "", fmt.Errorf("invalid file uri, must be for %s", JDT_CLASS_FILE_URI_PREFIX) + } + snip, err := p.scanFile(u.Filename(), loc) + if err != nil { + return "", err + } + return snip, nil +} + +func (p *javaProvider) scanFile(path string, loc engine.Location) (string, error) { + readFile, err := os.Open(path) + if err != nil { + p.Log.V(5).Error(err, "Unable to read file") + return "", err + } + defer readFile.Close() + + scanner := bufio.NewScanner(readFile) + lineNumber := 0 + codeSnip := "" + paddingSize := len(strconv.Itoa(loc.EndPosition.Line + p.contextLines)) + for scanner.Scan() { + if (lineNumber - p.contextLines) == loc.EndPosition.Line { + codeSnip = codeSnip + fmt.Sprintf("%*d %v", paddingSize, lineNumber+1, scanner.Text()) + break + } + if (lineNumber + p.contextLines) >= loc.StartPosition.Line { + codeSnip = codeSnip + fmt.Sprintf("%*d %v\n", paddingSize, lineNumber+1, scanner.Text()) + } + lineNumber += 1 + } + return codeSnip, nil +} diff --git a/pkg/java_external_provider/util.go b/pkg/java_external_provider/util.go new file mode 100644 index 0000000..dcff629 --- /dev/null +++ b/pkg/java_external_provider/util.go @@ -0,0 +1,610 @@ +package java + +import ( + "archive/zip" + "bufio" + "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" + "text/template" + + "github.com/go-logr/logr" + "github.com/konveyor/analyzer-lsp/tracing" + "go.opentelemetry.io/otel/attribute" +) + +const javaProjectPom = ` + + 4.0.0 + + io.konveyor + java-project + 1.0-SNAPSHOT + + java-project + http://www.konveyor.io + + + UTF-8 + + + +{{range .}} + + {{.GroupId}} + {{.ArtifactId}} + {{.Version}} + +{{end}} + + + + + +` + +type javaArtifact struct { + foundOnline bool + packaging string + GroupId string + ArtifactId string + Version string + sha1 string +} + +type decompileFilter interface { + shouldDecompile(javaArtifact) bool +} + +type alwaysDecompileFilter bool + +func (a alwaysDecompileFilter) shouldDecompile(j javaArtifact) bool { + return bool(a) +} + +type decompileJob struct { + inputPath string + outputPath string + artifact javaArtifact + m2RepoPath string +} + +// decompile decompiles files submitted via a list of decompileJob concurrently +// if a .class file is encountered, it will be decompiled to output path right away +// if a .jar file is encountered, it will be decompiled as a whole, then exploded to project path +func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, workerCount int, jobs []decompileJob, projectPath string) error { + wg := &sync.WaitGroup{} + jobChan := make(chan decompileJob) + + workerCount = int(math.Min(float64(len(jobs)), float64(workerCount))) + // init workers + for i := 0; i < workerCount; i++ { + logger := log.WithName(fmt.Sprintf("decompileWorker-%d", i)) + wg.Add(1) + go func(log logr.Logger, workerId int) { + defer log.V(6).Info("shutting down decompile worker") + defer wg.Done() + log.V(6).Info("init decompile worker") + for job := range jobChan { + // TODO (pgaikwad): when we move to external provider, inherit context from parent + jobCtx, span := tracing.StartNewSpan(ctx, "decomp-job", + attribute.Key("worker").Int(workerId)) + // apply decompile filter + if !filter.shouldDecompile(job.artifact) { + continue + } + if _, err := os.Stat(job.outputPath); err == nil { + // already decompiled, duplicate... + continue + } + outputPathDir := filepath.Dir(job.outputPath) + if err := os.MkdirAll(outputPathDir, 0755); err != nil { + log.V(3).Error(err, + "failed to create directories for decompiled file", "path", outputPathDir) + continue + } + // multiple java versions may be installed - chose $JAVA_HOME one + java := filepath.Join(os.Getenv("JAVA_HOME"), "bin", "java") + // -mpm (max processing method) is required to keep decomp time low + cmd := exec.CommandContext( + jobCtx, java, "-jar", "/bin/fernflower.jar", "-mpm=30", job.inputPath, outputPathDir) + err := cmd.Run() + if err != nil { + log.V(5).Error(err, "failed to decompile file", "file", job.inputPath, job.outputPath) + } else { + log.V(5).Info("decompiled file", "source", job.inputPath, "dest", job.outputPath) + } + // if we just decompiled a java archive, we need to + // explode it further and copy files to project + if job.artifact.packaging == JavaArchive && projectPath != "" { + _, _, _, err = explode(jobCtx, log, job.outputPath, projectPath, job.m2RepoPath) + if err != nil { + log.V(5).Error(err, "failed to explode decompiled jar", "path", job.inputPath) + } + } + span.End() + jobCtx.Done() + } + }(logger, i) + } + + seenJobs := map[string]bool{} + for _, job := range jobs { + jobKey := fmt.Sprintf("%s-%s", job.inputPath, job.outputPath) + if _, ok := seenJobs[jobKey]; !ok { + seenJobs[jobKey] = true + jobChan <- job + } + } + + close(jobChan) + + wg.Wait() + + return nil +} + +// decompileJava unpacks archive at archivePath, decompiles all .class files in it +// creates new java project and puts the java files in the tree of the project +// returns path to exploded archive, path to java project, and an error when encountered +func decompileJava(ctx context.Context, log logr.Logger, archivePath string, m2RepoPath string) (explodedPath, projectPath string, err error) { + ctx, span := tracing.StartNewSpan(ctx, "decompile") + defer span.End() + + projectPath = filepath.Join(filepath.Dir(archivePath), "java-project") + + decompFilter := alwaysDecompileFilter(true) + + explodedPath, decompJobs, deps, err := explode(ctx, log, archivePath, projectPath, m2RepoPath) + if err != nil { + log.Error(err, "failed to decompile archive", "path", archivePath) + return "", "", err + } + + err = createJavaProject(ctx, projectPath, deduplicateJavaArtifacts(deps)) + if err != nil { + log.Error(err, "failed to create java project", "path", projectPath) + return "", "", err + } + log.V(5).Info("created java project", "path", projectPath) + + err = decompile(ctx, log, decompFilter, 10, decompJobs, projectPath) + if err != nil { + log.Error(err, "failed to decompile", "path", archivePath) + return "", "", err + } + + return explodedPath, projectPath, err +} + +func deduplicateJavaArtifacts(artifacts []javaArtifact) []javaArtifact { + uniq := []javaArtifact{} + seen := map[string]bool{} + for _, a := range artifacts { + key := fmt.Sprintf("%s-%s-%s%s", + a.ArtifactId, a.GroupId, a.Version, a.packaging) + if _, ok := seen[key]; !ok { + seen[key] = true + uniq = append(uniq, a) + } + } + return uniq +} + +// explode explodes the given JAR, WAR or EAR archive, generates javaArtifact struct for given archive +// and identifies all .class found recursively. returns output path, a list of decompileJob for .class files +// it also returns a list of any javaArtifact we could interpret from jars +func explode(ctx context.Context, log logr.Logger, archivePath, projectPath string, m2Repo string) (string, []decompileJob, []javaArtifact, error) { + var dependencies []javaArtifact + fileInfo, err := os.Stat(archivePath) + if err != nil { + return "", nil, dependencies, err + } + + // Create the destDir directory using the same permissions as the Java archive file + // java.jar should become java-jar-exploded + destDir := filepath.Join(path.Dir(archivePath), strings.Replace(path.Base(archivePath), ".", "-", -1)+"-exploded") + // make sure execute bits are set so that fernflower can decompile + err = os.MkdirAll(destDir, fileInfo.Mode()|0111) + if err != nil { + return "", nil, dependencies, err + } + + archive, err := zip.OpenReader(archivePath) + if err != nil { + return "", nil, dependencies, err + } + defer archive.Close() + + decompileJobs := []decompileJob{} + + for _, f := range archive.File { + // Stop processing if our context is cancelled + select { + case <-ctx.Done(): + return "", decompileJobs, dependencies, ctx.Err() + default: + } + + filePath := filepath.Join(destDir, f.Name) + + // fernflower already deemed this unparsable, skip... + if strings.Contains(f.Name, "unparsable") || strings.Contains(f.Name, "NonParsable") { + log.V(8).Info("unable to parse file", "file", filePath) + continue + } + + if f.FileInfo().IsDir() { + // make sure execute bits are set so that fernflower can decompile + os.MkdirAll(filePath, f.Mode()|0111) + continue + } + + if err = os.MkdirAll(filepath.Dir(filePath), f.Mode()|0111); err != nil { + return "", decompileJobs, dependencies, err + } + + dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()|0111) + if err != nil { + return "", decompileJobs, dependencies, err + } + defer dstFile.Close() + + archiveFile, err := f.Open() + if err != nil { + return "", decompileJobs, dependencies, err + } + defer archiveFile.Close() + + if _, err := io.Copy(dstFile, archiveFile); err != nil { + return "", decompileJobs, dependencies, err + } + seenDirArtificat := map[string]interface{}{} + switch { + // when it's a .class file and it is in the web-inf, decompile it into java project + // This is the users code. + case strings.HasSuffix(f.Name, ClassFile) && + (strings.Contains(f.Name, "WEB-INF") || strings.Contains(f.Name, "META-INF")): + + // full path in the java project for the decompd file + destPath := filepath.Join( + projectPath, "src", "main", "java", + strings.Replace(filePath, destDir, "", -1)) + destPath = strings.ReplaceAll(destPath, "WEB-INF/classes", "") + destPath = strings.ReplaceAll(destPath, "META-INF/classes", "") + destPath = strings.TrimSuffix(destPath, ClassFile) + ".java" + decompileJobs = append(decompileJobs, decompileJob{ + inputPath: filePath, + outputPath: destPath, + artifact: javaArtifact{ + packaging: ClassFile, + }, + }) + // when it's a .class file and it is not in the web-inf, decompile it into java project + // This is some dependency that is not packaged as dependency. + case strings.HasSuffix(f.Name, ClassFile) && + !(strings.Contains(f.Name, "WEB-INF") || strings.Contains(f.Name, "META-INF")): + destPath := filepath.Join( + projectPath, "src", "main", "java", + strings.Replace(filePath, destDir, "", -1)) + destPath = strings.TrimSuffix(destPath, ClassFile) + ".java" + decompileJobs = append(decompileJobs, decompileJob{ + inputPath: filePath, + outputPath: destPath, + artifact: javaArtifact{ + packaging: ClassFile, + }, + }) + if _, ok := seenDirArtificat[filepath.Dir(f.Name)]; !ok { + dep, err := toFilePathDependency(ctx, f.Name) + if err != nil { + log.V(8).Error(err, "error getting dependcy for path", "path", destPath) + continue + } + dependencies = append(dependencies, dep) + seenDirArtificat[filepath.Dir(f.Name)] = nil + } + // when it's a java file, it's already decompiled, move it to project path + case strings.HasSuffix(f.Name, JavaFile): + destPath := filepath.Join( + projectPath, "src", "main", "java", + strings.Replace(filePath, destDir, "", -1)) + destPath = strings.ReplaceAll(destPath, "WEB-INF/classes", "") + destPath = strings.ReplaceAll(destPath, "META-INF/classes", "") + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + log.V(8).Error(err, "error creating directory for java file", "path", destPath) + continue + } + if err := moveFile(filePath, destPath); err != nil { + log.V(8).Error(err, "error moving decompiled file to project path", + "src", filePath, "dest", destPath) + continue + } + // decompile web archives + case strings.HasSuffix(f.Name, WebArchive): + // TODO(djzager): Should we add these deps to the pom? + _, nestedJobs, deps, err := explode(ctx, log, filePath, projectPath, m2Repo) + if err != nil { + log.Error(err, "failed to decompile file", "file", filePath) + } + decompileJobs = append(decompileJobs, nestedJobs...) + dependencies = append(dependencies, deps...) + // attempt to add nested jars as dependency before decompiling + case strings.HasSuffix(f.Name, JavaArchive): + dep, err := toDependency(ctx, filePath) + if err != nil { + log.V(3).Error(err, "failed to add dep", "file", filePath) + // when we fail to identify a dep we will fallback to + // decompiling it ourselves and adding as source + if (dep != javaArtifact{}) { + outputPath := filepath.Join( + filepath.Dir(filePath), fmt.Sprintf("%s-decompiled", + strings.TrimSuffix(f.Name, JavaArchive)), filepath.Base(f.Name)) + decompileJobs = append(decompileJobs, decompileJob{ + inputPath: filePath, + outputPath: outputPath, + artifact: javaArtifact{ + packaging: JavaArchive, + GroupId: dep.GroupId, + ArtifactId: dep.ArtifactId, + }, + }) + } + } + if (dep != javaArtifact{}) { + if dep.foundOnline { + dependencies = append(dependencies, dep) + // copy this into m2 repo to avoid downloading again + groupPath := filepath.Join(strings.Split(dep.GroupId, ".")...) + artifactPath := filepath.Join(strings.Split(dep.ArtifactId, ".")...) + destPath := filepath.Join(m2Repo, groupPath, artifactPath, + dep.Version, filepath.Base(filePath)) + if err := CopyFile(filePath, destPath); err != nil { + log.V(8).Error(err, "failed copying jar to m2 local repo") + } else { + log.V(8).Info("copied jar file", "src", filePath, "dest", destPath) + } + } else { + // when it isn't found online, decompile it + outputPath := filepath.Join( + filepath.Dir(filePath), fmt.Sprintf("%s-decompiled", + strings.TrimSuffix(f.Name, JavaArchive)), filepath.Base(f.Name)) + decompileJobs = append(decompileJobs, decompileJob{ + inputPath: filePath, + outputPath: outputPath, + artifact: javaArtifact{ + packaging: JavaArchive, + GroupId: dep.GroupId, + ArtifactId: dep.ArtifactId, + }, + }) + } + } + } + } + + return destDir, decompileJobs, dependencies, nil +} + +func createJavaProject(_ context.Context, dir string, dependencies []javaArtifact) error { + tmpl := template.Must(template.New("javaProjectPom").Parse(javaProjectPom)) + + err := os.MkdirAll(filepath.Join(dir, "src", "main", "java"), 0755) + if err != nil { + return err + } + + pom, err := os.OpenFile(filepath.Join(dir, "pom.xml"), os.O_CREATE|os.O_WRONLY, 0755) + if err != nil { + return err + } + + err = tmpl.Execute(pom, dependencies) + if err != nil { + return err + } + return nil +} + +func moveFile(srcPath string, destPath string) error { + err := CopyFile(srcPath, destPath) + if err != nil { + return err + } + err = os.Remove(srcPath) + if err != nil { + return err + } + return nil +} + +func CopyFile(srcPath string, destPath string) error { + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return err + } + inputFile, err := os.Open(srcPath) + if err != nil { + return err + } + defer inputFile.Close() + outputFile, err := os.Create(destPath) + if err != nil { + return err + } + defer outputFile.Close() + _, err = io.Copy(outputFile, inputFile) + if err != nil { + return err + } + return nil +} + +func AppendToFile(src string, dst string) error { + // Read the contents of the source file + content, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("error reading source file: %s", err) + } + + // Open the destination file in append mode + destFile, err := os.OpenFile(dst, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("error opening destination file: %s", err) + } + defer destFile.Close() + + // Append the content to the destination file + _, err = destFile.Write(content) + if err != nil { + return fmt.Errorf("error apending to destination file: %s", err) + } + + return nil +} + +// toDependency returns javaArtifact constructed for a jar +func toDependency(_ context.Context, jarFile string) (javaArtifact, error) { + // attempt to lookup java artifact in maven + dep, err := constructArtifactFromSHA(jarFile) + if err == nil { + return dep, nil + } + // if we fail to lookup on maven, construct it from pom + dep, err = constructArtifactFromPom(jarFile) + if err == nil { + return dep, nil + } + return dep, err +} + +func constructArtifactFromPom(jarFile string) (javaArtifact, error) { + dep := javaArtifact{} + jar, err := zip.OpenReader(jarFile) + if err != nil { + return dep, err + } + defer jar.Close() + + for _, file := range jar.File { + match, err := filepath.Match("META-INF/maven/*/*/pom.properties", file.Name) + if err != nil { + return dep, err + } + + if match { + // Open the file in the ZIP archive + rc, err := file.Open() + if err != nil { + return dep, err + } + defer rc.Close() + + // Read and process the lines in the properties file + scanner := bufio.NewScanner(rc) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "version=") { + dep.Version = strings.TrimSpace(strings.TrimPrefix(line, "version=")) + } else if strings.HasPrefix(line, "artifactId=") { + dep.ArtifactId = strings.TrimSpace(strings.TrimPrefix(line, "artifactId=")) + } else if strings.HasPrefix(line, "groupId=") { + dep.GroupId = strings.TrimSpace(strings.TrimPrefix(line, "groupId=")) + } + } + + return dep, err + } + } + return dep, fmt.Errorf("failed to construct artifact from pom properties") +} + +func constructArtifactFromSHA(jarFile string) (javaArtifact, error) { + dep := javaArtifact{} + // we look up the jar in maven + file, err := os.Open(jarFile) + if err != nil { + return dep, err + } + defer file.Close() + + hash := sha1.New() + _, err = io.Copy(hash, file) + if err != nil { + return dep, err + } + + sha1sum := hex.EncodeToString(hash.Sum(nil)) + + // Make an HTTP request to search.maven.org + searchURL := fmt.Sprintf("http://search.maven.org/solrsearch/select?q=1:%s&rows=20&wt=json", sha1sum) + resp, err := http.Get(searchURL) + if err != nil { + return dep, err + } + defer resp.Body.Close() + + // Read and parse the JSON response + body, err := io.ReadAll(resp.Body) + if err != nil { + return dep, err + } + + var data map[string]interface{} + err = json.Unmarshal(body, &data) + if err != nil { + return dep, err + } + + // Check if a single result is found + response, ok := data["response"].(map[string]interface{}) + if !ok { + return dep, err + } + + numFound, ok := response["numFound"].(float64) + if !ok { + return dep, err + } + + if numFound == 1 { + jarInfo := response["docs"].([]interface{})[0].(map[string]interface{}) + dep.GroupId = jarInfo["g"].(string) + dep.ArtifactId = jarInfo["a"].(string) + dep.Version = jarInfo["v"].(string) + dep.sha1 = sha1sum + dep.foundOnline = true + return dep, nil + } else if numFound > 1 { + dep, err = constructArtifactFromPom(jarFile) + if err == nil { + dep.foundOnline = true + return dep, nil + } + } + return dep, fmt.Errorf("failed to construct artifact from maven lookup") +} + +func toFilePathDependency(_ context.Context, filePath string) (javaArtifact, error) { + dep := javaArtifact{} + // Move up one level to the artifact. we are assuming that we get the full class file here. + // For instance the dir /org/springframework/boot/loader/jar/Something.class. + // in this cass the artificat is: Group: org.springframework.boot.loader, Artifact: Jar + dir := filepath.Dir(filePath) + dep.ArtifactId = filepath.Base(dir) + dep.GroupId = strings.Replace(filepath.Dir(dir), "/", ".", -1) + dep.Version = "0.0.0" + return dep, nil + +} diff --git a/pkg/java_external_provider/util_test.go b/pkg/java_external_provider/util_test.go new file mode 100644 index 0000000..68d347c --- /dev/null +++ b/pkg/java_external_provider/util_test.go @@ -0,0 +1,99 @@ +package java + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "testing" +) + +func TestRenderPom(t *testing.T) { + // Create a temporary directory for testing + tmpDir := t.TempDir() + + // Define some sample dependencies + dependencies := []javaArtifact{ + { + GroupId: "com.example", + ArtifactId: "example-artifact", + Version: "1.0.0", + }, + { + GroupId: "org.another", + ArtifactId: "another-artifact", + Version: "2.0.0", + }, + } + + // Call the function with the temporary directory and sample dependencies + err := createJavaProject(context.Background(), tmpDir, dependencies) + if err != nil { + t.Fatalf("createJavaProject returned an error: %v", err) + } + + // Verify that the project directory and pom.xml file were created + projectDir := filepath.Join(tmpDir, "src", "main", "java") + pomFile := filepath.Join(tmpDir, "pom.xml") + + if _, err := os.Stat(projectDir); os.IsNotExist(err) { + t.Errorf("Java source directory not created: %v", err) + } + + if _, err := os.Stat(pomFile); os.IsNotExist(err) { + t.Errorf("pom.xml file not created: %v", err) + } + + // Read the generated pom.xml content + pomContent, err := os.ReadFile(pomFile) + if err != nil { + t.Fatalf("error reading pom.xml file: %v", err) + } + + // Define the expected pom.xml content + expectedPom := ` + + 4.0.0 + + io.konveyor + java-project + 1.0-SNAPSHOT + + java-project + http://www.konveyor.io + + + UTF-8 + + + + + + com.example + example-artifact + 1.0.0 + + + + org.another + another-artifact + 2.0.0 + + + + + + + +` + + // Compare the generated pom.xml content with the expected content + if !bytes.Equal(pomContent, []byte(expectedPom)) { + t.Errorf("Generated pom.xml content does not match the expected content") + fmt.Println(string(pomContent)) + fmt.Println("expected POM") + fmt.Println(expectedPom) + } +} diff --git a/static-report.sh b/static-report.sh new file mode 100755 index 0000000..048c40c --- /dev/null +++ b/static-report.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -x + +HOMEDIR="" + +if [ "$(uname)" == "Linux" ] || [ "$(uname)" == "Darwin" ]; then + HOMEDIR="$HOME" +else + # windows + HOMEDIR="%USERPROFILE%" +fi + +# TODO test this on windows +(cd ${HOMEDIR}/.kantra/static-report && +npm clean-install && +CI=true PUBLIC_URL=. npm run build && +cp ${HOMEDIR}/.kantra/static-report/public/output.js ${HOMEDIR}/.kantra/static-report/build/output.js && +rm -rf ${HOMEDIR}/.kantra/static-report/static-report && +mv ${HOMEDIR}/.kantra/static-report/build ${HOMEDIR}/.kantra/static-report/static-report)