From b5cd6d215b30e661c8fc3caeb9a684654058a6c9 Mon Sep 17 00:00:00 2001 From: Richard North Date: Mon, 5 Aug 2024 09:23:17 +0100 Subject: [PATCH] Add file output to foreach command (#141) * Add file output to foreach command * Reorder * Simplify code --- cmd/foreach/foreach.go | 70 ++++++++++++++++++++++++++++-- cmd/foreach/foreach_test.go | 42 ++++++++++++++++++ internal/executor/fake_executor.go | 22 ++++++++++ internal/logging/activity.go | 4 ++ 4 files changed, 135 insertions(+), 3 deletions(-) diff --git a/cmd/foreach/foreach.go b/cmd/foreach/foreach.go index 14bf4ca..79dffc2 100644 --- a/cmd/foreach/foreach.go +++ b/cmd/foreach/foreach.go @@ -17,6 +17,7 @@ package foreach import ( "errors" + "fmt" "os" "path" "strings" @@ -34,7 +35,15 @@ import ( var exec executor.Executor = executor.NewRealExecutor() var ( - repoFile string = "repos.txt" + repoFile = "repos.txt" + + overallResultsDirectory string + + successfulResultsDirectory string + successfulReposFileName string + + failedResultsDirectory string + failedReposFileName string ) func formatArguments(arguments []string) string { @@ -49,8 +58,7 @@ func NewForeachCmd() *cobra.Command { cmd := &cobra.Command{ Use: "foreach [flags] -- COMMAND [ARGUMENT...]", Short: "Run COMMAND against each working copy", - Long: -`Run COMMAND against each working copy. Make sure to include a + Long: `Run COMMAND against each working copy. Make sure to include a double hyphen -- with space on both sides before COMMAND, as this marks that no further options should be interpreted by turbolift.`, RunE: runE, @@ -83,6 +91,10 @@ func runE(c *cobra.Command, args []string) error { // the user something they could copy and paste. prettyArgs := formatArguments(args) + setupOutputFiles(dir.Name, prettyArgs) + + logger.Printf("Logs for all executions will be stored under %s", overallResultsDirectory) + var doneCount, skippedCount, errorCount int for _, repo := range dir.Repos { repoDirPath := path.Join("work", repo.OrgName, repo.RepoName) // i.e. work/org/repo @@ -99,9 +111,11 @@ func runE(c *cobra.Command, args []string) error { err := exec.Execute(execActivity.Writer(), repoDirPath, args[0], args[1:]...) if err != nil { + emitOutcomeToFiles(repo, failedReposFileName, failedResultsDirectory, execActivity.Logs(), logger) execActivity.EndWithFailure(err) errorCount++ } else { + emitOutcomeToFiles(repo, successfulReposFileName, successfulResultsDirectory, execActivity.Logs(), logger) execActivity.EndWithSuccessAndEmitLogs() doneCount++ } @@ -113,5 +127,55 @@ func runE(c *cobra.Command, args []string) error { logger.Warnf("turbolift foreach completed with %s %s(%s, %s, %s)\n", colors.Red("errors"), colors.Normal(), colors.Green(doneCount, " OK"), colors.Yellow(skippedCount, " skipped"), colors.Red(errorCount, " errored")) } + logger.Printf("Logs for all executions have been stored under %s", overallResultsDirectory) + logger.Printf("Names of successful repos have been written to %s", successfulReposFileName) + logger.Printf("Names of failed repos have been written to %s", failedReposFileName) + return nil } + +// sets up a temporary directory to store success/failure logs etc +func setupOutputFiles(campaignName string, command string) { + overallResultsDirectory, _ = os.MkdirTemp("", fmt.Sprintf("turbolift-foreach-%s-", campaignName)) + successfulResultsDirectory = path.Join(overallResultsDirectory, "successful") + failedResultsDirectory = path.Join(overallResultsDirectory, "failed") + _ = os.MkdirAll(successfulResultsDirectory, 0755) + _ = os.MkdirAll(failedResultsDirectory, 0755) + + successfulReposFileName = path.Join(successfulResultsDirectory, "repos.txt") + failedReposFileName = path.Join(failedResultsDirectory, "repos.txt") + + // create the files + successfulReposFile, _ := os.Create(successfulReposFileName) + failedReposFile, _ := os.Create(failedReposFileName) + defer successfulReposFile.Close() + defer failedReposFile.Close() + + _, _ = successfulReposFile.WriteString(fmt.Sprintf("# This file contains the list of repositories that were successfully processed by turbolift foreach\n# for the command: %s\n", command)) + _, _ = failedReposFile.WriteString(fmt.Sprintf("# This file contains the list of repositories that failed to be processed by turbolift foreach\n# for the command: %s\n", command)) +} + +func emitOutcomeToFiles(repo campaign.Repo, reposFileName string, logsDirectoryParent string, executionLogs string, logger *logging.Logger) { + // write the repo name to the repos file + reposFile, _ := os.OpenFile(reposFileName, os.O_RDWR|os.O_APPEND, 0644) + defer reposFile.Close() + _, err := reposFile.WriteString(repo.FullRepoName + "\n") + if err != nil { + logger.Errorf("Failed to write repo name to %s: %s", reposFile.Name(), err) + } + + // write logs to a file under the logsParent directory, in a directory structure that mirrors that of the work directory + logsDir := path.Join(logsDirectoryParent, repo.FullRepoName) + logsFile := path.Join(logsDir, "logs.txt") + err = os.MkdirAll(logsDir, 0755) + if err != nil { + logger.Errorf("Failed to create directory %s: %s", logsDir, err) + } + + logs, _ := os.Create(logsFile) + defer logs.Close() + _, err = logs.WriteString(executionLogs) + if err != nil { + logger.Errorf("Failed to write logs to %s: %s", logsFile, err) + } +} diff --git a/cmd/foreach/foreach_test.go b/cmd/foreach/foreach_test.go index da97ece..9b8afbc 100644 --- a/cmd/foreach/foreach_test.go +++ b/cmd/foreach/foreach_test.go @@ -18,6 +18,7 @@ package foreach import ( "bytes" "os" + "regexp" "testing" "github.com/stretchr/testify/assert" @@ -162,6 +163,47 @@ func TestFormatArguments(t *testing.T) { } } +func TestItCreatesLogFiles(t *testing.T) { + fakeExecutor := executor.NewAlternatingSuccessFakeExecutor() + exec = fakeExecutor + + testsupport.PrepareTempCampaign(true, "org/repo1", "org/repo2") + + out, err := runCommand("--", "some", "command") + assert.NoError(t, err) + assert.Contains(t, out, "turbolift foreach completed") + assert.Contains(t, out, "1 OK, 0 skipped, 1 errored") + + // Logs should describe where output was written + r := regexp.MustCompile(`Logs for all executions have been stored under (.+)`) + matches := r.FindStringSubmatch(out) + assert.Len(t, matches, 2, "Expected to find the log directory path") + path := matches[1] + + // check that expected static directories and files exist + _, err = os.Stat(path) + assert.NoError(t, err, "Expected the log directory to exist") + + _, err = os.Stat(path + "/successful") + assert.NoError(t, err, "Expected the successful log directory to exist") + + _, err = os.Stat(path + "/failed") + assert.NoError(t, err, "Expected the failure log directory to exist") + + _, err = os.Stat(path + "/successful/repos.txt") + assert.NoError(t, err, "Expected the successful repos.txt file to exist") + + _, err = os.Stat(path + "/failed/repos.txt") + assert.NoError(t, err, "Expected the failure repos.txt file to exist") + + // check that the expected logs files exist + _, err = os.Stat(path + "/successful/org/repo1/logs.txt") + assert.NoError(t, err, "Expected the successful log file for org/repo1 to exist") + + _, err = os.Stat(path + "/failed/org/repo2/logs.txt") + assert.NoError(t, err, "Expected the failure log file for org/repo2 to exist") +} + func runCommand(args ...string) (string, error) { cmd := NewForeachCmd() outBuffer := bytes.NewBufferString("") diff --git a/internal/executor/fake_executor.go b/internal/executor/fake_executor.go index 68282eb..f31f336 100644 --- a/internal/executor/fake_executor.go +++ b/internal/executor/fake_executor.go @@ -70,3 +70,25 @@ func NewAlwaysFailsFakeExecutor() *FakeExecutor { return "", errors.New("synthetic error") }) } + +func NewAlternatingSuccessFakeExecutor() *FakeExecutor { + i := 0 + return NewFakeExecutor( + func(s string, s2 string, s3 ...string) error { + i++ + if i%2 == 1 { + return nil + } else { + return errors.New("synthetic error") + } + }, + func(s string, s2 string, s3 ...string) (string, error) { + i++ + if i%2 == 1 { + return "", nil + } else { + return "", errors.New("synthetic error") + } + }, + ) +} diff --git a/internal/logging/activity.go b/internal/logging/activity.go index 06e93e7..976d706 100644 --- a/internal/logging/activity.go +++ b/internal/logging/activity.go @@ -110,3 +110,7 @@ func (a *Activity) Writer() io.Writer { activity: a, } } + +func (a *Activity) Logs() string { + return strings.Join(a.logs, "\n") +}