GopherProxy

	Optimising by adding threading, combining HTML output to one file. - staticgit …
	Log
	Files
	Refs
	README
	---
	commit 89f13ce268930a8a931f4a82312653c96ac8d8ca
	parent 884ed087159dcf350421c692b8c530b7db260c28
	Author: Jay Scott <[email protected]>
	Date: Sat, 13 Jul 2024 19:37:19 +0100

	Optimising by adding threading, combining HTML output to one file.

	Diffstat:
	M Makefile \| 2 +-
	M main.go \| 452 ++++++++++++++---------------…

	2 files changed, 210 insertions(+), 244 deletions(-)
	---
	diff --git a/Makefile b/Makefile
	@@ -8,7 +8,7 @@ all: run

	run:
	@echo "Running $(APP_NAME)..."
	- @go run . -g -p ./git -o tmp -i .ssh,dotfiles
	+ @go run . -g -p ./git -o tmp -i .ssh,jay.scot,internal-docs

	build:
	@echo "Building $(APP_NAME) for local architecture..."
	diff --git a/main.go b/main.go
	@@ -10,23 +10,19 @@ import (
	"regexp"
	"sort"
	"strings"
	+ "sync"

	"github.com/go-git/go-git/v5"
	- "github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/object"
	)

	-type BranchInfo struct {
	- Name string
	- LastCommit string
	- LastCommitDate string
	-}
	-
	type CommitInfo struct {
	Hash string
	Author string
	Date string
	Message string
	+ Added int
	+ Removed int
	}

	type RepoInfo struct {
	@@ -36,109 +32,124 @@ type RepoInfo struct {
	}

	const (
	- baseTemplate = `
	+ base = `
	<!DOCTYPE html>
	<html>
	<head>
	-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
	-<meta name="viewport" content="width=device-width, initial-scale=1" />
	-<title>{{.Title}}</title>
	-<link rel="icon" type="image/png" href="{{.IconPath}}favicon.png" />
	-<link rel="stylesheet" type="text/css" href="{{.IconPath}}style.css" />
	+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
	+ <meta name="viewport" content="width=device-width, initial-scale=1" />
	+ <title>{{.Title}}</title>
	+ <link rel="icon" type="image/png" href="{{.IconPath}}favicon.png" />
	+ <style>
	+ body{color:#000;background-color:#FFF;font-family:monospace}
	+ table{padding-left:20px}
	+ h1{font-size:2em;margin:10}
	+ h2{font-size:1.5em;margin:10}
	+ table td{padding:0 .4em}
	+ a{color:#000;text-decoration:none}
	+ a:hover{color:#333;font-weight:700}
	+ a:target{background-color:#CCC}
	+ .desc{color:#555;margin-bottom:1em}
	+ hr{border:0;border-top:1px solid #AAA;height:2px}
	+ table tr:hover{background-color:#EEE}
	+ </style>
	</head>
	<body>
	-<table>
	-<tr><td><img src="{{.IconPath}}logo.png" alt="" width="32" height="32" /></td>
	-<td><span class="desc">{{.Title}}</span></td></tr><tr><td></td><td>
	-</td></tr>
	-</table>
	-<hr/>
	-<div id="content">
	-{{template "content" .}}
	-</div>
	+ <table>
	+ <tr>
	+ <td>
	+ <img src="{{.IconPath}}logo.png" alt="" width="32" height="3…
	+ </td>
	+ <td>
	+ <span class="desc">{{.Title}}</span>
	+ </td>
	+ </tr>
	+ </table>
	+
	+ <hr/>
	+
	+ <div id="content">
	+ {{template "content" .}}
	+ </div>
	</body>
	</html>
	`
	-
	- branchesContent = `
	-{{define "content"}}
	-<h1>{{.RepoName}} - Branches</h1>
	-<table>
	-<thead>
	-<tr><td><b>Branch Name</b></td><td><b>Last Commit</b></td><td><b>Last Commit D…
	-</thead>
	-<tbody>
	-{{range .Branches}}
	-<tr><td>{{.Name}}</td><td>{{.LastCommit}}</td><td>{{.LastCommitDate}}</td></tr>
	-{{end}}
	-</tbody>
	-</table>
	-{{end}}
	-`
	-
	- commitHistoryContent = `
	+ details = `
	{{define "content"}}
	-<h1>{{.RepoName}} - Commit History</h1>
	-<table>
	-<thead>
	-<tr><td><b>Hash</b></td><td><b>Author</b></td><td><b>Date</b></td><td><b>Messa…
	-</thead>
	-<tbody>
	-{{range .Commits}}
	-<tr><td>{{.Hash}}</td><td>{{.Author}}</td><td>{{.Date}}</td><td>{{.Message}}</…
	-{{end}}
	-</tbody>
	-</table>
	+ <pre>{{.ReadmeContent}}</pre>
	+
	+ <h2>Commit History</h2>
	+ <hr>
	+ <table>
	+ <thead>
	+ <tr>
	+ <td><b>Date</b></td>
	+ <td><b>Message</b></td>
	+ <td><b>Author</b></td>
	+ <td><b>Hash</b></td>
	+ <td><b>Added</b></td>
	+ <td><b>Deleted</b></td>
	+ </tr>
	+ </thead>
	+ <tbody>
	+ {{range .Commits}}
	+ <tr>
	+ <td>{{.Date}}</td>
	+ <td>{{.Message}}</td>
	+ <td>{{.Author}}</td>
	+ <td>{{.Hash}}</td>
	+ <td style="color: green;">+{{.Added}}</td>
	+ <td style="color: red;">-{{.Removed}}</td>
	+ </tr>
	+ {{end}}
	+ </tbody>
	+ </table>
	{{end}}
	`

	- indexContent = `
	-{{define "content"}}
	-<table id="index">
	-<thead>
	-<tr><td><b>Name</b></td><td><b>Description</b></td><td><b>Last commit</b></td>…
	-</thead>
	-<tbody>
	-{{range $group, $repos := .Repos}}
	-<tr><td colspan="4"><b>{{if eq $group ""}} {{else}}<hr>{{end}}</b></td></tr>
	- {{range $repos}}
	- <tr>
	- <td>{{.Name}}</td>
	- <td>{{.Description}}</td>
	- <td>\| {{.LastCommit}} \| </td>
	- <td>
	- <a href="{{.Name}}/README.html">readme</a> \|
	- <a href="{{.Name}}/commits.html">commits</a> \|
	- <a href="{{.Name}}/branches.html">branches</a>
	- </td>
	- </tr>
	- {{end}}
	-{{end}}
	-</tbody>
	-</table>
	-{{end}}
	-`
	- readmeContent = `
	+ index = `
	{{define "content"}}
	-<h1>{{.RepoName}}</h1>
	-<pre>{{.ReadmeContent}}</pre>
	+ <table id="index">
	+ <thead>
	+ <tr>
	+ <td><b>Name</b></td>
	+ <td><b>Description</b></td>
	+ <td><b>Last commit</b></td>
	+ </tr>
	+ </thead>
	+ <tbody>
	+ {{range $group, $repos := .Repos}}
	+ <tr>
	+ <td colspan="4">{{if eq $group ""}}{{else}}<hr>{{end}}</td>
	+ </tr>
	+ {{range $repos}}
	+ <tr>
	+ <td><a href="{{.Name}}/index.html">{{.Name}}</a></td>
	+ <td>{{.Description}}</td>
	+ <td>{{.LastCommit}}</td>
	+ </tr>
	+ {{end}}
	+ {{end}}
	+ </tbody>
	+ </table>
	{{end}}
	`
	)

	var (
	- reposPath string
	- groupFlag bool
	- ignoreDirs map[string]bool
	- outputRoot string
	-
	- branchesTmpl = template.Must(template.New("base").Parse(baseTempl…
	- commitHistoryTmpl = template.Must(template.New("base").Parse(baseTempl…
	- indexTmpl = template.Must(template.New("base").Parse(baseTempl…
	- readmeTmpl = template.Must(template.New("base").Parse(baseTempl…
	+ templates = map[string]*template.Template{
	+ "index": template.Must(template.New("base").Parse(base + ind…
	+ "details": template.Must(template.New("base").Parse(base + det…
	+ }
	+
	+ reposPath string
	+ groupFlag bool
	+ ignoreDirs map[string]bool
	+ outputRoot string
	+ commitLimit int
	)

	-func generateIndexHTML(repoInfos []RepoInfo) error {
	+func generateIndex(repoInfos []RepoInfo) error {
	groupedRepos := groupRepos(repoInfos)
	indexOutputPath := filepath.Join(outputRoot, "index.html")

	@@ -148,64 +159,95 @@ func generateIndexHTML(repoInfos []RepoInfo) error {
	}
	defer indexFile.Close()

	- return indexTmpl.Execute(indexFile, struct {
	+ return templates["index"].Execute(indexFile, struct {
	Title string
	IconPath string
	Repos map[string][]RepoInfo
	}{
	- Title: "git clone [email protected]:<reponame>",
	+ Title: "repos for days!",
	IconPath: "./",
	Repos: groupedRepos,
	})
	}

	-func getBranchInfo(repo *git.Repository) ([]BranchInfo, error) {
	- branches, err := repo.Branches()
	+func generateRepo(repoName, repoPath, outputDir string) error {
	+ repo, err := git.PlainOpen(repoPath)
	if err != nil {
	- return nil, fmt.Errorf("failed to get branches: %w", err)
	+ return fmt.Errorf("failed to open git repository: %w", err)
	}

	- var branchInfos []BranchInfo
	- err = branches.ForEach(func(branch *plumbing.Reference) error {
	- commit, err := repo.CommitObject(branch.Hash())
	- if err != nil {
	- return fmt.Errorf("failed to get commit for branch %s:…
	- }
	-
	- branchInfos = append(branchInfos, BranchInfo{
	- Name: branch.Name().Short(),
	- LastCommit: commit.Hash.String()[:7],
	- LastCommitDate: commit.Author.When.Format("02 Jan 2006…
	- })
	- return nil
	- })
	+ readme, err := getReadme(repoPath)
	+ if err != nil {
	+ return fmt.Errorf("failed to get README: %w", err)
	+ }

	+ commits, err := getCommits(repo)
	if err != nil {
	- return nil, fmt.Errorf("failed to iterate over branches: %w", …
	+ return fmt.Errorf("failed to get commit history: %w", err)
	}

	- return branchInfos, nil
	-}
	+ outputPath := filepath.Join(outputDir, "index.html")

	-func getCommitHistory(repo *git.Repository) ([]CommitInfo, error) {
	- ref, err := repo.Head()
	+ f, err := os.Create(outputPath)
	if err != nil {
	- return nil, fmt.Errorf("failed to get HEAD reference: %w", err)
	+ return fmt.Errorf("failed to create details HTML file: %w", er…
	}
	+ defer f.Close()

	- commits, err := repo.Log(&git.LogOptions{From: ref.Hash()})
	+ return templates["details"].Execute(f, struct {
	+ Title string
	+ IconPath string
	+ RepoName string
	+ ReadmeContent string
	+ Commits []CommitInfo
	+ }{
	+ Title: "git clone [email protected]:" + repoName,
	+ IconPath: "../",
	+ RepoName: repoName,
	+ ReadmeContent: readme,
	+ Commits: commits,
	+ })
	+}
	+
	+func getCommits(repo *git.Repository) ([]CommitInfo, error) {
	+ commitIter, err := repo.CommitObjects()
	if err != nil {
	- return nil, fmt.Errorf("failed to get commit log: %w", err)
	+ return nil, fmt.Errorf("failed to get commit objects: %w", err)
	}

	var commitHistory []CommitInfo
	- err = commits.ForEach(func(c *object.Commit) error {
	+ count := 0
	+ reachedLimit := false
	+
	+ err = commitIter.ForEach(func(c *object.Commit) error {
	+ if reachedLimit {
	+ return nil
	+ }
	+
	+ stats, err := c.Stats()
	+ if err != nil {
	+ return fmt.Errorf("failed to get commit stats: %w", er…
	+ }
	+
	+ added, removed := 0, 0
	+ for _, stat := range stats {
	+ added += stat.Addition
	+ removed += stat.Deletion
	+ }
	+
	commitHistory = append(commitHistory, CommitInfo{
	Hash: c.Hash.String()[:7],
	Author: c.Author.Name,
	Date: c.Author.When.Format("02 Jan 2006 15:04:05"),
	- Message: strings.Split(c.Message, "\n")[0], // Get onl…
	+ Message: strings.Split(c.Message, "\n")[0],
	+ Added: added,
	+ Removed: removed,
	})
	+
	+ count++
	+ if count >= commitLimit {
	+ reachedLimit = true
	+ }
	return nil
	})

	@@ -261,7 +303,7 @@ func getReadme(repoPath string) (string, error) {
	return "No README found!", nil
	}

	-func getRepoInfo(repoPath string) (RepoInfo, error) {
	+func getRepo(repoPath string) (RepoInfo, error) {
	repo, err := git.PlainOpen(repoPath)
	if err != nil {
	return RepoInfo{}, fmt.Errorf("failed to open repository: %w",…
	@@ -313,7 +355,7 @@ func getGroup(description string) string {
	return ""
	}

	-func parseIgnoreDirs(ignoreDirs string) map[string]bool {
	+func parseIgnored(ignoreDirs string) map[string]bool {
	ignoreMap := make(map[string]bool)
	for _, dir := range strings.Split(ignoreDirs, ",") {
	if trimmedDir := strings.TrimSpace(dir); trimmedDir != "" {
	@@ -323,149 +365,73 @@ func parseIgnoreDirs(ignoreDirs string) map[string]bool {
	return ignoreMap
	}

	-func processBranches(repoName, repoPath, outputDir string) error {
	- repo, err := git.PlainOpen(repoPath)
	- if err != nil {
	- return fmt.Errorf("failed to open git repository: %w", err)
	- }
	-
	- branches, err := getBranchInfo(repo)
	- if err != nil {
	- return fmt.Errorf("failed to get branch information: %w", err)
	- }
	-
	- outputPath := filepath.Join(outputDir, "branches.html")
	-
	- f, err := os.Create(outputPath)
	- if err != nil {
	- return fmt.Errorf("failed to create branches HTML file: %w", e…
	- }
	- defer f.Close()
	-
	- return branchesTmpl.Execute(f, struct {
	- Title string
	- IconPath string
	- RepoName string
	- Branches []BranchInfo
	- }{
	- Title: repoName + " - Branches",
	- IconPath: "../",
	- RepoName: repoName,
	- Branches: branches,
	- })
	-}
	-
	-func processCommitHistory(repoName, repoPath, outputDir string) error {
	- repo, err := git.PlainOpen(repoPath)
	- if err != nil {
	- return fmt.Errorf("failed to open git repository: %w", err)
	- }
	-
	- commits, err := getCommitHistory(repo)
	- if err != nil {
	- return fmt.Errorf("failed to get commit history: %w", err)
	- }
	-
	- outputPath := filepath.Join(outputDir, "commits.html")
	-
	- f, err := os.Create(outputPath)
	- if err != nil {
	- return fmt.Errorf("failed to create commit history HTML file: …
	- }
	- defer f.Close()
	-
	- return commitHistoryTmpl.Execute(f, struct {
	- Title string
	- IconPath string
	- RepoName string
	- Commits []CommitInfo
	- }{
	- Title: repoName + " - History",
	- IconPath: "../",
	- RepoName: repoName,
	- Commits: commits,
	- })
	-}
	-
	-func processReadme(repoName, repoPath, outputDir string) error {
	- readme, err := getReadme(repoPath)
	+func processRepos() error {
	+ repos, err := os.ReadDir(reposPath)
	if err != nil {
	- return fmt.Errorf("failed to get README: %w", err)
	+ return fmt.Errorf("failed to read repos directory: %w", err)
	}

	- outputPath := filepath.Join(outputDir, "README.html")
	+ var wg sync.WaitGroup
	+ repoInfosChan := make(chan RepoInfo, len(repos))
	+ errorsChan := make(chan error, len(repos))

	- f, err := os.Create(outputPath)
	- if err != nil {
	- return fmt.Errorf("failed to create README HTML file: %w", err)
	+ for _, r := range repos {
	+ if r.IsDir() && !ignoreDirs[r.Name()] {
	+ wg.Add(1)
	+ go func(r os.DirEntry) {
	+ defer wg.Done()
	+ repoPath := filepath.Join(reposPath, r.Name())
	+ repoInfo, err := getRepo(repoPath)
	+ if err != nil {
	+ errorsChan <- fmt.Errorf("failed to ge…
	+ return
	+ }
	+ repoInfosChan <- repoInfo
	+
	+ outputDir := filepath.Join(outputRoot, r.Name(…
	+ if err := os.MkdirAll(outputDir, 0755); err !=…
	+ errorsChan <- fmt.Errorf("failed to cr…
	+ return
	+ }
	+
	+ if err := generateRepo(r.Name(), repoPath, out…
	+ errorsChan <- fmt.Errorf("failed to pr…
	+ }
	+ }(r)
	+ }
	}
	- defer f.Close()

	- return readmeTmpl.Execute(f, struct {
	- Title string
	- IconPath string
	- RepoName string
	- ReadmeContent string
	- }{
	- Title: repoName + " - Readme!",
	- IconPath: "../",
	- RepoName: repoName,
	- ReadmeContent: readme,
	- })
	-}
	+ go func() {
	+ wg.Wait()
	+ close(repoInfosChan)
	+ close(errorsChan)
	+ }()

	-func processRepositories() error {
	- repos, err := os.ReadDir(reposPath)
	- if err != nil {
	- return fmt.Errorf("failed to read repos directory: %w", err)
	+ var repoInfos []RepoInfo
	+ for repoInfo := range repoInfosChan {
	+ repoInfos = append(repoInfos, repoInfo)
	}

	- var repoInfos []RepoInfo
	- for _, r := range repos {
	- if r.IsDir() && !ignoreDirs[r.Name()] {
	- repoPath := filepath.Join(reposPath, r.Name())
	- repoInfo, err := getRepoInfo(repoPath)
	- if err != nil {
	- fmt.Printf("Failed to get info for repo %s: %v…
	- continue
	- }
	- repoInfos = append(repoInfos, repoInfo)
	-
	- outputDir := filepath.Join(outputRoot, r.Name())
	- if err := os.MkdirAll(outputDir, 0755); err != nil {
	- fmt.Printf("Failed to create output directory …
	- continue
	- }
	-
	- if err := processReadme(r.Name(), repoPath, outputDir)…
	- fmt.Printf("Failed to process README for repo …
	- }
	-
	- if err := processCommitHistory(r.Name(), repoPath, out…
	- fmt.Printf("Failed to process commit history f…
	- }
	-
	- if err := processBranches(r.Name(), repoPath, outputDi…
	- fmt.Printf("Failed to process branches for rep…
	- }
	- }
	+ for err := range errorsChan {
	+ fmt.Println(err)
	}

	- return generateIndexHTML(repoInfos)
	+ return generateIndex(repoInfos)
	}

	func main() {
	flag.StringVar(&reposPath, "p", "", "Path to the git repositories (req…
	+ flag.StringVar(&outputRoot, "o", ".", "Root path where output director…
	flag.BoolVar(&groupFlag, "g", false, "Group repositories based on desc…
	+ flag.IntVar(&commitLimit, "c", 100, "Limit for the number of commits t…
	ignoreFlag := flag.String("i", "", "Directories to ignore (comma-separ…
	- flag.StringVar(&outputRoot, "o", ".", "Root path where output director…

	flag.Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage: %s [options]\n\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "Options:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\nExample:\n")
	- fmt.Fprintf(os.Stderr, " %s -p /path/to/repos -g -i dir1,dir2…
	+ fmt.Fprintf(os.Stderr, " %s -p /path/to/repos -g -i dir1,dir2…
	}

	flag.Parse()
	@@ -475,9 +441,9 @@ func main() {
	os.Exit(1)
	}

	- ignoreDirs = parseIgnoreDirs(*ignoreFlag)
	+ ignoreDirs = parseIgnored(*ignoreFlag)

	- if err := processRepositories(); err != nil {
	+ if err := processRepos(); err != nil {
	log.Fatalf("Error processing repositories: %v", err)
	}
	}