Skip to content
This repository was archived by the owner on Sep 9, 2020. It is now read-only.

Commit 54ccce4

Browse files
ibrashoibrasho
committed
WIP: implement pruning
Signed-off-by: Ibrahim AshShohail <ibra.sho@gmail.com>
1 parent 5c34ae8 commit 54ccce4

File tree

4 files changed

+395
-72
lines changed

4 files changed

+395
-72
lines changed

‎internal/gps/prune.go‎

Lines changed: 285 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,300 @@
44

55
package gps
66

7+
import (
8+
"fmt"
9+
"io/ioutil"
10+
"log"
11+
"os"
12+
"path/filepath"
13+
"sort"
14+
"strings"
15+
16+
"github.com/pkg/errors"
17+
)
18+
719
// PruneOptions represents the pruning options used to write the dependecy tree.
820
type PruneOptions uint8
921

1022
const (
1123
// PruneNestedVendorDirs indicates if nested vendor directories should be pruned.
1224
PruneNestedVendorDirs = 1 << iota
25+
// PruneUnusedPackages indicates if unused Go packages should be pruned.
26+
PruneUnusedPackages
1327
// PruneNonGoFiles indicates if non-Go files should be pruned.
1428
// LICENSE & COPYING files are kept for convience.
1529
PruneNonGoFiles
1630
// PruneGoTestFiles indicates if Go test files should be pruned.
1731
PruneGoTestFiles
18-
// PruneUnusedPackages indicates if unused Go packages should be pruned.
19-
PruneUnusedPackages
2032
)
33+
34+
var (
35+
preservedNonGoFiles = []string{
36+
"LICENSE",
37+
"COPYING",
38+
}
39+
)
40+
41+
func Prune(baseDir string, l Lock, options PruneOptions, logger *log.Logger) error {
42+
if (options & PruneNestedVendorDirs) != 0 {
43+
if err := pruneNestedVendorDirs(baseDir); err != nil {
44+
return err
45+
}
46+
}
47+
48+
if (options & PruneUnusedPackages) != 0 {
49+
if l == nil {
50+
return errors.New("pruning unused packages requires passing non-nil Lock")
51+
}
52+
if err := pruneUnusedPackages(baseDir, l, logger); err != nil {
53+
return err
54+
}
55+
}
56+
57+
// if (options & PruneNonGoFiles) != 0 {
58+
// if err := pruneNonGoFiles(baseDir, logger); err != nil {
59+
// return err
60+
// }
61+
// }
62+
63+
// if (options & PruneGoTestFiles) != 0 {
64+
// if err := pruneGoTestFiles(baseDir, logger); err != nil {
65+
// return err
66+
// }
67+
// }
68+
69+
// Delete all empty directories.
70+
return errors.Wrap(pruneEmptyDirs(baseDir, logger), "could not prune empty dirs")
71+
}
72+
73+
func pruneNestedVendorDirs(baseDir string) error {
74+
return filepath.Walk(baseDir, stripNestedVendorDirs(baseDir))
75+
}
76+
77+
func pruneUnusedPackages(baseDir string, l Lock, logger *log.Logger) error {
78+
unused, err := calculateUnusedPackages(baseDir, l, logger)
79+
if err != nil {
80+
return errors.Wrap(err, "could not strip unused packages")
81+
}
82+
83+
for _, pkg := range unused {
84+
pkgPath := filepath.Join(baseDir, pkg)
85+
86+
files, err := ioutil.ReadDir(pkgPath)
87+
if err != nil {
88+
return err
89+
}
90+
91+
// Delete *.go files in the package directory.
92+
for _, file := range files {
93+
// Skip directories and files that don't have a .go suffix.
94+
if file.IsDir() || !strings.HasSuffix(file.Name(), ".go") {
95+
continue
96+
}
97+
98+
if err := os.Remove(filepath.Join(pkgPath, file.Name())); err != nil {
99+
return err
100+
}
101+
}
102+
}
103+
104+
return nil
105+
}
106+
107+
func calculateUnusedPackages(baseDir string, l Lock, logger *log.Logger) ([]string, error) {
108+
imported := calculateImportedPackages(l)
109+
sort.Strings(imported)
110+
111+
var unused []string
112+
113+
if logger != nil {
114+
logger.Println("Calculating unused packages to prune. Checking the following packages:")
115+
}
116+
117+
err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
118+
if err != nil {
119+
return err
120+
}
121+
122+
fmt.Println(info.Name(), info.IsDir(), info.Mode())
123+
124+
// Ignore baseDir and anything that's not a directory.
125+
if path == baseDir || !info.IsDir() {
126+
return nil
127+
}
128+
129+
pkg := strings.TrimPrefix(path, baseDir+string(filepath.Separator))
130+
if logger != nil {
131+
logger.Printf(" %s", pkg)
132+
}
133+
134+
// If pkg is not a parent of an imported package, add it to the unused list.
135+
i := sort.Search(len(imported), func(i int) bool {
136+
return pkg <= imported[i]
137+
})
138+
if i >= len(imported) || !strings.HasPrefix(imported[i], pkg) {
139+
unused = append(unused, path)
140+
}
141+
142+
return nil
143+
})
144+
fmt.Println("err", err)
145+
146+
return unused, err
147+
}
148+
149+
func calculateImportedPackages(l Lock) []string {
150+
var imported []string
151+
152+
for _, project := range l.Projects() {
153+
projectRoot := string(project.Ident().ProjectRoot)
154+
for _, pkg := range project.Packages() {
155+
imported = append(imported, filepath.Join(projectRoot, pkg))
156+
}
157+
}
158+
return imported
159+
}
160+
161+
func pruneNonGoFiles(baseDir string, logger *log.Logger) error {
162+
files, err := calculateNonGoFiles(baseDir)
163+
if err != nil {
164+
return errors.Wrap(err, "could not prune non-Go files")
165+
}
166+
167+
if err := deleteFiles(files); err != nil {
168+
return err
169+
}
170+
171+
return nil
172+
}
173+
174+
func calculateNonGoFiles(baseDir string) ([]string, error) {
175+
var files []string
176+
177+
err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
178+
if err != nil {
179+
return err
180+
}
181+
182+
// Ignore directories.
183+
if info.IsDir() {
184+
return nil
185+
}
186+
187+
// Ignore all Go files.
188+
if strings.HasSuffix(info.Name(), ".go") {
189+
return nil
190+
}
191+
192+
// Ignore preserved non-Go files. We check for prefix incase the file
193+
// has an extension. For example: LICENSE.md.
194+
for _, prefix := range preservedNonGoFiles {
195+
if strings.HasPrefix(info.Name(), prefix) {
196+
return nil
197+
}
198+
}
199+
200+
files = append(files, path)
201+
202+
return nil
203+
})
204+
205+
return files, err
206+
}
207+
208+
func pruneGoTestFiles(baseDir string, logger *log.Logger) error {
209+
files, err := calculateGoTestFiles(baseDir)
210+
if err != nil {
211+
return errors.Wrap(err, "could not prune Go test files")
212+
}
213+
214+
if err := deleteFiles(files); err != nil {
215+
return err
216+
}
217+
218+
return nil
219+
}
220+
221+
func calculateGoTestFiles(baseDir string) ([]string, error) {
222+
var files []string
223+
224+
err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
225+
if err != nil {
226+
return err
227+
}
228+
229+
// Ignore directories.
230+
if info.IsDir() {
231+
return nil
232+
}
233+
234+
// Ignore any files that is not a Go test file.
235+
if !strings.HasSuffix(info.Name(), "_test.go") {
236+
return nil
237+
}
238+
239+
files = append(files, path)
240+
241+
return nil
242+
})
243+
244+
return files, err
245+
}
246+
247+
func deleteFiles(paths []string) error {
248+
for _, path := range paths {
249+
if err := os.Remove(path); err != nil {
250+
return err
251+
}
252+
}
253+
return nil
254+
}
255+
256+
func pruneEmptyDirs(baseDir string, logger *log.Logger) error {
257+
empty, err := calculateEmptyDirs(baseDir)
258+
if err != nil {
259+
return err
260+
}
261+
262+
if logger != nil {
263+
logger.Println("Deleting empty directories:")
264+
}
265+
266+
for _, dir := range empty {
267+
if logger != nil {
268+
logger.Printf(" %s\n", strings.TrimPrefix(dir, baseDir+string(os.PathSeparator)))
269+
}
270+
if err := os.Remove(dir); err != nil {
271+
return err
272+
}
273+
}
274+
275+
return nil
276+
}
277+
func calculateEmptyDirs(baseDir string) ([]string, error) {
278+
var empty []string
279+
280+
err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
281+
if err != nil {
282+
return nil
283+
}
284+
285+
if !info.IsDir() {
286+
return nil
287+
}
288+
289+
// TODO(ibrasho) should we use fs.IsNonEmptyDir instead?
290+
files, err := ioutil.ReadDir(path)
291+
if err != nil {
292+
return err
293+
}
294+
295+
if len(files) == 0 {
296+
empty = append(empty, path)
297+
}
298+
299+
return nil
300+
})
301+
302+
return empty, err
303+
}

‎internal/gps/source_manager.go‎

Lines changed: 33 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"context"
99
"encoding/hex"
1010
"fmt"
11+
"log"
1112
"os"
1213
"os/signal"
1314
"path/filepath"
@@ -486,41 +487,50 @@ func (sm *SourceMgr) ExportProject(id ProjectIdentifier, v Version, to string) e
486487
return srcg.exportVersionTo(context.TODO(), v, to)
487488
}
488489

489-
// WriteDepTree takes a basedir, a Lock and a PruneOptions, and exports all the
490-
// projects listed in the lock to the appropriate target location within basedir.
490+
// WriteDepTree takes a baseDir, a Lock and a PruneOptions, and exports all the
491+
// projects listed in the lock to the appropriate target location within baseDir.
491492
//
492-
// If the goal is to populate a vendor directory, basedir should be the absolute
493+
// If the goal is to populate a vendor directory, baseDir should be the absolute
493494
// path to that vendor directory, not its parent (a project root, typically).
494-
func (sm *SourceMgr) WriteDepTree(basedir string, l Lock, options PruneOptions) error {
495+
func (sm *SourceMgr) WriteDepTree(baseDir string, l Lock, options PruneOptions) error {
495496
if l == nil {
496497
return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
497498
}
498499

499-
if err := os.MkdirAll(basedir, 0777); err != nil {
500+
if err := os.MkdirAll(baseDir, 0777); err != nil {
500501
return err
501502
}
502503

503-
// TODO(sdboyer) parallelize
504+
var wg sync.WaitGroup
505+
errCh := make(chan error, len(l.Projects()))
506+
504507
for _, p := range l.Projects() {
505-
to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
508+
wg.Add(1)
509+
go func(p LockedProject) {
510+
to := filepath.FromSlash(filepath.Join(baseDir, string(p.Ident().ProjectRoot)))
506511

507-
if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
508-
removeAll(basedir)
509-
return fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err)
510-
}
512+
if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
513+
removeAll(baseDir)
514+
errCh <- errors.Wrapf(err, "error while exporting %s: %s", p.Ident().ProjectRoot)
515+
}
511516

512-
if (options & PruneNestedVendorDirs) != 0 {
513-
filepath.Walk(to, stripVendor)
514-
}
515-
if (options & PruneNonGoFiles) != 0 {
516-
// TODO: prune non Go files
517-
}
518-
if (options & PruneGoTestFiles) != 0 {
519-
// TODO: prune Go test files
520-
}
521-
if (options & PruneUnusedPackages) != 0 {
522-
// TODO: prune unused packages
523-
}
517+
wg.Done()
518+
}(p)
519+
}
520+
521+
wg.Wait()
522+
523+
// TODO(ibrasho) handle multiple errors
524+
if len(errCh) > 0 {
525+
return <-errCh
526+
}
527+
528+
// TODO(ibrasho) pass a proper logger?
529+
logger := log.New(os.Stdout, "", 0)
530+
531+
if err := Prune(baseDir, l, options, logger); err != nil {
532+
// removeAll(baseDir)
533+
return err
524534
}
525535

526536
return nil

0 commit comments

Comments
 (0)