From a1bd64719ce1c8017bea6a7e50f0d6a055666478 Mon Sep 17 00:00:00 2001 From: "yinxuran.lucky" Date: Thu, 25 Dec 2025 20:38:11 +0800 Subject: [PATCH] fix: cgo file length calculate wrong --- lang/golang/parser/ctx.go | 10 +++++++- lang/golang/parser/parser.go | 42 ++++++++++++++++++++++++------- lang/golang/parser/pkg.go | 49 ++++++++++++++++++++++++++++-------- 3 files changed, 80 insertions(+), 21 deletions(-) diff --git a/lang/golang/parser/ctx.go b/lang/golang/parser/ctx.go index 761b19e..89bdb37 100644 --- a/lang/golang/parser/ctx.go +++ b/lang/golang/parser/ctx.go @@ -86,7 +86,15 @@ func (p *GoParser) referCodes(ctx *fileContext, id *Identity, depth int) (err er if pkg == nil { return fmt.Errorf("cannot find package %s", id.PkgPath) } - for _, fpath := range pkg.GoFiles { + + var files []string + if len(p.cgoPkgs) > 0 { + files = pkg.CompiledGoFiles + } else { + files = pkg.GoFiles + } + + for _, fpath := range files { bs := p.getFileBytes(fpath) file, err := parser.ParseFile(pkg.Fset, fpath, bs, parser.ParseComments) if err != nil { diff --git a/lang/golang/parser/parser.go b/lang/golang/parser/parser.go index 24d9b99..e45ff02 100644 --- a/lang/golang/parser/parser.go +++ b/lang/golang/parser/parser.go @@ -47,6 +47,7 @@ type GoParser struct { types map[types.Type]Identity files map[string][]byte exclues []*regexp.Regexp + cgoPkgs map[string]bool // CGO packages } type moduleInfo struct { @@ -98,6 +99,7 @@ func newGoParser(name string, homePageDir string, opts Options) *GoParser { func (p *GoParser) collectGoMods(startDir string) error { hasGoWork := false deps := map[string]string{} + var cgoPkgs map[string]bool err := filepath.Walk(startDir, func(path string, info fs.FileInfo, err error) error { if err != nil || !strings.HasSuffix(path, "go.mod") { return nil @@ -115,10 +117,18 @@ func (p *GoParser) collectGoMods(startDir string) error { p.repo.Modules[name] = newModule(name, rel) p.modules = append(p.modules, newModuleInfo(name, rel, name)) - deps, hasGoWork, err = getDeps(filepath.Dir(path), hasGoWork) + deps, hasGoWork, cgoPkgs, err = getDeps(filepath.Dir(path), hasGoWork) if err != nil { return err } + if p.cgoPkgs == nil { + p.cgoPkgs = make(map[string]bool) + } + for pkgPath := range cgoPkgs { + if strings.HasPrefix(pkgPath, name) { + p.cgoPkgs[pkgPath] = true + } + } for k, v := range deps { p.repo.Modules[name].Dependencies[k] = v p.modules = append(p.modules, newModuleInfo(k, "", v)) @@ -148,18 +158,29 @@ type dep struct { Dir string `json:"Dir"` GoMod string `json:"GoMod"` } `json:"Module"` + CgoFiles []string `json:"CgoFiles"` } -func getDeps(dir string, goWork bool) (a map[string]string, hasGoWork bool, err error) { +func getDeps(dir string, goWork bool) (a map[string]string, hasGoWork bool, cgoPkgs map[string]bool, err error) { + cgoPkgs = make(map[string]bool) // run go mod tidy first to ensure all dependencies are resolved cmd := exec.Command("go", "mod", "tidy", "-e") cmd.Dir = dir + cmd.Env = append(os.Environ(), "GONOSUMDB=*") output, err := cmd.CombinedOutput() if err != nil { - return nil, hasGoWork, fmt.Errorf("failed to execute 'go mod tidy', err: %v, output: %s", err, string(output)) + fmt.Fprintf(os.Stderr, "failed to execute 'go mod tidy', err: %v, output: %s, remove go.sum file reexecute\n", err, string(output)) + os.Remove(filepath.Join(dir, "go.sum")) + cmd = exec.Command("go", "mod", "tidy", "-e") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOSUMDB=off") + output, err = cmd.CombinedOutput() + if err != nil { + return nil, hasGoWork, cgoPkgs, fmt.Errorf("failed to execute 'go mod tidy', err: %v, output: %s", err, string(output)) + } } if hasNoDeps(filepath.Join(dir, "go.mod")) { - return map[string]string{}, hasGoWork, nil + return map[string]string{}, hasGoWork, cgoPkgs, nil } // -mod=mod to use go mod when go mod is inconsistent with go vendor // if go.work exist, it's no need to set -mod=mod @@ -170,14 +191,15 @@ func getDeps(dir string, goWork bool) (a map[string]string, hasGoWork bool, err cmd = exec.Command("go", "list", "-e", "-json", "-mod=mod", "all") } cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOSUMDB=off") output, err = cmd.CombinedOutput() if err != nil { - return nil, hasGoWork, fmt.Errorf("failed to execute 'go list -json all', err: %v, output: %s, cmd string: %s, dir: %s", err, string(output), cmd.String(), dir) + return nil, hasGoWork, cgoPkgs, fmt.Errorf("failed to execute 'go list -json all', err: %v, output: %s, cmd string: %s, dir: %s", err, string(output), cmd.String(), dir) } // ignore content until first open index := strings.Index(string(output), "{") if index == -1 { - return nil, hasGoWork, fmt.Errorf("failed to find '{' in output, output: %s", string(output)) + return nil, hasGoWork, cgoPkgs, fmt.Errorf("failed to find '{' in output, output: %s", string(output)) } if index > 0 { log.Info("go list skip prefix, output: %s", string(output[:index])) @@ -191,13 +213,16 @@ func getDeps(dir string, goWork bool) (a map[string]string, hasGoWork bool, err if err.Error() == "EOF" { break } - return nil, hasGoWork, fmt.Errorf("failed to decode json: %v, output: %s", err, string(output)) + return nil, hasGoWork, cgoPkgs, fmt.Errorf("failed to decode json: %v, output: %s", err, string(output)) } module := mod.Module // golang internal package, ignore it. if module.Path == "" { continue } + if len(mod.CgoFiles) > 0 { + cgoPkgs[module.Path] = true + } if module.Replace != nil { deps[module.Path] = module.Replace.Path + "@" + module.Replace.Version } else { @@ -214,8 +239,7 @@ func getDeps(dir string, goWork bool) (a map[string]string, hasGoWork bool, err } } } - - return deps, hasGoWork, nil + return deps, hasGoWork, cgoPkgs, nil } // ParseRepo parse the entiry repo from homePageDir recursively until end diff --git a/lang/golang/parser/pkg.go b/lang/golang/parser/pkg.go index 9efe558..e044397 100644 --- a/lang/golang/parser/pkg.go +++ b/lang/golang/parser/pkg.go @@ -172,24 +172,44 @@ func (p *GoParser) loadPackages(mod *Module, dir string, pkgPath PkgPath) (err e fmt.Fprintf(os.Stderr, "[loadPackages] mod: %s, dir: %s, pkgPath: %s\n", mod.Name, dir, pkgPath) fset := token.NewFileSet() loadCount++ - // slow-path: load packages in the dir, including sub pakcages - opts := packages.NeedFiles | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedImports + + baseOpts := packages.NeedFiles | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedImports + if p.opts.ReferCodeDepth != 0 { + baseOpts |= packages.NeedDeps + } + if p.opts.NeedTest { + baseOpts |= packages.NeedForTest + } + cfg := &packages.Config{ - Mode: opts, + Mode: baseOpts, Fset: fset, Dir: dir, } - if p.opts.ReferCodeDepth != 0 { - opts |= packages.NeedDeps - } + if p.opts.NeedTest { - opts |= packages.NeedForTest cfg.Tests = true } + pkgs, err := packages.Load(cfg, pkgPath) if err != nil { return fmt.Errorf("load path '%s' failed: %v", dir, err) } + + hasCGO := false + if len(p.cgoPkgs) > 0 { + hasCGO = true + } + fmt.Fprintf(os.Stderr, "[loadPackages] mod: %s, dir: %s, pkgPath: %s, hasCGO: %v\n", mod.Name, dir, pkgPath, hasCGO) + if hasCGO { + baseOpts |= packages.NeedCompiledGoFiles + cfg.Mode = baseOpts + pkgs, err = packages.Load(cfg, pkgPath) + if err != nil { + return fmt.Errorf("load path '%s' with CGO failed: %v", dir, err) + } + } + for _, pkg := range pkgs { if mm := p.repo.Modules[mod.Name]; mm != nil && (*mm).Packages[pkg.ID] != nil { continue @@ -198,11 +218,18 @@ func (p *GoParser) loadPackages(mod *Module, dir string, pkgPath PkgPath) (err e continue } for idx, file := range pkg.Syntax { - if idx >= len(pkg.GoFiles) { - fmt.Fprintf(os.Stderr, "skip file %s by loader\n", file.Name) - continue + var filePath string + if hasCGO { + // Cgo file path is tmp file path, like: /Users/bytedance/Library/Caches/go-build/61/6150fdadd44b9dca151737e261abf95697ba13b799e8dbdd464c0c27b443792a-d. + // We should get it through CompiledGoFiles + if idx >= len(pkg.CompiledGoFiles) { + fmt.Fprintf(os.Stderr, "skip file %s by loader\n", file.Name) + continue + } + filePath = pkg.CompiledGoFiles[idx] + } else { + filePath = fset.Position(file.Pos()).Filename } - filePath := pkg.GoFiles[idx] var skip bool for _, exclude := range p.exclues { if exclude.MatchString(filePath) {