package git import ( "context" "strconv" "strings" "sync" "time" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" lru "github.com/hashicorp/golang-lru" "github.com/rs/zerolog/log" "golang.org/x/sync/singleflight" ) const ( repositoryCacheSize = 4 repositoryCacheTTL = 5 * time.Minute ) type RepoManager interface { Download(ctx context.Context, dst string, opt *git.CloneOptions) error LatestCommitID(ctx context.Context, repositoryUrl, referenceName string, opt *git.ListOptions) (string, error) ListRefs(ctx context.Context, repositoryUrl string, opt *git.ListOptions) ([]string, error) ListFiles(ctx context.Context, dirOnly bool, opt *git.CloneOptions) ([]string, error) } // Service represents a service for managing Git. type Service struct { shutdownCtx context.Context azure RepoManager git RepoManager timerStopped bool mut sync.Mutex cacheEnabled bool // Cache the result of repository refs, key is repository URL repoRefCache *lru.Cache // Cache the result of repository file tree, key is the concatenated string of repository URL and ref value repoFileCache *lru.Cache } // NewService initializes a new service. func NewService(ctx context.Context) *Service { return newService(ctx, repositoryCacheSize, repositoryCacheTTL) } func newService(ctx context.Context, cacheSize int, cacheTTL time.Duration) *Service { service := &Service{ shutdownCtx: ctx, azure: NewAzureClient(), git: NewGitClient(false), timerStopped: false, cacheEnabled: cacheSize > 0, } if service.cacheEnabled { var err error service.repoRefCache, err = lru.New(cacheSize) if err != nil { log.Debug().Err(err).Msg("failed to create ref cache") } service.repoFileCache, err = lru.New(cacheSize) if err != nil { log.Debug().Err(err).Msg("failed to create file cache") } if cacheTTL > 0 { go service.startCacheCleanTimer(cacheTTL) } } return service } // startCacheCleanTimer starts a timer to purge caches periodically func (service *Service) startCacheCleanTimer(d time.Duration) { ticker := time.NewTicker(d) for { select { case <-ticker.C: service.purgeCache() case <-service.shutdownCtx.Done(): ticker.Stop() service.mut.Lock() service.timerStopped = true service.mut.Unlock() return } } } // timerHasStopped shows the CacheClean timer state with thread-safe way func (service *Service) timerHasStopped() bool { service.mut.Lock() defer service.mut.Unlock() ret := service.timerStopped return ret } // CloneRepository clones a git repository using the specified URL in the specified // destination folder. func (service *Service) CloneRepository( destination, repositoryURL, referenceName, username, password string, tlsSkipVerify bool, ) error { gitOptions := &git.CloneOptions{ URL: repositoryURL, Depth: 1, InsecureSkipTLS: tlsSkipVerify, Auth: GetBasicAuth(username, password), Tags: git.NoTags, } if referenceName != "" { gitOptions.ReferenceName = plumbing.ReferenceName(referenceName) } return service.repoManager(repositoryURL).Download(context.TODO(), destination, gitOptions) } func (service *Service) repoManager(repositoryURL string) RepoManager { repoManager := service.git if IsAzureUrl(repositoryURL) { repoManager = service.azure } return repoManager } // LatestCommitID returns SHA1 of the latest commit of the specified reference func (service *Service) LatestCommitID( repositoryURL, referenceName, username, password string, tlsSkipVerify bool, ) (string, error) { listOptions := &git.ListOptions{ Auth: GetBasicAuth(username, password), InsecureSkipTLS: tlsSkipVerify, } return service.repoManager(repositoryURL).LatestCommitID(context.TODO(), repositoryURL, referenceName, listOptions) } // ListRefs will list target repository's references without cloning the repository func (service *Service) ListRefs( repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool, ) ([]string, error) { refCacheKey := generateCacheKey(repositoryURL, username, password, strconv.FormatBool(tlsSkipVerify)) if service.cacheEnabled && hardRefresh { // Should remove the cache explicitly, so that the following normal list can show the correct result service.repoRefCache.Remove(refCacheKey) // Remove file caches pointed to the same repository for _, fileCacheKey := range service.repoFileCache.Keys() { if key, ok := fileCacheKey.(string); ok && strings.HasPrefix(key, repositoryURL) { service.repoFileCache.Remove(key) } } } if service.repoRefCache != nil { // Lookup the refs cache first if cache, ok := service.repoRefCache.Get(refCacheKey); ok { if refs, ok := cache.([]string); ok { return refs, nil } } } options := &git.ListOptions{ Auth: GetBasicAuth(username, password), InsecureSkipTLS: tlsSkipVerify, } refs, err := service.repoManager(repositoryURL).ListRefs(context.TODO(), repositoryURL, options) if err != nil { return nil, err } if service.cacheEnabled && service.repoRefCache != nil { service.repoRefCache.Add(refCacheKey, refs) } return refs, nil } var singleflightGroup = &singleflight.Group{} // ListFiles will list all the files of the target repository with specific extensions. // If extension is not provided, it will list all the files under the target repository func (service *Service) ListFiles( repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool, ) ([]string, error) { repoKey := generateCacheKey( repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly), ) fs, err, _ := singleflightGroup.Do(repoKey, func() (any, error) { return service.listFiles( repositoryURL, referenceName, username, password, dirOnly, hardRefresh, tlsSkipVerify, ) }) return filterFiles(fs.([]string), includedExts), err } func (service *Service) listFiles( repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, tlsSkipVerify bool, ) ([]string, error) { repoKey := generateCacheKey( repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly), ) if service.cacheEnabled && hardRefresh { // Should remove the cache explicitly, so that the following normal list can show the correct result service.repoFileCache.Remove(repoKey) } if service.repoFileCache != nil { // lookup the files cache first if cache, ok := service.repoFileCache.Get(repoKey); ok { if files, ok := cache.([]string); ok { return files, nil } } } cloneOption := &git.CloneOptions{ URL: repositoryURL, NoCheckout: true, Depth: 1, SingleBranch: true, ReferenceName: plumbing.ReferenceName(referenceName), Auth: GetBasicAuth(username, password), InsecureSkipTLS: tlsSkipVerify, Tags: git.NoTags, } files, err := service.repoManager(repositoryURL).ListFiles(context.TODO(), dirOnly, cloneOption) if err != nil { return nil, err } if service.cacheEnabled && service.repoFileCache != nil { service.repoFileCache.Add(repoKey, files) } return files, nil } func (service *Service) purgeCache() { if service.repoRefCache != nil { service.repoRefCache.Purge() } if service.repoFileCache != nil { service.repoFileCache.Purge() } } func generateCacheKey(names ...string) string { return strings.Join(names, "-") } func matchExtensions(target string, exts []string) bool { if len(exts) == 0 { return true } for _, ext := range exts { if strings.HasSuffix(target, ext) { return true } } return false } func filterFiles(paths []string, includedExts []string) []string { if len(includedExts) == 0 { return paths } var includedFiles []string for _, filename := range paths { // Filter out the filenames with non-included extension if matchExtensions(filename, includedExts) { includedFiles = append(includedFiles, filename) } } return includedFiles } func GetBasicAuth(username, password string) *githttp.BasicAuth { if password != "" { if username == "" { username = "token" } return &githttp.BasicAuth{ Username: username, Password: password, } } return nil }