feat(cache): add additional_caches config for custom asset cache paths

cmd/project/ci.go

@@ -142,6 +142,7 @@ var projectCI = &cobra.Command{
 		lookingForExtensionsSection.End(cmd.Context())
 
 		assetCfg := extension.AssetBuildConfig{
+			EnableAssetCaching:           shopCfg.Build.AssetCaching,
 			CleanupNodeModules:           true,
 			ShopwareRoot:                 args[0],
 			ShopwareVersion:              shopwareConstraint,

internal/asset/source.go

@@ -7,4 +7,10 @@ type Source struct {
 	StorefrontEsbuildCompatible bool
 	DisableSass                 bool
 	NpmStrict                   bool
+	AdditionalCaches            []AdditionalCache
+}
+
+type AdditionalCache struct {
+	Path        string
+	SourcePaths []string
 }

internal/extension/asset.go

@@ -8,6 +8,22 @@ import (
 	"github.com/shopware/shopware-cli/logging"
 )
 
+func convertAdditionalCaches(configCaches []ConfigBuildZipAssetsAdditionalCache) []asset.AdditionalCache {
+	if len(configCaches) == 0 {
+		return nil
+	}
+
+	caches := make([]asset.AdditionalCache, len(configCaches))
+	for i, cp := range configCaches {
+		caches[i] = asset.AdditionalCache{
+			Path:        cp.Path,
+			SourcePaths: cp.SourcePaths,
+		}
+	}
+
+	return caches
+}
+
 func ConvertExtensionsToSources(ctx context.Context, extensions []Extension) []asset.Source {
 	sources := make([]asset.Source, 0)
 
@@ -25,6 +41,7 @@ func ConvertExtensionsToSources(ctx context.Context, extensions []Extension) []a
 			StorefrontEsbuildCompatible: ext.GetExtensionConfig().Build.Zip.Assets.EnableESBuildForStorefront,
 			DisableSass:                 ext.GetExtensionConfig().Build.Zip.Assets.DisableSass,
 			NpmStrict:                   ext.GetExtensionConfig().Build.Zip.Assets.NpmStrict,
+			AdditionalCaches:            convertAdditionalCaches(ext.GetExtensionConfig().Build.Zip.Assets.AdditionalCaches),
 		})
 
 		extConfig := ext.GetExtensionConfig()

internal/extension/asset_cache.go

@@ -4,30 +4,29 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
+	"path"
 	"slices"
 
+	"github.com/cespare/xxhash/v2"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/shopware/shopware-cli/internal/system"
 	"github.com/shopware/shopware-cli/logging"
 )
 
-var experimentalCachingEnabled bool
-
-func init() {
-	experimentalCachingEnabled = os.Getenv("SHOPWARE_CLI_EXPERIMENTAL_ASSET_CACHING") == "1"
+func hashCacheKeySuffix(p string) string {
+	return fmt.Sprintf("%x", xxhash.Sum64String(p))
 }
 
 func restoreAssetCaches(ctx context.Context, sources ExtensionAssetConfig, assetCfg AssetBuildConfig) error {
-	if !experimentalCachingEnabled {
+	if !assetCfg.EnableAssetCaching {
 		return nil
 	}
 
 	var errgrp errgroup.Group
 
 	for name, source := range sources {
-		if source.RequiresBuild() && !slices.Contains(assetCfg.ForceExtensionBuild, name) {
+		if (source.RequiresBuild() || len(source.AdditionalCaches) > 0) && !slices.Contains(assetCfg.ForceExtensionBuild, name) {
 			errgrp.Go(func() error {
 				return restoreAssetCache(ctx, source, assetCfg)
 			})
@@ -38,14 +37,14 @@ func restoreAssetCaches(ctx context.Context, sources ExtensionAssetConfig, asset
 }
 
 func storeAssetCaches(ctx context.Context, sources ExtensionAssetConfig, assetCfg AssetBuildConfig) error {
-	if !experimentalCachingEnabled {
+	if !assetCfg.EnableAssetCaching {
 		return nil
 	}
 
 	var errgrp errgroup.Group
 
 	for name, source := range sources {
-		if source.RequiresBuild() && !slices.Contains(assetCfg.ForceExtensionBuild, name) {
+		if (source.RequiresBuild() || len(source.AdditionalCaches) > 0) && !slices.Contains(assetCfg.ForceExtensionBuild, name) {
 			errgrp.Go(func() error {
 				return storeAssetCache(ctx, source, assetCfg)
 			})
@@ -68,32 +67,43 @@ func restoreAssetCache(ctx context.Context, source *ExtensionAssetConfigEntry, a
 
 	if source.Administration.EntryFilePath != nil {
 		if err := system.GetDefaultCache().RestoreFolderCache(ctx, cacheKey+"-administration", source.GetOutputAdminPath()); err != nil {
-			if errors.Is(err, system.ErrCacheNotFound) {
-				return nil
+			if !errors.Is(err, system.ErrCacheNotFound) {
+				return err
 			}
+		} else {
+			logging.FromContext(ctx).Infof("Restored administration assets for %s from cache", source.TechnicalName)
 
-			return err
+			source.Administration.EntryFilePath = nil
+			source.Administration.Webpack = nil
 		}
-
-		logging.FromContext(ctx).Infof("Restored administration assets for %s from cache", source.TechnicalName)
-
-		source.Administration.EntryFilePath = nil
-		source.Administration.Webpack = nil
 	}
 
 	if source.Storefront.EntryFilePath != nil {
 		if err := system.GetDefaultCache().RestoreFolderCache(ctx, cacheKey+"-storefront", source.GetOutputStorefrontPath()); err != nil {
-			if errors.Is(err, system.ErrCacheNotFound) {
-				return nil
+			if !errors.Is(err, system.ErrCacheNotFound) {
+				return err
 			}
+		} else {
+			logging.FromContext(ctx).Infof("Restored storefront assets for %s from cache", source.TechnicalName)
 
-			return err
+			source.Storefront.EntryFilePath = nil
+			source.Storefront.Webpack = nil
 		}
+	}
+
+	for _, cachePath := range source.AdditionalCaches {
+		outputPath := path.Join(source.BasePath, cachePath.Path)
+		suffix := hashCacheKeySuffix(cachePath.Path)
 
-		logging.FromContext(ctx).Infof("Restored storefront assets for %s from cache", source.TechnicalName)
+		if err := system.GetDefaultCache().RestoreFolderCache(ctx, cacheKey+"-"+suffix, outputPath); err != nil {
+			if !errors.Is(err, system.ErrCacheNotFound) {
+				return err
+			}
+
+			continue
+		}
 
-		source.Storefront.EntryFilePath = nil
-		source.Storefront.Webpack = nil
+		logging.FromContext(ctx).Infof("Restored additional cache path %s for %s from cache", cachePath.Path, source.TechnicalName)
 	}
 
 	return nil
@@ -122,5 +132,14 @@ func storeAssetCache(ctx context.Context, source *ExtensionAssetConfigEntry, ass
 		}
 	}
 
+	for _, cachePath := range source.AdditionalCaches {
+		outputPath := path.Join(source.BasePath, cachePath.Path)
+		suffix := hashCacheKeySuffix(cachePath.Path)
+
+		if err := system.GetDefaultCache().StoreFolderCache(ctx, cacheKey+"-"+suffix, outputPath); err != nil {
+			return fmt.Errorf("additional_caches path %q for extension %s: %w", cachePath.Path, source.TechnicalName, err)
+		}
+	}
+
 	return nil
 }

internal/extension/asset_config.go

@@ -33,6 +33,7 @@ const (
 )
 
 type AssetBuildConfig struct {
+	EnableAssetCaching           bool
 	CleanupNodeModules           bool
 	DisableAdminBuild            bool
 	DisableStorefrontBuild       bool
@@ -158,6 +159,7 @@ type ExtensionAssetConfigEntry struct {
 	EnableESBuildForStorefront bool
 	DisableSass                bool
 	NpmStrict                  bool
+	AdditionalCaches           []ConfigBuildZipAssetsAdditionalCache
 
 	// internal cache
 	cachedPossibleNodePaths []string
@@ -280,6 +282,21 @@ func (e *ExtensionAssetConfigEntry) GetContentHash() (string, error) {
 		}
 	}
 
+	// Include asset build config in hash so any config change invalidates the cache
+	if _, err := fmt.Fprintf(hasher, "%v%v%v%v", e.EnableESBuildForAdmin, e.EnableESBuildForStorefront, e.DisableSass, e.NpmStrict); err != nil {
+		return "", err
+	}
+	for _, cache := range e.AdditionalCaches {
+		if _, err := fmt.Fprintf(hasher, "%s", cache.Path); err != nil {
+			return "", err
+		}
+		for _, sp := range cache.SourcePaths {
+			if _, err := fmt.Fprintf(hasher, "%s", sp); err != nil {
+				return "", err
+			}
+		}
+	}
+
 	e.sumOfFiles = fmt.Sprintf("%x", hasher.Sum64())
 	return e.sumOfFiles, nil
 }
@@ -319,6 +336,16 @@ func (e *ExtensionAssetConfigEntry) collectFilesForHashing() ([]string, error) {
 		}
 	}
 
+	// Collect files from additional cache source paths
+	for _, cachePath := range e.AdditionalCaches {
+		for _, sourcePath := range cachePath.SourcePaths {
+			absSourcePath := path.Join(e.BasePath, sourcePath)
+			if err := e.collectFilesFromDir(absSourcePath, &files); err != nil {
+				return nil, fmt.Errorf("failed to collect custom cache source files from %s: %w", sourcePath, err)
+			}
+		}
+	}
+
 	// Add package.json files
 	files = append(files, e.getPossibleNodePaths()...)
 
@@ -336,8 +363,12 @@ func (e *ExtensionAssetConfigEntry) collectFilesFromDir(dir string, files *[]str
 			return err
 		}
 
-		// Skip directories and node_modules
-		if info.IsDir() || strings.Contains(path, "node_modules") {
+		// Skip node_modules directories entirely
+		if info.IsDir() && info.Name() == "node_modules" {
+			return filepath.SkipDir
+		}
+
+		if info.IsDir() {
 			return nil
 		}
 
@@ -436,6 +467,7 @@ func BuildAssetConfigFromExtensions(ctx context.Context, sources []asset.Source,
 		sourceConfig.EnableESBuildForStorefront = source.StorefrontEsbuildCompatible
 		sourceConfig.DisableSass = source.DisableSass
 		sourceConfig.NpmStrict = source.NpmStrict
+		sourceConfig.AdditionalCaches = convertSourceAdditionalCaches(source.AdditionalCaches)
 
 		if assetCfg.SkipExtensionsWithBuildFiles {
 			expectedAdminCompiledFile := path.Join(source.Path, "Resources", "public", "administration", "js", esbuild.ToKebabCase(source.Name)+".js")
@@ -542,3 +574,19 @@ func createConfigFromPath(entryPointName string, extensionRoot string) *Extensio
 	}
 	return &cfg
 }
+
+func convertSourceAdditionalCaches(paths []asset.AdditionalCache) []ConfigBuildZipAssetsAdditionalCache {
+	if len(paths) == 0 {
+		return nil
+	}
+
+	result := make([]ConfigBuildZipAssetsAdditionalCache, len(paths))
+	for i, cp := range paths {
+		result[i] = ConfigBuildZipAssetsAdditionalCache{
+			Path:        cp.Path,
+			SourcePaths: cp.SourcePaths,
+		}
+	}
+
+	return result
+}