ANE-1036: Glob file matching for exclusion filters in .fossa.yml

Changelog.md

@@ -1,5 +1,9 @@
 # FOSSA CLI Changelog
 
+## 3.17.6
+
+- Config: `paths.only` and `paths.exclude` in `.fossa.yml` now accept glob patterns. ([#1703](https://github.com/fossas/fossa-cli/pull/1703))
+
 ## 3.17.5
 
 - Vendetta: Debug bundles now include per-file component match data from Vendetta scans, making it easier to diagnose why a vendored dependency was or wasn't detected. ([#1706](https://github.com/fossas/fossa-cli/pull/1706))

docs/references/files/fossa-yml.md

@@ -334,6 +334,28 @@ The list of paths to exclude from scanning in your directory.
 
 This section is intended to be used as the inverse to `paths.only`. If you have a certain directory such as `development` you wish to exclude, `paths.exclude` enables you to do this.
 
+#### Glob patterns
+
+Entries in `paths.only` and `paths.exclude` may also be glob patterns. An entry is treated as a glob if it contains `*`; other entries keep their existing semantics (match the directory and all of its children). Glob matching follows [`System.FilePattern`][filepattern] semantics: `*` matches any sequence of characters within a single path segment, and `**` matches any number of segments.
+
+Patterns use forward slashes (`/`) as path separators; backslashes are normalized so Windows-native patterns also work.
+
+```yaml
+paths:
+  exclude:
+    - "**/vendor/**"
+    - "**/node_modules/**"
+    - "build/generated/*"
+```
+
+Each example above excludes a different shape of directory:
+
+- `**/vendor/**` skips Go-style vendored trees at any depth, e.g. `services/billing/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/`.
+- `**/node_modules/**` skips installed npm packages wherever they appear, e.g. `apps/web-frontend/node_modules/@babel/preset-env/lib/plugins/syntax-dynamic-import/`.
+- `build/generated/*` is anchored at the repo root and matches *direct* children of `build/generated/` only. `build/generated/proto-go/` matches; the walker then prunes its entire subtree (e.g. `build/generated/proto-go/v1/messagepb/`).
+
+[filepattern]: https://hackage.haskell.org/package/filepattern
+
 ### Analysis target configuration
 Analysis target configuration allows you to select a very specific subset of your directory for scanning. The `targets` and `paths` sections allow users to configure which targets and directories should be scanned. This is useful if you have a custom test directory or development projects within the root project.
 

docs/references/files/fossa-yml.v3.schema.json

@@ -528,18 +528,18 @@
         },
         "paths": {
             "type": "object",
-            "description": "The paths filtering section allows you to specify which paths should be scanned and which should not. The paths should be listed as their location from the root of your project.",
+            "description": "The paths filtering section allows you to specify which paths should be scanned and which should not. Entries may be concrete directory paths (which match the directory and all of its children) or glob patterns. An entry is treated as a glob if it contains `*`. Glob syntax follows System.FilePattern: `*` matches any sequence of characters within a single path segment, and `**` matches any number of segments.",
             "properties": {
                 "only": {
                     "type": "array",
-                    "description": "The list of paths to only allow scanning within.",
+                    "description": "The list of paths or glob patterns to only allow scanning within.",
                     "items": {
                         "type": "string"
                     }
                 },
                 "exclude": {
                     "type": "array",
-                    "description": "The list of paths to exclude from scanning in your directory.",
+                    "description": "The list of paths or glob patterns to exclude from scanning in your directory.",
                     "items": {
                         "type": "string"
                     }

src/App/Fossa/Analyze.hs

@@ -121,7 +121,7 @@ import Data.Aeson qualified as Aeson
 import Data.ByteString.Lazy qualified as BL
 import Data.Error (createBody)
 import Data.Flag (Flag, fromFlag)
-import Data.Foldable (traverse_)
+import Data.Foldable (for_, traverse_)
 import Data.Functor (($>))
 import Data.List.NonEmpty qualified as NE
 import Data.Map qualified as Map
@@ -132,8 +132,9 @@ import Data.Traversable (for)
 import Diag.Diagnostic as DI
 import Diag.Result (resultToMaybe)
 import Discovery.Archive qualified as Archive
-import Discovery.Filters (AllFilters, MavenScopeFilters, applyFilters, filterIsVSIOnly, ignoredPaths, isDefaultNonProductionPath)
+import Discovery.Filters (AllFilters (..), MavenScopeFilters, applyFilters, combinedPathGlobs, combinedPaths, filterIsVSIOnly, ignoredPaths, isDefaultNonProductionPath)
 import Discovery.Projects (withDiscoveredProjects)
+import Discovery.Walk (enumeratePrunedSubtrees)
 import Effect.Exec (Exec)
 import Effect.Logger (
   Logger,
@@ -297,6 +298,32 @@ runAnalyzers allowedTactics filters withoutDefaultFilters basedir pathPrefix = d
   where
     single (DiscoverFunc f) = withDiscoveredProjects f basedir (runDependencyAnalysis basedir filters withoutDefaultFilters pathPrefix allowedTactics)
 
+-- | Walk the tree once at startup and surface every directory the path
+-- filters will prune. Each prune is logged once at info level here, instead
+-- of emitting per-strategy duplicates from inside the walker (~28 strategies
+-- would otherwise each report the same prune). Short-circuits when no path
+-- filters are configured so the extra walk is only paid for when it can
+-- produce output.
+logPrunedSubtrees ::
+  ( Has Logger sig m
+  , Has ReadFS sig m
+  , Has Diag.Diagnostics sig m
+  ) =>
+  AllFilters ->
+  Path Abs Dir ->
+  m ()
+logPrunedSubtrees filters basedir =
+  unless (noPathFilters filters) $ do
+    pruned <- enumeratePrunedSubtrees filters basedir
+    for_ pruned $ \p ->
+      logInfo $ "Skipping path " <> viaShow p <> " (excluded by paths filter)"
+  where
+    noPathFilters AllFilters{includeFilters = i, excludeFilters = e} =
+      null (combinedPaths i)
+        && null (combinedPathGlobs i)
+        && null (combinedPaths e)
+        && null (combinedPathGlobs e)
+
 analyze ::
   ( Has Debug sig m
   , Has Diag.Diagnostics sig m
@@ -331,6 +358,12 @@ analyze cfg = Diag.context "fossa-analyze" $ do
       withoutDefaultFilters = Config.withoutDefaultFilters cfg
       enableSnippetScan = Config.snippetScan cfg
       enableVendetta = Config.xVendetta cfg
+      -- Discovery runs with `mempty` when `--no-discovery-exclusion` is set
+      -- (see definition further down). Log against the same filter set so the
+      -- startup output matches what discovery actually applies.
+      discoveryFilters = if fromFlag NoDiscoveryExclusion noDiscoveryExclusion then mempty else filters
+
+  logPrunedSubtrees discoveryFilters basedir
 
   manualDepsResult <-
     Diag.errorBoundaryIO . diagToDebug $
@@ -434,7 +467,6 @@ analyze cfg = Diag.context "fossa-analyze" $ do
           pure Nothing
         else Diag.context "first-party-scans" . runStickyLogger SevInfo $ runFirstPartyScan basedir maybeApiOpts cfg
   let firstPartyScanResults = join . resultToMaybe $ maybeFirstPartyScanResults
-  let discoveryFilters = if fromFlag NoDiscoveryExclusion noDiscoveryExclusion then mempty else filters
   let strategyCfg =
         (Config.strategyConfig cfg)
           { Config.useGitBackedCargoLocators = Config.UseGitBackedCargoLocators $ maybe True orgSupportsGitBackedCargoLocators orgInfo

src/App/Fossa/Config/Common.hs

@@ -120,7 +120,7 @@ import Data.String (IsString)
 import Data.String.Conversion (ToText (toText))
 import Data.Text (Text, null, strip, toLower)
 import Diag.Result (Result (Failure, Success), renderFailure)
-import Discovery.Filters (AllFilters (AllFilters), MavenScopeFilters (..), comboExclude, comboInclude, setExclude, setInclude, targetFilterParser)
+import Discovery.Filters (AllFilters (AllFilters), MavenScopeFilters (..), comboExcludeWithGlobs, comboIncludeWithGlobs, partitionPathFilters, setExclude, setInclude, targetFilterParser)
 import Effect.Exec (Exec)
 import Effect.Logger (Logger, logDebug, logInfo, renderIt, vsep)
 import Effect.ReadFS (ReadFS, doesDirExist, doesFileExist)
@@ -624,11 +624,13 @@ collectConfigFileFilters configFile = do
   let pullFromFile :: (a -> [b]) -> (ConfigFile -> Maybe a) -> [b]
       pullFromFile field section = maybe [] field (section configFile)
       onlyT = pullFromFile targetsOnly configTargets
-      onlyP = pullFromFile pathsOnly configPaths
+      (onlyP, onlyG) = partitionPathFilters $ pullFromFile pathsOnly configPaths
       excludeT = pullFromFile targetsExclude configTargets
-      excludeP = pullFromFile pathsExclude configPaths
+      (excludeP, excludeG) = partitionPathFilters $ pullFromFile pathsExclude configPaths
 
-  AllFilters (comboInclude onlyT onlyP) (comboExclude excludeT excludeP)
+  AllFilters
+    (comboIncludeWithGlobs onlyT onlyP onlyG)
+    (comboExcludeWithGlobs excludeT excludeP excludeG)
 
 collectConfigMavenScopeFilters :: ConfigFile -> MavenScopeFilters
 collectConfigMavenScopeFilters configFile = do

src/App/Fossa/Config/ConfigFile.hs

@@ -11,6 +11,7 @@ module App.Fossa.Config.ConfigFile (
   ConfigRevision (..),
   ConfigTargets (..),
   ConfigPaths (..),
+  PathFilter (..),
   ConfigTelemetry (..),
   ConfigTelemetryScope (..),
   ExperimentalConfigs (..),
@@ -57,6 +58,7 @@ import Data.Set qualified as Set
 import Data.String.Conversion (ToString (toString), ToText (toText))
 import Data.Text (Text, strip, toLower)
 import Diag.Diagnostic (ToDiagnostic (..))
+import Discovery.Filters (PathFilter (..))
 import Effect.Logger (
   Logger,
   logDebug,
@@ -257,8 +259,8 @@ data ConfigTargets = ConfigTargets
   deriving (Eq, Ord, Show)
 
 data ConfigPaths = ConfigPaths
-  { pathsOnly :: [Path Rel Dir]
-  , pathsExclude :: [Path Rel Dir]
+  { pathsOnly :: [PathFilter]
+  , pathsExclude :: [PathFilter]
   }
   deriving (Eq, Ord, Show)