Add recursive glob support.

Sources/XcodeGenKit/Glob.swift

@@ -0,0 +1,227 @@
+//
+//  Created by Eric Firestone on 3/22/16.
+//  Copyright © 2016 Square, Inc. All rights reserved.
+//  Released under the Apache v2 License.
+//
+//  Adapted from https://gist.github.com/blakemerryman/76312e1cbf8aec248167
+//  Adapted from https://gist.github.com/efirestone/ce01ae109e08772647eb061b3bb387c3
+
+
+import Foundation
+
+
+public let GlobBehaviorBashV3 = Glob.Behavior(
+    supportsGlobstar: false,
+    includesFilesFromRootOfGlobstar: false,
+    includesDirectoriesInResults: true,
+    includesFilesInResultsIfTrailingSlash: false
+)
+public let GlobBehaviorBashV4 = Glob.Behavior(
+    supportsGlobstar: true, // Matches Bash v4 with "shopt -s globstar" option
+    includesFilesFromRootOfGlobstar: true,
+    includesDirectoriesInResults: true,
+    includesFilesInResultsIfTrailingSlash: false
+)
+public let GlobBehaviorGradle = Glob.Behavior(
+    supportsGlobstar: true,
+    includesFilesFromRootOfGlobstar: true,
+    includesDirectoriesInResults: false,
+    includesFilesInResultsIfTrailingSlash: true
+)
+
+
+/**
+ Finds files on the file system using pattern matching.
+ */
+public class Glob: Collection {
+
+    /**
+     * Different glob implementations have different behaviors, so the behavior of this
+     * implementation is customizable.
+     */
+    public struct Behavior {
+        // If true then a globstar ("**") causes matching to be done recursively in subdirectories.
+        // If false then "**" is treated the same as "*"
+        let supportsGlobstar: Bool
+
+        // If true the results from the directory where the globstar is declared will be included as well.
+        // For example, with the pattern "dir/**/*.ext" the fie "dir/file.ext" would be included if this
+        // property is true, and would be omitted if it's false.
+        let includesFilesFromRootOfGlobstar: Bool
+
+        // If false then the results will not include directory entries. This does not affect recursion depth.
+        let includesDirectoriesInResults: Bool
+
+        // If false and the last characters of the pattern are "**/" then only directories are returned in the results.
+        let includesFilesInResultsIfTrailingSlash: Bool
+    }
+
+    public static var defaultBehavior = GlobBehaviorBashV4
+
+    public static let defaultBlacklistedDirectories = ["node_modules", "Pods"]
+
+    private var isDirectoryCache = [String: Bool]()
+
+    public let behavior: Behavior
+    public let blacklistedDirectories: [String]
+    var paths = [String]()
+    public var startIndex: Int { return paths.startIndex }
+    public var endIndex: Int   { return paths.endIndex   }
+
+    /// Initialize a glob
+    ///
+    /// - Parameters:
+    ///   - pattern: The pattern to use when building the list of matching directories.
+    ///   - behavior: See individual descriptions on `Glob.Behavior` values.
+    ///   - blacklistedDirectories: An array of directories to ignore at the root level of the project.
+    public init(pattern: String, behavior: Behavior = Glob.defaultBehavior, blacklistedDirectories: [String] = defaultBlacklistedDirectories) {
+
+        self.behavior = behavior
+        self.blacklistedDirectories = blacklistedDirectories
+
+        var adjustedPattern = pattern
+        let hasTrailingGlobstarSlash = pattern.hasSuffix("**/")
+        var includeFiles = !hasTrailingGlobstarSlash
+
+        if behavior.includesFilesInResultsIfTrailingSlash {
+            includeFiles = true
+            if hasTrailingGlobstarSlash {
+                // Grab the files too.
+                adjustedPattern += "*"
+            }
+        }
+
+        let patterns = behavior.supportsGlobstar ? expandGlobstar(pattern: adjustedPattern) : [adjustedPattern]
+
+        for pattern in patterns {
+            var gt = glob_t()
+            if executeGlob(pattern: pattern, gt: &gt) {
+                populateFiles(gt: gt, includeFiles: includeFiles)
+            }
+
+            globfree(&gt)
+        }
+
+        paths = Array(Set(paths)).sorted { lhs, rhs in
+            lhs.compare(rhs) != ComparisonResult.orderedDescending
+        }
+
+        clearCaches()
+    }
+
+    // MARK: Subscript Support
+
+    public subscript(i: Int) -> String {
+        return paths[i]
+    }
+
+    // MARK: Protocol of IndexableBase
+
+    public func index(after i: Glob.Index) -> Glob.Index {
+        return i + 1
+    }
+
+    // MARK: Private
+
+    private var globalFlags = GLOB_TILDE | GLOB_BRACE | GLOB_MARK
+
+    private func executeGlob(pattern: UnsafePointer<CChar>, gt: UnsafeMutablePointer<glob_t>) -> Bool {
+        return 0 == glob(pattern, globalFlags, nil, gt)
+    }
+
+    private func expandGlobstar(pattern: String) -> [String] {
+        guard pattern.contains("**") else {
+            return [pattern]
+        }
+
+        var results = [String]()
+        var parts = pattern.components(separatedBy: "**")
+        let firstPart = parts.removeFirst()
+        var lastPart = parts.joined(separator: "**")
+
+        let fileManager = FileManager.default
+
+        var directories: [String]
+
+        do {
+            directories = try fileManager.contentsOfDirectory(atPath: firstPart).compactMap { subpath -> [String]? in
+                if blacklistedDirectories.contains(subpath) {
+                    return nil
+                }
+                let firstLevelPath = NSString(string: firstPart).appendingPathComponent(subpath)
+                if isDirectory(path: firstLevelPath) {
+                    var subDirs: [String] = try fileManager.subpathsOfDirectory(atPath: firstLevelPath).compactMap { subpath -> String? in
+                        let fullPath = NSString(string: firstLevelPath).appendingPathComponent(subpath)
+                        return isDirectory(path: fullPath) ? fullPath : nil
+                    }
+                    subDirs.append(firstLevelPath)
+                    return subDirs
+                } else {
+                    return nil
+                }
+            }.joined().array()
+        } catch {
+            directories = []
+            print("Error parsing file system item: \(error)")
+        }
+
+        if behavior.includesFilesFromRootOfGlobstar {
+            // Check the base directory for the glob star as well.
+            directories.insert(firstPart, at: 0)
+
+            // Include the globstar root directory ("dir/") in a pattern like "dir/**" or "dir/**/"
+            if lastPart.isEmpty {
+                results.append(firstPart)
+            }
+        }
+
+        if lastPart.isEmpty {
+            lastPart = "*"
+        }
+        for directory in directories {
+            let partiallyResolvedPattern = NSString(string: directory).appendingPathComponent(lastPart)
+            results.append(contentsOf: expandGlobstar(pattern: partiallyResolvedPattern))
+        }
+
+        return results
+    }
+
+    private func isDirectory(path: String) -> Bool {
+        if let isDirectory = isDirectoryCache[path] {
+            return isDirectory
+        }
+
+        var isDirectoryBool = ObjCBool(false)
+        let isDirectory = FileManager.default.fileExists(atPath: path, isDirectory: &isDirectoryBool) && isDirectoryBool.boolValue
+        isDirectoryCache[path] = isDirectory
+
+        return isDirectory
+    }
+
+    private func clearCaches() {
+        isDirectoryCache.removeAll()
+    }
+
+    private func populateFiles(gt: glob_t, includeFiles: Bool) {
+        let includeDirectories = behavior.includesDirectoriesInResults
+
+        for i in 0..<Int(gt.gl_matchc) {
+            if let path = String(validatingUTF8: gt.gl_pathv[i]!) {
+                if !includeFiles || !includeDirectories {
+                    let isDirectory = self.isDirectory(path: path)
+                    if (!includeFiles && !isDirectory) || (!includeDirectories && isDirectory) {
+                        continue
+                    }
+                }
+
+                paths.append(path)
+            }
+        }
+    }
+}
+
+private extension Sequence {
+    func array() -> [Element] {
+        return Array(self)
+    }
+}

Sources/XcodeGenKit/SourceGenerator.swift

@@ -303,7 +303,7 @@ class SourceGenerator {
 
         return Set(
             targetSource.excludes.map {
-                Path.glob("\(rootSourcePath)/\($0)")
+                return expandPattern("\(rootSourcePath)/\($0)")
                     .map {
                         guard $0.isDirectory else {
                             return [$0]
@@ -317,6 +317,21 @@ class SourceGenerator {
         )
     }
 
+    private func expandPattern(_ pattern: String) -> [Path] {
+        let filePaths = listFilePaths(pattern: pattern)
+        let urls = filePaths.map { Path($0) }
+
+        return urls
+    }
+
+    private func listFilePaths(pattern: String) -> [String] {
+        guard !pattern.isEmpty else {
+            return []
+        }
+
+        return Glob(pattern: pattern).paths
+    }
+
     /// Checks whether the path is not in any default or TargetSource excludes
     func isIncludedPath(_ path: Path) -> Bool {
         return !defaultExcludedFiles.contains(where: { path.lastComponent.contains($0) })

Tests/XcodeGenKitTests/GlobTests.swift

@@ -0,0 +1,279 @@
+//
+//  Created by Eric Firestone on 3/22/16.
+//  Copyright © 2016 Square, Inc. All rights reserved.
+//  Released under the Apache v2 License.
+//
+//  Adapted from https://gist.github.com/blakemerryman/76312e1cbf8aec248167
+//  Adapted from https://gist.github.com/efirestone/ce01ae109e08772647eb061b3bb387c3
+
+import XCTest
+@testable import XcodeGenKit
+
+class GlobTests : XCTestCase {
+
+    let tmpFiles = ["foo", "bar", "baz", "dir1/file1.ext", "dir1/dir2/dir3/file2.ext", "dir1/file1.extfoo"]
+    var tmpDir = ""
+
+    override func setUp() {
+        super.setUp()
+
+        self.tmpDir = newTmpDir()
+
+        let flags = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH
+        mkdir("\(tmpDir)/dir1/", flags)
+        mkdir("\(tmpDir)/dir1/dir2", flags)
+        mkdir("\(tmpDir)/dir1/dir2/dir3", flags)
+
+        for file in tmpFiles {
+            close(open("\(tmpDir)/\(file)", O_CREAT))
+        }
+    }
+
+    override func tearDown() {
+        for file in tmpFiles {
+            unlink("\(tmpDir)/\(file)")
+        }
+        rmdir("\(tmpDir)/dir1/dir2/dir3")
+        rmdir("\(tmpDir)/dir1/dir2")
+        rmdir("\(tmpDir)/dir1")
+        rmdir(self.tmpDir)
+
+        super.tearDown()
+    }
+
+    private func newTmpDir() -> String {
+        var tmpDirTmpl = "/tmp/glob-test.XXXXX".cString(using: .utf8)!
+        return String(validatingUTF8: mkdtemp(&tmpDirTmpl))!
+    }
+
+    func testBraces() {
+        let pattern = "\(tmpDir)/ba{r,y,z}"
+        let glob = Glob(pattern: pattern)
+        var contents = [String]()
+        for file in glob {
+            contents.append(file)
+        }
+        XCTAssertEqual(contents, ["\(tmpDir)/bar", "\(tmpDir)/baz"], "matching with braces failed")
+    }
+
+    func testNothingMatches() {
+        let pattern = "\(tmpDir)/nothing"
+        let glob = Glob(pattern: pattern)
+        var contents = [String]()
+        for file in glob {
+            contents.append(file)
+        }
+        XCTAssertEqual(contents, [], "expected empty list of files")
+    }
+
+    func testDirectAccess() {
+        let pattern = "\(tmpDir)/ba{r,y,z}"
+        let glob = Glob(pattern: pattern)
+        XCTAssertEqual(glob.paths, ["\(tmpDir)/bar", "\(tmpDir)/baz"], "matching with braces failed")
+    }
+
+    func testIterateTwice() {
+        let pattern = "\(tmpDir)/ba{r,y,z}"
+        let glob = Glob(pattern: pattern)
+        var contents1 = [String]()
+        var contents2 = [String]()
+        for file in glob {
+            contents1.append(file)
+        }
+        let filesAfterOnce = glob.paths
+        for file in glob {
+            contents2.append(file)
+        }
+        XCTAssertEqual(contents1, contents2, "results for calling for-in twice are the same")
+        XCTAssertEqual(glob.paths, filesAfterOnce, "calling for-in twice doesn't only memoizes once")
+    }
+
+    func testIndexing() {
+        let pattern = "\(tmpDir)/ba{r,y,z}"
+        let glob = Glob(pattern: pattern)
+        XCTAssertEqual(glob[0], "\(tmpDir)/bar", "indexing")
+    }
+
+    // MARK: - Globstar - Bash v3
+
+    func testGlobstarBashV3NoSlash() {
+        // Should be the equivalent of "ls -d -1 /(tmpdir)/**"
+        let pattern = "\(tmpDir)/**"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3)
+        XCTAssertEqual(glob.paths, ["\(tmpDir)/bar", "\(tmpDir)/baz", "\(tmpDir)/dir1/", "\(tmpDir)/foo"])
+    }
+
+    func testGlobstarBashV3WithSlash() {
+        // Should be the equivalent of "ls -d -1 /(tmpdir)/**/"
+        let pattern = "\(tmpDir)/**/"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3)
+        XCTAssertEqual(glob.paths, ["\(tmpDir)/dir1/"])
+    }
+
+    func testGlobstarBashV3WithSlashAndWildcard() {
+        // Should be the equivalent of "ls -d -1 /(tmpdir)/**/*"
+        let pattern = "\(tmpDir)/**/*"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3)
+        XCTAssertEqual(glob.paths, ["\(tmpDir)/dir1/dir2/", "\(tmpDir)/dir1/file1.ext", "\(tmpDir)/dir1/file1.extfoo"])
+    }
+
+    func testDoubleGlobstarBashV3() {
+        let pattern = "\(tmpDir)/**/dir2/**/*"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3)
+        XCTAssertEqual(glob.paths, ["\(tmpDir)/dir1/dir2/dir3/file2.ext"])
+    }
+
+    // MARK: - Globstar - Bash v4
+
+    func testGlobstarBashV4NoSlash() {
+        // Should be the equivalent of "ls -d -1 /(tmpdir)/**"
+        let pattern = "\(tmpDir)/**"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4)
+        XCTAssertEqual(glob.paths, [
+            "\(tmpDir)/",
+            "\(tmpDir)/bar",
+            "\(tmpDir)/baz",
+            "\(tmpDir)/dir1/",
+            "\(tmpDir)/dir1/dir2/",
+            "\(tmpDir)/dir1/dir2/dir3/",
+            "\(tmpDir)/dir1/dir2/dir3/file2.ext",
+            "\(tmpDir)/dir1/file1.ext",
+            "\(tmpDir)/dir1/file1.extfoo",
+            "\(tmpDir)/foo"
+        ])
+    }
+
+    func testGlobstarBashV4WithSlash() {
+        // Should be the equivalent of "ls -d -1 /(tmpdir)/**/"
+        let pattern = "\(tmpDir)/**/"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4)
+        XCTAssertEqual(glob.paths, [
+            "\(tmpDir)/",
+            "\(tmpDir)/dir1/",
+            "\(tmpDir)/dir1/dir2/",
+            "\(tmpDir)/dir1/dir2/dir3/",
+        ])
+    }
+
+    func testGlobstarBashV4WithSlashAndWildcard() {
+        // Should be the equivalent of "ls -d -1 /(tmpdir)/**/*"
+        let pattern = "\(tmpDir)/**/*"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4)
+        XCTAssertEqual(glob.paths, [
+            "\(tmpDir)/bar",
+            "\(tmpDir)/baz",
+            "\(tmpDir)/dir1/",
+            "\(tmpDir)/dir1/dir2/",
+            "\(tmpDir)/dir1/dir2/dir3/",
+            "\(tmpDir)/dir1/dir2/dir3/file2.ext",
+            "\(tmpDir)/dir1/file1.ext",
+            "\(tmpDir)/dir1/file1.extfoo",
+            "\(tmpDir)/foo",
+        ])
+    }
+
+    func testDoubleGlobstarBashV4() {
+        let pattern = "\(tmpDir)/**/dir2/**/*"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4)
+        XCTAssertEqual(glob.paths, [
+            "\(tmpDir)/dir1/dir2/dir3/",
+            "\(tmpDir)/dir1/dir2/dir3/file2.ext",
+        ])
+    }
+
+    func testDoubleGlobstarBashV4WithFileExtension() {
+        // Should be the equivalent of "ls -d -1 /(tmpdir)/**/*.ext"
+        // Should not find "\(tmpDir)/dir1/file1.extfoo" which the file extension prefix is .ext
+        let pattern = "\(tmpDir)/**/*.ext"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4)
+        XCTAssertEqual(glob.paths, [
+            "\(tmpDir)/dir1/dir2/dir3/file2.ext",
+            "\(tmpDir)/dir1/file1.ext"
+        ])
+    }
+
+    // MARK: - Globstar - Gradle
+
+    func testGlobstarGradleNoSlash() {
+        // Should be the equivalent of
+        // FileTree tree = project.fileTree((Object)'/tmp') {
+        //   include 'glob-test.7m0Lp/**'
+        // }
+        //
+        // Note that the sort order currently matches Bash and not Gradle
+        let pattern = "\(tmpDir)/**"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle)
+        XCTAssertEqual(glob.paths, [
+            "\(tmpDir)/bar",
+            "\(tmpDir)/baz",
+            "\(tmpDir)/dir1/dir2/dir3/file2.ext",
+            "\(tmpDir)/dir1/file1.ext",
+            "\(tmpDir)/dir1/file1.extfoo",
+            "\(tmpDir)/foo",
+        ])
+    }
+
+    func testGlobstarGradleWithSlash() {
+        // Should be the equivalent of
+        // FileTree tree = project.fileTree((Object)'/tmp') {
+        //   include 'glob-test.7m0Lp/**/'
+        // }
+        //
+        // Note that the sort order currently matches Bash and not Gradle
+        let pattern = "\(tmpDir)/**/"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle)
+        XCTAssertEqual(glob.paths, [
+            "\(tmpDir)/bar",
+            "\(tmpDir)/baz",
+            "\(tmpDir)/dir1/dir2/dir3/file2.ext",
+            "\(tmpDir)/dir1/file1.ext",
+            "\(tmpDir)/dir1/file1.extfoo",
+            "\(tmpDir)/foo",
+        ])
+    }
+
+    func testGlobstarGradleWithSlashAndWildcard() {
+        // Should be the equivalent of
+        // FileTree tree = project.fileTree((Object)'/tmp') {
+        //   include 'glob-test.7m0Lp/**/*'
+        // }
+        //
+        // Note that the sort order currently matches Bash and not Gradle
+        let pattern = "\(tmpDir)/**/*"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle)
+        XCTAssertEqual(glob.paths, [
+            "\(tmpDir)/bar",
+            "\(tmpDir)/baz",
+            "\(tmpDir)/dir1/dir2/dir3/file2.ext",
+            "\(tmpDir)/dir1/file1.ext",
+            "\(tmpDir)/dir1/file1.extfoo",
+            "\(tmpDir)/foo",
+        ])
+    }
+
+    func testDoubleGlobstarGradle() {
+        // Should be the equivalent of
+        // FileTree tree = project.fileTree((Object)'/tmp') {
+        //   include 'glob-test.7m0Lp/**/dir2/**/*'
+        // }
+        //
+        // Note that the sort order currently matches Bash and not Gradle
+        let pattern = "\(tmpDir)/**/dir2/**/*"
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle)
+        XCTAssertEqual(glob.paths, [
+            "\(tmpDir)/dir1/dir2/dir3/file2.ext",
+        ])
+    }
+
+    func testBlacklistedDirectories() {
+        let pattern = "\(tmpDir)/**/*"
+
+        let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle, blacklistedDirectories: ["dir1"])
+
+        XCTAssertEqual(glob.paths, [
+            "\(tmpDir)/bar",
+            "\(tmpDir)/baz",
+            "\(tmpDir)/foo",
+        ])
+    }
+}

Tests/XcodeGenKitTests/SourceGeneratorTests.swift

@@ -222,7 +222,9 @@ class SourceGeneratorTests: XCTestCase {
                     - B:
                       - b.swift
                       - b.ignored
+                      - b.alsoIgnored
                     - a.ignored
+                    - a.alsoIgnored
                   - B:
                     - b.swift
                   - D:
@@ -272,8 +274,8 @@ class SourceGeneratorTests: XCTestCase {
                     "*.ignored",
                     "*.xcodeproj",
                     "*.playground",
-                    // not supported
-                    // "**/*.ignored",
+                    "**/*.ignored",
+                    "A/B/**/*.alsoIgnored",
                 ]
 
                 let target = Target(name: "Test", type: .application, platform: .iOS, sources: [TargetSource(path: "Sources", excludes: excludes)])
@@ -283,6 +285,7 @@ class SourceGeneratorTests: XCTestCase {
                     let project = Project(basePath: directoryPath, name: "Test", targets: [target], options: options)
                     let pbxProj = try project.generatePbxProj()
                     try pbxProj.expectFile(paths: ["Sources", "A", "a.swift"])
+                    try pbxProj.expectFile(paths: ["Sources", "A", "a.alsoIgnored"])
                     try pbxProj.expectFile(paths: ["Sources", "D", "d.h"])
                     try pbxProj.expectFile(paths: ["Sources", "D", "d.m"])
                     try pbxProj.expectFile(paths: ["Sources", "E", "e.jpg"])
@@ -303,9 +306,9 @@ class SourceGeneratorTests: XCTestCase {
                     try pbxProj.expectFileMissing(paths: ["Sources", "ignore.file"])
                     try pbxProj.expectFileMissing(paths: ["Sources", "project.xcodeproj"])
                     try pbxProj.expectFileMissing(paths: ["Sources", "a.playground"])
-                    // not supported: "**/*.ignored"
-                    // try pbxProj.expectFileMissing(paths: ["Sources", "A", "a.ignored"])
-                    // try pbxProj.expectFileMissing(paths: ["Sources", "A", "B", "b.ignored"])
+                    try pbxProj.expectFileMissing(paths: ["Sources", "A", "a.ignored"])
+                    try pbxProj.expectFileMissing(paths: ["Sources", "A", "B", "b.ignored"])
+                    try pbxProj.expectFileMissing(paths: ["Sources", "A", "B", "b.alsoIgnored"])
                 }
 
                 try test(generateEmptyDirectories: false)