From bcdbad4a10749a1254a1356503e8aeb525f7d7ba Mon Sep 17 00:00:00 2001 From: Brian Clymer Date: Wed, 28 Aug 2019 13:39:05 -0500 Subject: [PATCH] Add recursive glob support. --- Sources/XcodeGenKit/Glob.swift | 227 ++++++++++++++ Sources/XcodeGenKit/SourceGenerator.swift | 17 +- Tests/XcodeGenKitTests/GlobTests.swift | 279 ++++++++++++++++++ .../SourceGeneratorTests.swift | 13 +- 4 files changed, 530 insertions(+), 6 deletions(-) create mode 100644 Sources/XcodeGenKit/Glob.swift create mode 100644 Tests/XcodeGenKitTests/GlobTests.swift diff --git a/Sources/XcodeGenKit/Glob.swift b/Sources/XcodeGenKit/Glob.swift new file mode 100644 index 00000000..23db65d9 --- /dev/null +++ b/Sources/XcodeGenKit/Glob.swift @@ -0,0 +1,227 @@ +// +// Created by Eric Firestone on 3/22/16. +// Copyright © 2016 Square, Inc. All rights reserved. +// Released under the Apache v2 License. +// +// Adapted from https://gist.github.com/blakemerryman/76312e1cbf8aec248167 +// Adapted from https://gist.github.com/efirestone/ce01ae109e08772647eb061b3bb387c3 + + +import Foundation + + +public let GlobBehaviorBashV3 = Glob.Behavior( + supportsGlobstar: false, + includesFilesFromRootOfGlobstar: false, + includesDirectoriesInResults: true, + includesFilesInResultsIfTrailingSlash: false +) +public let GlobBehaviorBashV4 = Glob.Behavior( + supportsGlobstar: true, // Matches Bash v4 with "shopt -s globstar" option + includesFilesFromRootOfGlobstar: true, + includesDirectoriesInResults: true, + includesFilesInResultsIfTrailingSlash: false +) +public let GlobBehaviorGradle = Glob.Behavior( + supportsGlobstar: true, + includesFilesFromRootOfGlobstar: true, + includesDirectoriesInResults: false, + includesFilesInResultsIfTrailingSlash: true +) + + +/** + Finds files on the file system using pattern matching. + */ +public class Glob: Collection { + + /** + * Different glob implementations have different behaviors, so the behavior of this + * implementation is customizable. + */ + public struct Behavior { + // If true then a globstar ("**") causes matching to be done recursively in subdirectories. + // If false then "**" is treated the same as "*" + let supportsGlobstar: Bool + + // If true the results from the directory where the globstar is declared will be included as well. + // For example, with the pattern "dir/**/*.ext" the fie "dir/file.ext" would be included if this + // property is true, and would be omitted if it's false. + let includesFilesFromRootOfGlobstar: Bool + + // If false then the results will not include directory entries. This does not affect recursion depth. + let includesDirectoriesInResults: Bool + + // If false and the last characters of the pattern are "**/" then only directories are returned in the results. + let includesFilesInResultsIfTrailingSlash: Bool + } + + public static var defaultBehavior = GlobBehaviorBashV4 + + public static let defaultBlacklistedDirectories = ["node_modules", "Pods"] + + private var isDirectoryCache = [String: Bool]() + + public let behavior: Behavior + public let blacklistedDirectories: [String] + var paths = [String]() + public var startIndex: Int { return paths.startIndex } + public var endIndex: Int { return paths.endIndex } + + /// Initialize a glob + /// + /// - Parameters: + /// - pattern: The pattern to use when building the list of matching directories. + /// - behavior: See individual descriptions on `Glob.Behavior` values. + /// - blacklistedDirectories: An array of directories to ignore at the root level of the project. + public init(pattern: String, behavior: Behavior = Glob.defaultBehavior, blacklistedDirectories: [String] = defaultBlacklistedDirectories) { + + self.behavior = behavior + self.blacklistedDirectories = blacklistedDirectories + + var adjustedPattern = pattern + let hasTrailingGlobstarSlash = pattern.hasSuffix("**/") + var includeFiles = !hasTrailingGlobstarSlash + + if behavior.includesFilesInResultsIfTrailingSlash { + includeFiles = true + if hasTrailingGlobstarSlash { + // Grab the files too. + adjustedPattern += "*" + } + } + + let patterns = behavior.supportsGlobstar ? expandGlobstar(pattern: adjustedPattern) : [adjustedPattern] + + for pattern in patterns { + var gt = glob_t() + if executeGlob(pattern: pattern, gt: >) { + populateFiles(gt: gt, includeFiles: includeFiles) + } + + globfree(>) + } + + paths = Array(Set(paths)).sorted { lhs, rhs in + lhs.compare(rhs) != ComparisonResult.orderedDescending + } + + clearCaches() + } + + // MARK: Subscript Support + + public subscript(i: Int) -> String { + return paths[i] + } + + // MARK: Protocol of IndexableBase + + public func index(after i: Glob.Index) -> Glob.Index { + return i + 1 + } + + // MARK: Private + + private var globalFlags = GLOB_TILDE | GLOB_BRACE | GLOB_MARK + + private func executeGlob(pattern: UnsafePointer, gt: UnsafeMutablePointer) -> Bool { + return 0 == glob(pattern, globalFlags, nil, gt) + } + + private func expandGlobstar(pattern: String) -> [String] { + guard pattern.contains("**") else { + return [pattern] + } + + var results = [String]() + var parts = pattern.components(separatedBy: "**") + let firstPart = parts.removeFirst() + var lastPart = parts.joined(separator: "**") + + let fileManager = FileManager.default + + var directories: [String] + + do { + directories = try fileManager.contentsOfDirectory(atPath: firstPart).compactMap { subpath -> [String]? in + if blacklistedDirectories.contains(subpath) { + return nil + } + let firstLevelPath = NSString(string: firstPart).appendingPathComponent(subpath) + if isDirectory(path: firstLevelPath) { + var subDirs: [String] = try fileManager.subpathsOfDirectory(atPath: firstLevelPath).compactMap { subpath -> String? in + let fullPath = NSString(string: firstLevelPath).appendingPathComponent(subpath) + return isDirectory(path: fullPath) ? fullPath : nil + } + subDirs.append(firstLevelPath) + return subDirs + } else { + return nil + } + }.joined().array() + } catch { + directories = [] + print("Error parsing file system item: \(error)") + } + + if behavior.includesFilesFromRootOfGlobstar { + // Check the base directory for the glob star as well. + directories.insert(firstPart, at: 0) + + // Include the globstar root directory ("dir/") in a pattern like "dir/**" or "dir/**/" + if lastPart.isEmpty { + results.append(firstPart) + } + } + + if lastPart.isEmpty { + lastPart = "*" + } + for directory in directories { + let partiallyResolvedPattern = NSString(string: directory).appendingPathComponent(lastPart) + results.append(contentsOf: expandGlobstar(pattern: partiallyResolvedPattern)) + } + + return results + } + + private func isDirectory(path: String) -> Bool { + if let isDirectory = isDirectoryCache[path] { + return isDirectory + } + + var isDirectoryBool = ObjCBool(false) + let isDirectory = FileManager.default.fileExists(atPath: path, isDirectory: &isDirectoryBool) && isDirectoryBool.boolValue + isDirectoryCache[path] = isDirectory + + return isDirectory + } + + private func clearCaches() { + isDirectoryCache.removeAll() + } + + private func populateFiles(gt: glob_t, includeFiles: Bool) { + let includeDirectories = behavior.includesDirectoriesInResults + + for i in 0.. [Element] { + return Array(self) + } +} diff --git a/Sources/XcodeGenKit/SourceGenerator.swift b/Sources/XcodeGenKit/SourceGenerator.swift index 2ab9a4a4..1d58a5ae 100644 --- a/Sources/XcodeGenKit/SourceGenerator.swift +++ b/Sources/XcodeGenKit/SourceGenerator.swift @@ -303,7 +303,7 @@ class SourceGenerator { return Set( targetSource.excludes.map { - Path.glob("\(rootSourcePath)/\($0)") + return expandPattern("\(rootSourcePath)/\($0)") .map { guard $0.isDirectory else { return [$0] @@ -317,6 +317,21 @@ class SourceGenerator { ) } + private func expandPattern(_ pattern: String) -> [Path] { + let filePaths = listFilePaths(pattern: pattern) + let urls = filePaths.map { Path($0) } + + return urls + } + + private func listFilePaths(pattern: String) -> [String] { + guard !pattern.isEmpty else { + return [] + } + + return Glob(pattern: pattern).paths + } + /// Checks whether the path is not in any default or TargetSource excludes func isIncludedPath(_ path: Path) -> Bool { return !defaultExcludedFiles.contains(where: { path.lastComponent.contains($0) }) diff --git a/Tests/XcodeGenKitTests/GlobTests.swift b/Tests/XcodeGenKitTests/GlobTests.swift new file mode 100644 index 00000000..4fb6a70d --- /dev/null +++ b/Tests/XcodeGenKitTests/GlobTests.swift @@ -0,0 +1,279 @@ +// +// Created by Eric Firestone on 3/22/16. +// Copyright © 2016 Square, Inc. All rights reserved. +// Released under the Apache v2 License. +// +// Adapted from https://gist.github.com/blakemerryman/76312e1cbf8aec248167 +// Adapted from https://gist.github.com/efirestone/ce01ae109e08772647eb061b3bb387c3 + +import XCTest +@testable import XcodeGenKit + +class GlobTests : XCTestCase { + + let tmpFiles = ["foo", "bar", "baz", "dir1/file1.ext", "dir1/dir2/dir3/file2.ext", "dir1/file1.extfoo"] + var tmpDir = "" + + override func setUp() { + super.setUp() + + self.tmpDir = newTmpDir() + + let flags = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH + mkdir("\(tmpDir)/dir1/", flags) + mkdir("\(tmpDir)/dir1/dir2", flags) + mkdir("\(tmpDir)/dir1/dir2/dir3", flags) + + for file in tmpFiles { + close(open("\(tmpDir)/\(file)", O_CREAT)) + } + } + + override func tearDown() { + for file in tmpFiles { + unlink("\(tmpDir)/\(file)") + } + rmdir("\(tmpDir)/dir1/dir2/dir3") + rmdir("\(tmpDir)/dir1/dir2") + rmdir("\(tmpDir)/dir1") + rmdir(self.tmpDir) + + super.tearDown() + } + + private func newTmpDir() -> String { + var tmpDirTmpl = "/tmp/glob-test.XXXXX".cString(using: .utf8)! + return String(validatingUTF8: mkdtemp(&tmpDirTmpl))! + } + + func testBraces() { + let pattern = "\(tmpDir)/ba{r,y,z}" + let glob = Glob(pattern: pattern) + var contents = [String]() + for file in glob { + contents.append(file) + } + XCTAssertEqual(contents, ["\(tmpDir)/bar", "\(tmpDir)/baz"], "matching with braces failed") + } + + func testNothingMatches() { + let pattern = "\(tmpDir)/nothing" + let glob = Glob(pattern: pattern) + var contents = [String]() + for file in glob { + contents.append(file) + } + XCTAssertEqual(contents, [], "expected empty list of files") + } + + func testDirectAccess() { + let pattern = "\(tmpDir)/ba{r,y,z}" + let glob = Glob(pattern: pattern) + XCTAssertEqual(glob.paths, ["\(tmpDir)/bar", "\(tmpDir)/baz"], "matching with braces failed") + } + + func testIterateTwice() { + let pattern = "\(tmpDir)/ba{r,y,z}" + let glob = Glob(pattern: pattern) + var contents1 = [String]() + var contents2 = [String]() + for file in glob { + contents1.append(file) + } + let filesAfterOnce = glob.paths + for file in glob { + contents2.append(file) + } + XCTAssertEqual(contents1, contents2, "results for calling for-in twice are the same") + XCTAssertEqual(glob.paths, filesAfterOnce, "calling for-in twice doesn't only memoizes once") + } + + func testIndexing() { + let pattern = "\(tmpDir)/ba{r,y,z}" + let glob = Glob(pattern: pattern) + XCTAssertEqual(glob[0], "\(tmpDir)/bar", "indexing") + } + + // MARK: - Globstar - Bash v3 + + func testGlobstarBashV3NoSlash() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**" + let pattern = "\(tmpDir)/**" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3) + XCTAssertEqual(glob.paths, ["\(tmpDir)/bar", "\(tmpDir)/baz", "\(tmpDir)/dir1/", "\(tmpDir)/foo"]) + } + + func testGlobstarBashV3WithSlash() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**/" + let pattern = "\(tmpDir)/**/" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3) + XCTAssertEqual(glob.paths, ["\(tmpDir)/dir1/"]) + } + + func testGlobstarBashV3WithSlashAndWildcard() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**/*" + let pattern = "\(tmpDir)/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3) + XCTAssertEqual(glob.paths, ["\(tmpDir)/dir1/dir2/", "\(tmpDir)/dir1/file1.ext", "\(tmpDir)/dir1/file1.extfoo"]) + } + + func testDoubleGlobstarBashV3() { + let pattern = "\(tmpDir)/**/dir2/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV3) + XCTAssertEqual(glob.paths, ["\(tmpDir)/dir1/dir2/dir3/file2.ext"]) + } + + // MARK: - Globstar - Bash v4 + + func testGlobstarBashV4NoSlash() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**" + let pattern = "\(tmpDir)/**" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/", + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/dir1/", + "\(tmpDir)/dir1/dir2/", + "\(tmpDir)/dir1/dir2/dir3/", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext", + "\(tmpDir)/dir1/file1.extfoo", + "\(tmpDir)/foo" + ]) + } + + func testGlobstarBashV4WithSlash() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**/" + let pattern = "\(tmpDir)/**/" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/", + "\(tmpDir)/dir1/", + "\(tmpDir)/dir1/dir2/", + "\(tmpDir)/dir1/dir2/dir3/", + ]) + } + + func testGlobstarBashV4WithSlashAndWildcard() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**/*" + let pattern = "\(tmpDir)/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/dir1/", + "\(tmpDir)/dir1/dir2/", + "\(tmpDir)/dir1/dir2/dir3/", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext", + "\(tmpDir)/dir1/file1.extfoo", + "\(tmpDir)/foo", + ]) + } + + func testDoubleGlobstarBashV4() { + let pattern = "\(tmpDir)/**/dir2/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/dir1/dir2/dir3/", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + ]) + } + + func testDoubleGlobstarBashV4WithFileExtension() { + // Should be the equivalent of "ls -d -1 /(tmpdir)/**/*.ext" + // Should not find "\(tmpDir)/dir1/file1.extfoo" which the file extension prefix is .ext + let pattern = "\(tmpDir)/**/*.ext" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorBashV4) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext" + ]) + } + + // MARK: - Globstar - Gradle + + func testGlobstarGradleNoSlash() { + // Should be the equivalent of + // FileTree tree = project.fileTree((Object)'/tmp') { + // include 'glob-test.7m0Lp/**' + // } + // + // Note that the sort order currently matches Bash and not Gradle + let pattern = "\(tmpDir)/**" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext", + "\(tmpDir)/dir1/file1.extfoo", + "\(tmpDir)/foo", + ]) + } + + func testGlobstarGradleWithSlash() { + // Should be the equivalent of + // FileTree tree = project.fileTree((Object)'/tmp') { + // include 'glob-test.7m0Lp/**/' + // } + // + // Note that the sort order currently matches Bash and not Gradle + let pattern = "\(tmpDir)/**/" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext", + "\(tmpDir)/dir1/file1.extfoo", + "\(tmpDir)/foo", + ]) + } + + func testGlobstarGradleWithSlashAndWildcard() { + // Should be the equivalent of + // FileTree tree = project.fileTree((Object)'/tmp') { + // include 'glob-test.7m0Lp/**/*' + // } + // + // Note that the sort order currently matches Bash and not Gradle + let pattern = "\(tmpDir)/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + "\(tmpDir)/dir1/file1.ext", + "\(tmpDir)/dir1/file1.extfoo", + "\(tmpDir)/foo", + ]) + } + + func testDoubleGlobstarGradle() { + // Should be the equivalent of + // FileTree tree = project.fileTree((Object)'/tmp') { + // include 'glob-test.7m0Lp/**/dir2/**/*' + // } + // + // Note that the sort order currently matches Bash and not Gradle + let pattern = "\(tmpDir)/**/dir2/**/*" + let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle) + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/dir1/dir2/dir3/file2.ext", + ]) + } + + func testBlacklistedDirectories() { + let pattern = "\(tmpDir)/**/*" + + let glob = Glob(pattern: pattern, behavior: GlobBehaviorGradle, blacklistedDirectories: ["dir1"]) + + XCTAssertEqual(glob.paths, [ + "\(tmpDir)/bar", + "\(tmpDir)/baz", + "\(tmpDir)/foo", + ]) + } +} diff --git a/Tests/XcodeGenKitTests/SourceGeneratorTests.swift b/Tests/XcodeGenKitTests/SourceGeneratorTests.swift index bac0e043..5faf8305 100644 --- a/Tests/XcodeGenKitTests/SourceGeneratorTests.swift +++ b/Tests/XcodeGenKitTests/SourceGeneratorTests.swift @@ -222,7 +222,9 @@ class SourceGeneratorTests: XCTestCase { - B: - b.swift - b.ignored + - b.alsoIgnored - a.ignored + - a.alsoIgnored - B: - b.swift - D: @@ -272,8 +274,8 @@ class SourceGeneratorTests: XCTestCase { "*.ignored", "*.xcodeproj", "*.playground", - // not supported - // "**/*.ignored", + "**/*.ignored", + "A/B/**/*.alsoIgnored", ] let target = Target(name: "Test", type: .application, platform: .iOS, sources: [TargetSource(path: "Sources", excludes: excludes)]) @@ -283,6 +285,7 @@ class SourceGeneratorTests: XCTestCase { let project = Project(basePath: directoryPath, name: "Test", targets: [target], options: options) let pbxProj = try project.generatePbxProj() try pbxProj.expectFile(paths: ["Sources", "A", "a.swift"]) + try pbxProj.expectFile(paths: ["Sources", "A", "a.alsoIgnored"]) try pbxProj.expectFile(paths: ["Sources", "D", "d.h"]) try pbxProj.expectFile(paths: ["Sources", "D", "d.m"]) try pbxProj.expectFile(paths: ["Sources", "E", "e.jpg"]) @@ -303,9 +306,9 @@ class SourceGeneratorTests: XCTestCase { try pbxProj.expectFileMissing(paths: ["Sources", "ignore.file"]) try pbxProj.expectFileMissing(paths: ["Sources", "project.xcodeproj"]) try pbxProj.expectFileMissing(paths: ["Sources", "a.playground"]) - // not supported: "**/*.ignored" - // try pbxProj.expectFileMissing(paths: ["Sources", "A", "a.ignored"]) - // try pbxProj.expectFileMissing(paths: ["Sources", "A", "B", "b.ignored"]) + try pbxProj.expectFileMissing(paths: ["Sources", "A", "a.ignored"]) + try pbxProj.expectFileMissing(paths: ["Sources", "A", "B", "b.ignored"]) + try pbxProj.expectFileMissing(paths: ["Sources", "A", "B", "b.alsoIgnored"]) } try test(generateEmptyDirectories: false)