Merge pull request #151 from MobileNativeFoundation/issues_limit

Add a flag to truncate number of issues in individual tasks

README.md

@@ -126,7 +126,7 @@ An example output has been omitted for brevity since it can contain a lot of inf
 
 Parses the build information from a `xcactivitylog` and converts it into different representations such as a [JSON file](#JSON-Reporter), [flat JSON file](#FlatJson-Reporter), [summary JSON file](#SummaryJson-Reporter), [issues JSON file](#Issues-Reporter), [Chrome Tracer file](#ChromeTracer-Reporter) or a static [HTML page](#HTML-Reporter).
 
-This command supports parsing additional data if some flags are passed to Xcode/xcodebuild: 
+This command supports parsing additional data if some flags are passed to Xcode/xcodebuild:
 
 1. `swiftc` reported compilation times. For using that feature, you need to build your project with the options `-Xfrontend -debug-time-expression-type-checking` and `-Xfrontend -debug-time-function-bodies`.
 2. ld64's statistics output. The statistics info can be generated by adding `-Xlinker -print_statistics` to Xcode's "Other Linker Flags" and it's useful for tracking linking time regression.
@@ -161,6 +161,7 @@ Example output available in the [reporters](#reporters) section.
   | `--machine_name`  | If specified, the machine name will be used to create the `buildIdentifier`. If it is not specified, the host name will be used.  | No |
   | `--omit_warnings` | Omit the warnings details in the final report. This is useful if there are too many of them and the report's size is too big with them. | No |
   | `--omit_notes` | Omit the notes details in the final report. This is useful if there are too many of them and the report's size is too big with them. | No |
+  | `--trunc_large_issues` | If an individual task has more than a 100 issues (Warnings, notes, errors) it truncates them to be 100. This is useful to reduce the amount of memory used. | No |
 
   >No *: One of `--file`, `--project`, `--workspace`, `--xcodeproj` parameters is required.
 

Sources/XCLogParser/commands/ActionOptions.swift

@@ -52,14 +52,19 @@ public struct ActionOptions {
     /// If true, the parse command won't add the Notes details to the output.
     public let omitNotesDetails: Bool
 
+    /// If true, tasks with more than a 100 issues (warnings, errors, notes) will be truncated to a 100
+    public let truncLargeIssues: Bool
+
     public init(reporter: Reporter,
                 outputPath: String,
                 redacted: Bool,
                 withoutBuildSpecificInformation: Bool,
                 machineName: String? = nil,
                 rootOutput: String = "",
                 omitWarningsDetails: Bool = false,
-                omitNotesDetails: Bool = false) {
+                omitNotesDetails: Bool = false,
+                truncLargeIssues: Bool = false
+    ) {
         self.reporter = reporter
         self.outputPath = outputPath
         self.redacted = redacted
@@ -68,5 +73,6 @@ public struct ActionOptions {
         self.rootOutput = rootOutput
         self.omitWarningsDetails = omitWarningsDetails
         self.omitNotesDetails = omitNotesDetails
+        self.truncLargeIssues = truncLargeIssues
     }
 }

Sources/XCLogParser/commands/CommandHandler.swift

@@ -65,7 +65,8 @@ public struct CommandHandler {
 
         let buildParser = ParserBuildSteps(machineName: options.machineName,
                                            omitWarningsDetails: options.omitWarningsDetails,
-                                           omitNotesDetails: options.omitNotesDetails)
+                                           omitNotesDetails: options.omitNotesDetails,
+                                           truncLargeIssues: options.truncLargeIssues)
         let buildSteps = try buildParser.parse(activityLog: activityLog)
         let reporterOutput = ReporterOutputFactory.makeReporterOutput(path: options.outputPath)
         let logReporter = options.reporter.makeLogReporter()

Sources/XCLogParser/commands/Version.swift

@@ -21,6 +21,6 @@ import Foundation
 
 public struct Version {
 
-    public static let current = "0.2.30"
+    public static let current = "0.2.31"
 
 }

Sources/XCLogParser/parser/IDEActivityLogSection+Builders.swift

@@ -0,0 +1,46 @@
+// Copyright (c) 2021 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+
+extension IDEActivityLogSection {
+
+    func with(messages newMessages: [IDEActivityLogMessage]) -> IDEActivityLogSection {
+        return IDEActivityLogSection(sectionType: self.sectionType,
+                                     domainType: self.domainType,
+                                     title: self.title,
+                                     signature: self.signature,
+                                     timeStartedRecording: self.timeStartedRecording,
+                                     timeStoppedRecording: self.timeStoppedRecording,
+                                     subSections: self.subSections,
+                                     text: self.text,
+                                     messages: newMessages,
+                                     wasCancelled: self.wasCancelled,
+                                     isQuiet: self.isQuiet,
+                                     wasFetchedFromCache: self.wasFetchedFromCache,
+                                     subtitle: self.subtitle,
+                                     location: self.location,
+                                     commandDetailDesc: self.commandDetailDesc,
+                                     uniqueIdentifier: self.uniqueIdentifier,
+                                     localizedResultString: self.localizedResultString,
+                                     xcbuildSignature: self.xcbuildSignature,
+                                     unknown: self.unknown)
+    }
+
+}

Sources/XCLogParser/parser/Notice+Parser.swift

@@ -27,9 +27,16 @@ extension Notice {
     /// For CLANG warnings, it parses the `IDEActivityLogSection` text property looking for a *-W-warning-name* pattern
     /// - parameter logSection: An `IDEActivityLogSection`
     /// - parameter forType: The `DetailStepType` of the logSection
+    /// - parameter truncLargeIssues: If true, if a task have more than 100 `Notice`, will be truncated to 100
     /// - returns: An Array of `Notice`
-    public static func parseFromLogSection(_ logSection: IDEActivityLogSection, forType type: DetailStepType)
+    public static func parseFromLogSection(_ logSection: IDEActivityLogSection,
+                                           forType type: DetailStepType,
+                                           truncLargeIssues: Bool)
         -> [Notice] {
+        var logSection = logSection
+        if truncLargeIssues && logSection.messages.count > 100 {
+            logSection = self.logSectionWithTruncatedIssues(logSection: logSection)
+        }
         // we look for clangWarnings parsing the text of the logSection
         let clangWarningsFlags = self.parseClangWarningFlags(text: logSection.text)
         let clangWarnings = self.parseClangWarnings(clangFlags: clangWarningsFlags, logSection: logSection)
@@ -187,4 +194,28 @@ extension Notice {
         }
         return false
     }
+
+    private static func logSectionWithTruncatedIssues(logSection: IDEActivityLogSection) -> IDEActivityLogSection {
+        let issuesKept = min(99, logSection.messages.count)
+        var truncatedMessages = Array(logSection.messages[0..<issuesKept])
+        truncatedMessages.append(getTruncatedIssuesWarning(logSection: logSection, issuesKept: issuesKept))
+        return logSection.with(messages: truncatedMessages)
+    }
+
+    private static func getTruncatedIssuesWarning(logSection: IDEActivityLogSection, issuesKept: Int)
+    -> IDEActivityLogMessage {
+        let title = "Warning: \(logSection.messages.count - issuesKept) issues were truncated"
+        return IDEActivityLogMessage(title: title,
+                                     shortTitle: "",
+                                     timeEmitted: 0,
+                                     rangeEndInSectionText: 0,
+                                     rangeStartInSectionText: 0,
+                                     subMessages: [],
+                                     severity: 0,
+                                     type: "",
+                                     location: DVTDocumentLocation(documentURLString: "", timestamp: 0),
+                                     categoryIdent: "Warning",
+                                     secondaryLocations: [],
+                                     additionalDescription: "")
+    }
 }

Sources/XCLogParser/parser/ParserBuildSteps.swift

@@ -42,6 +42,10 @@ public final class ParserBuildSteps {
     /// Usefult to save space.
     let omitNotesDetails: Bool
 
+    /// If true, tasks with more than a 100 issues will be
+    /// truncated to have only 100
+    let truncLargeIssues: Bool
+
     public lazy var dateFormatter: DateFormatter = {
         let formatter = DateFormatter()
         formatter.timeZone = TimeZone(abbreviation: "UTC")
@@ -79,16 +83,19 @@ public final class ParserBuildSteps {
     /// for the log. If `nil`, the host name will be used instead.
     /// - parameter omitWarningsDetails: if true, the Warnings won't be parsed
     /// - parameter omitNotesDetails: if true, the Notes won't be parsed
+    /// - parameter truncLargeIssues: if true, tasks with more than a 100 issues will be truncated to have a 100
     public init(machineName: String? = nil,
                 omitWarningsDetails: Bool,
-                omitNotesDetails: Bool) {
+                omitNotesDetails: Bool,
+                truncLargeIssues: Bool) {
         if let machineName = machineName {
             self.machineName = machineName
         } else {
             self.machineName = MacOSMachineNameReader().machineName ?? "unknown"
         }
         self.omitWarningsDetails = omitWarningsDetails
         self.omitNotesDetails = omitNotesDetails
+        self.truncLargeIssues = truncLargeIssues
     }
 
     /// Parses the content from an Xcode log into a `BuildStep`
@@ -298,7 +305,7 @@ public final class ParserBuildSteps {
 
     private func parseWarningsAndErrorsFromLogSection(_ logSection: IDEActivityLogSection, forType type: DetailStepType)
         -> [String: [Notice]]? {
-        let notices = Notice.parseFromLogSection(logSection, forType: type)
+        let notices = Notice.parseFromLogSection(logSection, forType: type, truncLargeIssues: truncLargeIssues)
         return ["warnings": notices.getWarnings(),
                 "errors": notices.getErrors(),
                 "notes": notices.getNotes()]

Sources/XCLogParserApp/commands/ParseCommand.swift

@@ -121,6 +121,14 @@ struct ParseCommand: ParsableCommand {
     """)
     var omitNotes: Bool = false
 
+    @Flag(name: .customLong("trunc_large_issues"),
+          help: """
+    If present, for tasks with more than a 100 issues (warnings, notes or errors)
+    Those will be truncated to a 100.
+    Useful to reduce the amount of memory used and the size of the report.
+    """)
+    var truncLargeIssues: Bool = false
+
     mutating func validate() throws {
         if !hasValidLogOptions() {
             throw ValidationError("""
@@ -163,7 +171,8 @@ struct ParseCommand: ParsableCommand {
                                           machineName: machineName,
                                           rootOutput: rootOutput ?? "",
                                           omitWarningsDetails: omitWarnings,
-                                          omitNotesDetails: omitNotes)
+                                          omitNotesDetails: omitNotes,
+                                          truncLargeIssues: truncLargeIssues)
         let action = Action.parse(options: actionOptions)
         let command = Command(logOptions: logOptions, action: action)
         try commandHandler.handle(command: command)

Tests/XCLogParserTests/ParserTests.swift

@@ -24,7 +24,8 @@ import XCTest
 class ParserTests: XCTestCase {
 
     let parser = ParserBuildSteps(omitWarningsDetails: false,
-                                  omitNotesDetails: false)
+                                  omitNotesDetails: false,
+                                  truncLargeIssues: false)
 
     func testDateFormatterUsesJSONFormat() {
         let jsonDateString = "2014-09-27T12:30:00.450000Z"
@@ -42,7 +43,8 @@ class ParserTests: XCTestCase {
         let timestamp = Date().timeIntervalSinceNow
         let parser = ParserBuildSteps(machineName: machineName,
                                       omitWarningsDetails: false,
-                                      omitNotesDetails: false)
+                                      omitNotesDetails: false,
+                                      truncLargeIssues: false)
         let fakeMainSection = IDEActivityLogSection(sectionType: 1,
                                                     domainType: "",
                                                     title: "Main",
@@ -70,7 +72,8 @@ class ParserTests: XCTestCase {
         if let hostName = Host.current().localizedName {
             let parserNoMachineName = ParserBuildSteps(machineName: nil,
                                                        omitWarningsDetails: false,
-                                                       omitNotesDetails: false)
+                                                       omitNotesDetails: false,
+                                                       truncLargeIssues: false)
             let buildStepNoMachineName = try parserNoMachineName.parse(activityLog: fakeActivityLog)
             XCTAssertEqual("\(hostName)_\(uniqueIdentifier)", buildStepNoMachineName.buildIdentifier)
         }
@@ -449,7 +452,8 @@ note: use 'updatedDoSomething' instead\r doSomething()\r        ^~~~~~~~~~~\r
                                                        andText: "This is deprecated, [-Wdeprecated-declarations]",
                                                        loc: textDocumentLocation)
         let parser = ParserBuildSteps(omitWarningsDetails: true,
-                                      omitNotesDetails: false)
+                                      omitNotesDetails: false,
+                                      truncLargeIssues: false)
         let build = try parser.parse(activityLog: fakeLog)
         XCTAssertEqual(0, build.warnings?.count ?? 0, "Warnings should be empty")
         XCTAssertEqual(1, build.warningCount, "Number of warnings should be reported")
@@ -484,11 +488,47 @@ note: use 'updatedDoSomething' instead\r doSomething()\r        ^~~~~~~~~~~\r
                                                        andText: "Log",
                                                        loc: textDocumentLocation)
         let parser = ParserBuildSteps(omitWarningsDetails: false,
-                                      omitNotesDetails: true)
+                                      omitNotesDetails: true,
+                                      truncLargeIssues: false)
         let build = try parser.parse(activityLog: fakeLog)
         XCTAssertEqual(0, build.notes?.count ?? 0, "Notes should be empty")
     }
 
+    func testParseTruncateLargeIssues() throws {
+        let timestamp = Date().timeIntervalSinceReferenceDate
+        let textDocumentLocation = DVTTextDocumentLocation(documentURLString: "file://project/file.swift",
+                                                           timestamp: timestamp,
+                                                           startingLineNumber: 10,
+                                                           startingColumnNumber: 11,
+                                                           endingLineNumber: 12,
+                                                           endingColumnNumber: 13,
+                                                           characterRangeEnd: 14,
+                                                           characterRangeStart: 15,
+                                                           locationEncoding: 16)
+        let aThousandWarnings = (0...999).map { _ in
+            IDEActivityLogMessage(title: "Swift Compiler Warning",
+                                  shortTitle: "",
+                                  timeEmitted: timestamp,
+                                  rangeEndInSectionText: 18446744073709551615,
+                                  rangeStartInSectionText: 0,
+                                  subMessages: [],
+                                  severity: 1,
+                                  type: "com.apple.dt.IDE.diagnostic",
+                                  location: textDocumentLocation,
+                                  categoryIdent: "",
+                                  secondaryLocations: [],
+                                  additionalDescription: "")
+        }
+        let fakeLog = getFakeIDEActivityLogWithMessages(aThousandWarnings,
+                                                       andText: "Swift Compiler Warning",
+                                                       loc: textDocumentLocation)
+        let parser = ParserBuildSteps(omitWarningsDetails: false,
+                                      omitNotesDetails: false,
+                                      truncLargeIssues: true)
+        let build = try parser.parse(activityLog: fakeLog)
+        XCTAssertEqual(100, build.warnings?.count ?? 0, "Warnings should be truncated up to 100")
+    }
+
     // swiftlint:disable line_length
     let commandDetailSwiftSteps = """
 CompileSwift normal x86_64 (in target 'Alamofire' from project 'Pods')