Merge pull request #4796 from bruntib/checker-coverage-cli

[feat] Adding Checker coverage statistics to CLI

Author: bruntib

web/client/codechecker_client/cmd_line_client.py

@@ -1536,6 +1536,13 @@ def get_statistics(
     def checker_count(checker_dict, key):
         return checker_dict.get(key, 0)
 
+    def formatted_guidelines(guideline_rules: Iterable[dict]) -> Optional[str]:
+        return ";  ".join(
+            f"{guideline_rule['guideline']}: "
+            f"{', '.join(guideline_rule['rules'])}"
+            for guideline_rule in guideline_rules
+        ) if guideline_rules else None
+
     client = setup_client(args.product_url)
 
     if 'component' in args:
@@ -1587,6 +1594,44 @@ def checker_count(checker_dict, key):
         })
         severity_total += count
 
+    # Get checker coverage statistic
+    checker_details = client.getCheckerStatusVerificationDetails(
+        run_ids, all_checkers_report_filter)
+
+    checkers = [ttypes.Checker(stat.analyzerName, stat.checkerName)
+                for stat in checker_details.values()]
+    checker_labels = client.getCheckerLabels(checkers)
+
+    all_guideline_rules = []
+    for labels in checker_labels:
+        guideline_entries = []
+        guidelines = [label.split("guideline:")[1]
+                      for label in labels if label.startswith("guideline")]
+
+        for guideline in guidelines:
+            guideline_entries.append({
+                "guideline": guideline,
+                "rules": [label.split(f"{guideline}:")[1]
+                          for label in labels
+                          if label.startswith(f"{guideline}:")]
+            })
+        all_guideline_rules.append(guideline_entries)
+
+    for checker_id, guideline_rules in zip(
+            checker_details, all_guideline_rules):
+        setattr(checker_details[checker_id], "guidelineRules", guideline_rules)
+
+    checker_coverage_stat = [{
+        "checker": stat.checkerName,
+        "severity": stat.severity,
+        "guidelineRules": stat.guidelineRules,
+        "enabledInAllRuns": len(stat.disabled) == 0,
+        "enabledRunLength": len(stat.enabled),
+        "disabledRunLength": len(stat.disabled),
+        "closed": stat.closed,
+        "outstanding": stat.outstanding,
+    } for stat in checker_details.values()]
+
     all_results = []
     total = defaultdict(int)
     for key, checker_data in sorted(list(all_checkers_dict.items()),
@@ -1639,14 +1684,28 @@ def checker_count(checker_dict, key):
 
         rows = []
         for stat in severities:
-            rows.append((stat['severity'],
-                         str(stat['reports'])))
+            rows.append((stat['severity'], str(stat['reports'])))
 
         rows.append(('Total', str(severity_total)))
 
         print(twodim.to_str(args.output_format, header, rows,
                             separate_footer=True))
 
+        # Print checker coverage stat
+        header = ["Checker Name", "guideline", "Severity",
+                  "Enabled in all selected runs", "Closed Reports",
+                  "Outstanding Reports",]
+
+        rows = [(stat["checker"],
+                 formatted_guidelines(stat["guidelineRules"]),
+                 ttypes.Severity._VALUES_TO_NAMES[stat["severity"]],
+                 stat["enabledInAllRuns"],
+                 str(stat["closed"]),
+                 str(stat["outstanding"])) for stat in checker_coverage_stat]
+
+        print(twodim.to_str(args.output_format, header, rows,
+                            separate_footer=False))
+
 
 def handle_remove_run_results(args):
 

web/client/codechecker_client/helpers/results.py

@@ -145,6 +145,14 @@ def getCheckerCounts(self, base_run_ids, reportFilter, cmpData, limit,
                          offset):
         pass
 
+    @thrift_client_call
+    def getCheckerStatusVerificationDetails(self, runIds, reportFilter):
+        pass
+
+    @thrift_client_call
+    def getCheckerLabels(self, checkers):
+        pass
+
     @thrift_client_call
     def exportData(self, runId):
         pass

web/server/codechecker_server/api/report_server.py

@@ -26,7 +26,7 @@
 
 import sqlalchemy
 from sqlalchemy.sql.expression import or_, and_, not_, func, \
-    asc, desc, union_all, select, bindparam, literal_column, case, cast, true
+    asc, desc, union_all, select, bindparam, literal_column, cast, true
 from sqlalchemy.orm import contains_eager
 from sqlalchemy.types import ARRAY, String
 
@@ -1384,48 +1384,6 @@ def get_run_id_expression(session, report_filter):
     return Run.id.label("run_id")
 
 
-def get_is_enabled_case(subquery):
-    """
-    Creating a case statement to decide the report
-    is enabled or not based on the detection status
-    """
-    detection_status_filters = subquery.c.detection_status.in_(list(
-        map(detection_status_str,
-            (DetectionStatus.OFF, DetectionStatus.UNAVAILABLE))
-    ))
-
-    return case(
-        (detection_status_filters, False),
-        else_=True
-    )
-
-
-def get_is_opened_case(subquery):
-    """
-    Creating a case statement to decide the report is opened or not
-    based on the detection status and the review status
-    """
-    detection_statuses = (
-        DetectionStatus.NEW,
-        DetectionStatus.UNRESOLVED,
-        DetectionStatus.REOPENED
-    )
-    review_statuses = (
-        API_ReviewStatus.UNREVIEWED,
-        API_ReviewStatus.CONFIRMED
-    )
-    detection_and_review_status_filters = [
-        subquery.c.detection_status.in_(list(map(
-            detection_status_str, detection_statuses))),
-        subquery.c.review_status.in_(list(map(
-            review_status_str, review_statuses)))
-    ]
-    return case(
-        (and_(*detection_and_review_status_filters), True),
-        else_=False
-    )
-
-
 def remove_reports(session: DBSession,
                    report_ids: Collection,
                    chunk_size: int = SQLITE_MAX_VARIABLE_NUMBER):
@@ -3345,18 +3303,15 @@ def getCheckerStatusVerificationDetails(self, run_ids, report_filter):
 
             subquery = subquery.subquery()
 
-            is_enabled_case = get_is_enabled_case(subquery)
-            is_opened_case = get_is_opened_case(subquery)
-
             query = (
                 session.query(
                     subquery.c.checker_id,
                     subquery.c.checker_name,
                     subquery.c.analyzer_name,
                     subquery.c.severity,
                     subquery.c.run_id,
-                    is_enabled_case.label("isEnabled"),
-                    is_opened_case.label("isOpened"),
+                    subquery.c.detection_status,
+                    subquery.c.review_status,
                     func.count(subquery.c.bug_id)
                 )
                 .group_by(
@@ -3365,8 +3320,8 @@ def getCheckerStatusVerificationDetails(self, run_ids, report_filter):
                     subquery.c.analyzer_name,
                     subquery.c.severity,
                     subquery.c.run_id,
-                    is_enabled_case,
-                    is_opened_case
+                    subquery.c.detection_status,
+                    subquery.c.review_status
                 )
             )
 
@@ -3377,8 +3332,8 @@ def getCheckerStatusVerificationDetails(self, run_ids, report_filter):
                 analyzer_name, \
                 severity, \
                 run_id_list, \
-                is_enabled, \
-                is_opened, \
+                detection_status, \
+                review_status, \
                 cnt \
                     in query.all():
 
@@ -3394,6 +3349,22 @@ def getCheckerStatusVerificationDetails(self, run_ids, report_filter):
                         outstanding=0
                     ))
 
+                is_enabled = detection_status not in map(
+                    detection_status_str,
+                    (DetectionStatus.OFF, DetectionStatus.UNAVAILABLE))
+
+                is_opened = \
+                    detection_status in map(
+                        detection_status_str,
+                        (DetectionStatus.NEW,
+                         DetectionStatus.UNRESOLVED,
+                         DetectionStatus.REOPENED)) \
+                    and \
+                    review_status in map(
+                        review_status_str,
+                        (API_ReviewStatus.UNREVIEWED,
+                         API_ReviewStatus.CONFIRMED))
+
                 if is_enabled:
                     for r in (run_id_list.split(",")
                               if isinstance(run_id_list, str)
@@ -3602,10 +3573,24 @@ def getReportStatusCounts(self, run_ids, report_filter, cmp_data):
             filter_expression, join_tables = process_report_filter(
                 session, run_ids, report_filter, cmp_data)
 
+            detection_and_review_status_filters = [
+                Report.detection_status.in_(list(map(
+                    detection_status_str,
+                    (DetectionStatus.NEW,
+                     DetectionStatus.UNRESOLVED,
+                     DetectionStatus.REOPENED)))),
+                Report.review_status.in_(list(map(
+                    review_status_str,
+                    (API_ReviewStatus.UNREVIEWED,
+                     API_ReviewStatus.CONFIRMED))))
+            ]
+
             extended_table = session.query(
                 Report.review_status,
                 Report.detection_status,
-                Report.bug_id
+                Report.bug_id,
+                and_(*detection_and_review_status_filters)
+                .label("isOutstanding")
             )
 
             if report_filter.annotations is not None:
@@ -3620,19 +3605,16 @@ def getReportStatusCounts(self, run_ids, report_filter, cmp_data):
 
             extended_table = extended_table.subquery()
 
-            is_outstanding_case = get_is_opened_case(extended_table)
-            case_label = "isOutstanding"
-
             if report_filter.isUnique:
                 q = session.query(
-                    is_outstanding_case.label(case_label),
+                    extended_table.c.isOutstanding,
                     func.count(extended_table.c.bug_id.distinct())) \
-                    .group_by(is_outstanding_case)
+                    .group_by(extended_table.c.isOutstanding)
             else:
                 q = session.query(
-                    is_outstanding_case.label(case_label),
+                    extended_table.c.isOutstanding,
                     func.count(extended_table.c.bug_id)) \
-                    .group_by(is_outstanding_case)
+                    .group_by(extended_table.c.isOutstanding)
 
             results = {
                 report_status_enum(

web/tests/functional/cmdline/test_cmdline.py

@@ -37,6 +37,7 @@ def run_cmd(cmd, environ=None):
 
     out, err = proc.communicate()
     print(out)
+    print(err)
     return proc.returncode, out, err