Improve testlog squasher and add BEP output

tf_sig_build_dockerfiles/devel.usertools/cpu.bazelrc

@@ -36,6 +36,11 @@ build --copt=-mavx --host_copt=-mavx
 # See https://docs.bazel.build/versions/main/skylark/performance.html#performance-profiling
 build --profile=/tf/pkg/profile.json
 
+# Store BEP logs for generating and storing test status and flakiness data
+build --build_event_json_file=/tf/pkg/bep.json
+build --build_event_text_file=/tf/pkg/bep.txt
+build --build_event_binary_file=/tf/pkg/bep.pb
+
 # Use the NVCC toolchain to compile for manylinux2010
 build --crosstool_top=@sigbuild-r2.9_config_cuda//crosstool:toolchain
 

tf_sig_build_dockerfiles/devel.usertools/gpu.bazelrc

@@ -36,6 +36,11 @@ build --copt=-mavx --host_copt=-mavx
 # See https://docs.bazel.build/versions/main/skylark/performance.html#performance-profiling
 build --profile=/tf/pkg/profile.json
 
+# Store BEP logs for generating and storing test status and flakiness data
+build --build_event_json_file=/tf/pkg/bep.json
+build --build_event_text_file=/tf/pkg/bep.txt
+build --build_event_binary_file=/tf/pkg/bep.pb
+
 # CUDA: Set up compilation CUDA version and paths
 build --@local_config_cuda//:enable_cuda
 build --repo_env TF_NEED_CUDA=1

tf_sig_build_dockerfiles/devel.usertools/squash_testlogs.py

@@ -9,6 +9,7 @@
 # uses this to generate a simple overview of an entire pip and nonpip test
 # invocation, since the normal logs that Bazel creates are too large for the
 # internal invocation viewer.
+import collections
 import glob
 import os
 import sys
@@ -25,7 +26,7 @@ except subprocess.CalledProcessError as e:
   exit(0)
 
 # For test cases, only show the ones that failed that have text (a log)
-seen = set()
+seen = collections.Counter()
 runfiles_matcher = re.compile(r"(/.*\.runfiles/)")
 
 for f in files.strip().splitlines():
@@ -50,23 +51,32 @@ for f in files.strip().splitlines():
       key = p.getparent().get("name", "") + p.text
       if key in seen:
         testsuite._elem.remove(p.getparent())
-      else:
-        seen.add(key)
-    # Include helpful notes
-    for p in testsuite._elem.xpath('.//error | .//failure'):
-      short_name = re.search(r'/(bazel_pip|tensorflow)/.*', f.decode("utf-8")).group(0)
-      p.text += f"\nNOTE: From /{short_name}"
-      p.text = runfiles_matcher.sub("[testroot]/", p.text)
-      if "bazel_pip" in short_name:
-        p.text += "\nNOTE: This is a --config=pip test. Remove 'bazel_pip' to find the file."
-      p.text += f"\nNOTE: The list of failures from the XML includes flakes and attempts as well."
-      p.text += f"\n      The error(s) that caused the invocation to fail may not include this testcase."
+      seen[key] += 1
     # Remove this testsuite if it doesn't have anything in it any more
     if len(testsuite) == 0:
       r._elem.remove(testsuite._elem)
   if len(r) > 0:
     result += r
 
+# Insert the number of failures for each test to help identify flaikes
+for p in result._elem.xpath('.//error | .//failure'):
+  short_name = re.search(r'/(bazel_pip|tensorflow)/.*', f.decode("utf-8")).group(0)
+  key = p.getparent().get("name", "") + p.text
+  p.text += f"\nNOTE: From /{short_name}"
+  p.text = runfiles_matcher.sub("[testroot]/", p.text)
+  if "bazel_pip" in short_name:
+    p.text += "\nNOTE: This is a --config=pip test. Remove 'bazel_pip' to find the file."
+  n_failures = seen[key]
+  p.text += f"\nNOTE: Number of failures for this test: {seen[key]}."
+  p.text += f"\n      Most TF jobs run tests three times to root out flakes."
+  if seen[key] == 3:
+    p.text += f"\n      Since there were three failures, this is not flaky, and it"
+    p.text += f"\n      probably caused the Kokoro invocation to fail."
+  else:
+    p.text += f"\n      Since there were not three failures, this is probably a flake."
+    p.text += f"\n      Flakes make this pkg/pip_and_nonpip_tests target show as failing,"
+    p.text += f"\n      but do not make the Kokoro invocation fail."
+
 os.makedirs(os.path.dirname(sys.argv[2]), exist_ok=True)
 result.update_statistics()
 result.write(sys.argv[2])