From 7bea56c640ca372a4dda892e1228c0eed9c1dbbd Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Mon, 7 Jun 2021 11:26:25 -0700
Subject: [PATCH] Merge Dockerfiles directory and Build environments
directories
---
CODEOWNERS | 1 +
README.md | 6 ++----
experimental_official_bazelrcs/README.md | 12 ------------
.../README.md | 17 ++++++++++++++---
.../master/linux}/Dockerfile | 2 ++
.../master/linux}/bashrc | 0
.../master/linux}/builder.packages.txt | 0
.../master/linux}/devel.packages.txt | 0
.../master/linux}/devel.requirements.txt | 0
.../linux}/devel.user_tools/build_addons.sh | 0
.../master/linux}/devel.user_tools/build_tf.sh | 0
.../master/linux/devel.user_tools/cpu.bazelrc | 8 ++++++++
.../master/linux/devel.user_tools/gpu.bazelrc | 8 ++++++++
.../linux/devel.user_tools/mounted.bazelrc | 2 ++
.../linux}/devel.user_tools/preliminary.bats | 0
.../master/linux}/devel.user_tools/pylintrc | 0
.../linux}/devtoolset/build_devtoolset.sh | 0
.../master/linux}/devtoolset/fixlinks.sh | 0
.../master/linux}/devtoolset/rpm-patch.sh | 0
.../master/linux}/devtoolset/update.sh | 0
.../master/linux}/runtime.requirements.txt | 0
.../master/linux}/setup.cuda.sh | 0
.../master/linux}/setup.devel.sh | 0
.../master/linux}/setup.packages.sh | 0
.../master/linux}/setup.python.sh | 0
.../master/linux}/setup.sources.sh | 0
.../master/linux}/setup.virtualenv.sh | 0
27 files changed, 37 insertions(+), 19 deletions(-)
delete mode 100644 experimental_official_bazelrcs/README.md
rename {experimental_official_dockerfiles => official_build_environments}/README.md (70%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/Dockerfile (96%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/bashrc (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/builder.packages.txt (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/devel.packages.txt (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/devel.requirements.txt (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/devel.user_tools/build_addons.sh (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/devel.user_tools/build_tf.sh (100%)
rename experimental_official_bazelrcs/linux.cpu.bazelrc => official_build_environments/master/linux/devel.user_tools/cpu.bazelrc (66%)
rename experimental_official_bazelrcs/linux.gpu.bazelrc => official_build_environments/master/linux/devel.user_tools/gpu.bazelrc (77%)
create mode 100644 official_build_environments/master/linux/devel.user_tools/mounted.bazelrc
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/devel.user_tools/preliminary.bats (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/devel.user_tools/pylintrc (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/devtoolset/build_devtoolset.sh (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/devtoolset/fixlinks.sh (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/devtoolset/rpm-patch.sh (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/devtoolset/update.sh (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/runtime.requirements.txt (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/setup.cuda.sh (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/setup.devel.sh (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/setup.packages.sh (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/setup.python.sh (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/setup.sources.sh (100%)
rename {experimental_official_dockerfiles/master => official_build_environments/master/linux}/setup.virtualenv.sh (100%)
diff --git a/CODEOWNERS b/CODEOWNERS
index d375a69..6e0c586 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -6,6 +6,7 @@
/directory_template @angerson @perfinion
/golang_install_guide @wamuir
/manylinux_2014_docker_images @sub-mod
+/official_build_environments @angerson @perfinion
/ppc64le @wdirons
/tekton @perfinion
/wsl2_gpu_guide @Avditvs
diff --git a/README.md b/README.md
index 0861c38..ab67033 100644
--- a/README.md
+++ b/README.md
@@ -59,10 +59,8 @@ Want to add your own project to this list? It's easy: check out
### WIP / Other
-* [**(Experimental) Official Dockerfiles**](experimental_official_dockerfiles):
- Rework of TensorFlow's Dockerfiles
-* [**(Experimental) Official Bazelrcs**](experimental_official_bazelrcs):
- Standard configurations for TensorFlow builds
+* [**Official Build Environments**](official_build_environments):
+ (Experimental) Standard Dockerfiles and configurations for TensorFlow builds
* [**Directory Template**](directory_template): Example short description.
* [**Tekton CI**](tekton): perfinion's experimental directory for using Tekton
CI with TensorFlow
diff --git a/experimental_official_bazelrcs/README.md b/experimental_official_bazelrcs/README.md
deleted file mode 100644
index b106efd..0000000
--- a/experimental_official_bazelrcs/README.md
+++ /dev/null
@@ -1,12 +0,0 @@
-# [Experimental] Official BazelRCs
-
-Standard configurations for TensorFlow builds
-
-Maintainer: @angerson (TensorFlow, SIG Build)
-
-* * *
-
-WIP Bazelrcs for building and using TensorFlow and related packages.
-
-This directory is intended to contain concrete `bazelrc` files for use with
-TensorFlow's CI, and for users to replicate TensorFlow's CI.
diff --git a/experimental_official_dockerfiles/README.md b/official_build_environments/README.md
similarity index 70%
rename from experimental_official_dockerfiles/README.md
rename to official_build_environments/README.md
index 1676c91..6f22f87 100644
--- a/experimental_official_dockerfiles/README.md
+++ b/official_build_environments/README.md
@@ -1,16 +1,26 @@
-# [Experimental] Official Dockerfiles
+# Official Dockerfiles
-Rework of TensorFlow's Dockerfiles
+**Experimental.**
+
+Standard Dockerfiles and configurations for TensorFlow builds.
Maintainer: @angerson (TensorFlow, SIG Build)
* * *
-WIP Dockerfiles for building and using TensorFlow and related packages.
+This directory contains experimental work on standardizing TF's development
+environment for ease of maintenance and contribution.
+
+Each directory is for a TF branch, and its subdirectories are for files relating
+to each platform (Windows, MacOS, Linux). Only linux has progress right now.
Nothing here is constant! If you see something that doesn't make sense or could
be better, chances are it could be improved.
+## Dockerfiles
+
+Dockerfiles for building and using TensorFlow and related packages.
+
The idea here is to create an easy-to-maintain Docker image that can build
TensorFlow and related tools. Challenges include reducing image size (CUDA for
instance is very big) and supporting manylinux2010 (which is why it uses
@@ -26,6 +36,7 @@ Initial versions of the `devel` images are available at
You can build the images yourself like this:
```
+$ cd master/linux
$ DOCKER_BUILDKIT=1 docker build --target=devel --build-arg PYTHON_VERSION=python3.8 -t tensorflow-build-devel .
```
diff --git a/experimental_official_dockerfiles/master/Dockerfile b/official_build_environments/master/linux/Dockerfile
similarity index 96%
rename from experimental_official_dockerfiles/master/Dockerfile
rename to official_build_environments/master/linux/Dockerfile
index 2ebd005..2958f6f 100644
--- a/experimental_official_dockerfiles/master/Dockerfile
+++ b/official_build_environments/master/linux/Dockerfile
@@ -46,6 +46,8 @@ RUN /setup.python.sh $PYTHON_VERSION devel.requirements.txt
# Setup build
COPY devel.user_tools /user_tools
+COPY ../../experimental_official_bazelrcs/master /bazelrcs
+RUN touch /bazelrcs/mounted.bazelrc
################################################################################
FROM nvidia/cuda:11.2.1-cudnn8-runtime-ubuntu20.04 as runtime
diff --git a/experimental_official_dockerfiles/master/bashrc b/official_build_environments/master/linux/bashrc
similarity index 100%
rename from experimental_official_dockerfiles/master/bashrc
rename to official_build_environments/master/linux/bashrc
diff --git a/experimental_official_dockerfiles/master/builder.packages.txt b/official_build_environments/master/linux/builder.packages.txt
similarity index 100%
rename from experimental_official_dockerfiles/master/builder.packages.txt
rename to official_build_environments/master/linux/builder.packages.txt
diff --git a/experimental_official_dockerfiles/master/devel.packages.txt b/official_build_environments/master/linux/devel.packages.txt
similarity index 100%
rename from experimental_official_dockerfiles/master/devel.packages.txt
rename to official_build_environments/master/linux/devel.packages.txt
diff --git a/experimental_official_dockerfiles/master/devel.requirements.txt b/official_build_environments/master/linux/devel.requirements.txt
similarity index 100%
rename from experimental_official_dockerfiles/master/devel.requirements.txt
rename to official_build_environments/master/linux/devel.requirements.txt
diff --git a/experimental_official_dockerfiles/master/devel.user_tools/build_addons.sh b/official_build_environments/master/linux/devel.user_tools/build_addons.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/devel.user_tools/build_addons.sh
rename to official_build_environments/master/linux/devel.user_tools/build_addons.sh
diff --git a/experimental_official_dockerfiles/master/devel.user_tools/build_tf.sh b/official_build_environments/master/linux/devel.user_tools/build_tf.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/devel.user_tools/build_tf.sh
rename to official_build_environments/master/linux/devel.user_tools/build_tf.sh
diff --git a/experimental_official_bazelrcs/linux.cpu.bazelrc b/official_build_environments/master/linux/devel.user_tools/cpu.bazelrc
similarity index 66%
rename from experimental_official_bazelrcs/linux.cpu.bazelrc
rename to official_build_environments/master/linux/devel.user_tools/cpu.bazelrc
index c2c1ac9..22991db 100644
--- a/experimental_official_bazelrcs/linux.cpu.bazelrc
+++ b/official_build_environments/master/linux/devel.user_tools/cpu.bazelrc
@@ -1,6 +1,14 @@
# This bazelrc can build a CPU-supporting TF package.
# Hopefully it's compatible with manylinux2010.
+# Convenient cache configurations
+# Use a cache directory mounted to /tf/cache. Very useful!
+build:sigbuild_local_cache --disk_cache=/tf/cache
+# Use the public-access TF DevInfra cache (read only)
+build:sigbuild_remote_cache --remote_cache="https://storage.googleapis.com/tensorflow-devinfra-bazel-cache" --remote_upload_local_results=false
+# Write to the TF DevInfra cache (only works for internal TF CI)
+build:sigbuild_remote_cache_push --remote_cache="https://storage.googleapis.com/tensorflow-devinfra-bazel-cache" --google_default_credentials
+
# Use Python 3.X as installed in container image
build --action_env PYTHON_BIN_PATH="/usr/bin/python3"
build --action_env PYTHON_LIB_PATH="/usr/lib/tf_python"
diff --git a/experimental_official_bazelrcs/linux.gpu.bazelrc b/official_build_environments/master/linux/devel.user_tools/gpu.bazelrc
similarity index 77%
rename from experimental_official_bazelrcs/linux.gpu.bazelrc
rename to official_build_environments/master/linux/devel.user_tools/gpu.bazelrc
index d7b8f03..0146e01 100644
--- a/experimental_official_bazelrcs/linux.gpu.bazelrc
+++ b/official_build_environments/master/linux/devel.user_tools/gpu.bazelrc
@@ -1,6 +1,14 @@
# This bazelrc can build a GPU-supporting TF package.
# Hopefully it's compatible with manylinux2010.
+# Convenient cache configurations
+# Use a cache directory mounted to /tf/cache. Very useful!
+build:sigbuild_local_cache --disk_cache=/tf/cache
+# Use the public-access TF DevInfra cache (read only)
+build:sigbuild_remote_cache --remote_cache="https://storage.googleapis.com/tensorflow-devinfra-bazel-cache" --remote_upload_local_results=false
+# Write to the TF DevInfra cache (only works for internal TF CI)
+build:sigbuild_remote_cache_push --remote_cache="https://storage.googleapis.com/tensorflow-devinfra-bazel-cache" --google_default_credentials
+
# Use Python 3.X as installed in container image
build --action_env PYTHON_BIN_PATH="/usr/bin/python3"
build --action_env PYTHON_LIB_PATH="/usr/lib/tf_python"
diff --git a/official_build_environments/master/linux/devel.user_tools/mounted.bazelrc b/official_build_environments/master/linux/devel.user_tools/mounted.bazelrc
new file mode 100644
index 0000000..2c1e242
--- /dev/null
+++ b/official_build_environments/master/linux/devel.user_tools/mounted.bazelrc
@@ -0,0 +1,2 @@
+# You can mount your own bazelrc over this file in Docker with
+# -v $HOME/my/bazelrc:/user_tools/mounted.bazelrc
diff --git a/experimental_official_dockerfiles/master/devel.user_tools/preliminary.bats b/official_build_environments/master/linux/devel.user_tools/preliminary.bats
similarity index 100%
rename from experimental_official_dockerfiles/master/devel.user_tools/preliminary.bats
rename to official_build_environments/master/linux/devel.user_tools/preliminary.bats
diff --git a/experimental_official_dockerfiles/master/devel.user_tools/pylintrc b/official_build_environments/master/linux/devel.user_tools/pylintrc
similarity index 100%
rename from experimental_official_dockerfiles/master/devel.user_tools/pylintrc
rename to official_build_environments/master/linux/devel.user_tools/pylintrc
diff --git a/experimental_official_dockerfiles/master/devtoolset/build_devtoolset.sh b/official_build_environments/master/linux/devtoolset/build_devtoolset.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/devtoolset/build_devtoolset.sh
rename to official_build_environments/master/linux/devtoolset/build_devtoolset.sh
diff --git a/experimental_official_dockerfiles/master/devtoolset/fixlinks.sh b/official_build_environments/master/linux/devtoolset/fixlinks.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/devtoolset/fixlinks.sh
rename to official_build_environments/master/linux/devtoolset/fixlinks.sh
diff --git a/experimental_official_dockerfiles/master/devtoolset/rpm-patch.sh b/official_build_environments/master/linux/devtoolset/rpm-patch.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/devtoolset/rpm-patch.sh
rename to official_build_environments/master/linux/devtoolset/rpm-patch.sh
diff --git a/experimental_official_dockerfiles/master/devtoolset/update.sh b/official_build_environments/master/linux/devtoolset/update.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/devtoolset/update.sh
rename to official_build_environments/master/linux/devtoolset/update.sh
diff --git a/experimental_official_dockerfiles/master/runtime.requirements.txt b/official_build_environments/master/linux/runtime.requirements.txt
similarity index 100%
rename from experimental_official_dockerfiles/master/runtime.requirements.txt
rename to official_build_environments/master/linux/runtime.requirements.txt
diff --git a/experimental_official_dockerfiles/master/setup.cuda.sh b/official_build_environments/master/linux/setup.cuda.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/setup.cuda.sh
rename to official_build_environments/master/linux/setup.cuda.sh
diff --git a/experimental_official_dockerfiles/master/setup.devel.sh b/official_build_environments/master/linux/setup.devel.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/setup.devel.sh
rename to official_build_environments/master/linux/setup.devel.sh
diff --git a/experimental_official_dockerfiles/master/setup.packages.sh b/official_build_environments/master/linux/setup.packages.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/setup.packages.sh
rename to official_build_environments/master/linux/setup.packages.sh
diff --git a/experimental_official_dockerfiles/master/setup.python.sh b/official_build_environments/master/linux/setup.python.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/setup.python.sh
rename to official_build_environments/master/linux/setup.python.sh
diff --git a/experimental_official_dockerfiles/master/setup.sources.sh b/official_build_environments/master/linux/setup.sources.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/setup.sources.sh
rename to official_build_environments/master/linux/setup.sources.sh
diff --git a/experimental_official_dockerfiles/master/setup.virtualenv.sh b/official_build_environments/master/linux/setup.virtualenv.sh
similarity index 100%
rename from experimental_official_dockerfiles/master/setup.virtualenv.sh
rename to official_build_environments/master/linux/setup.virtualenv.sh
--
GitLab