[Breaking] Make NativeLink configurable via K8s-compatible yaml

Cargo.toml

@@ -10,6 +10,8 @@ rust-version = "1.84.0"
 [profile.release]
 lto = true
 opt-level = 3
+debug = true
+strip = false
 
 [profile.dev]
 lto = false
@@ -35,10 +37,14 @@ enable_tokio_console = [
 nix = [
   "nativelink-worker/nix"
 ]
+crd = [
+  "nativelink-config/crd"
+]
 
 [dependencies]
 nativelink-error = { path = "nativelink-error" }
 nativelink-config = { path = "nativelink-config" }
+nativelink-controller = { path = "nativelink-controller" }
 nativelink-scheduler = { path = "nativelink-scheduler" }
 nativelink-service = { path = "nativelink-service" }
 nativelink-store = { path = "nativelink-store" }

MODULE.bazel

@@ -37,6 +37,7 @@ crate.from_cargo(
     manifests = [
         "//:Cargo.toml",
         "//nativelink-config:Cargo.toml",
+        "//nativelink-controller:Cargo.toml",
         "//nativelink-error:Cargo.toml",
         "//nativelink-macro:Cargo.toml",
         "//nativelink-metric:Cargo.toml",

deployment-examples/docker-compose/local-storage-cas.json5

@@ -12,12 +12,12 @@
           "lz4": {}
         },
         "backend": {
+          "name": "fs-cas",
           "filesystem": {
             "content_path": "~/.cache/nativelink/content_path-cas",
             "temp_path": "~/.cache/nativelink/tmp_path-cas",
             "eviction_policy": {
-              // 10gb.
-              "max_bytes": 10000000000
+              "max_bytes": "10Gb",
             }
           }
         }
@@ -28,8 +28,7 @@
         "content_path": "~/.cache/nativelink/content_path-ac",
         "temp_path": "~/.cache/nativelink/tmp_path-ac",
         "eviction_policy": {
-          // 500mb.
-          "max_bytes": 500000000
+          "max_bytes": "500Mb"
         }
       }
     }

deployment-examples/docker-compose/scheduler.json5

@@ -1,8 +1,8 @@
 {
   "stores": [
     {
-      "name": "GRPC_LOCAL_STORE",
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_STORE",
       "grpc": {
         "instance_name": "",
         "endpoints": [
@@ -11,8 +11,8 @@
         "store_type": "cas"
       }
     }, {
-      "name": "GRPC_LOCAL_AC_STORE",
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_AC_STORE",
       "grpc": {
         "instance_name": "",
         "endpoints": [

deployment-examples/docker-compose/worker.json5

@@ -3,6 +3,7 @@
     {
       "name": "GRPC_LOCAL_STORE",
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_STORE",
       "grpc": {
         "instance_name": "",
         "endpoints": [
@@ -11,8 +12,8 @@
         "store_type": "cas"
       }
     }, {
-      "name": "GRPC_LOCAL_AC_STORE",
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_AC_STORE",
       "grpc": {
         "instance_name": "",
         "endpoints": [
@@ -22,20 +23,21 @@
       }
     }, {
       "name": "WORKER_FAST_SLOW_STORE",
-      "fast_slow": {
+      "fastSlow": {
         "fast": {
+          "name": "fs-worker",
           "filesystem": {
             "content_path": "/root/.cache/nativelink/data-worker-test/content_path-cas",
             "temp_path": "/root/.cache/nativelink/data-worker-test/tmp_path-cas",
             "eviction_policy": {
-              // 10gb.
-              "max_bytes": 10000000000
+              "max_bytes": "10Gb",
             }
           }
         },
         "slow": {
-          "ref_store": {
-            "name": "GRPC_LOCAL_STORE"
+          "name": "ref-worker",
+          "ref": {
+            "name": "GRPC_LOCAL_STORE",
           }
         }
       }

flake.nix

@@ -131,6 +131,15 @@
           (craneLibFor p).buildPackage ((commonArgsFor p)
             // {
               cargoArtifacts = cargoArtifactsFor p;
+              dontStrip = true;
+            });
+
+        nativelinkControllerFor = p:
+          (craneLibFor p).buildPackage ((commonArgsFor p)
+            // {
+              pname = "nativelink-controller";
+              cargoExtraArgs = "--package=nativelink-controller";
+              cargoArtifacts = cargoArtifactsFor p;
             });
 
         nativeTargetPkgs =
@@ -155,6 +164,18 @@
             cargoExtraArgs = "--features enable_tokio_console";
           });
 
+        # TODO(aaronmondal): Enable.
+        # nativelink-controller = nativelinkControllerFor nativeTargetPkgs;
+
+        # These two can be built by all build platforms. This is not true for
+        # darwin targets which are only buildable via native compilation.
+        nativelink-controller-aarch64-linux = nativelinkControllerFor pkgs.pkgsCross.aarch64-multiplatform-musl;
+        nativelink-controller-x86_64-linux = nativelinkControllerFor pkgs.pkgsCross.musl64;
+
+        publish-ghcr = pkgs.callPackage ./tools/publish-ghcr.nix {};
+
+        local-image-test = pkgs.callPackage ./tools/local-image-test.nix {};
+
         nativelink-is-executable-test = pkgs.callPackage ./tools/nativelink-is-executable-test.nix {inherit nativelink;};
 
         generate-toolchains = pkgs.callPackage ./tools/generate-toolchains.nix {};
@@ -201,6 +222,35 @@
             };
           };
 
+        nativelink-controller-image = let
+          nativelinkControllerForImage =
+            if pkgs.stdenv.isx86_64
+            then nativelink-controller-x86_64-linux
+            else nativelink-controller-aarch64-linux;
+        in
+          buildImage {
+            name = "nativelink-controller";
+            copyToRoot = [
+              (pkgs.buildEnv {
+                name = "nativelink-buildEnv";
+                paths = [nativelinkControllerForImage];
+                pathsToLink = ["/bin"];
+              })
+            ];
+            config = {
+              Entrypoint = [(pkgs.lib.getExe' nativelinkControllerForImage "nativelink-controller")];
+              Labels = {
+                "org.opencontainers.image.description" = "Controller for NativeLink CRDs.";
+                "org.opencontainers.image.documentation" = "https://github.com/TraceMachina/nativelink";
+                "org.opencontainers.image.licenses" = "Apache-2.0";
+                "org.opencontainers.image.revision" = "${self.rev or self.dirtyRev or "dirty"}";
+                "org.opencontainers.image.source" = "https://github.com/TraceMachina/nativelink";
+                "org.opencontainers.image.title" = "NativeLink Controller";
+                "org.opencontainers.image.vendor" = "Trace Machina, Inc.";
+              };
+            };
+          };
+
         nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;};
 
         createWorker = pkgs.nativelink-tools.lib.createWorker self;
@@ -323,6 +373,7 @@
               nativelink-aarch64-linux
               nativelink-debug
               nativelink-image
+              nativelink-controller-image
               nativelink-is-executable-test
               nativelink-worker-init
               nativelink-x86_64-linux
@@ -429,6 +480,7 @@
               pkgs.fluxcd
               pkgs.go
               pkgs.kustomize
+              pkgs.grpcurl
               pkgs.kubectx
 
               # Web

kubernetes/components/worker/worker.json5

@@ -1,8 +1,8 @@
 {
   "stores": [
     {
-      "name": "GRPC_LOCAL_STORE",
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_STORE",
       "grpc": {
         "instance_name": "",
         "endpoints": [
@@ -11,8 +11,8 @@
         "store_type": "cas"
       }
     }, {
-      "name": "GRPC_LOCAL_AC_STORE",
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_AC_STORE",
       "grpc": {
         "instance_name": "",
         "endpoints": [
@@ -22,8 +22,9 @@
       }
     }, {
       "name": "WORKER_FAST_SLOW_STORE",
-      "fast_slow": {
+      "fastSlow": {
         "fast": {
+          "name": "fs-worker",
           "filesystem": {
             "content_path": "~/.cache/nativelink/data-worker-test/content_path-cas",
             "temp_path": "~/.cache/nativelink/data-worker-test/tmp_path-cas",
@@ -33,8 +34,9 @@
           }
         },
         "slow": {
-          "ref_store": {
-            "name": "GRPC_LOCAL_STORE"
+          "name": "ref-worker",
+          "ref": {
+            "name": "GRPC_LOCAL_STORE",
           }
         }
       }

nativelink-config/BUILD.bazel

@@ -22,7 +22,9 @@ rust_library(
     deps = [
         "@crates//:byte-unit",
         "@crates//:humantime",
+        "@crates//:schemars",
         "@crates//:serde",
+        "@crates//:serde_json",
         "@crates//:shellexpand",
     ],
 )

nativelink-config/Cargo.toml

@@ -3,12 +3,17 @@ name = "nativelink-config"
 version = "0.5.4"
 edition = "2021"
 
+[features]
+crd = []
+
 [dependencies]
 byte-unit = { version = "5.1.6", default-features = false, features = ["byte"] }
 humantime = "2.1.0"
 serde = { version = "1.0.217", default-features = false, features = ["derive"] }
+serde_json = "1.0.132"
 serde_json5 = "0.1.0"
 shellexpand = { version = "3.1.0", default-features = false, features = ["base-0"] }
+schemars = "0.8.21"
 
 [dev-dependencies]
 pretty_assertions = { version = "1.4.1", features = ["std"] }

nativelink-config/examples/filesystem_cas.json5

@@ -12,12 +12,12 @@
           "lz4": {}
         },
         "backend": {
+          "name": "fs-conent",
           "filesystem": {
             "content_path": "/tmp/nativelink/data/content_path-cas",
             "temp_path": "/tmp/nativelink/data/tmp_path-cas",
             "eviction_policy": {
-              // 2gb.
-              "max_bytes": 2000000000
+              "max_bytes": "2Gb",
             }
           }
         }
@@ -32,43 +32,48 @@
           // general build content, since many objects are quite small and by
           // putting this size distinguish store in place will prevent 1+ index
           // read/write per small object request.
-          "size_partitioning": {
+          "name": "cas-parts",
+          "sizePartitioning": {
             "size": 262144, // 256k.
             "lower_store": {
-              "ref_store": {
+              "name": "cas-lower",
+              "ref": {
                 "name": "FS_CONTENT_STORE"
               }
             },
             "upper_store": {
+              "name": "cas-upper",
               "dedup": {
                 "index_store": {
                   // Since our index store is queried so much, we use a fast_slow
                   // store so it will keep in memory objects that are accessed
                   // frequently before going to disk.
                   // Note: indexes are generally quite small, but accessed frequently.
-                  "fast_slow": {
+                  "name": "cas-index",
+                  "fastSlow": {
                     "fast": {
+                      "name": "cas-index-fast",
                       "memory": {
                         "eviction_policy": {
-                          // 10mb.
-                          "max_bytes": 10000000
+                          "max_bytes": "10Mb",
                         }
                       }
                     },
                     "slow": {
+                      "name": "cas-index-slow",
                       "filesystem": {
                         "content_path": "/tmp/nativelink/data/content_path-index",
                         "temp_path": "/tmp/nativelink/data/tmp_path-index",
                         "eviction_policy": {
-                          // 500mb.
-                          "max_bytes": 500000000
+                          "max_bytes": "500Mb",
                         }
                       }
                     }
                   }
                 },
                 "content_store": {
-                  "ref_store": {
+                  "name": "cas-content",
+                  "ref": {
                     "name": "FS_CONTENT_STORE"
                   }
                 }
@@ -85,8 +90,7 @@
         "content_path": "/tmp/nativelink/data/content_path-ac",
         "temp_path": "/tmp/nativelink/data/tmp_path-ac",
         "eviction_policy": {
-          // 500mb.
-          "max_bytes": 500000000
+          "max_bytes": "500Mb",
         }
       }
     }