From 743bc4be3bf38988fc0c4477fa0a8d98632782e5 Mon Sep 17 00:00:00 2001 From: Jeremie Fraeys Date: Wed, 4 Mar 2026 13:23:30 -0500 Subject: [PATCH] cli: update Zig CLI build and native hash integration - Update build.zig configuration - Improve queue command implementation - Enhance native hash support --- cli/README.md | 21 +++++++++++++++++++++ cli/build.zig | 21 +++++++++++++-------- cli/src/commands/queue.zig | 26 ++++++++++++++++++++++++++ cli/src/native/hash.zig | 38 +++++++++++++++++++++++++++++++++++--- 4 files changed, 95 insertions(+), 11 deletions(-) diff --git a/cli/README.md b/cli/README.md index b9e673e..638b1eb 100644 --- a/cli/README.md +++ b/cli/README.md @@ -2,6 +2,27 @@ Fast CLI tool for managing ML experiments. Supports both **local mode** (SQLite) and **server mode** (WebSocket). +## Build Policy + +**Native C++ libraries** (dataset_hash, etc.) are available when building natively on any platform. Cross-compilation is supported for development on non-native targets but disables native library features. + +| Build Type | Target | Native Libraries | Purpose | +|------------|--------|-------------------|---------| +| Native | Host platform (Linux, macOS) | Yes | Dev, staging, production | +| Cross-compile | Different arch/OS | Stubbed | Testing on foreign targets | + +### Native Build (Recommended) +Builds on the host platform with full native library support: +```bash +zig build -Doptimize=ReleaseSmall +``` + +### Cross-Compile (Dev Only) +For testing on different architectures without native library support: +```bash +zig build -Dtarget=x86_64-linux-gnu # from macOS/Windows +``` + ## Architecture The CLI follows a modular 3-layer architecture for maintainability: diff --git a/cli/build.zig b/cli/build.zig index 74aa3f7..ad62770 100644 --- a/cli/build.zig +++ b/cli/build.zig @@ -1,6 +1,9 @@ const std = @import("std"); -// Clean build configuration for optimized CLI (Zig 0.15 std.Build API) +// Build Policy: +// - Native C++ libraries: Available when building natively (not cross-compiling) +// - Cross-compiling: Dev-only, native library stubs used + pub fn build(b: *std.Build) void { // Standard target options const target = b.standardTargetOptions(.{}); @@ -16,6 +19,13 @@ pub fn build(b: *std.Build) void { const arch = target.result.cpu.arch; const os_tag = target.result.os.tag; + // Check if we're cross-compiling (target differs from host) + const host_target = b.graph.host; + const is_cross_compiling = (target.result.os.tag != host_target.query.os_tag) or + (target.result.cpu.arch != host_target.query.cpu_arch); + + options.addOption(bool, "is_cross_compiling", is_cross_compiling); + const arch_str: []const u8 = switch (arch) { .x86_64 => "x86_64", .aarch64 => "arm64", @@ -99,11 +109,6 @@ pub fn build(b: *std.Build) void { // LTO disabled: requires LLD linker which may not be available // exe.want_lto = true; - // Check if we're cross-compiling (target differs from host) - const host_target = b.graph.host; - const is_cross_compiling = (target.result.os.tag != host_target.query.os_tag) or - (target.result.cpu.arch != host_target.query.cpu_arch); - // Link native dataset_hash library (only when not cross-compiling) exe.linkLibC(); if (!is_cross_compiling) { @@ -111,8 +116,8 @@ pub fn build(b: *std.Build) void { exe.linkSystemLibrary("dataset_hash"); exe.addIncludePath(b.path("../native/dataset_hash")); } else { - // Cross-compiling: native library not available, skip it - std.log.warn("Cross-compiling detected - skipping native library linking", .{}); + // Cross-compiling: dev-only, native library not available + std.log.warn("Cross-compiling (dev-only): native libraries disabled", .{}); } // SQLite setup: embedded for ReleaseSmall only, system lib for dev diff --git a/cli/src/commands/queue.zig b/cli/src/commands/queue.zig index 460df2d..a007947 100644 --- a/cli/src/commands/queue.zig +++ b/cli/src/commands/queue.zig @@ -55,6 +55,12 @@ pub const QueueOptions = struct { network_mode: ?[]const u8 = null, read_only: bool = false, secrets: std.ArrayList([]const u8), + // Scheduler options + reservation_id: ?[]const u8 = null, + gang_size: ?u32 = null, + max_wait_time: ?u32 = null, + preemptible: bool = false, + preferred_worker: ?[]const u8 = null, }; fn resolveCommitHexOrPrefix(allocator: std.mem.Allocator, base_path: []const u8, input: []const u8) ![]u8 { @@ -327,6 +333,20 @@ fn executeQueue(allocator: std.mem.Allocator, args: []const []const u8, config: } else if (std.mem.eql(u8, arg, "--secret") and i + 1 < pre.len) { try options.secrets.append(allocator, pre[i + 1]); i += 1; + } else if (std.mem.eql(u8, arg, "--reservation") and i + 1 < pre.len) { + options.reservation_id = pre[i + 1]; + i += 1; + } else if (std.mem.eql(u8, arg, "--gang-size") and i + 1 < pre.len) { + options.gang_size = try std.fmt.parseInt(u32, pre[i + 1], 10); + i += 1; + } else if (std.mem.eql(u8, arg, "--max-wait") and i + 1 < pre.len) { + options.max_wait_time = try std.fmt.parseInt(u32, pre[i + 1], 10); + i += 1; + } else if (std.mem.eql(u8, arg, "--preemptible")) { + options.preemptible = true; + } else if (std.mem.eql(u8, arg, "--worker") and i + 1 < pre.len) { + options.preferred_worker = pre[i + 1]; + i += 1; } } else { // This is a job name @@ -706,6 +726,12 @@ fn printUsage() !void { std.debug.print("\t--network \tNetwork mode: none, bridge, slirp4netns\n", .{}); std.debug.print("\t--read-only\t\tMount root filesystem as read-only\n", .{}); std.debug.print("\t--secret \t\tInject secret as env var (can repeat)\n", .{}); + std.debug.print("\nScheduler Options:\n", .{}); + std.debug.print("\t--reservation \tUse existing GPU reservation\n", .{}); + std.debug.print("\t--gang-size \t\tRequest gang scheduling for multi-node jobs\n", .{}); + std.debug.print("\t--max-wait \tMaximum wait time before failing\n", .{}); + std.debug.print("\t--preemptible\t\tAllow job to be preempted\n", .{}); + std.debug.print("\t--worker \t\tPrefer specific worker\n", .{}); std.debug.print("\nExamples:\n", .{}); std.debug.print("\tml queue my_job\t\t\t # Queue a job\n", .{}); std.debug.print("\tml queue my_job --dry-run\t # Preview submission\n", .{}); diff --git a/cli/src/native/hash.zig b/cli/src/native/hash.zig index c619dae..bc735ca 100644 --- a/cli/src/native/hash.zig +++ b/cli/src/native/hash.zig @@ -1,15 +1,37 @@ const std = @import("std"); -const c = @cImport({ - @cInclude("dataset_hash.h"); -}); +const build_options = @import("build_options"); pub const HashError = error{ ContextInitFailed, HashFailed, InvalidPath, OutOfMemory, + NotAvailable, }; +// Conditionally compile C imports only when not cross-compiling +const c = if (build_options.is_cross_compiling) + struct { + pub const fh_context_t = opaque {}; + pub fn fh_init(_: i32) ?*fh_context_t { + return null; + } + pub fn fh_hash_directory_combined(_: *fh_context_t, _: [*c]const u8) [*c]u8 { + return null; + } + pub fn fh_free_string(_: [*c]u8) void {} + pub fn fh_has_simd_sha256() i32 { + return 0; + } + pub fn fh_get_simd_impl_name() [*c]const u8 { + return @ptrCast(@alignCast("none")); + } + } +else + @cImport({ + @cInclude("dataset_hash.h"); + }); + // Global context for reuse across multiple hash operations var global_ctx: ?*c.fh_context_t = null; var ctx_initialized = std.atomic.Value(bool).init(false); @@ -17,6 +39,10 @@ var init_mutex = std.Thread.Mutex{}; /// Initialize global hash context once (thread-safe) pub fn init() !void { + if (build_options.is_cross_compiling) { + return HashError.NotAvailable; + } + if (ctx_initialized.load(.seq_cst)) return; init_mutex.lock(); @@ -39,6 +65,10 @@ pub fn init() !void { /// Hash a directory using the native library (reuses global context) /// Returns the hex-encoded SHA256 hash string pub fn hashDirectory(allocator: std.mem.Allocator, path: []const u8) ![]const u8 { + if (build_options.is_cross_compiling) { + return HashError.NotAvailable; + } + try init(); // Idempotent initialization const ctx = global_ctx.?; // Safe: init() guarantees non-null @@ -61,11 +91,13 @@ pub fn hashDirectory(allocator: std.mem.Allocator, path: []const u8) ![]const u8 /// Check if SIMD SHA256 is available pub fn hasSimdSha256() bool { + if (build_options.is_cross_compiling) return false; return c.fh_has_simd_sha256() == 1; } /// Get the name of the SIMD implementation being used pub fn getSimdImplName() []const u8 { + if (build_options.is_cross_compiling) return "none"; const name = c.fh_get_simd_impl_name(); return std.mem.span(name); }