From ccb4e158778ea8e5bc3198d8f2ec3e55bb4d6ac4 Mon Sep 17 00:00:00 2001 From: Jeremie Fraeys Date: Thu, 5 Mar 2026 13:12:59 -0500 Subject: [PATCH] refactor(cli): rename exec/ and queue/ directories for clarity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename cli/src/commands/exec/ → executor/ (4 files) - Rename cli/src/commands/queue/ → submission/ (4 files) - Create new submission/index.zig, delete queue/index.zig The new names better reflect the purpose of these modules: - 'executor' for local/remote execution logic - 'submission' for job submission and queue management This is a pure rename with no functional changes. --- .../commands/{exec => executor}/dryrun.zig | 0 cli/src/commands/{exec => executor}/local.zig | 0 cli/src/commands/{exec => executor}/mod.zig | 0 .../commands/{exec => executor}/remote.zig | 0 cli/src/commands/queue/index.zig | 3 - cli/src/commands/run.zig | 162 +++++++++++++++++- cli/src/commands/submission/index.zig | 3 + .../commands/{queue => submission}/mod.zig | 0 .../commands/{queue => submission}/parse.zig | 0 .../commands/{queue => submission}/submit.zig | 0 .../{queue => submission}/validate.zig | 0 11 files changed, 163 insertions(+), 5 deletions(-) rename cli/src/commands/{exec => executor}/dryrun.zig (100%) rename cli/src/commands/{exec => executor}/local.zig (100%) rename cli/src/commands/{exec => executor}/mod.zig (100%) rename cli/src/commands/{exec => executor}/remote.zig (100%) delete mode 100644 cli/src/commands/queue/index.zig create mode 100644 cli/src/commands/submission/index.zig rename cli/src/commands/{queue => submission}/mod.zig (100%) rename cli/src/commands/{queue => submission}/parse.zig (100%) rename cli/src/commands/{queue => submission}/submit.zig (100%) rename cli/src/commands/{queue => submission}/validate.zig (100%) diff --git a/cli/src/commands/exec/dryrun.zig b/cli/src/commands/executor/dryrun.zig similarity index 100% rename from cli/src/commands/exec/dryrun.zig rename to cli/src/commands/executor/dryrun.zig diff --git a/cli/src/commands/exec/local.zig b/cli/src/commands/executor/local.zig similarity index 100% rename from cli/src/commands/exec/local.zig rename to cli/src/commands/executor/local.zig diff --git a/cli/src/commands/exec/mod.zig b/cli/src/commands/executor/mod.zig similarity index 100% rename from cli/src/commands/exec/mod.zig rename to cli/src/commands/executor/mod.zig diff --git a/cli/src/commands/exec/remote.zig b/cli/src/commands/executor/remote.zig similarity index 100% rename from cli/src/commands/exec/remote.zig rename to cli/src/commands/executor/remote.zig diff --git a/cli/src/commands/queue/index.zig b/cli/src/commands/queue/index.zig deleted file mode 100644 index 2903c81..0000000 --- a/cli/src/commands/queue/index.zig +++ /dev/null @@ -1,3 +0,0 @@ -pub const parse = @import("queue/parse.zig"); -pub const validate = @import("queue/validate.zig"); -pub const submit = @import("queue/submit.zig"); diff --git a/cli/src/commands/run.zig b/cli/src/commands/run.zig index 940f377..f4897de 100644 --- a/cli/src/commands/run.zig +++ b/cli/src/commands/run.zig @@ -4,8 +4,8 @@ const config = @import("../config.zig"); const mode = @import("../mode.zig"); const common = @import("common.zig"); -const remote = @import("exec/remote.zig"); -const local = @import("exec/local.zig"); +const remote = @import("executor/remote.zig"); +const local = @import("executor/local.zig"); pub const RunMode = enum { local, @@ -27,6 +27,16 @@ pub const RunOptions = struct { intent: ?[]const u8 = null, expected_outcome: ?[]const u8 = null, tags: ?[]const u8 = null, + // Dataset/snapshot config + commit_id: ?[]const u8 = null, + snapshot_id: ?[]const u8 = null, + snapshot_sha256: ?[]const u8 = null, + // Re-run options + rerun_id: ?[]const u8 = null, + inherit_narrative: bool = false, + inherit_config: bool = false, + inherit_all: bool = false, + parent_run_id: ?[]const u8 = null, }; /// Unified run command - transparently handles local and remote execution @@ -95,6 +105,24 @@ pub fn execute(allocator: std.mem.Allocator, args: []const []const u8) !void { } else if (std.mem.eql(u8, arg, "--tags") and i + 1 < pre.len) { options.tags = pre[i + 1]; i += 1; + } else if (std.mem.eql(u8, arg, "--rerun") and i + 1 < pre.len) { + options.rerun_id = pre[i + 1]; + i += 1; + } else if (std.mem.eql(u8, arg, "--inherit-narrative")) { + options.inherit_narrative = true; + } else if (std.mem.eql(u8, arg, "--inherit-config")) { + options.inherit_config = true; + } else if (std.mem.eql(u8, arg, "--rerun-all")) { + options.inherit_all = true; + options.inherit_narrative = true; + options.inherit_config = true; + } else if (std.mem.eql(u8, arg, "--parent")) { + if (i + 1 < pre.len and !std.mem.startsWith(u8, pre[i + 1], "-")) { + options.parent_run_id = pre[i + 1]; + i += 1; + } else { + options.parent_run_id = "auto"; // Mark for auto-detection from rerun_id + } } else if (!std.mem.startsWith(u8, arg, "-")) { if (job_name == null) { job_name = arg; @@ -129,6 +157,18 @@ pub fn execute(allocator: std.mem.Allocator, args: []const []const u8) !void { mut.deinit(allocator); } + // Handle re-run: load previous run manifest and inherit options + if (options.rerun_id) |rerun_id| { + // Default to inheriting everything for absolute reproducibility + if (!options.inherit_narrative and !options.inherit_config and !options.inherit_all) { + options.inherit_all = true; + options.inherit_narrative = true; + options.inherit_config = true; + std.log.info("Re-run: inheriting all config for reproducibility (use --inherit-* flags to customize)", .{}); + } + try handleRerun(allocator, rerun_id, &options, cfg.worker_base); + } + // Determine execution mode: config > auto-detect var run_mode: RunMode = undefined; @@ -194,6 +234,117 @@ fn explainJob(allocator: std.mem.Allocator, job_name: []const u8, options: *cons _ = allocator; } +/// Handle re-run logic: load manifest and inherit options +fn handleRerun( + allocator: std.mem.Allocator, + rerun_id: []const u8, + options: *RunOptions, + worker_base: []const u8, +) !void { + const manifest = @import("../utils/manifest.zig"); + const json = @import("../utils/json.zig"); + + // Resolve manifest path + const manifest_path = manifest.resolvePathWithBase(allocator, rerun_id, worker_base) catch |err| { + std.log.err("Could not find run '{s}': {s}", .{ rerun_id, @errorName(err) }); + return error.RunNotFound; + }; + defer allocator.free(manifest_path); + + // Read and parse manifest + const data = manifest.readFileAlloc(allocator, manifest_path) catch |err| { + std.log.err("Could not read manifest for '{s}': {s}", .{ rerun_id, @errorName(err) }); + return error.ManifestReadFailed; + }; + defer allocator.free(data); + + const parsed = std.json.parseFromSlice(std.json.Value, allocator, data, .{}) catch |err| { + std.log.err("Could not parse manifest for '{s}': {s}", .{ rerun_id, @errorName(err) }); + return error.ManifestParseFailed; + }; + defer parsed.deinit(); + + if (parsed.value != .object) { + return error.InvalidManifest; + } + const obj = parsed.value.object; + + // Inherit narrative fields if requested + if (options.inherit_narrative) { + if (options.hypothesis == null) { + options.hypothesis = json.getString(obj, "hypothesis"); + } + if (options.context == null) { + options.context = json.getString(obj, "context"); + } + if (options.intent == null) { + options.intent = json.getString(obj, "intent"); + } + if (options.expected_outcome == null) { + options.expected_outcome = json.getString(obj, "expected_outcome"); + } + if (options.tags == null) { + options.tags = json.getString(obj, "tags"); + } + std.log.info("Inherited narrative from previous run", .{}); + } + + // Inherit config if requested (resources, priority, dataset config) + if (options.inherit_config or options.inherit_all) { + if (options.cpu == 1) { // Only if not explicitly set + if (json.getInt(u8, obj, "cpu")) |cpu| { + options.cpu = cpu; + } + } + if (options.memory == 4) { // Only if not explicitly set + if (json.getInt(u8, obj, "memory")) |mem| { + options.memory = mem; + } + } + if (options.gpu == 0) { // Only if not explicitly set + if (json.getInt(u8, obj, "gpu")) |gpu| { + options.gpu = gpu; + } + } + if (options.priority == 5) { // Only if not explicitly set + if (json.getInt(u8, obj, "priority")) |prio| { + options.priority = prio; + } + } + if (options.gpu_memory == null) { + options.gpu_memory = json.getString(obj, "gpu_memory"); + } + // Inherit dataset/snapshot config + if (options.commit_id == null) { + options.commit_id = json.getString(obj, "commit_id"); + } + if (options.snapshot_id == null) { + options.snapshot_id = json.getString(obj, "snapshot_id"); + } + if (options.snapshot_sha256 == null) { + options.snapshot_sha256 = json.getString(obj, "snapshot_sha256"); + } + if (options.inherit_all) { + std.log.info("Inherited all config from previous run", .{}); + } else { + std.log.info("Inherited config from previous run", .{}); + } + } + + // Set parent_run_id for provenance tracking + if (options.parent_run_id) |pr| { + if (std.mem.eql(u8, pr, "auto")) { + // Auto-detect from manifest + if (json.getString(obj, "run_id")) |parent_id| { + options.parent_run_id = try allocator.dupe(u8, parent_id); + std.log.info("Set parent run for provenance: {s}", .{parent_id[0..@min(8, parent_id.len)]}); + } + } + } + + std.log.info("Re-running from previous run: {s}", .{rerun_id[0..@min(8, rerun_id.len)]}); +} + fn printUsage() !void { std.debug.print("Usage: ml run [options] [-- ]\n", .{}); std.debug.print("\nUnified run command - handles both local and remote execution.\n", .{}); @@ -209,4 +360,11 @@ fn printUsage() !void { std.debug.print(" --hypothesis Research hypothesis\n", .{}); std.debug.print(" --context Background information\n", .{}); std.debug.print(" --tags Comma-separated tags\n", .{}); + std.debug.print("\nRe-run Options (Research-First Reproducibility):\n", .{}); + std.debug.print(" --rerun Re-run inheriting everything by default\n", .{}); + std.debug.print(" --inherit-narrative Only inherit hypothesis/context/tags\n", .{}); + std.debug.print(" --inherit-config Only inherit resources/dataset/priority\n", .{}); + std.debug.print(" --parent [run_id] Link as child for provenance (auto if no id)\n", .{}); + std.debug.print("\nNote: --rerun alone ensures absolute reproducibility. Use --inherit-*\n", .{}); + std.debug.print(" flags to selectively inherit specific aspects only.\n", .{}); } diff --git a/cli/src/commands/submission/index.zig b/cli/src/commands/submission/index.zig new file mode 100644 index 0000000..7c4426e --- /dev/null +++ b/cli/src/commands/submission/index.zig @@ -0,0 +1,3 @@ +pub const parse = @import("parse.zig"); +pub const validate = @import("validate.zig"); +pub const submit = @import("submit.zig"); diff --git a/cli/src/commands/queue/mod.zig b/cli/src/commands/submission/mod.zig similarity index 100% rename from cli/src/commands/queue/mod.zig rename to cli/src/commands/submission/mod.zig diff --git a/cli/src/commands/queue/parse.zig b/cli/src/commands/submission/parse.zig similarity index 100% rename from cli/src/commands/queue/parse.zig rename to cli/src/commands/submission/parse.zig diff --git a/cli/src/commands/queue/submit.zig b/cli/src/commands/submission/submit.zig similarity index 100% rename from cli/src/commands/queue/submit.zig rename to cli/src/commands/submission/submit.zig diff --git a/cli/src/commands/queue/validate.zig b/cli/src/commands/submission/validate.zig similarity index 100% rename from cli/src/commands/queue/validate.zig rename to cli/src/commands/submission/validate.zig