parallelize markdown processing

Signed-off-by: Robert Günzler <r@gnzler.io>

1 files changed, 57 insertions, 39 deletions

diff --git a/src/exe.zig b/src/exe.zig
index 2af8b66..d5bc239 100644
--- a/src/exe.zig
+++ b/src/exe.zig
@@ -8,50 +8,68 @@ pub fn run_pandoc(ctx: *Context) !void {
     ctx.progress_start("Processing markdown files");
     defer ctx.progress_end();
 
-    for (ctx.markdown.items) |page| {
-        const source_path = try std.fs.path.join(ctx.allocator(), &.{ Context.params.PAGE_PATH, page });
-        defer ctx.allocator().free(source_path);
-        const target_path = try fs.replace_extension(ctx.allocator(), source_path, "htm");
-        defer ctx.allocator().free(target_path);
-
-        const f = try fs.FileSwapper.createAt(ctx.allocator(), target_path);
-        defer f.close();
-
-        try f.writer().print(
-            \\<!-- Generated file, you may edit and discard {s} :) -->
-            \\
-        ,
-            .{page},
-        );
-
-        // get mtime of the source document
-        const mtime = try fs.get_mtime(ctx.allocator(), source_path);
-        const mtime_str = try fs.format_mtime(ctx.allocator(), mtime, "%s");
-        defer ctx.allocator().free(mtime_str);
-
-        // make pandoc runs reproducable
-        var env_map = std.process.EnvMap.init(ctx.allocator());
-        defer env_map.deinit();
-        try env_map.put("SOURCE_DATE_EPOCH", mtime_str);
-
-        try capture_stdout(ctx, &.{
-            Context.params.PANDOC,
-            "--from",
-            "markdown+wikilinks_title_after_pipe+autolink_bare_uris",
-            "--to",
-            "html5",
-            "--lua-filter",
-            // TODO: embed the preproc filter
-            Context.params.PANDOC_PREPROC,
-            "--strip-comments",
-            "--output=-",
-            source_path,
-        }, &env_map, f.writer().any());
+    var pool: std.Thread.Pool = undefined;
+    try pool.init(.{
+        .n_jobs = (try std.Thread.getCpuCount()) - 1,
+        .allocator = ctx.allocator(),
+    });
+    defer pool.deinit();
+
+    for (ctx.markdown.items) |*page| {
+        // the inline struct trick is a workardoun for Thread.Pool::spawn not
+        // accepting functions that can error:
+        // https://github.com/ziglang/zig/issues/18810
+        try pool.spawn(struct {
+            fn worker(worker_ctx: *Context, worker_page: *const []const u8) void {
+                pandoc_worker(worker_ctx, worker_page.*) catch |err| std.log.err("pandoc worker failed with: {}", .{err});
+            }
+        }.worker, .{ ctx, page });
     }
 
     log.info("Processed {d} markdown files.", .{ctx.markdown.items.len});
 }
 
+fn pandoc_worker(ctx: *Context, page: []const u8) !void {
+    const source_path = try std.fs.path.join(ctx.allocator(), &.{ Context.params.PAGE_PATH, page });
+    defer ctx.allocator().free(source_path);
+    const target_path = try fs.replace_extension(ctx.allocator(), source_path, "htm");
+    defer ctx.allocator().free(target_path);
+
+    const f = try fs.FileSwapper.createAt(ctx.allocator(), target_path);
+    defer f.close();
+
+    try f.writer().print(
+        \\<!-- Generated file, you may edit and discard {s} :) -->
+        \\
+    ,
+        .{page},
+    );
+
+    // get mtime of the source document
+    const mtime = try fs.get_mtime(ctx.allocator(), source_path);
+    const mtime_str = try fs.format_mtime(ctx.allocator(), mtime, "%s");
+    defer ctx.allocator().free(mtime_str);
+
+    // make pandoc runs reproducable
+    var env_map = std.process.EnvMap.init(ctx.allocator());
+    defer env_map.deinit();
+    try env_map.put("SOURCE_DATE_EPOCH", mtime_str);
+
+    try capture_stdout(ctx, &.{
+        Context.params.PANDOC,
+        "--from",
+        "markdown+wikilinks_title_after_pipe+autolink_bare_uris",
+        "--to",
+        "html5",
+        "--lua-filter",
+        // TODO: embed the preproc filter
+        Context.params.PANDOC_PREPROC,
+        "--strip-comments",
+        "--output=-",
+        source_path,
+    }, &env_map, f.writer().any());
+}
+
 pub fn capture_stdout(ctx: *Context, argv: []const []const u8, env_map: ?*std.process.EnvMap, buf: std.io.AnyWriter) !void {
     const command = try std.mem.join(ctx.allocator(), " ", argv);
     defer ctx.allocator().free(command);