perf: MD outline ~15× + full parse ~8× via line scanner

.gitignore

@@ -12,3 +12,4 @@ _tags
 /dune-workspace
 examples/syntax.html
 js/package/index.js
+examples/logseq_large.md

bench/bench.ml

@@ -1,3 +1,4 @@
+open Mldoc
 open Mldoc.Parser
 open Mldoc.Conf
 open Core
@@ -26,6 +27,8 @@ let config =
   }
 
 let outline_config = { config with parse_outline_only = true }
+let md_config = { config with format = Markdown }
+let md_outline_config = { md_config with parse_outline_only = true }
 
 let main () =
   Command_unix.run
@@ -41,9 +44,9 @@ let main () =
        ; Bench.Test.create ~name:"Mldoc Org mode parser (outline only)"
            (fun () -> ignore (parse outline_config doc_org))
        ; Bench.Test.create ~name:"Mldoc Markdown parser" (fun () ->
-             ignore (parse config syntax_md))
+             ignore (parse md_config syntax_md))
        ; Bench.Test.create ~name:"Mldoc Markdown parser (outline only)"
-           (fun () -> ignore (parse outline_config syntax_md))
+           (fun () -> ignore (parse md_outline_config syntax_md))
        ])
 
 let () = main ()

bench/dune

@@ -2,6 +2,10 @@
  (name bench)
  (libraries angstrom mldoc core core_bench core_unix.command_unix))
 
+(executable
+ (name time_parse)
+ (libraries unix mldoc))
+
 (alias
  (name bench)
- (deps bench.exe))
+ (deps bench.exe time_parse.exe))

bench/time_parse.ml

@@ -0,0 +1,103 @@
+(** Wall-clock timing focused on Markdown (Logseq) workloads. *)
+open Mldoc.Parser
+
+open Mldoc.Conf
+
+let ensure_logseq_large path =
+  if not (Sys.file_exists path) then (
+    let buf = Buffer.create 1_200_000 in
+    for i = 0 to 3999 do
+      Buffer.add_string buf
+        (Printf.sprintf "- Block title %d with [[page %d]] and #tag%d\n" i
+           (i mod 50) (i mod 20));
+      if i mod 3 = 0 then
+        Buffer.add_string buf
+          (Printf.sprintf "  id:: %08x-xxxx-xxxx-xxxx-%012x\n" i i);
+      if i mod 5 = 0 then (
+        Buffer.add_string buf
+          (Printf.sprintf "  - child of %d with ((%08x-xxxx-xxxx-xxxx-%012x))\n"
+             i i i);
+        Buffer.add_string buf
+          (Printf.sprintf "    more plain text line without markup %d\n" i)
+      );
+      if i mod 7 = 0 then
+        Buffer.add_string buf
+          (Printf.sprintf
+             "  plain paragraph under block %d word word word word word\n" i);
+      if i mod 11 = 0 then
+        Buffer.add_string buf (Printf.sprintf "  + unordered item %d\n" i);
+      if i mod 13 = 0 then
+        Buffer.add_string buf
+          (Printf.sprintf "  ```\n  code line %d\n  ```\n" i)
+    done;
+    let content = Buffer.contents buf in
+    (* Grow to ~1.2MB so benches stay comparable across revisions. *)
+    let pieces = ref [] in
+    let len = ref 0 in
+    while !len < 1_200_000 do
+      pieces := content :: !pieces;
+      len := !len + String.length content
+    done;
+    let grown = String.concat "" (List.rev !pieces) in
+    let oc = open_out path in
+    output_string oc (String.sub grown 0 1_200_000);
+    close_out oc
+  )
+
+let () = ensure_logseq_large "./examples/logseq_large.md"
+let doc_org = load_file "./examples/doc.org"
+let syntax_md = load_file "./examples/syntax.md"
+let logseq_md = load_file "./examples/logseq_large.md"
+
+let base =
+  { toc = true
+  ; parse_outline_only = false
+  ; heading_number = true
+  ; keep_line_break = false
+  ; format = Markdown
+  ; heading_to_list = false
+  ; exporting_keep_properties = false
+  ; inline_type_with_pos = false
+  ; inline_skip_macro = false
+  ; export_md_indent_style = Dashes
+  ; export_md_remove_options = []
+  ; hiccup_in_block = true
+  ; enable_drawers = true
+  ; parse_marker = true
+  ; parse_priority = true
+  }
+
+let avg ~n f =
+  ignore (f ());
+  let t0 = Unix.gettimeofday () in
+  for _ = 1 to n do
+    ignore (f ())
+  done;
+  let t1 = Unix.gettimeofday () in
+  (t1 -. t0) /. float n
+
+let () =
+  let n = 3 in
+  let md_full = avg ~n (fun () -> parse base logseq_md) in
+  let md_outline =
+    avg ~n (fun () -> parse { base with parse_outline_only = true } logseq_md)
+  in
+  let syn_full = avg ~n (fun () -> parse base syntax_md) in
+  let syn_outline =
+    avg ~n (fun () -> parse { base with parse_outline_only = true } syntax_md)
+  in
+  let org = { base with format = Org; parse_outline_only = false } in
+  let org_full = avg ~n (fun () -> parse org doc_org) in
+  let org_outline =
+    avg ~n (fun () -> parse { org with parse_outline_only = true } doc_org)
+  in
+  Printf.printf "iterations=%d (avg seconds)\n" n;
+  Printf.printf "MD logseq_large full:         %.4f\n" md_full;
+  Printf.printf "MD logseq_large outline_only: %.4f  (%.1fx vs full)\n"
+    md_outline (md_full /. md_outline);
+  Printf.printf "MD syntax.md full:            %.4f\n" syn_full;
+  Printf.printf "MD syntax.md outline_only:    %.4f  (%.1fx vs full)\n"
+    syn_outline (syn_full /. syn_outline);
+  Printf.printf "Org doc.org full:             %.4f\n" org_full;
+  Printf.printf "Org doc.org outline_only:     %.4f  (%.1fx vs full)\n"
+    org_outline (org_full /. org_outline)

lib/dune

@@ -1,9 +1,9 @@
 (include_subdirs unqualified)
 
-(env
+ (env
  (dev
   (flags
-   (:standard -warn-error -A))))
+   (:standard -warn-error -A -alert -deprecated))))
 
 (library
  (name mldoc)

lib/export/conf.ml

@@ -42,23 +42,25 @@ type t =
     (* hiccup: bool; *)
     toc : bool [@default false]
   ; parse_outline_only : bool [@default false]
+        (** Fast path (esp. Markdown): block structure + properties, and in
+            content only node refs ([[page]] / ((block))), tags (#tag).
+            Skips emphasis/code/timestamps and full Inline.parse. *)
   ; heading_number : bool [@default false]
   ; keep_line_break : bool (* FIXME: is this option deprecated? *)
   ; format : format
   ; heading_to_list : bool [@default false] (* export heading as list *)
   ; exporting_keep_properties : bool
         [@default false] (* keep properties when exporting *)
   ; inline_type_with_pos : bool [@default false]
-  ; inline_skip_macro: bool [@default false]
+  ; inline_skip_macro : bool [@default false]
   ; export_md_indent_style : indent_style [@default Dashes]
   ; export_md_remove_options : meta_chars list [@default []]
   ; hiccup_in_block : bool [@default true]
   ; enable_drawers : bool [@default true]
-  ; parse_marker: bool [@default true]
-  ; parse_priority: bool [@default true]
+  ; parse_marker : bool [@default true]
+  ; parse_priority : bool [@default true]
   }
 [@@deriving yojson]
 
 let is_markdown t = t.format = Markdown
-
 let is_org t = t.format = Org

lib/mldoc.ml

@@ -3,6 +3,7 @@
 module Document = Document
 module Block = Type_parser.Block
 module Inline = Inline
+module Outline_inline = Outline_inline
 module Nested_link = Nested_link
 module Pos = Pos
 module Exporters = Exporter.Exporters