diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76cba2c0..97a52a36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,6 @@ jobs: uses: ocaml/setup-ocaml@v3 with: ocaml-compiler: ${{ matrix.ocaml-compiler }} - dune-cache: true - name: Install dependencies run: opam install . --deps-only --with-doc --with-test diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index e74f7710..89f9a881 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -16,7 +16,6 @@ jobs: uses: ocaml/setup-ocaml@v3 with: ocaml-compiler: 4.14.x - dune-cache: true - name: Install formatter run: opam install ocamlformat.0.26.2 dune diff --git a/bench/dune b/bench/dune index 7bf79085..1a60565e 100644 --- a/bench/dune +++ b/bench/dune @@ -6,6 +6,10 @@ (name time_parse) (libraries unix mldoc)) +(executable + (name time_graph) + (libraries unix mldoc)) + (alias (name bench) - (deps bench.exe time_parse.exe)) + (deps bench.exe time_parse.exe time_graph.exe)) diff --git a/bench/time_graph.ml b/bench/time_graph.ml new file mode 100644 index 00000000..683383d4 --- /dev/null +++ b/bench/time_graph.ml @@ -0,0 +1,142 @@ +(** Bench parsing a Logseq Markdown graph directory (many small files). *) +open Mldoc.Parser + +open Mldoc.Conf + +let rec md_files acc dir = + match Unix.opendir dir with + | exception Unix.Unix_error (Unix.ENOENT, _, _) -> acc + | dh -> + let rec loop acc = + match Unix.readdir dh with + | exception End_of_file -> + Unix.closedir dh; + acc + | "." + | ".." -> + loop acc + | name -> + let path = Filename.concat dir name in + let acc = + match Unix.stat path with + | { st_kind = S_DIR; _ } -> md_files acc path + | { st_kind = S_REG; _ } when Filename.check_suffix name ".md" -> + path :: acc + | _ -> acc + in + loop acc + in + loop acc + +let load_all dir = + let files = md_files [] dir in + let files = List.sort String.compare files in + List.map (fun path -> (path, load_file path)) files + +let base = + { toc = true + ; parse_outline_only = false + ; heading_number = true + ; keep_line_break = false + ; format = Markdown + ; heading_to_list = false + ; exporting_keep_properties = false + ; inline_type_with_pos = false + ; inline_skip_macro = false + ; export_md_indent_style = Dashes + ; export_md_remove_options = [] + ; hiccup_in_block = true + ; enable_drawers = true + ; parse_marker = true + ; parse_priority = true + } + +let avg ~n f = + ignore (f ()); + let t0 = Unix.gettimeofday () in + for _ = 1 to n do + ignore (f ()) + done; + let t1 = Unix.gettimeofday () in + (t1 -. t0) /. float n + +let property_values contents = + let acc = ref [] in + List.iter + (fun content -> + String.split_on_char '\n' content + |> List.iter (fun line -> + match String.index_opt line ':' with + | Some i when i + 1 < String.length line && line.[i + 1] = ':' -> + let v = + String.trim + (String.sub line (i + 2) (String.length line - i - 2)) + in + if v <> "" then acc := v :: !acc + | _ -> ())) + contents; + List.rev !acc + +let () = + let dir = + if Array.length Sys.argv > 1 then + Sys.argv.(1) + else + "/tmp/ls-movies-4k" + in + let pages = Filename.concat dir "pages" in + let journals = Filename.concat dir "journals" in + let loaded = + if Sys.file_exists pages then + load_all pages + @ + if Sys.file_exists journals then + load_all journals + else + [] + else + load_all dir + in + let n_files = List.length loaded in + let contents = List.map snd loaded in + let bytes = List.fold_left (fun s c -> s + String.length c) 0 contents in + let concat = String.concat "\n" contents in + let values = property_values contents in + let n = 3 in + Printf.printf "graph=%s files=%d bytes=%d avg_file=%.0f props=%d\n" dir + n_files bytes + (if n_files = 0 then + 0. + else + float bytes /. float n_files) + (List.length values); + let parse_each config docs = + List.iter (fun c -> ignore (parse config c)) docs + in + let full = avg ~n (fun () -> parse_each base contents) in + let outline = + avg ~n (fun () -> + parse_each { base with parse_outline_only = true } contents) + in + let concat_full = avg ~n (fun () -> parse base concat) in + let concat_outline = + avg ~n (fun () -> parse { base with parse_outline_only = true } concat) + in + let refs = + avg ~n (fun () -> + List.iter + (fun v -> ignore (Mldoc.Property.property_references base v)) + values) + in + Printf.printf "iterations=%d (avg seconds)\n" n; + Printf.printf "per-file full: %.4f (%.1f files/s)\n" full + (float n_files /. full); + Printf.printf "per-file outline_only: %.4f (%.1fx vs full)\n" outline + (full /. outline); + Printf.printf "concatenated full: %.4f\n" concat_full; + Printf.printf "concatenated outline_only: %.4f (%.1fx vs concat full)\n" + concat_outline + (concat_full /. concat_outline); + Printf.printf "property_references only: %.4f (%.0f%% of per-file full)\n" + refs + (100. *. refs /. full) diff --git a/lib/export/conf.ml b/lib/export/conf.ml index 9905bad0..6a0b782d 100644 --- a/lib/export/conf.ml +++ b/lib/export/conf.ml @@ -42,9 +42,10 @@ type t = (* hiccup: bool; *) toc : bool [@default false] ; parse_outline_only : bool [@default false] - (** Fast path (esp. Markdown): block structure + properties, and in - content only node refs ([[page]] / ((block))), tags (#tag). - Skips emphasis/code/timestamps and full Inline.parse. *) + (** Outline: headings keep title (Plain + [[page]] / ((block)) / #tag), + plus status/priority, properties, SCHEDULED/DEADLINE, and + front matter (first block only). Skips mixed markdown used for + block rendering (emphasis, code, autolinks). *) ; heading_number : bool [@default false] ; keep_line_break : bool (* FIXME: is this option deprecated? *) ; format : format diff --git a/lib/mldoc_parser.ml b/lib/mldoc_parser.ml index 269089a0..f33d304a 100644 --- a/lib/mldoc_parser.ml +++ b/lib/mldoc_parser.ml @@ -146,24 +146,35 @@ let build_md_outline_parsers config = let parse config input = let outline_only = Conf.(config.parse_outline_only) in let md = Conf.is_markdown config in - (* Markdown: line-oriented path for outline and full. *) - if md then + (* Outline markdown uses the line scanner. Full markdown stays on Angstrom so + mixed constructs (org blocks, drawers, definition lists, quotes) match + published mldoc / Logseq graph-parser. *) + if md && outline_only then let ast = Md_outline.parse config input in - if (not outline_only) || String.contains input '\\' then + if String.contains input '\\' then List.map (fun (t, pos) -> (Type_op.md_unescaped t, pos)) ast else ast else let parsers = - match outline_only with - | true -> build_choice_parsers org_outline_parsers config - | false -> build_choice_parsers org_full_parsers config + if md then + build_choice_parsers md_full_parsers config + else if outline_only then + build_choice_parsers org_outline_parsers config + else + build_choice_parsers org_full_parsers config in match parse_string ~consume:All parsers input with | Ok result -> let ast = Paragraph.concat_paragraph_lines config result in let ast = - if outline_only then + if md then + List.map (fun (t, pos) -> (Type_op.md_unescaped t, pos)) ast + else + ast + in + let ast = + if (not md) && outline_only then Prelude.remove (fun (t, _) -> match t with diff --git a/lib/syntax/heading0.ml b/lib/syntax/heading0.ml index e1aaa03b..3a90371e 100644 --- a/lib/syntax/heading0.ml +++ b/lib/syntax/heading0.ml @@ -147,12 +147,14 @@ struct explode (String.trim s) |> List.map map_char |> String.concat "" let outline_title config title = - if Outline_inline.may_have_outline_markup config title then + if title = "" then + [] + else if Outline_inline.may_have_outline_markup config title then match parse_string ~consume:All (Outline_inline.parse config) title with | Ok title -> title - | Error _ -> [] + | Error _ -> Type_op.inline_list_with_none_pos [ Inline.Plain title ] else - [] + Type_op.inline_list_with_none_pos [ Inline.Plain title ] let make_outline_heading ~level ~unordered ~size ~marker ~priority ~title = Heading diff --git a/lib/syntax/inline.ml b/lib/syntax/inline.ml index c57d6e7a..c816deae 100644 --- a/lib/syntax/inline.ml +++ b/lib/syntax/inline.ml @@ -1530,6 +1530,14 @@ let try_fast_md_inline s = true | _ -> false in + let is_tag_boundary = function + | ' ' + | '\t' + | '\n' + | '\r' -> + true + | _ -> false + in let tag_trail = function | ',' | ';' @@ -1586,18 +1594,34 @@ let try_fast_md_inline s = in while !i < n && not !complex do match s.[!i] with - | '\n' -> - flush_plain !i; - acc := Break_Line :: !acc; - incr i; - plain_start := !i + | '\n' | '\r' -> - flush_plain !i; - incr i; - if !i < n && s.[!i] = '\n' then incr i; - acc := Break_Line :: !acc; + (* Two trailing spaces before newline = CommonMark hard break. *) + let rec count_spaces k = + if k > !plain_start && s.[k - 1] = ' ' then + count_spaces (k - 1) + else + k + in + let sp = count_spaces !i in + if !i - sp >= 2 then ( + flush_plain sp; + acc := Hard_Break_Line :: !acc + ) else ( + flush_plain !i; + acc := Break_Line :: !acc + ); + if s.[!i] = '\r' then ( + incr i; + if !i < n && s.[!i] = '\n' then incr i + ) else + incr i; plain_start := !i - | '#' when !i + 1 < n && (not (is_ws s.[!i + 1])) && s.[!i + 1] <> '#' + | '#' + when !i + 1 < n + && (not (is_ws s.[!i + 1])) + && s.[!i + 1] <> '#' + && (!i = 0 || is_tag_boundary s.[!i - 1] || tag_trail s.[!i - 1]) -> flush_plain !i; let start = !i + 1 in diff --git a/lib/syntax/md_outline.ml b/lib/syntax/md_outline.ml index abbdba08..5ed158ba 100644 --- a/lib/syntax/md_outline.ml +++ b/lib/syntax/md_outline.ml @@ -1,14 +1,13 @@ -(* Fast Markdown document parser (outline_only + full). +(* Fast Markdown outline parser (parse_outline_only). Line-oriented; avoids Angstrom choice/backtracking on the Logseq hot path. - Outline: headings, properties, lists, quotes, footnotes, outline inline. - Full: same structure with Inline.parse, Src fences, latex env, anchors. *) + Outline: title (Plain + refs/tags), properties, timestamps, front matter + (first block only). Full mixed-markdown parse stays on Angstrom. *) open! Prelude open Type open Conf -let dummy = Pos.dummy_pos -let with_pos t = (t, dummy) +let plain_inlines s = Type_op.inline_list_with_none_pos [ Inline.Plain s ] let ensure_trailing_nl s = let n = String.length s in @@ -51,13 +50,6 @@ let is_space_char = function true | _ -> false -let rstrip_cr s = - let n = String.length s in - if n > 0 && s.[n - 1] = '\r' then - String.sub s 0 (n - 1) - else - s - let skip_spaces s i = let n = String.length s in let rec loop j = @@ -151,9 +143,9 @@ let outline_inlines config s = Angstrom.parse_string ~consume:All (Outline_inline.parse config) s with | Ok r -> r - | Error _ -> []) + | Error _ -> plain_inlines s) else - [] + plain_inlines s let full_inlines config s = if s = "" then @@ -449,10 +441,7 @@ let try_footnote_line config line = if body = "" then [] else if config.parse_outline_only then - if Outline_inline.may_have_outline_markup config body then - outline_inlines config body - else - Type_op.inline_list_with_none_pos [ Inline.Plain body ] + outline_inlines config body else content_inlines config body in @@ -498,6 +487,90 @@ let collect_properties config lines i = in loop i [] +let is_front_matter_fence line = String.trim line = "---" + +let parse_front_matter_directive line = + let n = String.length line in + let i = skip_spaces line 0 in + if i >= n then + None + else + let j = ref i in + while !j < n && line.[!j] <> ':' && line.[!j] <> '\n' do + incr j + done; + if !j > i && !j < n && line.[!j] = ':' then + let key = String.sub line i (!j - i) in + let rest_i = skip_spaces line (!j + 1) in + let value = + if rest_i >= n then + "" + else + String.sub line rest_i (n - rest_i) + in + Some (Directive (key, value)) + else + None + +(** YAML/Jekyll front matter is only valid as the first block. *) +let collect_front_matter lines = + let n = Array.length lines in + if n = 0 || not (is_front_matter_fence lines.(0)) then + None + else + let rec find_close j = + if j >= n then + None + else if is_front_matter_fence lines.(j) then + Some j + else + find_close (j + 1) + in + match find_close 1 with + | None -> None + | Some close -> + let rec loop j acc = + if j >= close then + List.rev acc + else + match parse_front_matter_directive lines.(j) with + | Some dir -> loop (j + 1) ((j, dir) :: acc) + | None -> loop (j + 1) acc + in + Some (loop 1 [], close + 1) + +let starts_with_ci s prefix = + let plen = String.length prefix in + let n = String.length s in + n >= plen + && + let rec loop k = + if k = plen then + true + else if Char.lowercase_ascii s.[k] = Char.lowercase_ascii prefix.[k] then + loop (k + 1) + else + false + in + loop 0 + +let is_timestamp_keyword_line line = + let t = String.trim line in + starts_with_ci t "SCHEDULED:" + || starts_with_ci t "DEADLINE:" + || starts_with_ci t "CLOSED:" + +let try_timestamp_line config line = + if not (is_timestamp_keyword_line line) then + None + else + match + Angstrom.parse_string ~consume:All (Inline.parse config) + (String.trim line) + with + | Ok inlines -> Some (Paragraph inlines) + | Error _ -> None + let is_block_boundary config line = is_blank_line line || try_dash_heading config line <> None @@ -507,6 +580,7 @@ let is_block_boundary config line = || try_md_property config line <> None || try_org_style_prop line <> None || try_footnote_line config line <> None + || (config.parse_outline_only && is_timestamp_keyword_line line) let collect_paragraph_lines config lines i = let n = Array.length lines in @@ -762,9 +836,21 @@ let rec parse_list_items config lines i min_indent = (List.rev !items, !j) let parse config input = - let raw_lines = String.split_on_char '\n' input in - let lines = Array.of_list (List.map rstrip_cr raw_lines) in + let raw_split = String.split_on_char '\n' input in + let decoded = + List.map + (fun s -> + let n = String.length s in + if n > 0 && s.[n - 1] = '\r' then + (String.sub s 0 (n - 1), n) + else + (s, n)) + raw_split + in + let lines = Array.of_list (List.map fst decoded) in + let raw_lens = Array.of_list (List.map snd decoded) in let n = Array.length lines in + let input_len = String.length input in let line_starts = let arr = Array.make (max n 1) 0 in let pos = ref 0 in @@ -776,17 +862,30 @@ let parse config input = else 0 in - pos := !pos + String.length lines.(idx) + nl + pos := !pos + raw_lens.(idx) + nl done; arr in + let pos_range i j = + let start_pos = + if i < n then + line_starts.(i) + else + input_len + in + let end_pos = + if j < n then + line_starts.(j) + else + input_len + in + { Pos.start_pos; end_pos } + in + let with_range i j t = (t, pos_range i j) in let src_end_pos body_i = let rec find j = if j >= n then - if n = 0 then - 0 - else - line_starts.(n - 1) + String.length lines.(n - 1) + input_len else if is_fence_line lines.(j) then line_starts.(j) else @@ -796,6 +895,14 @@ let parse config input = in let acc = ref [] in let i = ref 0 in + (* Front matter is only valid as the first block. *) + (match collect_front_matter lines with + | Some (dirs, j) -> + List.iter + (fun (line_i, dir) -> acc := with_range line_i (line_i + 1) dir :: !acc) + dirs; + i := j + | None -> ()); while !i < n do let line = lines.(!i) in if is_blank_line line then @@ -803,7 +910,8 @@ let parse config input = else match try_dash_heading config line with | Some (h, rest) -> ( - acc := with_pos h :: !acc; + let h_i = !i in + acc := with_range h_i (h_i + 1) h :: !acc; incr i; match rest with | Nothing -> () @@ -816,73 +924,78 @@ let parse config input = if body_i < n then line_starts.(body_i) else - line_starts.(!i - 1) + String.length lines.(!i - 1) + 1 + input_len in let body_end_pos = src_end_pos body_i in let src, j = collect_src_from_header ~body_start_pos ~body_end_pos lines body_i hdr in - acc := with_pos src :: !acc; + acc := with_range h_i j src :: !acc; i := j | Quote_line qline -> - if config.parse_outline_only then - () - else - let q, j = collect_quote config ~first_line:qline lines !i in - acc := with_pos q :: !acc; - i := j) + let q_i = h_i in + let q, j = collect_quote config ~first_line:qline lines !i in + acc := with_range q_i j q :: !acc; + i := j) | None -> ( match try_atx_heading config line with | Some h -> - acc := with_pos h :: !acc; + acc := with_range !i (!i + 1) h :: !acc; incr i | None -> ( match try_footnote_line config line with | Some fn -> - acc := with_pos fn :: !acc; + acc := with_range !i (!i + 1) fn :: !acc; incr i | None -> ( match collect_properties_drawer config lines !i with | Some (kvs, j) -> - acc := with_pos (Property_Drawer kvs) :: !acc; + acc := with_range !i j (Property_Drawer kvs) :: !acc; i := j | None -> ( match collect_properties config lines !i with | (_ :: _ as kvs), j -> - acc := with_pos (Property_Drawer kvs) :: !acc; + acc := with_range !i j (Property_Drawer kvs) :: !acc; i := j | [], _ -> ( if is_fence_line line then ( if config.parse_outline_only then i := skip_fence lines !i else + let start_i = !i in let src, j = collect_src ~line_starts lines !i in - acc := with_pos src :: !acc; + acc := with_range start_i j src :: !acc; i := j ) else if is_quote_line line then ( + let start_i = !i in let q, j = collect_quote config lines !i in - acc := with_pos q :: !acc; + acc := with_range start_i j q :: !acc; i := j ) else if is_list_item_prefix line then ( + let start_i = !i in let items, j = parse_list_items config lines !i (indent_len line) in - acc := with_pos (List items) :: !acc; + acc := with_range start_i j (List items) :: !acc; i := j ) else match if config.parse_outline_only then - None + try_timestamp_line config line else try_latex_environment line with - | Some latex -> - acc := with_pos latex :: !acc; + | Some (Latex_Environment _ as latex) -> + acc := with_range !i (!i + 1) latex :: !acc; + incr i + | Some ts -> + acc := with_range !i (!i + 1) ts :: !acc; incr i | None -> + let start_i = !i in let p, j = collect_paragraph_lines config lines !i in - acc := with_pos p :: !acc; + acc := with_range start_i j p :: !acc; i := j))))) done; List.rev !acc diff --git a/lib/syntax/outline_inline.ml b/lib/syntax/outline_inline.ml index 9619837a..8aeebb43 100644 --- a/lib/syntax/outline_inline.ml +++ b/lib/syntax/outline_inline.ml @@ -1,8 +1,8 @@ open! Prelude open Angstrom -open Parsers -(** Outline mode only extracts node refs and tags (properties are block-level). *) +(** Outline mode keeps Plain text plus node refs and tags (properties are + block-level). Emphasis/code/timestamps are left as Plain. *) let is_outline_special = function | '#' | '[' @@ -22,9 +22,6 @@ let may_have_outline_markup _config s = in loop 0 -let skip_plain_run = - skip_while (fun c -> (not (is_outline_special c)) && not (is_whitespace c)) - let interesting config : Inline.t Angstrom.t = peek_char_fail >>= function | '#' -> Inline.hash_tag config @@ -32,20 +29,15 @@ let interesting config : Inline.t Angstrom.t = | '(' -> Inline.block_reference config | _ -> fail "not outline inline" -let inline_choices config : Inline.t_with_pos option Angstrom.t = - peek_char_fail >>= function - | c when is_whitespace c -> any_char *> return None - | _ -> - interesting config - >>| (fun t -> Some (t, None)) - <|> any_char *> skip_plain_run *> return None +let inline_choices config : Inline.t_with_pos Angstrom.t = + interesting config + >>| (fun t -> (t, None)) + <|> ( take_while1 (fun c -> not (is_outline_special c)) >>| fun s -> + (Inline.Plain s, None) ) + <|> (any_char >>| fun c -> (Inline.Plain (String.make 1 c), None)) let parse_angstrom config = - many1 (inline_choices config) - >>| (fun l -> - let l = List.filter_map (fun x -> x) l in - Inline.concat_plains l) - "outline inline" + many1 (inline_choices config) >>| Inline.concat_plains "outline inline" let is_ws = function | ' ' @@ -133,8 +125,8 @@ let strip_tag_trail raw = in strip raw -(** Fast path for #tag / [[page]] / ((block)). Returns None when markdown - links or nested-page hashtags need the angstrom parser. +(** Fast path for Plain + #tag / [[page]] / ((block)). Returns None when + markdown links or nested-page hashtags need the angstrom parser. Scans [s] from [off] with length [len] (no need to sub the whole title). *) let try_fast_scan_range s off len = if len < 0 || off < 0 || off + len > String.length s then @@ -148,11 +140,21 @@ let try_fast_scan_range s off len = let end_ = off + len in let acc = ref [] in let i = ref off in + let plain_start = ref off in let complex = ref false in + let flush_plain stop = + if stop > !plain_start then + acc := + Inline.Plain (String.sub s !plain_start (stop - !plain_start)) :: !acc + in while !i < end_ && not !complex do match s.[!i] with - | '#' when !i + 1 < end_ && (not (is_ws s.[!i + 1])) && s.[!i + 1] <> '#' - -> + | '#' + when !i + 1 < end_ + && (not (is_ws s.[!i + 1])) + && s.[!i + 1] <> '#' + && (!i = off || is_ws s.[!i - 1] || tag_trail s.[!i - 1]) -> + flush_plain !i; let start = !i + 1 in let j = ref start in let has_bracket = ref false in @@ -163,9 +165,20 @@ let try_fast_scan_range s off len = if !has_bracket then complex := true else - let name = strip_tag_trail (String.sub s start (!j - start)) in - if name <> "" then acc := Inline.Tag [ Inline.Plain name ] :: !acc; - i := !j + let raw = String.sub s start (!j - start) in + let name = strip_tag_trail raw in + if name = "" then + complex := true + else ( + acc := Inline.Tag [ Inline.Plain name ] :: !acc; + let nl = String.length name in + if nl < String.length raw then + acc := + Inline.Plain (String.sub raw nl (String.length raw - nl)) + :: !acc; + i := !j; + plain_start := !j + ) | '[' when !i + 1 < end_ && s.[!i + 1] = '[' -> ( (* find_page_ref_end walks to string end; clamp by checking within range *) match find_page_ref_end s !i with @@ -175,24 +188,32 @@ let try_fast_scan_range s off len = if String.contains name '[' then complex := true else ( + flush_plain !i; acc := page_ref_link name :: !acc; - i := e + i := e; + plain_start := e ) | _ -> complex := true) | '[' -> complex := true | '(' when !i + 1 < end_ && s.[!i + 1] = '(' -> ( match find_block_ref_end s !i with | Some e when e <= end_ -> + flush_plain !i; let id = String.sub s (!i + 2) (e - !i - 4) in acc := block_ref_link id :: !acc; - i := e + i := e; + plain_start := e | _ -> incr i) | _ -> incr i done; if !complex then None - else - Some (Type_op.inline_list_with_none_pos (List.rev !acc)) + else ( + flush_plain end_; + Some + (Inline.concat_plains + (Type_op.inline_list_with_none_pos (List.rev !acc))) + ) let try_fast_scan s = try_fast_scan_range s 0 (String.length s) diff --git a/lib/syntax/paragraph.ml b/lib/syntax/paragraph.ml index 59290f80..d4803870 100644 --- a/lib/syntax/paragraph.ml +++ b/lib/syntax/paragraph.ml @@ -31,9 +31,9 @@ let parse_lines config lines pos1 pos2 = parse_string ~consume:All (Outline_inline.parse config) content with | Ok result -> Paragraph result - | Error _ -> Paragraph [] + | Error _ -> plain_paragraph content else - Paragraph [] + plain_paragraph content else match parse_string ~consume:All (Inline.parse config) content with | Ok result -> Paragraph result diff --git a/lib/syntax/property.ml b/lib/syntax/property.ml index 3c29b701..eab2cd18 100644 --- a/lib/syntax/property.ml +++ b/lib/syntax/property.ml @@ -16,24 +16,51 @@ open Conf 2. if there's no links, check whether it's separated by `,` *) +let keep_refs result = + List.filter + (fun e -> + match e with + | Inline.Tag _ -> true + | Inline.Link _ -> true + | Inline.Nested_link _ -> true + | _ -> false) + result + +(** Page refs, tags, block refs, macros-with-refs. Autolinks/emphasis are not + property references. *) +let may_have_property_refs s = + let n = String.length s in + let rec loop i = + if i >= n then + false + else + match s.[i] with + | '#' + | '[' + | '{' -> + true + | '(' when i + 1 < n && s.[i + 1] = '(' -> true + | _ -> loop (i + 1) + in + loop 0 + +let parse_refs_inline config s = + match parse_string ~consume:All (Inline.parse config) s with + | Ok result -> keep_refs (List.map fst result) + | Error _ -> [] + let property_references config s = let config = { config with inline_skip_macro = true } in let end_quoted = match last_char s with | Some '"' -> true - | _ -> false in + | _ -> false + in if s = "" || (s.[0] == '"' && end_quoted) then [] + else if not (may_have_property_refs s) then + [] else - match parse_string ~consume:All (Inline.parse config) s with - | Ok result -> - let result = List.map fst result in - List.filter - (fun e -> - match e with - | Inline.Tag _ -> true - | Inline.Link _ -> true - | Inline.Nested_link _ -> true - | _ -> false) - result - | Error _ -> [] + match Outline_inline.try_fast_scan s with + | Some result -> keep_refs (List.map fst result) + | None -> parse_refs_inline config s diff --git a/test/test_markdown.ml b/test/test_markdown.ml index fbfe099b..80a19081 100644 --- a/test/test_markdown.ml +++ b/test/test_markdown.ml @@ -23,6 +23,13 @@ let check_aux source expect = let result = Mldoc.Parser.parse default_config source |> List.hd |> fst in fun _ -> check_mldoc_type expect result +let logseq_md_config : Conf.t = + { default_config with + toc = false + ; heading_number = false + ; keep_line_break = true + } + let check_mldoc_type2 = Alcotest.check (Alcotest.testable @@ -38,6 +45,10 @@ let check_aux2 source expect = let result = List.map fst (Mldoc.Parser.parse default_config source) in fun _ -> check_mldoc_type2 expect result +let check_aux2_with config source expect = + let result = List.map fst (Mldoc.Parser.parse config source) in + fun _ -> check_mldoc_type2 expect result + let testcases = List.map (fun (case, level, f) -> Alcotest.test_case case level f) @@ -521,6 +532,41 @@ let inline = ; metadata = "" } ]) ) + ; ( "page ref with underscore" + , `Quick + , check_aux "see [[foo_bar]] and [[baz]]" + (paragraph + [ I.Plain "see " + ; I.Link + { url = I.Page_ref "foo_bar" + ; label = [ Plain "" ] + ; title = None + ; full_text = "[[foo_bar]]" + ; metadata = "" + } + ; I.Plain " and " + ; I.Link + { url = I.Page_ref "baz" + ; label = [ Plain "" ] + ; title = None + ; full_text = "[[baz]]" + ; metadata = "" + } + ]) ) + ; ( "page ref then emphasis" + , `Quick + , check_aux "[[foo_bar]] *ok*" + (paragraph + [ I.Link + { url = I.Page_ref "foo_bar" + ; label = [ Plain "" ] + ; title = None + ; full_text = "[[foo_bar]]" + ; metadata = "" + } + ; I.Plain " " + ; I.Emphasis (`Italic, [ I.Plain "ok" ]) + ]) ) ; ( "image link" , `Quick , check_aux "![lab[el]](url-part)" @@ -868,6 +914,10 @@ let inline = , `Quick , check_aux "#tag,.?" (paragraph [ I.Tag [ I.Plain "tag" ]; I.Plain ",.?" ]) ) + ; ( "#test hello" + , `Quick + , check_aux "#test hello" + (paragraph [ I.Tag [ I.Plain "test" ]; I.Plain " hello" ]) ) ; ( "with '.'" , `Quick , check_aux "#a.b.c" (paragraph [ I.Tag [ I.Plain "a.b.c" ] ]) ) @@ -1107,6 +1157,23 @@ let block = ; unordered = true ; size = None }) ) + ; ( "#test hello is a tag" + , `Quick + , check_aux "- #test hello" + (Type.Heading + { Type.title = + Type_op.inline_list_with_none_pos + [ Inline.Tag [ I.Plain "test" ]; I.Plain " hello" ] + ; tags = [] + ; marker = None + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "hello" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + }) ) ; ( "drawer" , `Quick , check_aux "a:: 1\n#+b: 2" @@ -1159,6 +1226,327 @@ let block = } ] ) ] ) + ; ( "todo scheduled quote" + , testcases + [ ( "TODO keeps plain title" + , `Quick + , check_aux "- TODO todo item" + (Type.Heading + { Type.title = + Type_op.inline_list_with_none_pos + [ Inline.Plain "todo item" ] + ; tags = [] + ; marker = Some "TODO" + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "todo_item" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + }) ) + ; ( "scheduled after TODO heading" + , `Quick + , check_aux2 + "- TODO wish [[name]] a happy birthday\n\ + SCHEDULED: <2025-11-01 Sat 08:00 .+1y>" + [ Type.Heading + { title = + Type_op.inline_list_with_none_pos + [ Inline.Plain "wish " + ; Inline.Link + { url = Inline.Page_ref "name" + ; label = [ Inline.Plain "" ] + ; title = None + ; full_text = "[[name]]" + ; metadata = "" + } + ; Inline.Plain " a happy birthday" + ] + ; tags = [] + ; marker = Some "TODO" + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "wish__a_happy_birthday" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ; paragraph + [ Inline.Timestamp + (Inline.Scheduled + { Timestamp.date = { year = 2025; month = 11; day = 1 } + ; wday = "Sat" + ; time = Some { hour = 8; min = 0 } + ; repetition = + Some (Timestamp.Dotted, Timestamp.Year, 1) + ; active = true + }) + ] + ] ) + ; ( "quote with email" + , `Quick + , check_aux2 "- > \"CachyOS \"" + [ Type.Heading + { title = [] + ; tags = [] + ; marker = None + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ; Type.Quote + [ paragraph + [ Inline.Plain "\"CachyOS " + ; Inline.Email + { Email_address.local_part = "admin" + ; domain = "cachyos.org" + } + ; Inline.Plain "\"" + ; Inline.Break_Line + ] + ] + ] ) + ; ( "nested heading positions" + , `Quick + , fun _ -> + let source = "- a\n - b\n - c" in + let got = + Mldoc.Parser.parse default_config source + |> List.map (fun (_, p) -> (p.Pos.start_pos, p.Pos.end_pos)) + in + Alcotest.(check (list (pair int int))) + "heading byte ranges" + [ (0, 4); (4, 10); (10, 17) ] + got ) + ; ( "hashtag in tags property" + , `Quick + , check_aux "tags:: [[foo]], #generated-page" + (Type.Property_Drawer + [ ( "tags" + , "[[foo]], #generated-page" + , [ Inline.Link + { url = Inline.Page_ref "foo" + ; label = [ Inline.Plain "" ] + ; title = None + ; full_text = "[[foo]]" + ; metadata = "" + } + ; Inline.Tag [ Inline.Plain "generated-page" ] + ] ) + ]) ) + ; ( "property wiki links" + , `Quick + , check_aux "genre:: [[Comedy]], [[Drama]], [[Romance]]" + (Type.Property_Drawer + [ ( "genre" + , "[[Comedy]], [[Drama]], [[Romance]]" + , [ Inline.Link + { url = Inline.Page_ref "Comedy" + ; label = [ Inline.Plain "" ] + ; title = None + ; full_text = "[[Comedy]]" + ; metadata = "" + } + ; Inline.Link + { url = Inline.Page_ref "Drama" + ; label = [ Inline.Plain "" ] + ; title = None + ; full_text = "[[Drama]]" + ; metadata = "" + } + ; Inline.Link + { url = Inline.Page_ref "Romance" + ; label = [ Inline.Plain "" ] + ; title = None + ; full_text = "[[Romance]]" + ; metadata = "" + } + ] ) + ]) ) + ; ( "property page-ref with underscore" + , `Quick + , check_aux "ref:: [[foo_bar]]" + (Type.Property_Drawer + [ ( "ref" + , "[[foo_bar]]" + , [ Inline.Link + { url = Inline.Page_ref "foo_bar" + ; label = [ Inline.Plain "" ] + ; title = None + ; full_text = "[[foo_bar]]" + ; metadata = "" + } + ] ) + ]) ) + ; ( "property autolink url" + , `Quick + , check_aux "url:: http://example.com/a" + (Type.Property_Drawer [ ("url", "http://example.com/a", []) ]) ) + ; ( "property url fragment is not a tag" + , `Quick + , check_aux "url:: http://example.com/a#type" + (Type.Property_Drawer [ ("url", "http://example.com/a#type", []) ]) + ) + ; ( "property tags after punctuation" + , `Quick + , check_aux "prop:: #foo: '#bar'" + (Type.Property_Drawer + [ ( "prop" + , "#foo: '#bar'" + , [ Inline.Tag [ Inline.Plain "foo" ] + ; Inline.Tag [ Inline.Plain "bar" ] + ] ) + ]) ) + ; ( "property macro value stays a drawer" + , `Quick + , check_aux "url:: {{docs-base-url url}}" + (Type.Property_Drawer [ ("url", "{{docs-base-url url}}", []) ]) ) + ; ( "definition list" + , `Quick + , check_aux "term\n: definition" + (Type.List + [ { content = [ paragraph [ Inline.Plain "definition" ] ] + ; items = [] + ; number = None + ; name = + Type_op.inline_list_with_none_pos [ Inline.Plain "term" ] + ; checkbox = None + ; indent = 0 + ; ordered = false + } + ]) ) + ; ( "src with leading whitespace" + , `Quick + , check_aux2_with logseq_md_config + "\n ```\n hello\n world\n ```\n" + [ paragraph [ Inline.Break_Line ] + ; Type.Src + { lines = [ " hello"; "\n"; " world"; "\n" ] + ; language = None + ; options = None + ; pos_meta = { Pos.start_pos = 7; end_pos = 25 } + } + ] ) + ; ( "logbook drawer" + , `Quick + , check_aux2 + "- DOING logbook block\n\ + \ :LOGBOOK:\n\ + \ CLOCK: [2024-08-07 Wed 11:47:50]\n\ + \ :END:" + [ Type.Heading + { title = + Type_op.inline_list_with_none_pos + [ Inline.Plain "logbook block" ] + ; tags = [] + ; marker = Some "DOING" + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "logbook_block" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ; Type.Drawer ("logbook", [ " CLOCK: [2024-08-07 Wed 11:47:50]" ]) + ] ) + ; ( "org begin quote" + , `Quick + , check_aux2 + "- From Inception:\n\ + \ #+BEGIN_QUOTE\n\ + \ Saito: Cobb?\n\ + \ #+END_QUOTE" + [ Type.Heading + { title = + Type_op.inline_list_with_none_pos + [ Inline.Plain "From Inception:" ] + ; tags = [] + ; marker = None + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "From_Inception-3a-" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ; Type.Quote + [ paragraph [ Inline.Plain "Saito: Cobb?"; Inline.Break_Line ] + ] + ] ) + ; ( "org quote blank line is hard break" + , `Quick + , check_aux2_with logseq_md_config + "- #+BEGIN_QUOTE\n it's a\n \n org blockquote\n #+END_QUOTE" + [ Type.Heading + { title = [] + ; tags = [] + ; marker = None + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ; Type.Quote + [ paragraph + [ Inline.Plain "it's a" + ; Inline.Break_Line + ; Inline.Hard_Break_Line + ; Inline.Plain "org blockquote" + ; Inline.Break_Line + ] + ] + ] ) + ; ( "org begin query" + , `Quick + , fun _ -> + match + Mldoc.Parser.parse default_config + "- Text before\n\ + \ #+BEGIN_QUERY\n\ + \ {:query (task todo)}\n\ + \ #+END_QUERY" + |> List.map fst + with + | Type.Heading _ :: Type.Custom ("query", _, _, _) :: _ -> () + | other -> + Alcotest.fail + ("expected heading + custom query, got " + ^ String.concat "; " + (List.map + (fun t -> Yojson.Safe.to_string (Type.to_yojson t)) + other)) ) + ; ( "front matter first block" + , `Quick + , check_aux2 "---\ntitle: Hello\n---\n- keep title" + [ Type.Directive ("title", "Hello") + ; paragraph [ Inline.Break_Line ] + ; Type.Heading + { title = + Type_op.inline_list_with_none_pos + [ Inline.Plain "keep title" ] + ; tags = [] + ; marker = None + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "keep_title" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ] ) + ] ) ] let () = Alcotest.run "mldoc" @@ List.concat [ inline; block ] diff --git a/test/test_outline_markdown.ml b/test/test_outline_markdown.ml index 9b0e72bb..2edc1b26 100644 --- a/test/test_outline_markdown.ml +++ b/test/test_outline_markdown.ml @@ -20,6 +20,7 @@ let check_mldoc_type = Alcotest.check (Alcotest.testable Type.pp ( = )) "check mldoc type" let paragraph l = Type.Paragraph (Type_op.inline_list_with_none_pos l) +let plain s = paragraph [ Inline.Plain s ] let check_aux source expect = let result = Mldoc.Parser.parse default_config source in @@ -56,7 +57,10 @@ let inline = let module I = Inline in [ ( "inline-link" , testcases - [ ("normal", `Quick, check_aux "http://testtest/asdasd" (paragraph [])) + [ ( "normal" + , `Quick + , check_aux "http://testtest/asdasd" (plain "http://testtest/asdasd") + ) ; ( "link with page alias" , `Quick , check_aux "[foo](bar)" @@ -83,19 +87,20 @@ let inline = ]) ) ; ( "include brackets" , `Quick - , check_aux "http://test/(foo)bar" (paragraph []) ) + , check_aux "http://test/(foo)bar" (plain "http://test/(foo)bar") ) ; ( "include brackets (2)" , `Quick - , check_aux "http://test/[(foo)b]ar" (paragraph []) ) + , check_aux "http://test/[(foo)b]ar" (plain "http://test/[(foo)b]ar") + ) ; ( "include brackets (3)" , `Quick - , check_aux "http://test/[foo)b]ar" (paragraph []) ) + , check_aux "http://test/[foo)b]ar" (plain "http://test/[foo)b]ar") ) ; ( "include brackets (4)" , `Quick - , check_aux "http://te(s)t/foobar" (paragraph []) ) + , check_aux "http://te(s)t/foobar" (plain "http://te(s)t/foobar") ) ; ( "include brackets (5)" , `Quick - , check_aux "http://test/foo{bar}" (paragraph []) ) + , check_aux "http://test/foo{bar}" (plain "http://test/foo{bar}") ) ] ) ; ( "link" , testcases @@ -117,7 +122,8 @@ let inline = , `Quick , check_aux "[not label][label](url)" (paragraph - [ I.Link + [ I.Plain "[not label]" + ; I.Link { url = I.Search "url" ; label = [ Plain "label" ] ; title = None @@ -295,7 +301,8 @@ let inline = , `Quick , check_aux "![lab[el]](url-part)" (paragraph - [ I.Link + [ I.Plain "!" + ; I.Link { url = I.Search "url-part" ; label = [ Plain "lab[el]" ] ; title = None @@ -404,71 +411,90 @@ let inline = , check_aux ":PROPERTIES:\r\n:END:\r\n" (Property_Drawer []) ) ; ( "no drawer in quote" , `Quick - , check_aux "> a:: b" (Quote [ paragraph [] ]) ) + , check_aux "> a:: b" (Quote [ plain "a:: b" ]) ) ] ) ; ( "inline-code" , testcases - [ ("normal", `Quick, check_aux "`codes here`" (paragraph [])) - ; ("overlap-with-emphasis", `Quick, check_aux "*aa`*`" (paragraph [])) + [ ("normal", `Quick, check_aux "`codes here`" (plain "`codes here`")) + ; ("overlap-with-emphasis", `Quick, check_aux "*aa`*`" (plain "*aa`*`")) ; ( "overlap-with-emphasis-2" , `Quick - , check_aux "**aa`**`" (paragraph []) ) - ; ("overlap-with-emphasis-3", `Quick, check_aux "_a`_`" (paragraph [])) - ; ("overlap-with-emphasis-4", `Quick, check_aux "__a`__`" (paragraph [])) - ; ("overlap-with-emphasis-5", `Quick, check_aux "`as*d`*" (paragraph [])) + , check_aux "**aa`**`" (plain "**aa`**`") ) + ; ("overlap-with-emphasis-3", `Quick, check_aux "_a`_`" (plain "_a`_`")) + ; ( "overlap-with-emphasis-4" + , `Quick + , check_aux "__a`__`" (plain "__a`__`") ) + ; ( "overlap-with-emphasis-5" + , `Quick + , check_aux "`as*d`*" (plain "`as*d`*") ) ; ( "overlap-with-link" , `Quick - , check_aux "[as`d](`http://dwdw)" (paragraph []) ) + , check_aux "[as`d](`http://dwdw)" (plain "[as`d](`http://dwdw)") ) ; ( "overlap-with-link-2" , `Quick - , check_aux "[as`d](http://dwdw)`" (paragraph []) ) + , check_aux "[as`d](http://dwdw)`" (plain "[as`d](http://dwdw)`") ) ] ) ; ( "emphasis" , testcases - [ ("normal", `Quick, check_aux "*abc*" (paragraph [])) - ; ("normal-2", `Quick, check_aux "**abc**" (paragraph [])) - ; ("normal-3", `Quick, check_aux "_a_," (paragraph [])) - ; ("inline-code-inside", `Quick, check_aux "*asd`qwe`*" (paragraph [])) + [ ("normal", `Quick, check_aux "*abc*" (plain "*abc*")) + ; ("normal-2", `Quick, check_aux "**abc**" (plain "**abc**")) + ; ("normal-3", `Quick, check_aux "_a_," (plain "_a_,")) + ; ( "inline-code-inside" + , `Quick + , check_aux "*asd`qwe`*" (plain "*asd`qwe`*") ) ; ( "inline-code-inside-2" , `Quick - , check_aux "***asd`qwe`***" (paragraph []) ) - ; ("not emphasis (1)", `Quick, check_aux "a * b*" (paragraph [])) - ; ("not emphasis (2)", `Quick, check_aux "a_b_c" (paragraph [])) - ; ("contains underline", `Quick, check_aux "_a _ a_" (paragraph [])) - ; ("contains star", `Quick, check_aux "*a * a*" (paragraph [])) + , check_aux "***asd`qwe`***" (plain "***asd`qwe`***") ) + ; ("not emphasis (1)", `Quick, check_aux "a * b*" (plain "a * b*")) + ; ("not emphasis (2)", `Quick, check_aux "a_b_c" (plain "a_b_c")) + ; ("contains underline", `Quick, check_aux "_a _ a_" (plain "_a _ a_")) + ; ("contains star", `Quick, check_aux "*a * a*" (plain "*a * a*")) ; ( "left flanking delimiter" , `Quick - , check_aux "hello_world_" (paragraph []) ) + , check_aux "hello_world_" (plain "hello_world_") ) ; ( "left flanking delimiter (2)" , `Quick - , check_aux "hello,_world_" (paragraph []) ) - ; ("highlight (1)", `Quick, check_aux "111==text==222" (paragraph [])) - ; ("highlight (2)", `Quick, check_aux "111== text==222" (paragraph [])) + , check_aux "hello,_world_" (plain "hello,_world_") ) + ; ( "highlight (1)" + , `Quick + , check_aux "111==text==222" (plain "111==text==222") ) + ; ( "highlight (2)" + , `Quick + , check_aux "111== text==222" (plain "111== text==222") ) ] ) ; ( "tag" , testcases [ ( "endwith '.'" , `Quick - , check_aux "#tag." (paragraph [ I.Tag [ I.Plain "tag" ] ]) ) + , check_aux "#tag." + (paragraph [ I.Tag [ I.Plain "tag" ]; I.Plain "." ]) ) ; ( "endwith ','" , `Quick - , check_aux "#tag," (paragraph [ I.Tag [ I.Plain "tag" ] ]) ) + , check_aux "#tag," + (paragraph [ I.Tag [ I.Plain "tag" ]; I.Plain "," ]) ) ; ( "endwith '\"'" , `Quick - , check_aux "#tag\"" (paragraph [ I.Tag [ I.Plain "tag" ] ]) ) + , check_aux "#tag\"" + (paragraph [ I.Tag [ I.Plain "tag" ]; I.Plain "\"" ]) ) ; ( "endwith several periods" , `Quick - , check_aux "#tag,.?" (paragraph [ I.Tag [ I.Plain "tag" ] ]) ) + , check_aux "#tag,.?" + (paragraph [ I.Tag [ I.Plain "tag" ]; I.Plain ",.?" ]) ) + ; ( "#test hello" + , `Quick + , check_aux "#test hello" + (paragraph [ I.Tag [ I.Plain "test" ]; I.Plain " hello" ]) ) ; ( "with '.'" , `Quick , check_aux "#a.b.c" (paragraph [ I.Tag [ I.Plain "a.b.c" ] ]) ) ; ( "with '.' and endwith '.'" , `Quick - , check_aux "#a.b.c." (paragraph [ I.Tag [ I.Plain "a.b.c" ] ]) ) + , check_aux "#a.b.c." + (paragraph [ I.Tag [ I.Plain "a.b.c" ]; I.Plain "." ]) ) ; ( "with '.' and endwith '.' (2)" , `Quick - , check_aux "#a.b.c. defg" (paragraph [ I.Tag [ I.Plain "a.b.c" ] ]) - ) + , check_aux "#a.b.c. defg" + (paragraph [ I.Tag [ I.Plain "a.b.c" ]; I.Plain ". defg" ]) ) ; ( "with page-ref" , `Quick , check_aux "#a.[[b c d ]].e." @@ -484,6 +510,7 @@ let inline = } ; I.Plain ".e" ] + ; I.Plain "." ]) ) ] ) ; ( "footnote-reference" @@ -492,7 +519,8 @@ let inline = , `Quick , check_aux "[^1][label](url)" (paragraph - [ I.Link + [ I.Plain "[^1]" + ; I.Link { url = I.Search "url" ; label = [ I.Plain "label" ] ; title = None @@ -503,10 +531,10 @@ let inline = ] ) ; ( "escape metachars" , testcases - [ ("emphasis(1)", `Quick, check_aux "*a\\*b*" (paragraph [])) - ; ("emphasis(2)", `Quick, check_aux "*a\\\\\\*b*" (paragraph [])) - ; ("code", `Quick, check_aux "`a\\``" (paragraph [])) - ; ("nested emphasis", `Quick, check_aux "_a*b\\*_" (paragraph [])) + [ ("emphasis(1)", `Quick, check_aux "*a\\*b*" (plain "*a*b*")) + ; ("emphasis(2)", `Quick, check_aux "*a\\\\\\*b*" (plain "*a\\*b*")) + ; ("code", `Quick, check_aux "`a\\``" (plain "`a``")) + ; ("nested emphasis", `Quick, check_aux "_a*b\\*_" (plain "_a*b*_")) ; ( "link (1)" , `Quick , check_aux "[[\\]]]" @@ -546,31 +574,92 @@ let inline = ] ) ; ( "Timestamps" , testcases - [ (* Outline mode skips timestamps; only refs/tags/properties. *) - ( "scheduled" + [ ( "scheduled" , `Quick - , check_aux "SCHEDULED: <2004-12-25 Sat>" (paragraph []) ) + , check_aux "SCHEDULED: <2004-12-25 Sat>" + (paragraph + [ Inline.Timestamp + (Inline.Scheduled + { Timestamp.date = { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = None + ; repetition = None + ; active = true + }) + ]) ) ; ( "scheduled with time" , `Quick - , check_aux "SCHEDULED: <2004-12-25 Sat 10:00>" (paragraph []) ) + , check_aux "SCHEDULED: <2004-12-25 Sat 10:00>" + (paragraph + [ Inline.Timestamp + (Inline.Scheduled + { Timestamp.date = { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = Some { hour = 10; min = 0 } + ; repetition = None + ; active = true + }) + ]) ) ; ( "scheduled with a repeater" , `Quick - , check_aux "SCHEDULED: <2004-12-25 Sat +1m>" (paragraph []) ) + , check_aux "SCHEDULED: <2004-12-25 Sat +1m>" + (paragraph + [ Inline.Timestamp + (Inline.Scheduled + { Timestamp.date = { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = None + ; repetition = Some (Timestamp.Plus, Timestamp.Month, 1) + ; active = true + }) + ]) ) ; ( "scheduled after some text" , `Quick - , check_aux "blabla SCHEDULED: <2004-12-25 Sat>" (paragraph []) ) + , check_aux "blabla SCHEDULED: <2004-12-25 Sat>" + (plain "blabla SCHEDULED: <2004-12-25 Sat>") ) ; ( "deadline" , `Quick - , check_aux "DEADLINE: <2004-12-25 Sat>" (paragraph []) ) + , check_aux "DEADLINE: <2004-12-25 Sat>" + (paragraph + [ Inline.Timestamp + (Inline.Deadline + { Timestamp.date = { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = None + ; repetition = None + ; active = true + }) + ]) ) ; ( "deadline with time" , `Quick - , check_aux "DEADLINE: <2004-12-25 Sat 10:00>" (paragraph []) ) + , check_aux "DEADLINE: <2004-12-25 Sat 10:00>" + (paragraph + [ Inline.Timestamp + (Inline.Deadline + { Timestamp.date = { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = Some { hour = 10; min = 0 } + ; repetition = None + ; active = true + }) + ]) ) ; ( "deadline with a repeater" , `Quick - , check_aux "DEADLINE: <2004-12-25 Sat +1m>" (paragraph []) ) + , check_aux "DEADLINE: <2004-12-25 Sat +1m>" + (paragraph + [ Inline.Timestamp + (Inline.Deadline + { Timestamp.date = { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = None + ; repetition = Some (Timestamp.Plus, Timestamp.Month, 1) + ; active = true + }) + ]) ) ; ( "deadline after some text" , `Quick - , check_aux "blabla DEADLINE: <2004-12-25 Sat>" (paragraph []) ) + , check_aux "blabla DEADLINE: <2004-12-25 Sat>" + (plain "blabla DEADLINE: <2004-12-25 Sat>") ) ] ) ] @@ -588,14 +677,14 @@ let block = , testcases [ ( "multi lines" , `Quick - , check_aux ">foo\n>bar" (Quote [ paragraph [] ]) ) + , check_aux ">foo\n>bar" (Quote [ plain "foo\nbar" ]) ) ] ) ; ( "latex_env" , testcases [ ( "one-line" , `Quick , check_aux "\\begin{equation}[a,b,c] x=\\sqrt{b} \\end{equation}" - (paragraph []) ) + (plain "\\begin{equation}[a,b,c] x=\\sqrt{b} \\end{equation}") ) ] ) ; ( "list" , testcases @@ -603,7 +692,7 @@ let block = , `Quick , check_aux "+ line1\n - heading" (List - [ { content = [ paragraph [] ] + [ { content = [ plain "line1" ] ; items = [] ; number = None ; name = [] @@ -616,7 +705,7 @@ let block = , `Quick , check_aux "+ line1\n -" (List - [ { content = [ paragraph [] ] + [ { content = [ plain "line1" ] ; items = [] ; number = None ; name = [] @@ -632,7 +721,8 @@ let block = , `Quick , check_aux "- ## TODO text" (Type.Heading - { Type.title = [] + { Type.title = + Type_op.inline_list_with_none_pos [ Inline.Plain "text" ] ; tags = [] ; marker = Some "TODO" ; level = 1 @@ -658,6 +748,23 @@ let block = ; unordered = true ; size = Some 2 }) ) + ; ( "TODO keeps plain title" + , `Quick + , check_aux "- TODO todo item" + (Type.Heading + { Type.title = + Type_op.inline_list_with_none_pos + [ Inline.Plain "todo item" ] + ; tags = [] + ; marker = Some "TODO" + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + }) ) ; ( "followed by #tag" , `Quick , check_aux "- #tag" @@ -675,6 +782,23 @@ let block = ; unordered = true ; size = None }) ) + ; ( "#test hello is a tag" + , `Quick + , check_aux "- #test hello" + (Type.Heading + { Type.title = + Type_op.inline_list_with_none_pos + [ Inline.Tag [ I.Plain "test" ]; I.Plain " hello" ] + ; tags = [] + ; marker = None + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + }) ) ; ( "drawer" , `Quick , check_aux "a:: 1\n#+b: 2" @@ -705,7 +829,8 @@ let block = , `Quick , check_aux2 "- test\nSCHEDULED: <2004-12-25 Sat>" [ Type.Heading - { title = [] + { title = + Type_op.inline_list_with_none_pos [ Inline.Plain "test" ] ; tags = [] ; marker = None ; level = 1 @@ -716,13 +841,24 @@ let block = ; unordered = true ; size = None } - ; paragraph [] + ; paragraph + [ I.Timestamp + (I.Scheduled + { Timestamp.date = + { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = None + ; repetition = None + ; active = true + }) + ] ] ) ; ( "a heading with a scheduled" , `Quick , check_aux2 "# test\nSCHEDULED: <2004-12-25 Sat>" [ Type.Heading - { title = [] + { title = + Type_op.inline_list_with_none_pos [ Inline.Plain "test" ] ; tags = [] ; marker = None ; level = 1 @@ -733,13 +869,24 @@ let block = ; unordered = false ; size = Some 1 } - ; paragraph [] + ; paragraph + [ I.Timestamp + (I.Scheduled + { Timestamp.date = + { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = None + ; repetition = None + ; active = true + }) + ] ] ) ; ( "a heading with a scheduled and some text" , `Quick , check_aux2 "# test\nSCHEDULED: <2004-12-25 Sat>\nsome [[page]]" [ Type.Heading - { title = [] + { title = + Type_op.inline_list_with_none_pos [ Inline.Plain "test" ] ; tags = [] ; marker = None ; level = 1 @@ -751,7 +898,19 @@ let block = ; size = Some 1 } ; paragraph - [ I.Link + [ I.Timestamp + (I.Scheduled + { Timestamp.date = + { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = None + ; repetition = None + ; active = true + }) + ] + ; paragraph + [ I.Plain "some " + ; I.Link { url = I.Page_ref "page" ; label = [ I.Plain "" ] ; title = None @@ -768,7 +927,8 @@ let block = DEADLINE: <2004-12-25 Sat>\n\ some [[page]]" [ Type.Heading - { title = [] + { title = + Type_op.inline_list_with_none_pos [ Inline.Plain "test" ] ; tags = [] ; marker = None ; level = 1 @@ -780,7 +940,30 @@ let block = ; size = Some 1 } ; paragraph - [ I.Link + [ I.Timestamp + (I.Scheduled + { Timestamp.date = + { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = None + ; repetition = None + ; active = true + }) + ] + ; paragraph + [ I.Timestamp + (I.Deadline + { Timestamp.date = + { year = 2004; month = 12; day = 25 } + ; wday = "Sat" + ; time = None + ; repetition = None + ; active = true + }) + ] + ; paragraph + [ I.Plain "some " + ; I.Link { url = I.Page_ref "page" ; label = [ I.Plain "" ] ; title = None @@ -789,6 +972,111 @@ let block = } ] ] ) + ; ( "nested headings keep titles" + , `Quick + , check_aux2 "- a\n - b\n - c" + [ Type.Heading + { title = + Type_op.inline_list_with_none_pos [ Inline.Plain "a" ] + ; tags = [] + ; marker = None + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ; Type.Heading + { title = + Type_op.inline_list_with_none_pos [ Inline.Plain "b" ] + ; tags = [] + ; marker = None + ; level = 3 + ; numbering = None + ; priority = None + ; anchor = "" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ; Type.Heading + { title = + Type_op.inline_list_with_none_pos [ Inline.Plain "c" ] + ; tags = [] + ; marker = None + ; level = 5 + ; numbering = None + ; priority = None + ; anchor = "" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ] ) + ; ( "quote with email" + , `Quick + , check_aux2 "- > \"CachyOS \"" + [ Type.Heading + { title = [] + ; tags = [] + ; marker = None + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ; Type.Quote [ plain "\"CachyOS \"" ] + ] ) + ; ( "front matter first block" + , `Quick + , check_aux2 "---\ntitle: Hello\n---\n- keep title [[page]]" + [ Type.Directive ("title", "Hello") + ; Type.Heading + { title = + Type_op.inline_list_with_none_pos + [ Inline.Plain "keep title " + ; Inline.Link + { url = Inline.Page_ref "page" + ; label = [ Inline.Plain "" ] + ; title = None + ; full_text = "[[page]]" + ; metadata = "" + } + ] + ; tags = [] + ; marker = None + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ] ) + ; ( "front matter only at first block" + , `Quick + , check_aux2 "- hello\n---\ntitle: no\n---" + [ Type.Heading + { title = + Type_op.inline_list_with_none_pos [ Inline.Plain "hello" ] + ; tags = [] + ; marker = None + ; level = 1 + ; numbering = None + ; priority = None + ; anchor = "" + ; meta = { Type.timestamps = []; properties = [] } + ; unordered = true + ; size = None + } + ; plain "---\ntitle: no\n---" + ] ) ] ) ]