bytes: document null termination

kklib/include/kklib/bytes.h

@@ -157,7 +157,10 @@ static inline const char* kk_bytes_cbuf_borrow(const kk_bytes_t b, kk_ssize_t* l
   return (const char*)kk_bytes_buf_borrow(b, len, ctx);
 }
 
-
+static inline int8_t kk_bytes_at(kk_bytes_t p, uint64_t i, kk_context_t* ctx){
+  const uint8_t* buf = kk_bytes_buf_borrow(p, NULL, ctx);
+  return (int8_t)buf[i];
+}
 
 /*--------------------------------------------------------------------------------------------------
   Length, compare
@@ -192,6 +195,19 @@ static inline kk_bytes_t kk_bytes_copy(kk_bytes_t b, kk_context_t* ctx) {
   }
 }
 
+static inline kk_bytes_t kk_bytes_set(kk_bytes_t bytes, uint64_t i, int8_t b, kk_context_t* ctx){
+  if (kk_datatype_ptr_is_unique(bytes, ctx)) {
+    uint8_t* buf = (uint8_t*)kk_bytes_buf_borrow(bytes, NULL, ctx);
+    buf[i] = (uint8_t)b;
+    return bytes;
+  } else {
+    kk_bytes_t bytes_new = kk_bytes_copy(bytes, ctx);
+    uint8_t* buf = (uint8_t*)kk_bytes_buf_borrow(bytes_new, NULL, ctx);
+    buf[i] = (uint8_t)b;
+    return bytes_new;
+  }
+}
+
 static inline bool kk_bytes_ptr_eq_borrow(kk_bytes_t b1, kk_bytes_t b2) {
   return (kk_datatype_eq(b1, b2));
 }

kklib/src/os.c

@@ -648,7 +648,15 @@ kk_decl_export int kk_os_run_system(kk_string_t cmd, kk_context_t* ctx) {
   }
   #else
   kk_with_string_as_qutf8_borrow(cmd, ccmd, ctx) {
-    exitcode = system(ccmd);
+    int status = system(ccmd);
+    if (WIFEXITED(status)) {
+      exitcode = WEXITSTATUS(status);
+    } else if (WIFSIGNALED(status)) { 
+      exitcode = -WTERMSIG(status); // Like Haskell, return negative code for signal termination
+    } else {
+      // Technically WIFSTOPPED(status) can happen, but only if the process is being traced, or the call was done with WUNTRACED.
+      kk_fatal_error(EINVAL, "kk_os_run_system: unexpected termination");
+    }
   }
   #endif
   kk_string_drop(cmd, ctx);

lib/std/core.kk

@@ -25,6 +25,8 @@ pub import std/core/order
 pub import std/core/char
 pub import std/core/int
 pub import std/core/vector
+pub import std/core/bytes
+pub import std/core/bslice
 pub import std/core/string
 pub import std/core/sslice
 pub import std/core/list

lib/std/core/bslice.kk

@@ -0,0 +1,132 @@
+/*---------------------------------------------------------------------------
+  Copyright 2024, Tim Whiting.
+
+  This is free software; you can redistribute it and/or modify it under the
+  terms of the Apache License, Version 2.0. A copy of the License can be
+  found in the LICENSE file at the root of this distribution.
+---------------------------------------------------------------------------*/
+
+// Byte slices
+module std/core/bslice
+import std/core/bytes
+import std/core/types
+import std/core/int
+import std/core/exn
+
+extern import 
+  c file "inline/bslice" 
+
+// A byte slice
+pub value struct bslice 
+  backing-bytes: bytes
+  start: int
+  len: int // The length of the slice
+  total-len: int // Cached length of the backing byte array
+
+// Create a byte slice from bytes
+pub fun slice(b: bytes): bslice
+  val len = b.length.int
+  Bslice(b, 0, len, len)
+
+extern external/bytes(b: bslice): bytes
+  c "kk_bslice_bytes"
+
+// Get the bytes referenced by the slice
+pub fun get/bytes(b: bslice): bytes
+  external/bytes(b)
+
+// If the slice is not empty,
+// return the first byte, and a new slice that is advanced by 1.
+pub fun next(b : bslice): maybe<(int8, bslice)>
+  val Bslice(bts, start, len, tl) = b
+  if start >= tl then Nothing
+  else Just((bts.unsafe-index(start.ssize_t), Bslice(bts, start + 1, len - 1, tl)))
+
+// O(1). Advance the start position of a byte slice by `count` bytes
+// up to the end of the byte array.
+// A negative `count` advances the start position backwards up to index 0.
+pub fun advance(b: bslice, n: int): bslice
+  val Bslice(bts, start, len, tl) = b
+  if n > 0 then
+    if n > len then // If advancing past the current length, advance to the end
+      Bslice(bts, start + len, 0, tl)
+    else
+      Bslice(bts, start + n, len - n, tl)
+  elif n < 0 then
+    if start + n < 0 then // If advancing past the beginning, advance to the beginning
+      Bslice(bts, 0, len + start, tl)
+    else 
+      Bslice(bts, start + n, len - n, tl)
+  else 
+    Bslice(bts, start, len, tl)
+
+// O(1). Drop the first `n` bytes of the slice
+// If there is not `n` bytes to drop returns the empty slice
+pub fun drop(b: bslice, n : int): bslice
+  val Bslice(bts, start, len, tl) = b 
+  if n >= len then Bslice(bts, tl, 0, tl)
+  else Bslice(bts, start + n, len - n, tl)
+
+// O(1). Get the subslice from `slice.start + start` to `slice.start + end`. 
+// If the new start is past the end of the slice, returns the empty slice
+// If the end is past the end of the slice, return just to the end.
+pub fun subslice(b : bslice, start: int, end: int): bslice
+  val Bslice(bts, s0, l, tl) = b
+  if start > l then Bslice(bts, tl, 0, tl)
+  else if end >= l then Bslice(bts, s0 + start, l - start, tl)
+  else Bslice(bts, s0 + start, end - start, tl)
+
+// O(1). Extend a byte slice by `count` bytes up to the end of the byte slice.
+// A negative `count` shrinks the slice up to the empty slice.
+pub fun extend(b: bslice, n: int): bslice
+  val Bslice(bts, start, len, tl) = b
+  if n > 0 then
+    if n > (tl - len) then // If extending past total end of bytes, extend to end of bytes
+      Bslice(bts, start, tl - start, tl)
+    else
+      Bslice(bts, start, len + n, tl)
+  elif n < 0 then
+    if len + n < 0 then // If extending past start of slice, extend to start point
+      Bslice(bts, start, 0, tl)
+    else
+      Bslice(bts, start, len + n, tl)
+  else 
+    Bslice(bts, start, len, tl)
+
+// Truncates the slice to length 0
+pub fun truncate(b: bslice): bslice 
+  val Bslice(bts, start, _, tl) = b
+  Bslice(bts, start, 0, tl)
+
+// O(1). Return the byte slice from the start of the byte array up to the
+// start of the `slice` argument.
+pub fun before(slice: bslice): bslice
+  val Bslice(bts, start, _, tl) = slice
+  Bslice(bts, 0, start, tl)
+
+// O(1). Return the byte slice from the end of the `slice` argument
+// to the end of the byte array.
+pub fun after(slice: bslice): bslice
+  val Bslice(bts, start, len, tl) = slice
+  val new-start = start + len
+  Bslice(bts, new-start, tl - new-start, tl)
+
+// Get's the byte at the offset `i`
+pub fun @index( ^b : bslice, i : int ) : exn int8
+  val Bslice(bts, start, len, _) = b
+  if i < 0 || i >= len then throw("index out of bounds", ExnRange)
+  bts.unsafe-index((start + i).ssize_t)
+
+// Assigns the bytes starting at offset `i` to `new-value`, throws if there is not enough space or the index is invalid
+pub fun assign( b : bytes, i : int, new-value : bslice ) : exn bytes
+  val l = b.length.int
+  val size = new-value.len
+  if i < 0 || i > l then 
+    throw("index out of bounds", ExnRange)
+  elif size + i > l then
+    throw("size overflow", ExnRange)
+  else
+    assign(b, i.ssize_t, new-value)
+
+extern slice/assign(b : bytes, i : ssize_t, new-value : bslice ): bytes
+  c "kk_bslice_assign"

lib/std/core/bytes.kk

@@ -0,0 +1,88 @@
+/*---------------------------------------------------------------------------
+  Copyright 2024, Tim Whiting.
+
+  This is free software; you can redistribute it and/or modify it under the
+  terms of the Apache License, Version 2.0. A copy of the License can be
+  found in the LICENSE file at the root of this distribution.
+---------------------------------------------------------------------------*/
+
+/*
+  Byte arrays
+
+  **Null Termination**:
+
+  The functions in this module maintain the following invariant for `bytes` values,
+  in order to efficiently convert between `string` and `bytes`:
+
+  - The physical buffer is at least one byte larger than the logical length
+  - The memory address directly after the end of the logical length is set to the null byte.
+
+  Code using the functions in this module does not need to worry about this invariant,
+  however low-level C code performing direct memory access may need to be careful to
+  maintain it.
+
+  Violating this invariant could lead to memory-safty issues, for example the `string`
+  function could potentially read past the end of a buffer with no terminating null byte.
+*/
+
+module std/core/bytes
+import std/core/types
+import std/core/exn
+import std/core/bool
+import std/core/int
+
+extern import 
+  js file "inline/bytes.js"
+
+// Create a new empty array of bytes
+pub extern empty(): bytes
+  c  inline "kk_bytes_empty()"
+  js inline "new Uint8Array(0)"
+
+// Allocate a new zero initialized array of `n` bytes.
+pub extern alloc( n : ssize_t ) : bytes
+  c  inline "kk_bytes_alloc_buf(#1, NULL, kk_context())"
+  js inline "new Uint8Array(#1).fill(0)"
+
+// Converts an array of bytes to a string.
+pub extern string( bytes : bytes ) : string
+  c  "kk_string_convert_from_qutf8"
+  js inline "String.fromCharCode.apply(null, #1)"
+
+// Gets a view of the string as raw bytes
+pub extern string/bytes( s : string ) : bytes
+  c  inline "#1.bytes"
+  js inline "(new TextEncoder()).encode(#1)"
+
+// Gets the length of the byte array
+pub extern length( ^b : bytes ) : ssize_t
+  c  "kk_bytes_len_borrow"
+  js inline "#1.length"
+
+// Adjusts the byte array to be the size of the `len` argument
+pub extern adjust-length(b: bytes, len: ssize_t): bytes
+  c  "kk_bytes_adjust_length"
+  js "#1"
+
+extern extern-append(b1: bytes, b2: bytes): bytes
+  c "kk_bytes_cat"
+
+// Concatenates two byte arrays
+pub fun (++)(b1 : bytes, b2: bytes): bytes
+  extern-append(b1, b2)
+
+// Get's the byte at the offset `i`
+pub fun @index(^b: bytes, i: int): exn int8 
+  if i < 0 || i >= b.length.int then throw("index out of bounds", ExnRange)
+  b.unsafe-index(i.ssize_t)
+
+// Get's the byte at the offset `i`, without checking the offset is valid
+// !! Attention !! Unsafe API
+pub extern unsafe-index( ^b : bytes, i : ssize_t ) : int8
+  c  "kk_bytes_at"
+  js inline "#1[#2]"
+
+// Assigns the byte at offset `i` to `new-value`
+pub extern byte/assign( b : bytes, i : ssize_t, new-value : int8 ) : bytes
+  c  "kk_bytes_set"
+  js "kk_bytes_assign"

lib/std/core/inline/bslice.c

@@ -0,0 +1,34 @@
+kk_bytes_t kk_bslice_assign( kk_bytes_t bytes, kk_ssize_t i, struct kk_std_core_bslice_Bslice bslice, kk_context_t* ctx ){
+  kk_ssize_t len;
+  const uint8_t* bslice_buf = kk_bytes_buf_borrow(bslice.backing_bytes, &len, ctx);
+  kk_ssize_t slice_len = kk_integer_clamp_ssize_t_borrow(bslice.len, ctx);
+  kk_ssize_t slice_start = kk_integer_clamp_ssize_t_borrow(bslice.start, ctx);
+  if (kk_datatype_ptr_is_unique(bytes, ctx)) {
+    // kk_info_message("kk_bslice_assign: unique\n");
+    uint8_t* buf = (uint8_t*)kk_bytes_buf_borrow(bytes, &len, ctx);
+    for (kk_ssize_t j = 0; j < slice_len; j++) {
+      buf[i + j] = bslice_buf[slice_start + j];
+    }
+    kk_std_core_bslice__bslice_drop(bslice,ctx);
+    return bytes;
+  } else {
+    // kk_info_message("kk_bslice_assign: not unique\n");
+    kk_bytes_t bytes_new = kk_bytes_copy(bytes, ctx);
+    uint8_t* buf_new = (uint8_t*)kk_bytes_buf_borrow(bytes_new, NULL, ctx);
+    for (kk_ssize_t j = 0; j < slice_len; j++) {
+      buf_new[i + j] = bslice_buf[slice_start + j];
+    }
+    kk_std_core_bslice__bslice_drop(bslice,ctx);
+    return bytes_new;
+  }
+}
+
+kk_bytes_t kk_bslice_bytes(struct kk_std_core_bslice_Bslice bslice, kk_context_t* ctx ){
+  kk_ssize_t slice_len = kk_integer_clamp_ssize_t_borrow(bslice.len, ctx);
+  kk_ssize_t slice_start = kk_integer_clamp_ssize_t_borrow(bslice.start, ctx);
+  kk_ssize_t total_len;
+  const uint8_t* bslice_buf = kk_bytes_buf_borrow(bslice.backing_bytes, &total_len, ctx);
+  kk_bytes_t bytes = kk_bytes_alloc_dupn(slice_len, bslice_buf + slice_start, ctx);
+  kk_std_core_bslice__bslice_drop(bslice, ctx); // TODO: Optimize if getting full slice, or shortening the slice from the beginning
+  return bytes;
+}

lib/std/core/inline/bslice.h

@@ -0,0 +1,11 @@
+
+/*---------------------------------------------------------------------------
+  Copyright 2020-2024, Microsoft Research, Daan Leijen. Tim Whiting.
+
+  This is free software; you can redistribute it and/or modify it under the
+  terms of the Apache License, Version 2.0. A copy of the License can be
+  found in the LICENSE file at the root of this distribution.
+---------------------------------------------------------------------------*/
+
+struct kk_std_core_bslice_Bslice;
+kk_bytes_t kk_bslice_assign( kk_bytes_t bytes, kk_ssize_t i, struct kk_std_core_bslice_Bslice bslice, kk_context_t* ctx );

lib/std/core/inline/bytes.js

@@ -0,0 +1,5 @@
+function kk_bytes_assign(b, i, new_value) {
+  const fresh = new Uint8Array(b);
+  fresh[i] = new_value;
+  return fresh;
+}

lib/std/core/sslice.kk

@@ -139,9 +139,9 @@ pub extern common-prefix(s : string, t : string, ^upto : int = -1 ) : sslice
 
 // O(`count`). Advance the start position of a string slice by `count` characters
 // up to the end of the string.
-// A negative `count` advances the start position backwards upto the first position
+// A negative `count` advances the start position backwards up to the first position
 // in a string.
-// Maintains the character count of the original slice upto the end of the string.
+// Maintains the character count of the original slice up to the end of the string.
 // For example:
 //
 // * `"abc".first.advance(1).string == "b"`,
@@ -166,16 +166,17 @@ pub extern extend( slice : sslice, ^count : int ) : sslice
   js "_sslice_extend"
 
 // O(1). Return the string slice from the start of a string up to the
-// start of `slice` argument.
+// start of the `slice` argument.
 pub fun before(slice : sslice) : sslice
   val Sslice(s,start,_len) = slice
   Sslice(s,0,start)
 
-// O(1). Return the string slice from the end of `slice` argument
+// O(1). Return the string slice from the end of the `slice` argument
 // to the end of the string.
 pub fun after(slice : sslice) : sslice
   val Sslice(s,start,len) = slice
-  Sslice(s,start+len,s.length - (start+len))
+  val new-start = start+len
+  Sslice(s,new-start,s.length - new-start)
 
 // O(n). Copy the `slice` argument into a fresh string.
 // Takes O(1) time if the slice covers the entire string.

lib/std/core/types.kk

@@ -101,6 +101,9 @@ pub value type float32
 // See `module std/core/vector` for vector operations.
 pub type vector<a>
 
+// A raw wrapper around a uint8 character array.
+pub type bytes
+
 // An any type. Used for external calls.
 pub type any
 

lib/std/core/unsafe.kk

@@ -10,6 +10,7 @@
 module std/core/unsafe
 
 import std/core/types
+import std/core/hnd
 
 // _Unsafe_. This function calls a function and pretends it did not have any effect at all.
 // Use with utmost care as it should not be used to dismiss user-defined effects that need