Add simple tokenizer

s8slice.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <string.h>
+#include <stdint.h>
+
+typedef struct s8_slice {
+    uint8_t* data;
+    int64_t  len;
+} S8Slice;
+
+#define S8(str) (S8Slice) {     \
+    .data = (uint8_t*)("" str), \
+    .len = sizeof(str)-1,       \
+}
+
+#define S8_error(str) (S8Slice) { \
+    .data = (uint8_t*)("" str),   \
+    .len = -1,                    \
+}
+
+static inline S8Slice s8slice_from_cstr(const char* cstr)
+{
+    return (S8Slice){
+        .len = strlen(cstr),
+        .data = (uint8_t*)cstr,
+    };
+}
+
+static inline S8Slice s8slice(const S8Slice* s, int64_t begin, int64_t end)
+{
+    if (end >= s->len) {
+        return S8_error("s8slice: `end` surpasses length of `s`");
+    }
+    const int64_t diff = end - begin;
+    return (S8Slice) {
+        .len = diff,
+        .data = &s->data[begin]
+    };
+}