Slendi/dcfg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source 
Signed-off-by: Slendi <slendi@socopon.com>
This commit is contained in:
Slendi
2025-07-24 21:25:16 +03:00
commit 123763f48c
11 changed files with 20194 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
[Bb]uild
.cache

37
CMakeLists.txt Normal file
View File

@@ -0,0 +1,37 @@
cmake_minimum_required(VERSION 3.30)
project(DCFG C)
set(CMAKE_C_STANDARD 23) # Would've done C99 but I need typeof (for now)
set(CMAKE_C_STANDARD_REQUIRED ON)
set(CMAKE_C_EXTENSIONS OFF)
option(DCFG_BUILD_SHARED "Build DCFG as a shared library" ON)
option(DCFG_BUILD_STATIC "Build DCFG as a static library" OFF)
find_package(Threads REQUIRED)
set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src)
set(INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
if(DCFG_BUILD_SHARED)
add_library(${PROJECT_NAME} SHARED ${SRC_DIR}/dcfg.c)
target_include_directories(${PROJECT_NAME} PUBLIC ${INCLUDE_DIR})
target_link_libraries(${PROJECT_NAME} PRIVATE Threads::Threads)
set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME "dcfg")
endif()
if(DCFG_BUILD_STATIC)
add_library(${PROJECT_NAME}_static STATIC ${SRC_DIR}/dcfg.c)
target_include_directories(${PROJECT_NAME}_static PUBLIC ${INCLUDE_DIR})
target_link_libraries(${PROJECT_NAME}_static PRIVATE Threads::Threads)
set_target_properties(${PROJECT_NAME}_static PROPERTIES OUTPUT_NAME "dcfg")
endif()
if(DCFG_BUILD_SHARED)
install(TARGETS ${PROJECT_NAME} DESTINATION lib)
endif()
if(DCFG_BUILD_STATIC)
install(TARGETS ${PROJECT_NAME}_static DESTINATION lib)
endif()
install(DIRECTORY ${INCLUDE_DIR}/ DESTINATION include)

61
flake.lock generated Normal file
View File

@@ -0,0 +1,61 @@
{
"nodes": {
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1753250450,
"narHash": "sha256-i+CQV2rPmP8wHxj0aq4siYyohHwVlsh40kV89f3nw1s=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "fc02ee70efb805d3b2865908a13ddd4474557ecf",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

34
flake.nix Normal file
View File

@@ -0,0 +1,34 @@
{
description = "Alice";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
flake-utils.url = "github:numtide/flake-utils";
};
outputs =
{
self,
nixpkgs,
flake-utils,
}:
flake-utils.lib.eachDefaultSystem (
system:
let
pkgs = import nixpkgs {
system = system;
};
in
{
devShells.default = pkgs.mkShell.override { stdenv = pkgs.llvmPackages_20.libcxxStdenv; } {
packages = with pkgs; [
cmake
ninja
clang-tools
lldb
pkg-config
];
};
}
);
}

96
include/dcfg.h Normal file
View File

@@ -0,0 +1,96 @@
#ifndef DCFG_H
#define DCFG_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct dcfg_Value dcfg_Value;
typedef struct dcfg_Instance dcfg_Instance;
// Type definitions
typedef struct dcfg_StringView {
char const *data;
size_t size;
} dcfg_StringView;
typedef void *(*dcfg_AllocFn)(size_t); // This should automatically zero memory.
typedef void (*dcfg_FreeFn)(void *);
typedef struct dcfg_InstanceCreateInfo {
dcfg_AllocFn alloc;
dcfg_FreeFn free;
} dcfg_InstanceCreateInfo;
typedef enum dcfg_ValueKind {
dcfg_ValueType_Nil,
dcfg_ValueType_Boolean,
dcfg_ValueType_Integer,
dcfg_ValueType_Real,
dcfg_ValueType_String,
dcfg_ValueType_Path,
dcfg_ValueType_Object,
dcfg_ValueType_Array,
dcfg_ValueType_Function,
dcfg_ValueType_FunctionCall,
} dcfg_ValueType;
dcfg_Instance *dcfg_make_instance(dcfg_InstanceCreateInfo const *create_info);
void dcfg_destroy_instance(dcfg_Instance *instance);
char const *dcfg_last_error(dcfg_Instance *instance);
// File path gets copied internally to instance to a Value * -> path hashmap for
// evaluation.
dcfg_Value *dcfg_parse(dcfg_Instance *instance,
dcfg_StringView const file_path);
void dcfg_destroy(dcfg_Value *value);
bool dcfg_serialize_value(dcfg_Value *value, dcfg_StringView *out_sv);
// Value type checking
dcfg_ValueType dcfg_Value_type_ex(dcfg_Value *value, bool evaluate);
static inline dcfg_ValueType dcfg_Value_type(dcfg_Value *value) {
return dcfg_Value_type_ex(value, true);
}
// Value getters
bool dcfg_Value_get_object_field_ex(dcfg_Value *value,
dcfg_StringView const key,
dcfg_Value **out_value,
bool const evaluate);
bool dcfg_Value_get_array_item_ex(dcfg_Value *value, size_t const index,
dcfg_Value **out_value, bool const evaluate);
bool dcfg_Value_get_boolean(dcfg_Value *value, bool *out_value);
bool dcfg_Value_get_integer(dcfg_Value *value, int64_t *out_value);
bool dcfg_Value_get_real(dcfg_Value *value, double *out_value);
bool dcfg_Value_get_string(dcfg_Value *value, dcfg_StringView *out_sv);
bool dcfg_Value_get_path(dcfg_Value *value, dcfg_StringView *out_sv);
bool dcfg_Value_get_object_keys(dcfg_Value *value, size_t const capacity,
size_t *out_count, dcfg_StringView *out_keys);
bool dcfg_Value_get_array_size(dcfg_Value *value, size_t *out_size);
static inline bool dcfg_Value_get_object_field(dcfg_Value *value,
dcfg_StringView const key,
dcfg_Value **out_value) {
return dcfg_Value_get_object_field_ex(value, key, out_value, true);
}
static inline bool dcfg_Value_get_array_item(dcfg_Value *value,
size_t const index,
dcfg_Value **out_value) {
return dcfg_Value_get_array_item_ex(value, index, out_value, true);
}
bool dcfg_call_function(dcfg_Value *function, dcfg_Value **args,
size_t arg_count, dcfg_Value **out_value);
bool dcfg_Value_evaluate(dcfg_Value *value, dcfg_Value **out_value);
#ifdef __cplusplus
}
#endif
#endif // DCFG_H

970
src/dcfg.c Normal file
View File

@@ -0,0 +1,970 @@
#include <dcfg.h>
#include <stdio.h>
#include "vendor/utf8proc.h"
#include "vendor/vec.h"
#define _POSIX_C_SOURCE 200809L
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
int64_t dcfg_strtoll(const char *s, char **end, int base) {
size_t n = strlen(s);
char *clean = malloc(n + 1);
if (!clean) {
errno = ENOMEM;
if (end)
*end = (char *)s;
return 0;
}
size_t w = 0;
for (size_t r = 0; r < n && s[r]; ++r)
if (s[r] != '_')
clean[w++] = s[r];
clean[w] = '\0';
char *tmp_end;
int64_t v = strtoll(clean, &tmp_end, base);
if (end) {
size_t consumed = (size_t)(tmp_end - clean);
size_t i = 0, c = 0;
while (i < n && c < consumed) {
if (s[i] != '_')
++c;
++i;
}
*end = (char *)s + i;
}
free(clean);
return v;
}
double dcfg_strtod(const char *s, char **end) {
size_t n = strlen(s);
char *clean = malloc(n + 1);
if (!clean) {
errno = ENOMEM;
if (end)
*end = (char *)s;
return 0.0;
}
size_t w = 0;
for (size_t r = 0; r < n && s[r]; ++r)
if (s[r] != '_')
clean[w++] = s[r];
clean[w] = '\0';
char *tmp_end;
double v = strtod(clean, &tmp_end);
if (end) {
size_t consumed = (size_t)(tmp_end - clean);
size_t i = 0, c = 0;
while (i < n && c < consumed) {
if (s[i] != '_')
++c;
++i;
}
*end = (char *)s + i;
}
free(clean);
return v;
}
typedef dcfg_Value Value;
typedef dcfg_Instance Instance;
typedef dcfg_StringView StringView;
#define SV(cstr) ((StringView){.data = cstr, .size = strlen(cstr)})
static inline bool sv_eq(StringView a, StringView b) {
if (a.size != b.size) {
return false;
}
return memcmp(a.data, b.data, a.size) == 0;
}
typedef struct {
int row, col;
} Location;
typedef struct {
Location begin, end;
} LocationRange;
typedef struct {
StringView fp;
LocationRange range;
} SourceLocation;
typedef struct {
StringView k;
Value *v;
} ValueObjectEntry;
typedef struct {
ValueObjectEntry *entryv;
size_t entryc;
} ValueObject;
typedef struct {
Value *valuev;
size_t valuec;
} ValueArray;
typedef struct {
bool is_builtin;
union {
Value *(*bi)(Value **argv, size_t argc);
Value *v;
} v;
} ValueFunction;
typedef struct {
Value *function;
Value **argv;
size_t argc;
} ValueFunctionCall;
struct dcfg_Value {
dcfg_ValueType type;
SourceLocation location;
union {
int64_t i;
double r;
StringView s;
StringView p;
ValueObject o;
ValueArray a;
ValueFunction f;
ValueFunctionCall c;
} v;
};
struct dcfg_Instance {
pthread_mutex_t mtx;
dcfg_AllocFn alloc;
dcfg_FreeFn free;
char last_error[256];
};
#define ALLOC(sz) (instance->alloc((sz)))
#define FREE(ptr) (instance->free((ptr)))
static void *alloc(size_t size) { return calloc(1, size); }
dcfg_Instance *dcfg_make_instance(dcfg_InstanceCreateInfo const *create_info) {
assert(create_info);
dcfg_Instance *instance = calloc(1, sizeof(*instance));
if (!instance) {
return NULL;
}
pthread_mutex_init(&instance->mtx, NULL);
instance->alloc = create_info->alloc;
instance->free = create_info->free;
if (!instance->alloc) {
instance->alloc = alloc;
}
if (!instance->free) {
instance->free = free;
}
return instance;
}
void dcfg_destroy_instance(dcfg_Instance *instance) {
assert(instance);
pthread_mutex_lock(&instance->mtx);
{ // De-init other instance things
}
pthread_mutex_unlock(&instance->mtx);
pthread_mutex_destroy(&instance->mtx);
free(instance);
}
char const *dcfg_last_error(dcfg_Instance *instance) {
assert(instance);
char const *ret = NULL;
pthread_mutex_lock(&instance->mtx);
{
if (!instance->last_error[0]) {
pthread_mutex_unlock(&instance->mtx);
return NULL;
}
ret = instance->last_error;
}
pthread_mutex_unlock(&instance->mtx);
return ret;
}
typedef enum {
TokenType_Identifier,
TokenType_Integer,
TokenType_Real,
TokenType_String,
TokenType_Path,
TokenType_LBracket,
TokenType_RBracket,
TokenType_LParen,
TokenType_RParen,
TokenType_LSquirly,
TokenType_RSquirly,
TokenType_Set,
TokenType_Dot,
TokenType_End,
TokenType_Error,
} TokenType;
typedef struct {
TokenType type;
SourceLocation location;
union {
StringView id;
int64_t i;
double r;
StringView s;
StringView p;
} v;
} Token;
typedef struct {
StringView source;
StringView fp;
int32_t ch, next;
int ch_len, next_len;
Location cursor;
int offset;
} Lexer;
static inline int32_t decode_cp(StringView src, int pos, int *len) {
if (pos >= (int)src.size) {
*len = 0;
return -1;
}
return (int32_t)utf8proc_iterate((const uint8_t *)src.data + pos,
(int)(src.size - pos), len);
}
static inline bool is_space_cp(int32_t cp) {
return (cp <= 0x7F &&
(cp == ' ' || cp == '\t' || cp == '\r' || cp == '\n')) ||
utf8proc_category(cp) == UTF8PROC_CATEGORY_ZS;
}
static inline bool is_alpha_cp(int32_t cp) {
return (cp <= 0x7F && isalpha(cp)) ||
(utf8proc_category(cp) >= UTF8PROC_CATEGORY_LU &&
utf8proc_category(cp) <= UTF8PROC_CATEGORY_LO);
}
static inline bool is_digit_cp(int32_t cp) {
return (cp <= 0x7F && isdigit(cp)) ||
utf8proc_category(cp) == UTF8PROC_CATEGORY_ND;
}
static inline bool is_alnum_cp(int32_t cp) {
return is_alpha_cp(cp) || is_digit_cp(cp);
}
static inline bool is_path_ch_cp(int32_t cp) {
return cp == '_' || cp == '.' || cp == '/' || cp == '-' || cp == ':' ||
is_alnum_cp(cp);
}
#define is_space is_space_cp
#define is_path_ch is_path_ch_cp
#define isalpha_cp is_alpha_cp
#define isdigit_cp is_digit_cp
#define isalnum_cp is_alnum_cp
static void lex_advance(Lexer *lx) {
if (lx->next == -1) {
lx->ch = -1;
return;
}
if (lx->ch == '\n') {
++lx->cursor.row;
lx->cursor.col = 1;
} else {
++lx->cursor.col;
}
lx->offset += lx->ch_len;
lx->ch = lx->next;
lx->ch_len = lx->next_len;
lx->next = decode_cp(lx->source, lx->offset + lx->ch_len, &lx->next_len);
}
static StringView lex_slice(Lexer *lx, int start, int end) {
StringView sv;
sv.data = lx->source.data + start;
sv.size = (size_t)(end - start);
return sv;
}
static void skip_ws_and_comments(Lexer *lx) {
for (;;) {
while (is_space(lx->ch))
lex_advance(lx);
if (lx->ch == '#') {
while (lx->ch != -1 && lx->ch != '\n')
lex_advance(lx);
continue;
}
break;
}
}
static Token make_token(TokenType t, Lexer *lx, int start, int end) {
Token tk = {0};
tk.type = t;
tk.location.fp = lx->fp;
tk.location.range.begin = lx->cursor;
tk.location.range.end = lx->cursor;
switch (t) {
case TokenType_Identifier:
tk.v.id = lex_slice(lx, start, end);
break;
case TokenType_Integer:
tk.v.i = dcfg_strtoll(lex_slice(lx, start, end).data, NULL, 10);
break;
case TokenType_Real:
tk.v.r = dcfg_strtod(lex_slice(lx, start, end).data, NULL);
break;
case TokenType_String:
tk.v.s = lex_slice(lx, start, end);
break;
case TokenType_Path:
tk.v.p = lex_slice(lx, start, end);
break;
default:
break;
}
return tk;
}
static Token lex_number(Lexer *lx) {
int start = lx->offset;
bool real = false;
while (isdigit_cp(lx->ch) || lx->ch == '_')
lex_advance(lx);
if (lx->ch == '.') {
real = true;
lex_advance(lx);
while (isdigit_cp(lx->ch) || lx->ch == '_')
lex_advance(lx);
}
return make_token(real ? TokenType_Real : TokenType_Integer, lx, start,
lx->offset);
}
static Token lex_identifier(Lexer *lx) {
int start = lx->offset;
while (isalnum_cp(lx->ch) || lx->ch == '_')
lex_advance(lx);
return make_token(TokenType_Identifier, lx, start, lx->offset);
}
static Token lex_string(Lexer *lx) {
int quote = lx->ch;
lex_advance(lx); // skip opening quote
int start = lx->offset;
while (lx->ch != quote && lx->ch != -1) {
if (lx->ch == '\\' && lx->next != -1)
lex_advance(lx); // simple escapes
lex_advance(lx);
}
int end = lx->offset;
if (lx->ch == quote)
lex_advance(lx); // skip closing quote
return make_token(TokenType_String, lx, start, end);
}
static Token lex_path(Lexer *lx) {
int start = lx->offset;
for (;;) {
if (lx->ch == '\\') {
lex_advance(lx);
if (lx->ch != -1)
lex_advance(lx);
} else if (is_path_ch(lx->ch)) {
lex_advance(lx);
} else
break;
}
return make_token(TokenType_Path, lx, start, lx->offset);
}
bool Lexer_init(Lexer *lx, StringView src, StringView fp) {
memset(lx, 0, sizeof *lx);
lx->source = src;
lx->fp = fp;
lx->cursor = (Location){1, 1};
lx->ch = decode_cp(src, 0, &lx->ch_len);
lx->next = decode_cp(src, lx->ch_len, &lx->next_len);
return true;
}
Token Lexer_next(Lexer *lx) {
skip_ws_and_comments(lx);
int start_off = lx->offset;
if (lx->ch == -1) {
Token tk = {.type = TokenType_End};
tk.location.fp = lx->fp;
return tk;
}
if (lx->ch == '/' // "/foo"
|| (lx->ch == '.' && lx->next == '/') // "./foo"
|| (lx->ch == '.' && lx->offset + 2 < (int)lx->source.size &&
lx->source.data[lx->offset + 1] == '.' &&
lx->source.data[lx->offset + 2] == '/') // "../foo"
|| (isalpha_cp(lx->ch) && lx->next == ':')) { // "C:/foo"
return lex_path(lx);
}
switch (lx->ch) {
case '[':
lex_advance(lx);
return make_token(TokenType_LBracket, lx, start_off, lx->offset);
case ']':
lex_advance(lx);
return make_token(TokenType_RBracket, lx, start_off, lx->offset);
case '(':
lex_advance(lx);
return make_token(TokenType_LParen, lx, start_off, lx->offset);
case ')':
lex_advance(lx);
return make_token(TokenType_RParen, lx, start_off, lx->offset);
case '{':
lex_advance(lx);
return make_token(TokenType_LSquirly, lx, start_off, lx->offset);
case '}':
lex_advance(lx);
return make_token(TokenType_RSquirly, lx, start_off, lx->offset);
case '=':
lex_advance(lx);
return make_token(TokenType_Set, lx, start_off, lx->offset);
case '.':
lex_advance(lx);
return make_token(TokenType_Dot, lx, start_off, lx->offset);
case '"':
case '\'':
return lex_string(lx);
default:
break;
}
if (isalpha_cp(lx->ch) || lx->ch == '_')
return lex_identifier(lx);
if (isdigit_cp(lx->ch))
return lex_number(lx);
Token err = {.type = TokenType_Error};
err.location.fp = lx->fp;
lex_advance(lx);
return err;
}
typedef enum {
ASTKind_Key, // Identifier + string combo
ASTKind_Path,
ASTKind_Boolean,
ASTKind_Integer,
ASTKind_Real,
ASTKind_Block,
ASTKind_Array,
ASTKind_Function,
ASTKind_MemberAccess,
ASTKind_FunctionCall,
} ASTKind;
typedef struct AST AST;
typedef struct {
StringView *argv;
AST *body;
} ASTFunction;
typedef struct {
AST **childv;
} ASTArray;
typedef struct {
AST *k; // Either MemberAccess or Key
AST *v;
} ASTBlock_Entry;
typedef struct {
ASTBlock_Entry *entryv;
} ASTBlock;
typedef struct {
StringView *accessv;
} ASTMemberAccess;
typedef struct {
AST *function;
AST **argv;
} ASTFunctionCall;
struct AST {
ASTKind kind;
SourceLocation location;
union {
int64_t i;
double r;
StringView s;
StringView p;
bool b;
ASTFunction f;
ASTArray a;
ASTBlock bl;
ASTMemberAccess m;
ASTFunctionCall fc;
} v;
};
typedef struct {
Lexer *lexer;
Instance *instance;
Token cur, next;
} Parser;
bool Parser_next(Parser *parser) {
parser->cur = parser->next;
parser->next = Lexer_next(parser->lexer);
bool ret = parser->next.type != TokenType_Error;
if (!ret) {
strcpy(parser->instance->last_error, "Failed to get parser token");
}
return ret;
}
bool Parser_init(Parser *out_parser, Lexer *lexer, Instance *instance) {
out_parser->lexer = lexer;
out_parser->instance = instance;
if (!Parser_next(out_parser)) {
return false;
}
if (!Parser_next(out_parser)) {
return false;
}
return true;
}
bool Parser_accept(Parser *parser, TokenType type,
SourceLocation *out_location) {
if (parser->cur.type == type) {
if (out_location) {
*out_location = parser->cur.location;
}
if (!Parser_next(parser)) {
return false;
}
return true;
}
return false;
}
#undef ALLOC
#undef FREE
#define ALLOC(sz) (parser->instance->alloc((sz)))
#define FREE(ptr) (parser->instance->free((ptr)))
void AST_free_parser(AST *ast, Parser *parser) {
assert(parser);
if (!ast) {
return;
}
switch (ast->kind) {
case ASTKind_Function:
AST_free_parser(ast->v.f.body, parser);
vector_free(ast->v.f.argv);
break;
case ASTKind_Array:
for (size_t i = 0; i < vector_size(ast->v.a.childv); i++)
AST_free_parser(ast->v.a.childv[i], parser);
vector_free(ast->v.a.childv);
break;
case ASTKind_Block:
for (size_t i = 0; i < vector_size(ast->v.bl.entryv); i++) {
AST_free_parser(ast->v.bl.entryv[i].k, parser);
AST_free_parser(ast->v.bl.entryv[i].v, parser);
}
vector_free(ast->v.bl.entryv);
break;
case ASTKind_MemberAccess:
vector_free(ast->v.m.accessv);
break;
case ASTKind_FunctionCall:
AST_free_parser(ast->v.fc.function, parser);
for (size_t i = 0; i < vector_size(ast->v.fc.argv); i++)
AST_free_parser(ast->v.fc.argv[i], parser);
vector_free(ast->v.fc.argv);
break;
default:
break;
}
FREE(ast);
}
AST *parser_parse_dot_access(Parser *parser, StringView *current) {
assert(parser);
assert(parser->cur.type == TokenType_Identifier ||
parser->cur.type == TokenType_String);
StringView *accessv = vector_create();
if (!accessv) {
strcpy(parser->instance->last_error, "Failed to allocate vector");
return NULL;
}
if (current) {
vector_add(&accessv, *current);
}
SourceLocation loc = parser->cur.location;
SourceLocation last_loc = parser->cur.location;
while (true) {
if (parser->cur.type != TokenType_Identifier &&
parser->cur.type != TokenType_String) {
break;
}
last_loc = parser->cur.location;
vector_add(&accessv, parser->cur.v.s);
if (!Parser_next(parser)) {
strcpy(parser->instance->last_error,
"Failed to get next token in dot access parsing");
vector_free(accessv);
return NULL;
}
if (!Parser_accept(parser, TokenType_Dot, NULL)) {
break;
}
}
assert(vector_size(accessv) != 0);
loc.range.end = last_loc.range.end;
AST *ast = ALLOC(sizeof(*ast));
ast->kind = ASTKind_MemberAccess;
ast->v.m.accessv = accessv;
ast->location = loc;
if (vector_size(accessv) == 1) {
ast->kind = ASTKind_Key;
ast->v.s = accessv[0];
vector_free(accessv);
}
return ast;
}
AST *parser_parse_value(Parser *parser) {
AST *ast = ALLOC(sizeof(*ast));
ast->location = parser->cur.location;
if (parser->cur.type == TokenType_Integer) {
ast->kind = ASTKind_Integer;
ast->v.i = parser->cur.v.i;
} else if (parser->cur.type == TokenType_Real) {
ast->kind = ASTKind_Real;
ast->v.r = parser->cur.v.r;
} else if (parser->cur.type == TokenType_String ||
parser->cur.type == TokenType_Identifier) {
ast->kind = ASTKind_Key;
ast->v.s = parser->cur.v.s;
if (parser->next.type == TokenType_Dot) {
FREE(ast);
return parser_parse_dot_access(parser, NULL);
} else if (parser->cur.type == TokenType_Identifier) {
if (sv_eq(ast->v.s, SV("fn"))) {
if (!Parser_next(parser)) {
strcpy(parser->instance->last_error, "Failed to advance fn keyword");
FREE(ast);
return NULL;
}
ast->kind = ASTKind_Function;
ast->v.f.argv = vector_create();
if (!ast->v.f.argv) {
strcpy(parser->instance->last_error, "Failed to allocate vector");
FREE(ast);
return NULL;
}
while (true) {
if (Parser_accept(parser, TokenType_Set, NULL)) {
break;
}
if (parser->cur.type != TokenType_Identifier) {
strcpy(parser->instance->last_error,
"Expected identifier for function argument");
vector_free(ast->v.f.argv);
FREE(ast);
return NULL;
}
vector_add(&ast->v.f.argv, parser->cur.v.s);
}
ast->v.f.body = parser_parse_value(parser);
if (!ast->v.f.body) {
vector_free(ast->v.f.argv);
FREE(ast);
return NULL;
}
ast->location.range.end = ast->v.f.body->location.range.end;
return ast;
} else if (sv_eq(ast->v.s, SV("on")) || sv_eq(ast->v.s, SV("true"))) {
ast->kind = ASTKind_Boolean;
ast->v.b = true;
} else if (sv_eq(ast->v.s, SV("off")) || sv_eq(ast->v.s, SV("false"))) {
ast->kind = ASTKind_Boolean;
ast->v.b = false;
}
}
} else if (parser->cur.type == TokenType_Path) {
ast->kind = ASTKind_Path;
ast->v.p = parser->cur.v.p;
} else if (Parser_accept(parser, TokenType_LBracket, NULL)) {
ast->kind = ASTKind_Array;
ast->v.a.childv = vector_create();
if (!ast->v.a.childv) {
strcpy(parser->instance->last_error, "Failed to allocate vector");
FREE(ast);
return NULL;
}
SourceLocation rbracket_loc;
while (true) {
if (Parser_accept(parser, TokenType_RBracket, &rbracket_loc)) {
break;
}
AST *value = parser_parse_value(parser);
if (!value) {
for (size_t i = 0; i < vector_size(ast->v.a.childv); i++) {
AST_free_parser(ast->v.a.childv[i], parser);
}
vector_free(ast->v.a.childv);
FREE(ast);
return NULL;
}
vector_add(&ast->v.a.childv, value);
}
ast->location.range.end = rbracket_loc.range.end;
return ast;
} else if (Parser_accept(parser, TokenType_LSquirly, NULL)) {
ast->kind = ASTKind_Block;
ast->v.bl.entryv = vector_create();
if (!ast->v.bl.entryv) {
strcpy(parser->instance->last_error, "Failed to allocate vector");
FREE(ast);
return NULL;
}
SourceLocation rsquirly_loc;
while (true) {
if (Parser_accept(parser, TokenType_RSquirly, &rsquirly_loc)) {
break;
}
if (parser->cur.type != TokenType_Identifier &&
parser->cur.type != TokenType_String) {
strcpy(parser->instance->last_error,
"Expected identifier or string for object key");
vector_free(ast->v.bl.entryv);
FREE(ast);
return NULL;
}
AST *key = parser_parse_dot_access(parser, NULL);
if (!Parser_accept(parser, TokenType_Set, NULL)) {
strcpy(parser->instance->last_error, "Expected = after object key");
vector_free(ast->v.bl.entryv);
FREE(ast);
return NULL;
}
AST *value = parser_parse_value(parser);
if (!value) {
AST_free_parser(key, parser);
vector_free(ast->v.bl.entryv);
FREE(ast);
return NULL;
}
assert(key->kind == ASTKind_MemberAccess || key->kind == ASTKind_Key);
ASTBlock_Entry entry = {
.k = key,
.v = value,
};
vector_add(&ast->v.bl.entryv, entry);
}
return ast;
} else if (Parser_accept(parser, TokenType_LParen, NULL)) {
ast->kind = ASTKind_FunctionCall;
ast->v.fc.function = parser_parse_value(parser);
ast->v.fc.argv = vector_create();
if (!ast->v.fc.argv) {
strcpy(parser->instance->last_error, "Failed to allocate vector");
FREE(ast);
return NULL;
}
SourceLocation rparen_loc;
while (true) {
if (Parser_accept(parser, TokenType_RParen, &rparen_loc)) {
break;
}
AST *value = parser_parse_value(parser);
if (!value) {
for (size_t i = 0; i < vector_size(ast->v.fc.argv); i++) {
AST_free_parser(ast->v.fc.argv[i], parser);
}
vector_free(ast->v.fc.argv);
FREE(ast);
return NULL;
}
vector_add(&ast->v.fc.argv, value);
}
ast->location.range.end = rparen_loc.range.end;
return ast;
} else {
if (parser->cur.type == TokenType_End) {
strcpy(parser->instance->last_error, "Expected value, got end of file");
} else {
strcpy(parser->instance->last_error, "Unexpected token for value");
}
FREE(ast);
ast = NULL;
}
if (!Parser_next(parser)) {
if (ast) {
FREE(ast);
}
return NULL;
}
return ast;
}
// TODO: Print SourceLocation in last error.
AST *Parser_parse(Parser *parser) { return parser_parse_value(parser); }
#undef ALLOC
#undef FREE
#define ALLOC(sz) (instance->alloc((sz)))
#define FREE(ptr) (instance->free((ptr)))
dcfg_Value *dcfg_parse(dcfg_Instance *instance,
dcfg_StringView const file_path) {
char path_buf[file_path.size + 1] = {};
memcpy(path_buf, file_path.data, file_path.size);
char *abs = realpath(path_buf, NULL);
if (!abs) {
snprintf(instance->last_error, sizeof(instance->last_error) - 1,
"realpath: %s", strerror(errno));
}
StringView abs_sv = SV(abs);
FILE *fp = fopen(abs, "r");
if (!fp) {
FREE(abs);
snprintf(instance->last_error, sizeof(instance->last_error) - 1,
"fopen: %s", strerror(errno));
return NULL;
}
if (fseek(fp, 0, SEEK_END)) {
FREE(abs);
fclose(fp);
snprintf(instance->last_error, sizeof(instance->last_error) - 1,
"fseek: %s", strerror(errno));
return NULL;
}
long const size = ftell(fp);
if (fseek(fp, 0, SEEK_SET)) {
FREE(abs);
fclose(fp);
snprintf(instance->last_error, sizeof(instance->last_error) - 1,
"fseek: %s", strerror(errno));
return NULL;
}
StringView str = {
.size = size,
};
str.data = ALLOC(size);
if (!fread((void *)str.data, str.size, 1, fp)) {
FREE(abs);
FREE((void *)str.data);
fclose(fp);
snprintf(instance->last_error, sizeof(instance->last_error) - 1,
"fread: %s", strerror(errno));
return NULL;
}
Lexer lexer;
if (!Lexer_init(&lexer, str, abs_sv)) {
FREE(abs);
FREE((void *)str.data);
fclose(fp);
snprintf(instance->last_error, sizeof(instance->last_error) - 1,
"Could not init lexer");
return NULL;
}
Parser parser;
Parser_init(&parser, &lexer, instance);
AST *ast = Parser_parse(&parser);
if (!ast) {
FREE(abs);
FREE((void *)str.data);
fclose(fp);
return NULL;
}
// FIXME: Get Value * from AST
}
void dcfg_destroy(dcfg_Value *value) {}
// Libraries
#include "vendor/utf8proc.c"
#include "vendor/vec.c"

823
src/vendor/utf8proc.c vendored Normal file
View File

@@ -0,0 +1,823 @@
/* -*- mode: c; c-basic-offset: 2; tab-width: 2; indent-tabs-mode: nil -*- */
/*
* Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
/*
* This library contains derived data from a modified version of the
* Unicode data files.
*
* The original data files are available at
* https://www.unicode.org/Public/UNIDATA/
*
* Please notice the copyright statement in the file "utf8proc_data.c".
*/
/*
* File name: utf8proc.c
*
* Description:
* Implementation of libutf8proc.
*/
#include "utf8proc.h"
#ifndef SSIZE_MAX
#define SSIZE_MAX ((size_t)SIZE_MAX/2)
#endif
#ifndef UINT16_MAX
# define UINT16_MAX 65535U
#endif
#include "utf8proc_data.c"
UTF8PROC_DLLEXPORT const utf8proc_int8_t utf8proc_utf8class[256] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 };
#define UTF8PROC_HANGUL_SBASE 0xAC00
#define UTF8PROC_HANGUL_LBASE 0x1100
#define UTF8PROC_HANGUL_VBASE 0x1161
#define UTF8PROC_HANGUL_TBASE 0x11A7
#define UTF8PROC_HANGUL_LCOUNT 19
#define UTF8PROC_HANGUL_VCOUNT 21
#define UTF8PROC_HANGUL_TCOUNT 28
#define UTF8PROC_HANGUL_NCOUNT 588
#define UTF8PROC_HANGUL_SCOUNT 11172
/* END is exclusive */
#define UTF8PROC_HANGUL_L_START 0x1100
#define UTF8PROC_HANGUL_L_END 0x115A
#define UTF8PROC_HANGUL_L_FILLER 0x115F
#define UTF8PROC_HANGUL_V_START 0x1160
#define UTF8PROC_HANGUL_V_END 0x11A3
#define UTF8PROC_HANGUL_T_START 0x11A8
#define UTF8PROC_HANGUL_T_END 0x11FA
#define UTF8PROC_HANGUL_S_START 0xAC00
#define UTF8PROC_HANGUL_S_END 0xD7A4
/* Should follow semantic-versioning rules (semver.org) based on API
compatibility. (Note that the shared-library version number will
be different, being based on ABI compatibility.): */
#define STRINGIZEx(x) #x
#define STRINGIZE(x) STRINGIZEx(x)
UTF8PROC_DLLEXPORT const char *utf8proc_version(void) {
return STRINGIZE(UTF8PROC_VERSION_MAJOR) "." STRINGIZE(UTF8PROC_VERSION_MINOR) "." STRINGIZE(UTF8PROC_VERSION_PATCH) "";
}
UTF8PROC_DLLEXPORT const char *utf8proc_unicode_version(void) {
return "16.0.0";
}
UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
switch (errcode) {
case UTF8PROC_ERROR_NOMEM:
return "Memory for processing UTF-8 data could not be allocated.";
case UTF8PROC_ERROR_OVERFLOW:
return "UTF-8 string is too long to be processed.";
case UTF8PROC_ERROR_INVALIDUTF8:
return "Invalid UTF-8 string";
case UTF8PROC_ERROR_NOTASSIGNED:
return "Unassigned Unicode code point found in UTF-8 string.";
case UTF8PROC_ERROR_INVALIDOPTS:
return "Invalid options for UTF-8 processing chosen.";
default:
return "An unknown error occurred while processing UTF-8 data.";
}
}
#define utf_cont(ch) (((ch) & 0xc0) == 0x80)
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
) {
utf8proc_int32_t uc;
const utf8proc_uint8_t *end;
*dst = -1;
if (!strlen) return 0;
end = str + ((strlen < 0) ? 4 : strlen);
uc = *str++;
if (uc < 0x80) {
*dst = uc;
return 1;
}
// Must be between 0xc2 and 0xf4 inclusive to be valid
if ((utf8proc_uint32_t)(uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
if (uc < 0xe0) { // 2-byte sequence
// Must have valid continuation character
if (str >= end || !utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
*dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
return 2;
}
if (uc < 0xf0) { // 3-byte sequence
if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
return UTF8PROC_ERROR_INVALIDUTF8;
// Check for surrogate chars
if (uc == 0xed && *str > 0x9f)
return UTF8PROC_ERROR_INVALIDUTF8;
uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
if (uc < 0x800)
return UTF8PROC_ERROR_INVALIDUTF8;
*dst = uc;
return 3;
}
// 4-byte sequence
// Must have 3 valid continuation characters
if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
return UTF8PROC_ERROR_INVALIDUTF8;
// Make sure in correct range (0x10000 - 0x10ffff)
if (uc == 0xf0) {
if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
} else if (uc == 0xf4) {
if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
}
*dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
return 4;
}
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
if (uc < 0x00) {
return 0;
} else if (uc < 0x80) {
dst[0] = (utf8proc_uint8_t) uc;
return 1;
} else if (uc < 0x800) {
dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 2;
// Note: we allow encoding 0xd800-0xdfff here, so as not to change
// the API, however, these are actually invalid in UTF-8
} else if (uc < 0x10000) {
dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 3;
} else if (uc < 0x110000) {
dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 4;
} else return 0;
}
/* internal version used for inserting 0xff bytes between graphemes */
static utf8proc_ssize_t charbound_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
if (uc < 0x00) {
if (uc == -1) { /* internal value used for grapheme breaks */
dst[0] = (utf8proc_uint8_t)0xFF;
return 1;
}
return 0;
} else if (uc < 0x80) {
dst[0] = (utf8proc_uint8_t)uc;
return 1;
} else if (uc < 0x800) {
dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 2;
} else if (uc < 0x10000) {
dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 3;
} else if (uc < 0x110000) {
dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 4;
} else return 0;
}
/* internal "unsafe" version that does not check whether uc is in range */
static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
/* ASSERT: uc >= 0 && uc < 0x110000 */
return utf8proc_properties + (
utf8proc_stage2table[
utf8proc_stage1table[uc >> 8] + (uc & 0xFF)
]
);
}
UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
}
/* return whether there is a grapheme break between boundclasses lbc and tbc
(according to the definition of extended grapheme clusters)
Rule numbering refers to TR29 Version 29 (Unicode 9.0.0):
http://www.unicode.org/reports/tr29/tr29-29.html
CAVEATS:
Please note that evaluation of GB10 (grapheme breaks between emoji zwj sequences)
and GB 12/13 (regional indicator code points) require knowledge of previous characters
and are thus not handled by this function. This may result in an incorrect break before
an E_Modifier class codepoint and an incorrectly missing break between two
REGIONAL_INDICATOR class code points if such support does not exist in the caller.
See the special support in grapheme_break_extended, for required bookkeeping by the caller.
*/
static utf8proc_bool grapheme_break_simple(int lbc, int tbc) {
return
(lbc == UTF8PROC_BOUNDCLASS_START) ? true : // GB1
(lbc == UTF8PROC_BOUNDCLASS_CR && // GB3
tbc == UTF8PROC_BOUNDCLASS_LF) ? false : // ---
(lbc >= UTF8PROC_BOUNDCLASS_CR && lbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB4
(tbc >= UTF8PROC_BOUNDCLASS_CR && tbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB5
(lbc == UTF8PROC_BOUNDCLASS_L && // GB6
(tbc == UTF8PROC_BOUNDCLASS_L || // ---
tbc == UTF8PROC_BOUNDCLASS_V || // ---
tbc == UTF8PROC_BOUNDCLASS_LV || // ---
tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false : // ---
((lbc == UTF8PROC_BOUNDCLASS_LV || // GB7
lbc == UTF8PROC_BOUNDCLASS_V) && // ---
(tbc == UTF8PROC_BOUNDCLASS_V || // ---
tbc == UTF8PROC_BOUNDCLASS_T)) ? false : // ---
((lbc == UTF8PROC_BOUNDCLASS_LVT || // GB8
lbc == UTF8PROC_BOUNDCLASS_T) && // ---
tbc == UTF8PROC_BOUNDCLASS_T) ? false : // ---
(tbc == UTF8PROC_BOUNDCLASS_EXTEND || // GB9
tbc == UTF8PROC_BOUNDCLASS_ZWJ || // ---
tbc == UTF8PROC_BOUNDCLASS_SPACINGMARK || // GB9a
lbc == UTF8PROC_BOUNDCLASS_PREPEND) ? false : // GB9b
(lbc == UTF8PROC_BOUNDCLASS_E_ZWG && // GB11 (requires additional handling below)
tbc == UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC) ? false : // ----
(lbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR && // GB12/13 (requires additional handling below)
tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR) ? false : // ----
true; // GB999
}
static utf8proc_bool grapheme_break_extended(int lbc, int tbc, int licb, int ticb, utf8proc_int32_t *state)
{
if (state) {
int state_bc, state_icb; /* boundclass and indic_conjunct_break state */
if (*state == 0) { /* state initialization */
state_bc = lbc;
state_icb = licb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT ? licb : UTF8PROC_INDIC_CONJUNCT_BREAK_NONE;
}
else { /* lbc and licb are already encoded in *state */
state_bc = *state & 0xff; // 1st byte of state is bound class
state_icb = *state >> 8; // 2nd byte of state is indic conjunct break
}
utf8proc_bool break_permitted = grapheme_break_simple(state_bc, tbc) &&
!(state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER
&& ticb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT); // GB9c
// Special support for GB9c. Don't break between two consonants
// separated 1+ linker characters and 0+ extend characters in any order.
// After a consonant, we enter LINKER state after at least one linker.
if (ticb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT
|| state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT
|| state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND)
state_icb = ticb;
else if (state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER)
state_icb = ticb == UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND ?
UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER : ticb;
// Special support for GB 12/13 made possible by GB999. After two RI
// class codepoints we want to force a break. Do this by resetting the
// second RI's bound class to UTF8PROC_BOUNDCLASS_OTHER, to force a break
// after that character according to GB999 (unless of course such a break is
// forbidden by a different rule such as GB9).
if (state_bc == tbc && tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR)
state_bc = UTF8PROC_BOUNDCLASS_OTHER;
// Special support for GB11 (emoji extend* zwj / emoji)
else if (state_bc == UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC) {
if (tbc == UTF8PROC_BOUNDCLASS_EXTEND) // fold EXTEND codepoints into emoji
state_bc = UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC;
else if (tbc == UTF8PROC_BOUNDCLASS_ZWJ)
state_bc = UTF8PROC_BOUNDCLASS_E_ZWG; // state to record emoji+zwg combo
else
state_bc = tbc;
}
else
state_bc = tbc;
*state = state_bc + (state_icb << 8);
return break_permitted;
}
else
return grapheme_break_simple(lbc, tbc);
}
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
utf8proc_int32_t c1, utf8proc_int32_t c2, utf8proc_int32_t *state) {
const utf8proc_property_t *p1 = utf8proc_get_property(c1);
const utf8proc_property_t *p2 = utf8proc_get_property(c2);
return grapheme_break_extended(p1->boundclass,
p2->boundclass,
p1->indic_conjunct_break,
p2->indic_conjunct_break,
state);
}
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
utf8proc_int32_t c1, utf8proc_int32_t c2) {
return utf8proc_grapheme_break_stateful(c1, c2, NULL);
}
static utf8proc_int32_t seqindex_decode_entry(const utf8proc_uint16_t **entry)
{
utf8proc_int32_t entry_cp = **entry;
if ((entry_cp & 0xF800) == 0xD800) {
*entry = *entry + 1;
entry_cp = ((entry_cp & 0x03FF) << 10) | (**entry & 0x03FF);
entry_cp += 0x10000;
}
return entry_cp;
}
static utf8proc_int32_t seqindex_decode_index(const utf8proc_uint32_t seqindex)
{
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex];
return seqindex_decode_entry(&entry);
}
static utf8proc_ssize_t seqindex_write_char_decomposed(utf8proc_uint16_t seqindex, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
utf8proc_ssize_t written = 0;
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex & 0x3FFF];
int len = seqindex >> 14;
if (len >= 3) {
len = *entry;
entry++;
}
for (; len >= 0; entry++, len--) {
utf8proc_int32_t entry_cp = seqindex_decode_entry(&entry);
written += utf8proc_decompose_char(entry_cp, dst+written,
(bufsize > written) ? (bufsize - written) : 0, options,
last_boundclass);
if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
}
return written;
}
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c)
{
utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_seqindex;
return cl != UINT16_MAX ? seqindex_decode_index((utf8proc_uint32_t)cl) : c;
}
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c)
{
utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_seqindex;
return cu != UINT16_MAX ? seqindex_decode_index((utf8proc_uint32_t)cu) : c;
}
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c)
{
utf8proc_int32_t cu = utf8proc_get_property(c)->titlecase_seqindex;
return cu != UINT16_MAX ? seqindex_decode_index((utf8proc_uint32_t)cu) : c;
}
UTF8PROC_DLLEXPORT int utf8proc_islower(utf8proc_int32_t c)
{
const utf8proc_property_t *p = utf8proc_get_property(c);
return p->lowercase_seqindex != p->uppercase_seqindex && p->lowercase_seqindex == UINT16_MAX;
}
UTF8PROC_DLLEXPORT int utf8proc_isupper(utf8proc_int32_t c)
{
const utf8proc_property_t *p = utf8proc_get_property(c);
return p->lowercase_seqindex != p->uppercase_seqindex && p->uppercase_seqindex == UINT16_MAX && p->category != UTF8PROC_CATEGORY_LT;
}
/* return a character width analogous to wcwidth (except portable and
hopefully less buggy than most system wcwidth functions). */
UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t c) {
return utf8proc_get_property(c)->charwidth;
}
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_charwidth_ambiguous(utf8proc_int32_t c) {
return utf8proc_get_property(c)->ambiguous_width;
}
UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t c) {
return (utf8proc_category_t) utf8proc_get_property(c)->category;
}
UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t c) {
static const char s[][3] = {"Cn","Lu","Ll","Lt","Lm","Lo","Mn","Mc","Me","Nd","Nl","No","Pc","Pd","Ps","Pe","Pi","Pf","Po","Sm","Sc","Sk","So","Zs","Zl","Zp","Cc","Cf","Cs","Co"};
return s[utf8proc_category(c)];
}
#define utf8proc_decompose_lump(replacement_uc) \
return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
options & ~(unsigned int)UTF8PROC_LUMP, last_boundclass)
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
const utf8proc_property_t *property;
utf8proc_propval_t category;
utf8proc_int32_t hangul_sindex;
if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
property = unsafe_get_property(uc);
category = property->category;
hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
utf8proc_int32_t hangul_tindex;
if (bufsize >= 1) {
dst[0] = UTF8PROC_HANGUL_LBASE +
hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
if (bufsize >= 2) dst[1] = UTF8PROC_HANGUL_VBASE +
(hangul_sindex % UTF8PROC_HANGUL_NCOUNT) / UTF8PROC_HANGUL_TCOUNT;
}
hangul_tindex = hangul_sindex % UTF8PROC_HANGUL_TCOUNT;
if (!hangul_tindex) return 2;
if (bufsize >= 3) dst[2] = UTF8PROC_HANGUL_TBASE + hangul_tindex;
return 3;
}
}
if (options & UTF8PROC_REJECTNA) {
if (!category) return UTF8PROC_ERROR_NOTASSIGNED;
}
if (options & UTF8PROC_IGNORE) {
if (property->ignorable) return 0;
}
if (options & UTF8PROC_STRIPNA) {
if (!category) return 0;
}
if (options & UTF8PROC_LUMP) {
if (category == UTF8PROC_CATEGORY_ZS) utf8proc_decompose_lump(0x0020);
if (uc == 0x2018 || uc == 0x2019 || uc == 0x02BC || uc == 0x02C8)
utf8proc_decompose_lump(0x0027);
if (category == UTF8PROC_CATEGORY_PD || uc == 0x2212)
utf8proc_decompose_lump(0x002D);
if (uc == 0x2044 || uc == 0x2215) utf8proc_decompose_lump(0x002F);
if (uc == 0x2236) utf8proc_decompose_lump(0x003A);
if (uc == 0x2039 || uc == 0x2329 || uc == 0x3008)
utf8proc_decompose_lump(0x003C);
if (uc == 0x203A || uc == 0x232A || uc == 0x3009)
utf8proc_decompose_lump(0x003E);
if (uc == 0x2216) utf8proc_decompose_lump(0x005C);
if (uc == 0x02C4 || uc == 0x02C6 || uc == 0x2038 || uc == 0x2303)
utf8proc_decompose_lump(0x005E);
if (category == UTF8PROC_CATEGORY_PC || uc == 0x02CD)
utf8proc_decompose_lump(0x005F);
if (uc == 0x02CB) utf8proc_decompose_lump(0x0060);
if (uc == 0x2223) utf8proc_decompose_lump(0x007C);
if (uc == 0x223C) utf8proc_decompose_lump(0x007E);
if ((options & UTF8PROC_NLF2LS) && (options & UTF8PROC_NLF2PS)) {
if (category == UTF8PROC_CATEGORY_ZL ||
category == UTF8PROC_CATEGORY_ZP)
utf8proc_decompose_lump(0x000A);
}
}
if (options & UTF8PROC_STRIPMARK) {
if (category == UTF8PROC_CATEGORY_MN ||
category == UTF8PROC_CATEGORY_MC ||
category == UTF8PROC_CATEGORY_ME) return 0;
}
if (options & UTF8PROC_CASEFOLD) {
if (property->casefold_seqindex != UINT16_MAX) {
return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
}
}
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
if (property->decomp_seqindex != UINT16_MAX &&
(!property->decomp_type || (options & UTF8PROC_COMPAT))) {
return seqindex_write_char_decomposed(property->decomp_seqindex, dst, bufsize, options, last_boundclass);
}
}
if (options & UTF8PROC_CHARBOUND) {
utf8proc_bool boundary;
boundary = grapheme_break_extended(0, property->boundclass, 0, property->indic_conjunct_break,
last_boundclass);
if (boundary) {
if (bufsize >= 1) dst[0] = -1; /* sentinel value for grapheme break */
if (bufsize >= 2) dst[1] = uc;
return 2;
}
}
if (bufsize >= 1) *dst = uc;
return 1;
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
) {
return utf8proc_decompose_custom(str, strlen, buffer, bufsize, options, NULL, NULL);
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
) {
/* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
utf8proc_ssize_t wpos = 0;
if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
return UTF8PROC_ERROR_INVALIDOPTS;
if ((options & UTF8PROC_STRIPMARK) &&
!(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
return UTF8PROC_ERROR_INVALIDOPTS;
{
utf8proc_int32_t uc;
utf8proc_ssize_t rpos = 0;
utf8proc_ssize_t decomp_result;
int boundclass = UTF8PROC_BOUNDCLASS_START;
while (1) {
if (options & UTF8PROC_NULLTERM) {
rpos += utf8proc_iterate(str + rpos, -1, &uc);
/* checking of return value is not necessary,
as 'uc' is < 0 in case of error */
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
if (uc == 0) break;
} else {
if (rpos >= strlen) break;
rpos += utf8proc_iterate(str + rpos, strlen - rpos, &uc);
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
}
if (custom_func != NULL) {
uc = custom_func(uc, custom_data); /* user-specified custom mapping */
}
decomp_result = utf8proc_decompose_char(
uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
&boundclass
);
if (decomp_result < 0) return decomp_result;
wpos += decomp_result;
/* prohibiting integer overflows due to too long strings: */
if (wpos < 0 ||
wpos > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2))
return UTF8PROC_ERROR_OVERFLOW;
}
}
if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
utf8proc_ssize_t pos = 0;
while (pos < wpos-1) {
utf8proc_int32_t uc1, uc2;
const utf8proc_property_t *property1, *property2;
uc1 = buffer[pos];
uc2 = buffer[pos+1];
property1 = unsafe_get_property(uc1);
property2 = unsafe_get_property(uc2);
if (property1->combining_class > property2->combining_class &&
property2->combining_class > 0) {
buffer[pos] = uc2;
buffer[pos+1] = uc1;
if (pos > 0) pos--; else pos++;
} else {
pos++;
}
}
}
return wpos;
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored */
if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
utf8proc_ssize_t rpos;
utf8proc_ssize_t wpos = 0;
utf8proc_int32_t uc;
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
if (uc == 0x000A || uc == 0x000D || uc == 0x0085 ||
((options & UTF8PROC_STRIPCC) && (uc == 0x000B || uc == 0x000C))) {
if (options & UTF8PROC_NLF2LS) {
if (options & UTF8PROC_NLF2PS) {
buffer[wpos++] = 0x000A;
} else {
buffer[wpos++] = 0x2028;
}
} else {
if (options & UTF8PROC_NLF2PS) {
buffer[wpos++] = 0x2029;
} else {
buffer[wpos++] = 0x0020;
}
}
} else if ((options & UTF8PROC_STRIPCC) &&
(uc < 0x0020 || (uc >= 0x007F && uc < 0x00A0))) {
if (uc == 0x0009) buffer[wpos++] = 0x0020;
} else {
buffer[wpos++] = uc;
}
}
length = wpos;
}
if (options & UTF8PROC_COMPOSE) {
utf8proc_int32_t *starter = NULL;
const utf8proc_property_t *starter_property = NULL;
utf8proc_propval_t max_combining_class = -1;
utf8proc_ssize_t rpos;
utf8proc_ssize_t wpos = 0;
for (rpos = 0; rpos < length; rpos++) {
utf8proc_int32_t current_char = buffer[rpos];
const utf8proc_property_t *current_property = unsafe_get_property(current_char);
if (starter && current_property->combining_class > max_combining_class) {
/* combination perhaps possible */
utf8proc_int32_t hangul_lindex;
utf8proc_int32_t hangul_sindex;
hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
utf8proc_int32_t hangul_vindex;
hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
*starter = UTF8PROC_HANGUL_SBASE +
(hangul_lindex * UTF8PROC_HANGUL_VCOUNT + hangul_vindex) *
UTF8PROC_HANGUL_TCOUNT;
starter_property = NULL;
continue;
}
}
hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
(hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
utf8proc_int32_t hangul_tindex;
hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
*starter += hangul_tindex;
starter_property = NULL;
continue;
}
}
if (!starter_property) {
starter_property = unsafe_get_property(*starter);
}
int idx = starter_property->comb_index;
if (idx < 0x3FF && current_property->comb_issecond) {
int len = starter_property->comb_length;
utf8proc_int32_t max_second = utf8proc_combinations_second[idx + len - 1];
if (current_char <= max_second) {
// TODO: binary search? arithmetic search?
for (int off = 0; off < len; ++off) {
utf8proc_int32_t second = utf8proc_combinations_second[idx + off];
if (current_char < second) {
/* not found */
break;
}
if (current_char == second) {
/* found */
utf8proc_int32_t composition = utf8proc_combinations_combined[idx + off];
*starter = composition;
starter_property = NULL;
break;
}
}
if (starter_property == NULL) {
/* found */
continue;
}
}
}
}
buffer[wpos] = current_char;
if (current_property->combining_class) {
if (current_property->combining_class > max_combining_class) {
max_combining_class = current_property->combining_class;
}
} else {
starter = buffer + wpos;
starter_property = NULL;
max_combining_class = -1;
}
wpos++;
}
length = wpos;
}
return length;
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
ASSERT: 'buffer' has one spare byte of free space at the end! */
length = utf8proc_normalize_utf32(buffer, length, options);
if (length < 0) return length;
{
utf8proc_ssize_t rpos, wpos = 0;
utf8proc_int32_t uc;
if (options & UTF8PROC_CHARBOUND) {
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
wpos += charbound_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
}
} else {
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
}
}
((utf8proc_uint8_t *)buffer)[wpos] = 0;
return wpos;
}
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
) {
return utf8proc_map_custom(str, strlen, dstptr, options, NULL, NULL);
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
) {
utf8proc_int32_t *buffer;
utf8proc_ssize_t result;
*dstptr = NULL;
result = utf8proc_decompose_custom(str, strlen, NULL, 0, options, custom_func, custom_data);
if (result < 0) return result;
buffer = (utf8proc_int32_t *) malloc(((utf8proc_size_t)result) * sizeof(utf8proc_int32_t) + 1);
if (!buffer) return UTF8PROC_ERROR_NOMEM;
result = utf8proc_decompose_custom(str, strlen, buffer, result, options, custom_func, custom_data);
if (result < 0) {
free(buffer);
return result;
}
result = utf8proc_reencode(buffer, result, options);
if (result < 0) {
free(buffer);
return result;
}
{
utf8proc_int32_t *newptr;
newptr = (utf8proc_int32_t *) realloc(buffer, (size_t)result+1);
if (newptr) buffer = newptr;
}
*dstptr = (utf8proc_uint8_t *)buffer;
return result;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_DECOMPOSE);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_COMPOSE);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC_Casefold(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD | UTF8PROC_IGNORE);
return retval;
}

789
src/vendor/utf8proc.h vendored Normal file
View File

@@ -0,0 +1,789 @@
/*
* Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
/**
* @mainpage
*
* utf8proc is a free/open-source (MIT/expat licensed) C library
* providing Unicode normalization, case-folding, and other operations
* for strings in the UTF-8 encoding, supporting up-to-date Unicode versions.
* See the utf8proc home page (http://julialang.org/utf8proc/)
* for downloads and other information, or the source code on github
* (https://github.com/JuliaLang/utf8proc).
*
* For the utf8proc API documentation, see: @ref utf8proc.h
*
* The features of utf8proc include:
*
* - Transformation of strings (utf8proc_map()) to:
* - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
* - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
* - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
* - case-folding (@ref UTF8PROC_CASEFOLD)
* - Unicode normalization: utf8proc_NFD(), utf8proc_NFC(), utf8proc_NFKD(), utf8proc_NFKC()
* - Detecting grapheme boundaries (utf8proc_grapheme_break() and @ref UTF8PROC_CHARBOUND)
* - Character-width computation: utf8proc_charwidth()
* - Classification of characters by Unicode category: utf8proc_category() and utf8proc_category_string()
* - Encode (utf8proc_encode_char()) and decode (utf8proc_iterate()) Unicode codepoints to/from UTF-8.
*/
/** @file */
#ifndef UTF8PROC_H
#define UTF8PROC_H
/** @name API version
*
* The utf8proc API version MAJOR.MINOR.PATCH, following
* semantic-versioning rules (http://semver.org) based on API
* compatibility.
*
* This is also returned at runtime by utf8proc_version(); however, the
* runtime version may append a string like "-dev" to the version number
* for prerelease versions.
*
* @note The shared-library version number in the Makefile
* (and CMakeLists.txt, and MANIFEST) may be different,
* being based on ABI compatibility rather than API compatibility.
*/
/** @{ */
/** The MAJOR version number (increased when backwards API compatibility is broken). */
#define UTF8PROC_VERSION_MAJOR 2
/** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
#define UTF8PROC_VERSION_MINOR 10
/** The PATCH version (increased for fixes that do not change the API). */
#define UTF8PROC_VERSION_PATCH 0
/** @} */
#include <stdlib.h>
#if defined(_MSC_VER) && _MSC_VER < 1800
// MSVC prior to 2013 lacked stdbool.h and stdint.h
typedef signed char utf8proc_int8_t;
typedef unsigned char utf8proc_uint8_t;
typedef short utf8proc_int16_t;
typedef unsigned short utf8proc_uint16_t;
typedef int utf8proc_int32_t;
typedef unsigned int utf8proc_uint32_t;
# ifdef _WIN64
typedef __int64 utf8proc_ssize_t;
typedef unsigned __int64 utf8proc_size_t;
# else
typedef int utf8proc_ssize_t;
typedef unsigned int utf8proc_size_t;
# endif
# ifndef __cplusplus
// emulate C99 bool
typedef unsigned char utf8proc_bool;
# ifndef __bool_true_false_are_defined
# define false 0
# define true 1
# define __bool_true_false_are_defined 1
# endif
# else
typedef bool utf8proc_bool;
# endif
#else
# include <stddef.h>
# include <stdbool.h>
# include <stdint.h>
typedef int8_t utf8proc_int8_t;
typedef uint8_t utf8proc_uint8_t;
typedef int16_t utf8proc_int16_t;
typedef uint16_t utf8proc_uint16_t;
typedef int32_t utf8proc_int32_t;
typedef uint32_t utf8proc_uint32_t;
typedef size_t utf8proc_size_t;
typedef ptrdiff_t utf8proc_ssize_t;
typedef bool utf8proc_bool;
#endif
#include <limits.h>
#ifdef UTF8PROC_STATIC
# define UTF8PROC_DLLEXPORT
#else
# ifdef _WIN32
# ifdef UTF8PROC_EXPORTS
# define UTF8PROC_DLLEXPORT __declspec(dllexport)
# else
# define UTF8PROC_DLLEXPORT __declspec(dllimport)
# endif
# elif __GNUC__ >= 4
# define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default")))
# else
# define UTF8PROC_DLLEXPORT
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
/**
* Option flags used by several functions in the library.
*/
typedef enum {
/** The given UTF-8 input is NULL terminated. */
UTF8PROC_NULLTERM = (1<<0),
/** Unicode Versioning Stability has to be respected. */
UTF8PROC_STABLE = (1<<1),
/** Compatibility decomposition (i.e. formatting information is lost). */
UTF8PROC_COMPAT = (1<<2),
/** Return a result with decomposed characters. */
UTF8PROC_COMPOSE = (1<<3),
/** Return a result with decomposed characters. */
UTF8PROC_DECOMPOSE = (1<<4),
/** Strip "default ignorable characters" such as SOFT-HYPHEN or ZERO-WIDTH-SPACE. */
UTF8PROC_IGNORE = (1<<5),
/** Return an error, if the input contains unassigned codepoints. */
UTF8PROC_REJECTNA = (1<<6),
/**
* Indicating that NLF-sequences (LF, CRLF, CR, NEL) are representing a
* line break, and should be converted to the codepoint for line
* separation (LS).
*/
UTF8PROC_NLF2LS = (1<<7),
/**
* Indicating that NLF-sequences are representing a paragraph break, and
* should be converted to the codepoint for paragraph separation
* (PS).
*/
UTF8PROC_NLF2PS = (1<<8),
/** Indicating that the meaning of NLF-sequences is unknown. */
UTF8PROC_NLF2LF = (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS),
/** Strips and/or convers control characters.
*
* NLF-sequences are transformed into space, except if one of the
* NLF2LS/PS/LF options is given. HorizontalTab (HT) and FormFeed (FF)
* are treated as a NLF-sequence in this case. All other control
* characters are simply removed.
*/
UTF8PROC_STRIPCC = (1<<9),
/**
* Performs unicode case folding, to be able to do a case-insensitive
* string comparison.
*/
UTF8PROC_CASEFOLD = (1<<10),
/**
* Inserts 0xFF bytes at the beginning of each sequence which is
* representing a single grapheme cluster (see UAX#29).
*/
UTF8PROC_CHARBOUND = (1<<11),
/** Lumps certain characters together.
*
* E.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-". See lump.md for details.
*
* If NLF2LF is set, this includes a transformation of paragraph and
* line separators to ASCII line-feed (LF).
*/
UTF8PROC_LUMP = (1<<12),
/** Strips all character markings.
*
* This includes non-spacing, spacing and enclosing (i.e. accents).
* @note This option works only with @ref UTF8PROC_COMPOSE or
* @ref UTF8PROC_DECOMPOSE
*/
UTF8PROC_STRIPMARK = (1<<13),
/**
* Strip unassigned codepoints.
*/
UTF8PROC_STRIPNA = (1<<14),
} utf8proc_option_t;
/** @name Error codes
* Error codes being returned by almost all functions.
*/
/** @{ */
/** Memory could not be allocated. */
#define UTF8PROC_ERROR_NOMEM -1
/** The given string is too long to be processed. */
#define UTF8PROC_ERROR_OVERFLOW -2
/** The given string is not a legal UTF-8 string. */
#define UTF8PROC_ERROR_INVALIDUTF8 -3
/** The @ref UTF8PROC_REJECTNA flag was set and an unassigned codepoint was found. */
#define UTF8PROC_ERROR_NOTASSIGNED -4
/** Invalid options have been used. */
#define UTF8PROC_ERROR_INVALIDOPTS -5
/** @} */
/* @name Types */
/** Holds the value of a property. */
typedef utf8proc_int16_t utf8proc_propval_t;
/** Struct containing information about a codepoint. */
typedef struct utf8proc_property_struct {
/**
* Unicode category.
* @see utf8proc_category_t.
*/
utf8proc_propval_t category;
utf8proc_propval_t combining_class;
/**
* Bidirectional class.
* @see utf8proc_bidi_class_t.
*/
utf8proc_propval_t bidi_class;
/**
* @anchor Decomposition type.
* @see utf8proc_decomp_type_t.
*/
utf8proc_propval_t decomp_type;
utf8proc_uint16_t decomp_seqindex;
utf8proc_uint16_t casefold_seqindex;
utf8proc_uint16_t uppercase_seqindex;
utf8proc_uint16_t lowercase_seqindex;
utf8proc_uint16_t titlecase_seqindex;
/**
* Character combining table.
*
* The character combining table is formally indexed by two
* characters, the first and second character that might form a
* combining pair. The table entry then contains the combined
* character. Most character pairs cannot be combined. There are
* about 1,000 characters that can be the first character in a
* combining pair, and for most, there are only a handful for
* possible second characters.
*
* The combining table is stored as sparse matrix in the CSR
* (compressed sparse row) format. That is, it is stored as two
* arrays, `utf8proc_uint32_t utf8proc_combinations_second[]` and
* `utf8proc_uint32_t utf8proc_combinations_combined[]`. These
* contain the second combining characters and the combined
* character of every combining pair.
*
* - `comb_index`: Index into the combining table if this character
* is the first character in a combining pair, else 0x3ff
*
* - `comb_length`: Number of table entries for this first character
*
* - `comb_is_second`: As optimization we also record whether this
* character is the second combining character in any pair. If
* not, we can skip the table lookup.
*
* A table lookup starts from a given character pair. It first
* checks whether the first character is stored in the table
* (checking whether the index is 0x3ff) and whether the second
* index is stored in the table (looking at `comb_is_second`). If
* so, the `comb_length` table entries will be checked sequentially
* for a match.
*/
utf8proc_uint16_t comb_index:10;
utf8proc_uint16_t comb_length:5;
utf8proc_uint16_t comb_issecond:1;
unsigned bidi_mirrored:1;
unsigned comp_exclusion:1;
/**
* Can this codepoint be ignored?
*
* Used by utf8proc_decompose_char() when @ref UTF8PROC_IGNORE is
* passed as an option.
*/
unsigned ignorable:1;
unsigned control_boundary:1;
/** The width of the codepoint. */
unsigned charwidth:2;
/** East Asian width class A */
unsigned ambiguous_width:1;
unsigned pad:1;
/**
* Boundclass.
* @see utf8proc_boundclass_t.
*/
unsigned boundclass:6;
unsigned indic_conjunct_break:2;
} utf8proc_property_t;
/** Unicode categories. */
typedef enum {
UTF8PROC_CATEGORY_CN = 0, /**< Other, not assigned */
UTF8PROC_CATEGORY_LU = 1, /**< Letter, uppercase */
UTF8PROC_CATEGORY_LL = 2, /**< Letter, lowercase */
UTF8PROC_CATEGORY_LT = 3, /**< Letter, titlecase */
UTF8PROC_CATEGORY_LM = 4, /**< Letter, modifier */
UTF8PROC_CATEGORY_LO = 5, /**< Letter, other */
UTF8PROC_CATEGORY_MN = 6, /**< Mark, nonspacing */
UTF8PROC_CATEGORY_MC = 7, /**< Mark, spacing combining */
UTF8PROC_CATEGORY_ME = 8, /**< Mark, enclosing */
UTF8PROC_CATEGORY_ND = 9, /**< Number, decimal digit */
UTF8PROC_CATEGORY_NL = 10, /**< Number, letter */
UTF8PROC_CATEGORY_NO = 11, /**< Number, other */
UTF8PROC_CATEGORY_PC = 12, /**< Punctuation, connector */
UTF8PROC_CATEGORY_PD = 13, /**< Punctuation, dash */
UTF8PROC_CATEGORY_PS = 14, /**< Punctuation, open */
UTF8PROC_CATEGORY_PE = 15, /**< Punctuation, close */
UTF8PROC_CATEGORY_PI = 16, /**< Punctuation, initial quote */
UTF8PROC_CATEGORY_PF = 17, /**< Punctuation, final quote */
UTF8PROC_CATEGORY_PO = 18, /**< Punctuation, other */
UTF8PROC_CATEGORY_SM = 19, /**< Symbol, math */
UTF8PROC_CATEGORY_SC = 20, /**< Symbol, currency */
UTF8PROC_CATEGORY_SK = 21, /**< Symbol, modifier */
UTF8PROC_CATEGORY_SO = 22, /**< Symbol, other */
UTF8PROC_CATEGORY_ZS = 23, /**< Separator, space */
UTF8PROC_CATEGORY_ZL = 24, /**< Separator, line */
UTF8PROC_CATEGORY_ZP = 25, /**< Separator, paragraph */
UTF8PROC_CATEGORY_CC = 26, /**< Other, control */
UTF8PROC_CATEGORY_CF = 27, /**< Other, format */
UTF8PROC_CATEGORY_CS = 28, /**< Other, surrogate */
UTF8PROC_CATEGORY_CO = 29, /**< Other, private use */
} utf8proc_category_t;
/** Bidirectional character classes. */
typedef enum {
UTF8PROC_BIDI_CLASS_L = 1, /**< Left-to-Right */
UTF8PROC_BIDI_CLASS_LRE = 2, /**< Left-to-Right Embedding */
UTF8PROC_BIDI_CLASS_LRO = 3, /**< Left-to-Right Override */
UTF8PROC_BIDI_CLASS_R = 4, /**< Right-to-Left */
UTF8PROC_BIDI_CLASS_AL = 5, /**< Right-to-Left Arabic */
UTF8PROC_BIDI_CLASS_RLE = 6, /**< Right-to-Left Embedding */
UTF8PROC_BIDI_CLASS_RLO = 7, /**< Right-to-Left Override */
UTF8PROC_BIDI_CLASS_PDF = 8, /**< Pop Directional Format */
UTF8PROC_BIDI_CLASS_EN = 9, /**< European Number */
UTF8PROC_BIDI_CLASS_ES = 10, /**< European Separator */
UTF8PROC_BIDI_CLASS_ET = 11, /**< European Number Terminator */
UTF8PROC_BIDI_CLASS_AN = 12, /**< Arabic Number */
UTF8PROC_BIDI_CLASS_CS = 13, /**< Common Number Separator */
UTF8PROC_BIDI_CLASS_NSM = 14, /**< Nonspacing Mark */
UTF8PROC_BIDI_CLASS_BN = 15, /**< Boundary Neutral */
UTF8PROC_BIDI_CLASS_B = 16, /**< Paragraph Separator */
UTF8PROC_BIDI_CLASS_S = 17, /**< Segment Separator */
UTF8PROC_BIDI_CLASS_WS = 18, /**< Whitespace */
UTF8PROC_BIDI_CLASS_ON = 19, /**< Other Neutrals */
UTF8PROC_BIDI_CLASS_LRI = 20, /**< Left-to-Right Isolate */
UTF8PROC_BIDI_CLASS_RLI = 21, /**< Right-to-Left Isolate */
UTF8PROC_BIDI_CLASS_FSI = 22, /**< First Strong Isolate */
UTF8PROC_BIDI_CLASS_PDI = 23, /**< Pop Directional Isolate */
} utf8proc_bidi_class_t;
/** Decomposition type. */
typedef enum {
UTF8PROC_DECOMP_TYPE_FONT = 1, /**< Font */
UTF8PROC_DECOMP_TYPE_NOBREAK = 2, /**< Nobreak */
UTF8PROC_DECOMP_TYPE_INITIAL = 3, /**< Initial */
UTF8PROC_DECOMP_TYPE_MEDIAL = 4, /**< Medial */
UTF8PROC_DECOMP_TYPE_FINAL = 5, /**< Final */
UTF8PROC_DECOMP_TYPE_ISOLATED = 6, /**< Isolated */
UTF8PROC_DECOMP_TYPE_CIRCLE = 7, /**< Circle */
UTF8PROC_DECOMP_TYPE_SUPER = 8, /**< Super */
UTF8PROC_DECOMP_TYPE_SUB = 9, /**< Sub */
UTF8PROC_DECOMP_TYPE_VERTICAL = 10, /**< Vertical */
UTF8PROC_DECOMP_TYPE_WIDE = 11, /**< Wide */
UTF8PROC_DECOMP_TYPE_NARROW = 12, /**< Narrow */
UTF8PROC_DECOMP_TYPE_SMALL = 13, /**< Small */
UTF8PROC_DECOMP_TYPE_SQUARE = 14, /**< Square */
UTF8PROC_DECOMP_TYPE_FRACTION = 15, /**< Fraction */
UTF8PROC_DECOMP_TYPE_COMPAT = 16, /**< Compat */
} utf8proc_decomp_type_t;
/** Boundclass property. (TR29) */
typedef enum {
UTF8PROC_BOUNDCLASS_START = 0, /**< Start */
UTF8PROC_BOUNDCLASS_OTHER = 1, /**< Other */
UTF8PROC_BOUNDCLASS_CR = 2, /**< Cr */
UTF8PROC_BOUNDCLASS_LF = 3, /**< Lf */
UTF8PROC_BOUNDCLASS_CONTROL = 4, /**< Control */
UTF8PROC_BOUNDCLASS_EXTEND = 5, /**< Extend */
UTF8PROC_BOUNDCLASS_L = 6, /**< L */
UTF8PROC_BOUNDCLASS_V = 7, /**< V */
UTF8PROC_BOUNDCLASS_T = 8, /**< T */
UTF8PROC_BOUNDCLASS_LV = 9, /**< Lv */
UTF8PROC_BOUNDCLASS_LVT = 10, /**< Lvt */
UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR = 11, /**< Regional indicator */
UTF8PROC_BOUNDCLASS_SPACINGMARK = 12, /**< Spacingmark */
UTF8PROC_BOUNDCLASS_PREPEND = 13, /**< Prepend */
UTF8PROC_BOUNDCLASS_ZWJ = 14, /**< Zero Width Joiner */
/* the following are no longer used in Unicode 11, but we keep
the constants here for backward compatibility */
UTF8PROC_BOUNDCLASS_E_BASE = 15, /**< Emoji Base */
UTF8PROC_BOUNDCLASS_E_MODIFIER = 16, /**< Emoji Modifier */
UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ = 17, /**< Glue_After_ZWJ */
UTF8PROC_BOUNDCLASS_E_BASE_GAZ = 18, /**< E_BASE + GLUE_AFTER_ZJW */
/* the Extended_Pictographic property is used in the Unicode 11
grapheme-boundary rules, so we store it in the boundclass field */
UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC = 19,
UTF8PROC_BOUNDCLASS_E_ZWG = 20, /* UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC + ZWJ */
} utf8proc_boundclass_t;
/** Indic_Conjunct_Break property. (TR44) */
typedef enum {
UTF8PROC_INDIC_CONJUNCT_BREAK_NONE = 0,
UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER = 1,
UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT = 2,
UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND = 3,
} utf8proc_indic_conjunct_break_t;
/**
* Function pointer type passed to utf8proc_map_custom() and
* utf8proc_decompose_custom(), which is used to specify a user-defined
* mapping of codepoints to be applied in conjunction with other mappings.
*/
typedef utf8proc_int32_t (*utf8proc_custom_func)(utf8proc_int32_t codepoint, void *data);
/**
* Array containing the byte lengths of a UTF-8 encoded codepoint based
* on the first byte.
*/
UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];
/**
* Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
* (http://semver.org format), possibly with a "-dev" suffix for
* development versions.
*/
UTF8PROC_DLLEXPORT const char *utf8proc_version(void);
/**
* Returns the utf8proc supported Unicode version as a string MAJOR.MINOR.PATCH.
*/
UTF8PROC_DLLEXPORT const char *utf8proc_unicode_version(void);
/**
* Returns an informative error string for the given utf8proc error code
* (e.g. the error codes returned by utf8proc_map()).
*/
UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
/**
* Reads a single codepoint from the UTF-8 sequence being pointed to by `str`.
* The maximum number of bytes read is `strlen`, unless `strlen` is
* negative (in which case up to 4 bytes are read).
*
* If a valid codepoint could be read, it is stored in the variable
* pointed to by `codepoint_ref`, otherwise that variable will be set to -1.
* In case of success, the number of bytes read is returned; otherwise, a
* negative error code is returned.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
/**
* Check if a codepoint is valid (regardless of whether it has been
* assigned a value by the current Unicode standard).
*
* @return 1 if the given `codepoint` is valid and otherwise return 0.
*/
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t codepoint);
/**
* Encodes the codepoint as an UTF-8 string in the byte array pointed
* to by `dst`. This array must be at least 4 bytes long.
*
* In case of success the number of bytes written is returned, and
* otherwise 0 is returned.
*
* This function does not check whether `codepoint` is valid Unicode.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t codepoint, utf8proc_uint8_t *dst);
/**
* Look up the properties for a given codepoint.
*
* @param codepoint The Unicode codepoint.
*
* @returns
* A pointer to a (constant) struct containing information about
* the codepoint.
* @par
* If the codepoint is unassigned or invalid, a pointer to a special struct is
* returned in which `category` is 0 (@ref UTF8PROC_CATEGORY_CN).
*/
UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint);
/** Decompose a codepoint into an array of codepoints.
*
* @param codepoint the codepoint.
* @param dst the destination buffer.
* @param bufsize the size of the destination buffer.
* @param options one or more of the following flags:
* - @ref UTF8PROC_REJECTNA - return an error `codepoint` is unassigned
* - @ref UTF8PROC_IGNORE - strip "default ignorable" codepoints
* - @ref UTF8PROC_CASEFOLD - apply Unicode casefolding
* - @ref UTF8PROC_COMPAT - replace certain codepoints with their
* compatibility decomposition
* - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
* - @ref UTF8PROC_LUMP - lump certain different codepoints together
* - @ref UTF8PROC_STRIPMARK - remove all character marks
* - @ref UTF8PROC_STRIPNA - remove unassigned codepoints
* @param last_boundclass
* Pointer to an integer variable containing
* the previous codepoint's (boundclass + indic_conjunct_break << 1) if the @ref UTF8PROC_CHARBOUND
* option is used. If the string is being processed in order, this can be initialized to 0 for
* the beginning of the string, and is thereafter updated automatically. Otherwise, this parameter is ignored.
*
* @return
* In case of success, the number of codepoints written is returned; in case
* of an error, a negative error code is returned (utf8proc_errmsg()).
* @par
* If the number of written codepoints would be bigger than `bufsize`, the
* required buffer size is returned, while the buffer will be overwritten with
* undefined data.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(
utf8proc_int32_t codepoint, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize,
utf8proc_option_t options, int *last_boundclass
);
/**
* The same as utf8proc_decompose_char(), but acts on a whole UTF-8
* string and orders the decomposed sequences correctly.
*
* If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
* will be stopped, when a NULL byte is encountered, otherwise `strlen`
* bytes are processed. The result (in the form of 32-bit unicode
* codepoints) is written into the buffer being pointed to by
* `buffer` (which must contain at least `bufsize` entries). In case of
* success, the number of codepoints written is returned; in case of an
* error, a negative error code is returned (utf8proc_errmsg()).
* See utf8proc_decompose_custom() to supply additional transformations.
*
* If the number of written codepoints would be bigger than `bufsize`, the
* required buffer size is returned, while the buffer will be overwritten with
* undefined data.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
);
/**
* The same as utf8proc_decompose(), but also takes a `custom_func` mapping function
* that is called on each codepoint in `str` before any other transformations
* (along with a `custom_data` pointer that is passed through to `custom_func`).
* The `custom_func` argument is ignored if it is `NULL`. See also utf8proc_map_custom().
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
);
/**
* Normalizes the sequence of `length` codepoints pointed to by `buffer`
* in-place (i.e., the result is also stored in `buffer`).
*
* @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
* @param length the length (in codepoints) of the buffer.
* @param options a bitwise or (`|`) of one or more of the following flags:
* - @ref UTF8PROC_NLF2LS - convert LF, CRLF, CR and NEL into LS
* - @ref UTF8PROC_NLF2PS - convert LF, CRLF, CR and NEL into PS
* - @ref UTF8PROC_NLF2LF - convert LF, CRLF, CR and NEL into LF
* - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
* - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
* codepoints
* - @ref UTF8PROC_STABLE - prohibit combining characters that would violate
* the unicode versioning stability
*
* @return
* In case of success, the length (in codepoints) of the normalized UTF-32 string is
* returned; otherwise, a negative error code is returned (utf8proc_errmsg()).
*
* @warning The entries of the array pointed to by `str` have to be in the
* range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
/**
* Reencodes the sequence of `length` codepoints pointed to by `buffer`
* UTF-8 data in-place (i.e., the result is also stored in `buffer`).
* Can optionally normalize the UTF-32 sequence prior to UTF-8 conversion.
*
* @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
* @param length the length (in codepoints) of the buffer.
* @param options a bitwise or (`|`) of one or more of the following flags:
* - @ref UTF8PROC_NLF2LS - convert LF, CRLF, CR and NEL into LS
* - @ref UTF8PROC_NLF2PS - convert LF, CRLF, CR and NEL into PS
* - @ref UTF8PROC_NLF2LF - convert LF, CRLF, CR and NEL into LF
* - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
* - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
* codepoints
* - @ref UTF8PROC_STABLE - prohibit combining characters that would violate
* the unicode versioning stability
* - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
*
* @return
* In case of success, the length (in bytes) of the resulting nul-terminated
* UTF-8 string is returned; otherwise, a negative error code is returned
* (utf8proc_errmsg()).
*
* @warning The amount of free space pointed to by `buffer` must
* exceed the amount of the input data by one byte, and the
* entries of the array pointed to by `str` have to be in the
* range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
/**
* Given a pair of consecutive codepoints, return whether a grapheme break is
* permitted between them (as defined by the extended grapheme clusters in UAX#29).
*
* @param codepoint1 The first codepoint.
* @param codepoint2 The second codepoint, occurring consecutively after `codepoint1`.
* @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires
* state to break graphemes. This state can be passed in as a pointer
* in the `state` argument and should initially be set to 0. If the
* state is not passed in (i.e. a null pointer is passed), UAX#29 rules
* GB10/12/13 which require this state will not be applied, essentially
* matching the rules in Unicode 8.0.0.
*
* @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must
* be called IN ORDER on ALL potential breaks in a string. However, it
* is safe to reset the state to zero after a grapheme break.
*/
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
/**
* Same as utf8proc_grapheme_break_stateful(), except without support for the
* Unicode 9 additions to the algorithm. Supported for legacy reasons.
*/
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* lower-case character, if any; otherwise (if there is no lower-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* upper-case character, if any; otherwise (if there is no upper-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* title-case character, if any; otherwise (if there is no title-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return `1` if the codepoint corresponds to a lower-case character
* and `0` otherwise.
*/
UTF8PROC_DLLEXPORT int utf8proc_islower(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return `1` if the codepoint corresponds to an upper-case character
* and `0` otherwise.
*/
UTF8PROC_DLLEXPORT int utf8proc_isupper(utf8proc_int32_t c);
/**
* Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
* except that a width of 0 is returned for non-printable codepoints
* instead of -1 as in `wcwidth`.
*
* @note
* If you want to check for particular types of non-printable characters,
* (analogous to `isprint` or `iscntrl`), use utf8proc_category(). */
UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
/**
* Given a codepoint, return whether it has East Asian width class A (Ambiguous)
*
* Codepoints with this property are considered to have charwidth 1 (if they are printable)
* but some East Asian fonts render them as double width.
*/
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_charwidth_ambiguous(utf8proc_int32_t codepoint);
/**
* Return the Unicode category for the codepoint (one of the
* @ref utf8proc_category_t constants.)
*/
UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
/**
* Return the two-letter (nul-terminated) Unicode category string for
* the codepoint (e.g. `"Lu"` or `"Co"`).
*/
UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoint);
/**
* Maps the given UTF-8 string pointed to by `str` to a new UTF-8
* string, allocated dynamically by `malloc` and returned via `dstptr`.
*
* If the @ref UTF8PROC_NULLTERM flag in the `options` field is set,
* the length is determined by a NULL terminator, otherwise the
* parameter `strlen` is evaluated to determine the string length, but
* in any case the result will be NULL terminated (though it might
* contain NULL characters with the string if `str` contained NULL
* characters). Other flags in the `options` field are passed to the
* functions defined above, and regarded as described. See also
* utf8proc_map_custom() to supply a custom codepoint transformation.
*
* In case of success the length of the new string is returned,
* otherwise a negative error code is returned.
*
* @note The memory of the new UTF-8 string will have been allocated
* with `malloc`, and should therefore be deallocated with `free`.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
);
/**
* Like utf8proc_map(), but also takes a `custom_func` mapping function
* that is called on each codepoint in `str` before any other transformations
* (along with a `custom_data` pointer that is passed through to `custom_func`).
* The `custom_func` argument is ignored if it is `NULL`.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
);
/** @name Unicode normalization
*
* Returns a pointer to newly allocated memory of a NFD, NFC, NFKD, NFKC or
* NFKC_Casefold normalized version of the null-terminated string `str`. These
* are shortcuts to calling utf8proc_map() with @ref UTF8PROC_NULLTERM
* combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
*/
/** @{ */
/** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str);
/** NFC normalization (@ref UTF8PROC_COMPOSE). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str);
/** NFKD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str);
/** NFKC normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str);
/**
* NFKC_Casefold normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT
* and @ref UTF8PROC_CASEFOLD and @ref UTF8PROC_IGNORE).
**/
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC_Casefold(const utf8proc_uint8_t *str);
/** @} */
#ifdef __cplusplus
}
#endif
#endif

17106
src/vendor/utf8proc_data.c vendored Normal file
View File

File diff suppressed because it is too large Load Diff

148
src/vendor/vec.c vendored Normal file
View File

@@ -0,0 +1,148 @@
/*
BSD 3-Clause License
Copyright (c) 2024, Mashpoe
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "vec.h"
#include <string.h>
typedef struct
{
vec_size_t size;
vec_size_t capacity;
unsigned char data[];
} vector_header;
vector_header* vector_get_header(vector vec) { return &((vector_header*)vec)[-1]; }
vector vector_create(void)
{
vector_header* h = (vector_header*)malloc(sizeof(vector_header));
h->capacity = 0;
h->size = 0;
return &h->data;
}
void vector_free(vector vec) { free(vector_get_header(vec)); }
vec_size_t vector_size(vector vec) { return vector_get_header(vec)->size; }
vec_size_t vector_capacity(vector vec) { return vector_get_header(vec)->capacity; }
vector_header* vector_realloc(vector_header* h, vec_type_t type_size)
{
vec_size_t new_capacity = (h->capacity == 0) ? 1 : h->capacity * 2;
vector_header* new_h = (vector_header*)realloc(h, sizeof(vector_header) + new_capacity * type_size);
new_h->capacity = new_capacity;
return new_h;
}
bool vector_has_space(vector_header* h)
{
return h->capacity - h->size > 0;
}
void* _vector_add_dst(vector* vec_addr, vec_type_t type_size)
{
vector_header* h = vector_get_header(*vec_addr);
if (!vector_has_space(h))
{
h = vector_realloc(h, type_size);
*vec_addr = h->data;
}
return &h->data[type_size * h->size++];
}
void* _vector_insert_dst(vector* vec_addr, vec_type_t type_size, vec_size_t pos)
{
vector_header* h = vector_get_header(*vec_addr);
vec_size_t new_length = h->size + 1;
// make sure there is enough room for the new element
if (!vector_has_space(h))
{
h = vector_realloc(h, type_size);
*vec_addr = h->data;
}
// move trailing elements
memmove(&h->data[(pos + 1) * type_size],
&h->data[pos * type_size],
(h->size - pos) * type_size);
h->size = new_length;
return &h->data[pos * type_size];
}
void _vector_erase(vector vec, vec_type_t type_size, vec_size_t pos, vec_size_t len)
{
vector_header* h = vector_get_header(vec);
memmove(&h->data[pos * type_size],
&h->data[(pos + len) * type_size],
(h->size - pos - len) * type_size);
h->size -= len;
}
void _vector_remove(vector vec, vec_type_t type_size, vec_size_t pos)
{
_vector_erase(vec, type_size, pos, 1);
}
void vector_pop(vector vec) { --vector_get_header(vec)->size; }
void _vector_reserve(vector* vec_addr, vec_type_t type_size, vec_size_t capacity)
{
vector_header* h = vector_get_header(*vec_addr);
if (h->capacity >= capacity)
{
return;
}
h = (vector_header*)realloc(h, sizeof(vector_header) + capacity * type_size);
h->capacity = capacity;
*vec_addr = &h->data;
}
vector _vector_copy(vector vec, vec_type_t type_size)
{
vector_header* h = vector_get_header(vec);
size_t alloc_size = sizeof(vector_header) + h->size * type_size;
vector_header* copy_h = (vector_header*)malloc(alloc_size);
memcpy(copy_h, h, alloc_size);
copy_h->capacity = copy_h->size;
return &copy_h->data;
}

127
src/vendor/vec.h vendored Normal file
View File

@@ -0,0 +1,127 @@
/*
BSD 3-Clause License
Copyright (c) 2024, Mashpoe
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef vec_h
#define vec_h
#ifdef __cpp_decltype
#include <type_traits>
#define typeof(T) std::remove_reference<std::add_lvalue_reference<decltype(T)>::type>::type
#endif
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdlib.h>
// generic type for internal use
typedef void* vector;
// number of elements in a vector
typedef size_t vec_size_t;
// number of bytes for a type
typedef size_t vec_type_t;
// TODO: more rigorous check for typeof support with different compilers
#if _MSC_VER == 0 || __STDC_VERSION__ >= 202311L || defined __cpp_decltype
// shortcut defines
// vec_addr is a vector* (aka type**)
#define vector_add_dst(vec_addr)\
((typeof(*vec_addr))(\
_vector_add_dst((vector*)vec_addr, sizeof(**vec_addr))\
))
#define vector_insert_dst(vec_addr, pos)\
((typeof(*vec_addr))(\
_vector_insert_dst((vector*)vec_addr, sizeof(**vec_addr), pos)))
#define vector_add(vec_addr, value)\
(*vector_add_dst(vec_addr) = value)
#define vector_insert(vec_addr, pos, value)\
(*vector_insert_dst(vec_addr, pos) = value)
#else
#define vector_add_dst(vec_addr, type)\
((type*)_vector_add_dst((vector*)vec_addr, sizeof(type)))
#define vector_insert_dst(vec_addr, type, pos)\
((type*)_vector_insert_dst((vector*)vec_addr, sizeof(type), pos))
#define vector_add(vec_addr, type, value)\
(*vector_add_dst(vec_addr, type) = value)
#define vector_insert(vec_addr, type, pos, value)\
(*vector_insert_dst(vec_addr, type, pos) = value)
#endif
// vec is a vector (aka type*)
#define vector_erase(vec, pos, len)\
(_vector_erase((vector)vec, sizeof(*vec), pos, len))
#define vector_remove(vec, pos)\
(_vector_remove((vector)vec, sizeof(*vec), pos))
#define vector_reserve(vec_addr, capacity)\
(_vector_reserve((vector*)vec_addr, sizeof(**vec_addr), capacity))
#define vector_copy(vec)\
(_vector_copy((vector)vec, sizeof(*vec)))
vector vector_create(void);
void vector_free(vector vec);
void* _vector_add_dst(vector* vec_addr, vec_type_t type_size);
void* _vector_insert_dst(vector* vec_addr, vec_type_t type_size, vec_size_t pos);
void _vector_erase(vector vec_addr, vec_type_t type_size, vec_size_t pos, vec_size_t len);
void _vector_remove(vector vec_addr, vec_type_t type_size, vec_size_t pos);
void vector_pop(vector vec);
void _vector_reserve(vector* vec_addr, vec_type_t type_size, vec_size_t capacity);
vector _vector_copy(vector vec, vec_type_t type_size);
vec_size_t vector_size(vector vec);
vec_size_t vector_capacity(vector vec);
// closing bracket for extern "C"
#ifdef __cplusplus
}
#endif
#endif /* vec_h */