From: Lukáš Jiřiště Date: Tue, 30 Jun 2026 18:35:46 +0000 (+0200) Subject: Refactor tokenization into separate file X-Git-Url: https://git.ljiriste.work/?a=commitdiff_plain;ds=sidebyside;p=KyciFTA Refactor tokenization into separate file --- diff --git a/Makefile b/Makefile index 871fe26..e642adb 100644 --- a/Makefile +++ b/Makefile @@ -17,9 +17,10 @@ INCLUDE := $(addprefix -I, $(INCDIR)) SRCDIR := src/ -SOURCES := \ - main.c \ - grammar.c \ +SOURCES := \ + main.c \ + grammar.c \ + parse_tree.c \ SOURCES := $(addprefix $(SRCDIR), $(SOURCES)) diff --git a/inc/grammar.h b/inc/grammar.h index 4c5b74f..1a0ed0b 100644 --- a/inc/grammar.h +++ b/inc/grammar.h @@ -1,6 +1,7 @@ #ifndef GRAMMAR_H # define GRAMMAR_H +# include "libft.h" // These strings should be regarded as constant // They are generated by xxd hence this type is forced @@ -11,4 +12,81 @@ extern unsigned int grammar_len; extern unsigned char parsing_table[]; extern unsigned int parsing_table_len; +typedef enum e_state +{ + WORKING, + FAILURE, +} t_state; + +typedef double (*t_prob_model)(double); + +typedef struct s_atom +{ + char *name; + t_state state; + double fault_prob; + t_prob_model model; +} t_atom; + +typedef enum e_gate_type +{ + AND, + OR, + NOOK, +} t_gate_type; + +typedef struct s_gate +{ + char *name; + t_state state; + t_gate_type type; + t_vec input_objects; +} t_gate; + +typedef union u_object +{ + t_gate gate; + t_atom atom; + struct + { + char *name; + t_state state; + }; +} t_object; + +typedef enum e_single_char_terminal +{ + DOT, + COMMA, + TIMES, + PLUS, + RIGHT_PARA, + LEFT_PARA, + MINUS, + + single_char_terminal_count +} t_single_char_terminal; + +typedef enum e_keyword +{ + EXP = single_char_terminal_count, + FROM, + ATOM, + N_OUT_OF_K, + SYSTEM, + INSTANTIATE, + + keyword_count +} t_keyword; + +typedef enum e_other_terminals +{ + WORD = keyword_count, + INT, + + terminal_count +} t_other_terminals; + +t_parse_tree_node *parse(const char *file); + #endif //GRAMMAR_H diff --git a/src/main.c b/src/main.c index 2a55105..8b72e33 100644 --- a/src/main.c +++ b/src/main.c @@ -1,331 +1,10 @@ #include "grammar.h" #include "libft.h" -#include -#include -#include static const char *g_help = "You need to invoke this command this way:\n" "\tcommand time step_num input_file\n"; -typedef double (*t_prob_model)(double); - -typedef enum e_state -{ - WORKING, - FAILURE, -} t_state; - -typedef struct s_atom -{ - char *name; - t_state state; - double fault_prob; - t_prob_model model; -} t_atom; - -typedef enum e_gate_type -{ - AND, - OR, - NOOK, -} t_gate_type; - -typedef struct s_gate -{ - char *name; - t_state state; - t_gate_type type; - t_vec input_objects; -} t_gate; - -typedef union u_object -{ - t_gate gate; - t_atom atom; - struct - { - char *name; - t_state state; - }; -} t_object; - -typedef enum e_single_char_terminal -{ - DOT, - COMMA, - TIMES, - PLUS, - RIGHT_PARA, - LEFT_PARA, - MINUS, - - single_char_terminal_count -} t_single_char_terminal; - -typedef enum e_keyword -{ - EXP = single_char_terminal_count, - FROM, - ATOM, - N_OUT_OF_K, - SYSTEM, - INSTANTIATE, - - keyword_count -} t_keyword; - -typedef enum e_other_terminals -{ - WORD = keyword_count, - INT, - - terminal_count -} t_other_terminals; - - -const char *g_keyword_strings[keyword_count] = { - [MINUS] = "-", - [DOT] = ".", - [COMMA] = ",", - [TIMES] = "*", - [PLUS] = "+", - [RIGHT_PARA] = ")", - [LEFT_PARA] = "(", - [EXP] = "e", - [FROM] = "FROM", - [ATOM] = "ATOM", - [N_OUT_OF_K] = "NOOK", - [SYSTEM] = "SYSTEM", - [INSTANTIATE] = "INST", -}; - -const char *g_token_strings[terminal_count] = { - [EXP] = "EXP", - [DOT] = "DOT", - [FROM] = "FROM", - [ATOM] = "ATOM", - [COMMA] = "COMMA", - [N_OUT_OF_K] = "N_OUT_OF_K", - [TIMES] = "TIMES", - [PLUS] = "PLUS", - [RIGHT_PARA] = "RIGHT_PARA", - [LEFT_PARA] = "LEFT_PARA", - [SYSTEM] = "SYSTEM", - [INSTANTIATE] = "INSTANTIATE", - [WORD] = "WORD", - [INT] = "INT", - [MINUS] = "MINUS", -}; - -int is_single_char_terminal(char c) -{ - for (t_single_char_terminal t = 0; t < single_char_terminal_count; ++t) - { - if (c == g_keyword_strings[t][0]) - { - return (1); - } - } - return (0); -} - -int is_in_context(char c) -{ - return (c && !ft_isspace(c) && !is_single_char_terminal(c) && !ft_isdigit(c)); -} - -const t_keyword *get_keyword(const char *line, size_t *i) -{ - static t_keyword keyword = 0; - size_t j; - - j = *i; - if (is_single_char_terminal(line[j])) - { - ++j; - } - else - { - while (line[j] && !ft_isspace(line[j]) && !is_single_char_terminal(line[j])) - { - ++j; - } - } - j -= *i; - for (keyword = 0; keyword < keyword_count; ++keyword) - { - if (ft_strncmp(line + *i, g_keyword_strings[keyword], j) == 0) - { - *i += j; - return (&keyword); - } - } - return (NULL); -} - -t_ft_stat add_keyword_token(t_keyword keyword, t_vec *tokens) -{ - t_token new_token; - - new_token.type = ft_strdup(g_token_strings[keyword]); - if (!new_token.type) - { - return (alloc_fail); - } - new_token.str = NULL; - return (ft_vec_append(tokens, &new_token)); -} - -t_ft_stat add_int_token(const char *line, size_t *i, t_vec *tokens) -{ - t_token new_token; - size_t j; - - j = 0; - while (ft_isdigit(line[*i + j])) - { - ++j; - } - new_token.type = ft_strdup(g_token_strings[INT]); - if (!new_token.type) - { - return (alloc_fail); - } - new_token.str = ft_strndup(line + *i, j); - if (!new_token.str) - { - return (alloc_fail); - } - *i += j; - return (ft_vec_append(tokens, &new_token)); -} - -t_ft_stat add_word_token(const char *line, size_t *i, t_vec *tokens) -{ - t_token new_token; - size_t j; - - j = *i; - while (line[j] && !ft_isspace(line[j]) && !is_single_char_terminal(line[j])) - { - ++j; - } - j -= *i; - new_token.type = ft_strdup(g_token_strings[WORD]); - new_token.str = ft_strndup(line + *i, j); - if (!new_token.type || !new_token.str) - { - free(new_token.type); - free(new_token.str); - return (alloc_fail); - } - *i += j; - return (ft_vec_append(tokens, &new_token)); -} - -t_ft_stat tokenize_line(const char *line, t_vec *tokens) -{ - size_t i; - const t_keyword *keyword; - - i = 0; - while (line[i]) - { - if (ft_isspace(line[i])) - { - ++i; - continue ; - } - keyword = get_keyword(line, &i); - if (keyword) - { - if (add_keyword_token(*keyword, tokens)) - { - return (alloc_fail); - } - } - else if (ft_isdigit(line[i])) - { - if (add_int_token(line, &i, tokens)) - { - return (alloc_fail); - } - } - else if (add_word_token(line, &i, tokens)) - { - return (alloc_fail); - } - } - return (success); -} - -t_ft_stat tokenize(const char *file, t_vec *tokens) -{ - int fd; - t_ft_stat res; - char *line; - - fd = open(file, O_RDONLY); - if (fd < 0) - { - return (file_error); - } - line = get_next_line(fd); - while (line) - { - res = tokenize_line(line, tokens); - if (res) - { - free(line); - get_next_line(-1); - close(fd); - return (res); - } - free(line); - line = get_next_line(fd); - } - close(fd); - return (success); -} - -void free_token(void *v_token) -{ - t_token *token; - - token = v_token; - free(token->str); - free(token->type); - return ; -} - -t_parse_tree_node *parse(const char *file) -{ - t_parsing_table table; - t_vec tokens; - t_parse_tree_node *parse_tree; - - ft_parsing_table_init(&table); - if (ft_parsing_table_load_str(&table, (char *) parsing_table, (char *) grammar)) - { - return (NULL); - } - if (ft_vec_init(&tokens, sizeof(t_token))) - { - ft_parsing_table_free(&table); - return (NULL); - } - if (tokenize(file, &tokens)) - { - ft_parsing_table_free(&table); - ft_vec_free(&tokens, free_token); - return (NULL); - } - parse_tree = ft_parse(&tokens, &table); - ft_parsing_table_free(&table); - ft_vec_free(&tokens, free_token); - return (parse_tree); -} int main(int argc, char **argv) { diff --git a/src/parse_tree.c b/src/parse_tree.c new file mode 100644 index 0000000..e69123c --- /dev/null +++ b/src/parse_tree.c @@ -0,0 +1,247 @@ +#include "grammar.h" +#include +#include +#include + +const char *g_keyword_strings[keyword_count] = { + [MINUS] = "-", + [DOT] = ".", + [COMMA] = ",", + [TIMES] = "*", + [PLUS] = "+", + [RIGHT_PARA] = ")", + [LEFT_PARA] = "(", + [EXP] = "e", + [FROM] = "FROM", + [ATOM] = "ATOM", + [N_OUT_OF_K] = "NOOK", + [SYSTEM] = "SYSTEM", + [INSTANTIATE] = "INST", +}; + +const char *g_token_strings[terminal_count] = { + [EXP] = "EXP", + [DOT] = "DOT", + [FROM] = "FROM", + [ATOM] = "ATOM", + [COMMA] = "COMMA", + [N_OUT_OF_K] = "N_OUT_OF_K", + [TIMES] = "TIMES", + [PLUS] = "PLUS", + [RIGHT_PARA] = "RIGHT_PARA", + [LEFT_PARA] = "LEFT_PARA", + [SYSTEM] = "SYSTEM", + [INSTANTIATE] = "INSTANTIATE", + [WORD] = "WORD", + [INT] = "INT", + [MINUS] = "MINUS", +}; + +static int is_single_char_terminal(char c) +{ + for (t_single_char_terminal t = 0; t < single_char_terminal_count; ++t) + { + if (c == g_keyword_strings[t][0]) + { + return (1); + } + } + return (0); +} + +static int is_in_context(char c) +{ + return (c && !ft_isspace(c) && !is_single_char_terminal(c)); +} + +static const t_keyword *get_keyword(const char *line, size_t *i) +{ + static t_keyword keyword = 0; + size_t j; + + j = *i; + if (is_single_char_terminal(line[j])) + { + ++j; + } + else + { + while (is_in_context(line[j])) + { + ++j; + } + } + j -= *i; + for (keyword = 0; keyword < keyword_count; ++keyword) + { + if (ft_strncmp(line + *i, g_keyword_strings[keyword], j) == 0) + { + *i += j; + return (&keyword); + } + } + return (NULL); +} + +static t_ft_stat add_keyword_token(t_keyword keyword, t_vec *tokens) +{ + t_token new_token; + + new_token.type = ft_strdup(g_token_strings[keyword]); + if (!new_token.type) + { + return (alloc_fail); + } + new_token.str = NULL; + return (ft_vec_append(tokens, &new_token)); +} + +static t_ft_stat add_int_token(const char *line, size_t *i, t_vec *tokens) +{ + t_token new_token; + size_t j; + + j = 0; + while (ft_isdigit(line[*i + j])) + { + ++j; + } + new_token.type = ft_strdup(g_token_strings[INT]); + if (!new_token.type) + { + return (alloc_fail); + } + new_token.str = ft_strndup(line + *i, j); + if (!new_token.str) + { + return (alloc_fail); + } + *i += j; + return (ft_vec_append(tokens, &new_token)); +} + +static t_ft_stat add_word_token(const char *line, size_t *i, t_vec *tokens) +{ + t_token new_token; + size_t j; + + j = *i; + while (is_in_context(line[j])) + { + ++j; + } + j -= *i; + new_token.type = ft_strdup(g_token_strings[WORD]); + new_token.str = ft_strndup(line + *i, j); + if (!new_token.type || !new_token.str) + { + free(new_token.type); + free(new_token.str); + return (alloc_fail); + } + *i += j; + return (ft_vec_append(tokens, &new_token)); +} + +static t_ft_stat tokenize_line(const char *line, t_vec *tokens) +{ + size_t i; + const t_keyword *keyword; + + i = 0; + while (line[i]) + { + if (ft_isspace(line[i])) + { + ++i; + continue ; + } + keyword = get_keyword(line, &i); + if (keyword) + { + if (add_keyword_token(*keyword, tokens)) + { + return (alloc_fail); + } + } + else if (ft_isdigit(line[i])) + { + if (add_int_token(line, &i, tokens)) + { + return (alloc_fail); + } + } + else if (add_word_token(line, &i, tokens)) + { + return (alloc_fail); + } + } + return (success); +} + +static t_ft_stat tokenize(const char *file, t_vec *tokens) +{ + int fd; + t_ft_stat res; + char *line; + + fd = open(file, O_RDONLY); + if (fd < 0) + { + return (file_error); + } + line = get_next_line(fd); + while (line) + { + res = tokenize_line(line, tokens); + if (res) + { + free(line); + get_next_line(-1); + close(fd); + return (res); + } + free(line); + line = get_next_line(fd); + } + close(fd); + return (success); +} + +static void free_token(void *v_token) +{ + t_token *token; + + token = v_token; + free(token->str); + free(token->type); + return ; +} + +t_parse_tree_node *parse(const char *file) +{ + t_parsing_table table; + t_vec tokens; + t_parse_tree_node *parse_tree; + + ft_parsing_table_init(&table); + if (ft_parsing_table_load_str(&table, (char *) parsing_table, (char *) grammar)) + { + return (NULL); + } + if (ft_vec_init(&tokens, sizeof(t_token))) + { + ft_parsing_table_free(&table); + return (NULL); + } + if (tokenize(file, &tokens)) + { + ft_parsing_table_free(&table); + ft_vec_free(&tokens, free_token); + return (NULL); + } + parse_tree = ft_parse(&tokens, &table); + ft_parsing_table_free(&table); + ft_vec_free(&tokens, free_token); + return (parse_tree); +}