commit 240f0babe1590f8e811293ec32bae534dce0bd7c Author: Philippe Gerum Date: Sun Dec 9 18:28:32 2018 +0100 evenless: nothing more Signed-off-by: Philippe Gerum diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..93e0a66 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +lib/git-stamp.h +include/uapi diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..964a694 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +include scripts/config.mk + +GOALS := all clean clobber mrproper install +TARGETS := include lib tests utils commands + +$(GOALS): + @for target in $(TARGETS); do \ + $(MAKE) -C $$target O=$(O_DIR)/$$target V=$(V) $@; \ + done + +$(TARGETS): + $(Q)$(MAKE) -C $@ O=$(O_DIR)/$@ V=$(V) + +lib: include + +tests utils: lib + +.PHONY: $(GOALS) $(TARGETS) diff --git a/README b/README new file mode 100644 index 0000000..7124c10 --- /dev/null +++ b/README @@ -0,0 +1,63 @@ + +Evenless (EVL) core library and utilities +========================================= + +What's that? +------------ + +Building +-------- + +Prerequisites: + +- a GCC toolchain with working thread-local storage support (TLS). + +- the UAPI headers from the target kernel fit with the EVL core. We + need access to the contents of include/uapi/asm/ and + include/uapi/evenless/ from the source Linux tree. + +Command: + +$ make [-C $SRCDIR] [ARCH=$cpu_arch] [CROSS_COMPILE=$toolchain] UAPI=$uapi_dir [OTHER_BUILD_VARS] [goal...] + +Make goals +~~~~~~~~~~ + +all generate binaries +clean remove build files +install copy generated binaries to $DESTDIR + +Main build variables +~~~~~~~~~~~~~~~~~~~~ + +$SRCDIR Path to this source tree +$cpu_arch CPU architecture you build for + (e.g. 'arm', 'arm64') +$toolchain Optional fixed part of the binutils filename + (e.g. 'aarch64-linux-gnu-', 'arm-linux-gnueabihf-') + +Other build variables +~~~~~~~~~~~~~~~~~~~~~ + +D={0|1} Disable|enable debug build, i.e. -g -O0 vs -O2 [0] +O=$output_dir Generate binary output files into $output_dir [.] +V={0|1} Set build verbosity level, 0 is terse [0] +DESTDIR=$install_dir Install library and binaries into $install_dir [/usr/evenless] + +Build command examples +~~~~~~~~~~~~~~~~~~~~~~ + +Say the library source code is located at ~/git/evenless, and the +kernel sources featuring the EVL core is located at +~/git/linux-evenless. You could build+install the EVL library and +utilities directly to a staging directory at +/nfsroot//usr/evenless as follows: + +$ mkdir /tmp/build-hikey && cd /tmp/build-hikey +$ make -C ~/git/evenless O=$PWD ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- UAPI=~/git/linux-evenless DESTDIR=/nfsroot/imx6q/usr/evenless install + +or, + +$ mkdir /tmp/build-hikey +$ cd ~/git/evenless +$ make O=/tmp/build-hikey ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- UAPI=~/git/linux-evenless DESTDIR=/nfsroot/hikey/usr/evenless install diff --git a/commands/Makefile b/commands/Makefile new file mode 100644 index 0000000..76d683f --- /dev/null +++ b/commands/Makefile @@ -0,0 +1,27 @@ +include ../scripts/config.mk + +CMDFILES := $(wildcard *.c) +BINARIES = $(CMDFILES:%.c=%) +TARGETS = $(CMDFILES:%.c=$(O_DIR)/%) +DEPFILES = $(CMDFILES:%.c=$(O_DIR)/%.d) + +CMD_CPPFLAGS := $(BASE_CPPFLAGS) -I. +CMD_CFLAGS := $(CMD_CPPFLAGS) $(BASE_CFLAGS) +override CFLAGS := $(CMD_CFLAGS) $(CFLAGS) + +all: $(TARGETS) + +install: all + $(Q)for bin in $(BINARIES); do \ + $(INSTALL) -D $(O_DIR)/$$bin $(DESTDIR)/$(bindir)/$$bin; \ + done + +clean clobber mrproper: + $(Q)$(RM) -f $(TARGETS) $(DEPFILES) + +$(O_DIR)/%: %.c + $(call ccld-cmd,$@,$(Q)$(CC) -o $(@) $< $(CFLAGS) $(LDFLAGS)) + +.PHONY: all install clean clobber mrproper + +-include $(DEPFILES) diff --git a/commands/evl-ps b/commands/evl-ps new file mode 100755 index 0000000..b66ab4d --- /dev/null +++ b/commands/evl-ps @@ -0,0 +1,26 @@ +#! /bin/sh +# SPDX-License-Identifier: MIT + +set -- $(getopt -n $(basename $0) el "$@") +if [ $? -ne 0 ]; then + echo >&2 "usage: $0 [-el]" + exit 1 +fi + +all=false +long=false + +for opt +do +case "$opt" in + -e) all=true + shift;; + -l) long=true + shift;; + --) shift; break;; + esac +done + +# TBD + +exit 0 diff --git a/commands/evl-trace b/commands/evl-trace new file mode 100755 index 0000000..c201345 --- /dev/null +++ b/commands/evl-trace @@ -0,0 +1,87 @@ +#! /bin/sh +# SPDX-License-Identifier: MIT + +if test \! -d $EVL_TRACEDIR; then + echo >&2 "no kernel support for tracing" + exit 2 +fi + +usage() { + echo >&2 "usage: $1 [-e [-s ]] [-d] [-p] [-f] [-?] [-c ]" +} + +set -- $(getopt -n $(basename $0) 'es:dc:pf?' "$@") +if [ $? -ne 0 ]; then + usage $0 + exit 1 +fi + +enable=false +disable=false +print=true +full=false +bufsz=64 +pcpu= +help=false + +for opt +do +case "$opt" in + -e) enable=true + disable=false + shift;; + -s) bufsz=$(eval echo $2) + shift;; + -d) enable=false + disable=true + shift;; + -p) print=true + shift;; + -f) full=true + shift;; + -c) pcpu=per_cpu/cpu$(eval echo $2)/ + shift;; + -?) help=true + shift;; + --) shift; break;; + esac +done + +if test x$help = xtrue; then + usage $0 + exit 0 +fi + +cd $EVL_TRACEDIR + +if test x$enable = xtrue; then + echo nop > current_tracer + echo 0 > snapshot + echo $bufsz > ${pcpu}buffer_size_kb + if test x$full = xfalse; then + echo 1 > events/irq/enable + echo 1 > events/power/cpu_idle/enable + echo 1 > events/evenless/enable + else + echo function > current_tracer + fi + echo 1 > ${pcpu}snapshot + echo 1 > events/evenless/evl_timer_shot/enable + echo \!snapshot > events/evenless/evl_trigger/trigger + echo snapshot > events/evenless/evl_trigger/trigger + echo 1 > events/evenless/evl_trigger/enable + echo 1 > events/evenless/evl_latspot/enable + echo "tracing enabled" + print=false +elif test x$disable = xtrue; then + echo nop > current_tracer + echo 0 > snapshot + echo "tracing disabled" + print=false +fi + +if test x$print = xtrue; then + cat ${pcpu}snapshot +fi + +exit 0 diff --git a/commands/evl.c b/commands/evl.c new file mode 100644 index 0000000..697a2fc --- /dev/null +++ b/commands/evl.c @@ -0,0 +1,132 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2019 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static char *find_command_dir(const char *arg0) +{ + char *cmddir; + int ret; + + cmddir = malloc(PATH_MAX); + if (cmddir == NULL) + error(1, ENOMEM, "malloc"); + + ret = readlink("/proc/self/exe", cmddir, PATH_MAX); + if (ret < 0) + error(1, errno, "readlink"); + + return dirname(cmddir); +} + +static void usage(void) +{ + fprintf(stderr, "usage: evl [options] []\n"); + fprintf(stderr, "-P --prefix= set command path prefix\n"); +} + +#define short_optlist "P:" + +static const struct option options[] = { + { + .name = "prefix", + .has_arg = required_argument, + .val = 'P' + }, + { /* Sentinel */ } +}; + +static int is_word(const char *s) +{ + while (*s && isalpha(*s)) + s++; + + return *s == '\0'; +} + +int main(int argc, char *const argv[]) +{ + char *cmddir = NULL, *cmdpath, **cmdargv, *cmd; + int ret, c, n, wcount, wpos = -1, cmdargc; + const char *arg0 = argv[0]; + + if (argc < 2) { + wpos = 0; + cmd = strdup("help"); + goto run; + } + + for (n = 1, wcount = 0; n < argc; n++) { + if (is_word(argv[n])) { + wpos = n; + if (++wcount > 1) + break; + } else + wcount = 0; + } + + if (wpos < 0) { + fprintf(stderr, "evl: no command word\n"); + usage(); + return 2; + } + + cmd = strdup(argv[wpos]); + *argv[wpos] = '\0'; + + opterr = 0; + + for (;;) { + c = getopt_long(wpos, argv, short_optlist, options, NULL); + if (c == EOF) + break; + + switch (c) { + case 0: + break; + case 'P': + cmddir = optarg; + break; + case '?': + default: + usage(); + return 2; + } + } +run: + if (cmddir == NULL) + cmddir = find_command_dir(arg0); + + ret = asprintf(&cmdpath, "%s/evl-%s", cmddir, cmd); + if (ret < 0) + error(1, ENOMEM, arg0); + + setenv("EVL_CMDDIR", cmddir, 1); + setenv("EVL_SYSDIR", "/sys/devices/virtual", 1); + setenv("EVL_TRACEDIR", "/sys/kernel/debug/tracing", 1); + + cmdargv = malloc(sizeof(char *) * (argc - wpos + 1)); + if (cmdargv == NULL) + error(1, ENOMEM, "malloc"); + cmdargv[0] = basename(cmdpath); + for (cmdargc = 1, n = wpos + 1; n < argc; n++) + cmdargv[cmdargc++] = argv[n]; + cmdargv[cmdargc] = NULL; + + execv(cmdpath, cmdargv); + fprintf(stderr, "evl: undefined command '%s'\n", cmd); + + return 1; +} diff --git a/include/Makefile b/include/Makefile new file mode 100644 index 0000000..4efb77e --- /dev/null +++ b/include/Makefile @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: MIT + +include ../scripts/config.mk + +O_UAPI = $(O_DIR)/uapi + +TARGETS := uapi + +uapi: $(O_DIR)/.uapi_stamp $(O_DIR)/uapi + +$(O_DIR)/.uapi_stamp $(O_DIR)/uapi: + $(Q)$(MKDIR_P) $(O_UAPI) + $(Q)$(RM) -f $(O_UAPI)/asm $(O_UAPI)/evenless + $(Q)if test -r $(UAPI)/Makefile; then \ + $(LN_S) $(UAPI)/arch/$(ARCH)/include/uapi/asm $(O_UAPI)/asm; \ + $(LN_S) $(UAPI)/include/uapi/evenless $(O_UAPI)/evenless; \ + else \ + $(LN_S) $(UAPI)/asm $(O_UAPI)/asm; \ + $(LN_S) $(UAPI)/evenless $(O_UAPI)/evenless; \ + fi; \ + touch $@ + +all: uapi + +clean clobber mrproper: + $(Q)$(RM) -f $(O_DIR)/.uapi_stamp $(O_UAPI)/asm $(O_UAPI)/evenless + +install: all + $(Q)$(MKDIR_P) $(DESTDIR)/$(includedir)/uapi + $(Q)(cd $(O_UAPI) && find -L evenless \! \( -name '*~' \) -type f | $(CPIO) -Lpdum $(DESTDIR)/$(includedir)/uapi) + $(Q)(find evenless \! \( -name '*~' \) -type f | $(CPIO) -Lpdum $(DESTDIR)/$(includedir)) + +.PHONY: clean clobber mrproper install diff --git a/include/evenless/atomic.h b/include/evenless/atomic.h new file mode 100644 index 0000000..7b47e9d --- /dev/null +++ b/include/evenless/atomic.h @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _EVENLESS_ATOMIC_H +#define _EVENLESS_ATOMIC_H + +#include + +typedef struct { __u32 val; } atomic_t; + +#define ATOMIC_INIT(__val) { (__val) } + +static inline int atomic_read(const atomic_t *ptr) +{ + return ptr->val; +} + +static inline void atomic_set(atomic_t *ptr, long val) +{ + ptr->val = val; +} + +#ifndef atomic_cmpxchg +#define atomic_cmpxchg(__ptr, __old, __new) \ + __sync_val_compare_and_swap(&(__ptr)->val, __old, __new) +#endif + +#ifndef smp_mb +#define smp_mb() __sync_synchronize() +#endif + +#define compiler_barrier() __asm__ __volatile__("": : :"memory") + +#endif /* _EVENLESS_ATOMIC_H */ diff --git a/include/evenless/clock.h b/include/evenless/clock.h new file mode 100644 index 0000000..8087086 --- /dev/null +++ b/include/evenless/clock.h @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _EVENLESS_CLOCK_H +#define _EVENLESS_CLOCK_H + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +static inline +int evl_read_clock(int efd, struct timespec *tp) +{ + extern int (*arch_clock_gettime)(clockid_t clk_id, + struct timespec *tp); + switch (efd) { + case -CLOCK_MONOTONIC: + case -CLOCK_REALTIME: + return arch_clock_gettime(-efd, tp) ? -errno : 0; + default: + return oob_ioctl(efd, EVL_CLKIOC_GET_TIME, tp) ? -errno : 0; + } +} + +int evl_set_clock(int efd, const struct timespec *tp); + +int evl_get_clock_resolution(int efd, struct timespec *tp); + +int evl_adjust_clock(int efd, struct timex *tx); + +int evl_delay(int efd, const struct timespec *timeout, + struct timespec *remain); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENLESS_CLOCK_H */ diff --git a/include/evenless/init.h b/include/evenless/init.h new file mode 100644 index 0000000..a82d3d4 --- /dev/null +++ b/include/evenless/init.h @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _EVENLESS_INIT_H +#define _EVENLESS_INIT_H + +#include + +#define __EVL__ 0 /* API revision */ + +#ifdef __cplusplus +extern "C" { +#endif + +int evl_init(void); + +int evl_sigshadow_handler(int sig, siginfo_t *si, void *ctxt); + +void evl_sigdebug_handler(int sig, siginfo_t *si, void *ctxt); + +unsigned int evl_detect_fpu(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENLESS_INIT_H */ diff --git a/include/evenless/logger.h b/include/evenless/logger.h new file mode 100644 index 0000000..cc49d60 --- /dev/null +++ b/include/evenless/logger.h @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _EVENLESS_LOGGER_H +#define _EVENLESS_LOGGER_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int evl_new_logger(int dstfd, size_t logsz, + const char *fmt, ...); + +ssize_t evl_write_logger(int efd, + const void *buf, size_t len); + +int evl_printf_logger(int efd, + const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENLESS_LOGGER_H */ diff --git a/include/evenless/mapper.h b/include/evenless/mapper.h new file mode 100644 index 0000000..7f45a91 --- /dev/null +++ b/include/evenless/mapper.h @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _EVENLESS_MAPPER_H +#define _EVENLESS_MAPPER_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int evl_new_mapper(int mapfd, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENLESS_MAPPER_H */ diff --git a/include/evenless/monitor.h b/include/evenless/monitor.h new file mode 100644 index 0000000..10243dd --- /dev/null +++ b/include/evenless/monitor.h @@ -0,0 +1,102 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _EVENLESS_MONITOR_H +#define _EVENLESS_MONITOR_H + +#include +#include +#include +#include +#include +#include + +struct evl_thread; + +struct evl_monitor { + unsigned int magic; + union { + struct { + fundle_t fundle; + struct evl_monitor_state *state; + int efd; + int type; + } active; + struct { + const char *name; + int clockfd; + unsigned int ceiling; + int type; + } uninit; + }; +}; + +#define __MONITOR_ACTIVE_MAGIC 0xab12ab12 +#define __MONITOR_UNINIT_MAGIC 0xfe11fe11 +#define __MONITOR_DEAD_MAGIC 0 + +#define EVL_GATE_INITIALIZER(__name, __clock, __ceiling) { \ + .magic = __MONITOR_UNINIT_MAGIC, \ + .uninit = { \ + .type = (__ceiling) ? EVL_MONITOR_PP : EVL_MONITOR_PI, \ + .name = (__name), \ + .clockfd = (__clockfd), \ + .ceiling = (__ceiling), \ + } \ + } + +#define EVL_EVENT_INITIALIZER(__name, __clockfd) { \ + .magic = __MONITOR_UNINIT_MAGIC, \ + .uninit = { \ + .type = EVL_MONITOR_EV, \ + .name = (__name), \ + .clockfd = (__clockfd), \ + } \ + } + +#ifdef __cplusplus +extern "C" { +#endif + +int evl_new_gate(struct evl_monitor *gate, + int type, int clockfd, + unsigned int ceiling, + const char *fmt, ...); + +int evl_new_event(struct evl_monitor *event, + int clockfd, + const char *fmt, ...); + +int evl_open_monitor(struct evl_monitor *mon, + const char *fmt, ...); + +int evl_release_monitor(struct evl_monitor *mon); + +int evl_enter_gate(struct evl_monitor *gate); + +int evl_exit_gate(struct evl_monitor *gate); + +int evl_set_gate_ceiling(struct evl_monitor *gate, + unsigned int ceiling); + +int evl_get_gate_ceiling(struct evl_monitor *gate); + +int evl_wait_event(struct evl_monitor *event, + struct evl_monitor *gate, + const struct timespec *timeout); + +int evl_signal_event(struct evl_monitor *event); + +int evl_signal_event_targeted(struct evl_monitor *event, + int thrfd); + +int evl_broadcast_event(struct evl_monitor *event); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENLESS_MONITOR_H */ diff --git a/include/evenless/sem.h b/include/evenless/sem.h new file mode 100644 index 0000000..5eb2048 --- /dev/null +++ b/include/evenless/sem.h @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _EVENLESS_SEM_H +#define _EVENLESS_SEM_H + +#include +#include +#include +#include +#include + +struct evl_thread; + +struct evl_sem { + unsigned int magic; + union { + struct { + fundle_t fundle; + struct evl_sem_state *state; + int efd; + } active; + struct { + const char *name; + int clockfd; + int flags; + int initval; + } uninit; + }; +}; + +#define __SEM_ACTIVE_MAGIC 0xcb13cb13 +#define __SEM_UNINIT_MAGIC 0xed15ed15 +#define __SEM_DEAD_MAGIC 0 + +#define EVL_SEM_INITIALIZER(__name, __clockfd, __flags, __initval) { \ + .magic = __SEM_UNINIT_MAGIC, \ + .uninit = { \ + .name = (__name), \ + .flags = (__flags), \ + .clockfd = (__clockfd), \ + .initval = (__initval), \ + } \ + } + +#ifdef __cplusplus +extern "C" { +#endif + +int evl_new_sem(struct evl_sem *sem, + int flags, int clockfd, int initval, + const char *fmt, ...); + +int evl_open_sem(struct evl_sem *sem, + const char *fmt, ...); + +int evl_release_sem(struct evl_sem *sem); + +int evl_get_sem(struct evl_sem *sem, + int count, + const struct timespec *timeout); + +int evl_put_sem(struct evl_sem *sem, int count); + +int evl_tryget_sem(struct evl_sem *sem, int count); + +int evl_broadcast_sem(struct evl_sem *sem); + +int evl_get_semval(struct evl_sem *sem); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENLESS_SEM_H */ diff --git a/include/evenless/syscall.h b/include/evenless/syscall.h new file mode 100644 index 0000000..c986466 --- /dev/null +++ b/include/evenless/syscall.h @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _EVENLESS_SYSCALL_H +#define _EVENLESS_SYSCALL_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +ssize_t oob_read(int efd, void *buf, size_t count); + +ssize_t oob_write(int efd, const void *buf, size_t count); + +int oob_ioctl(int efd, unsigned long request, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENLESS_SYSCALL_H */ diff --git a/include/evenless/thread.h b/include/evenless/thread.h new file mode 100644 index 0000000..a5058f7 --- /dev/null +++ b/include/evenless/thread.h @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _EVENLESS_THREAD_H +#define _EVENLESS_THREAD_H + +#include +#include +#include +#include +#include + +/* Enable dlopen() on libevenless.so. */ +#define EVL_TLS_MODEL "global-dynamic" + +#define EVL_STACK_DEFAULT \ + ({ \ + int __ret = PTHREAD_STACK_MIN; \ + if (__ret < 65536) \ + __ret = 65536; \ + __ret; \ + }) + +#ifdef __cplusplus +extern "C" { +#endif + +extern __thread __attribute__ ((tls_model (EVL_TLS_MODEL))) +struct evl_user_window *evl_current_window; + +static inline int evl_get_current_mode(void) +{ + return evl_current_window ? + evl_current_window->state : T_INBAND; +} + +static inline int evl_is_inband(void) +{ + return evl_get_current_mode() & T_INBAND; +} + +int evl_attach_self(const char *fmt, ...); + +int evl_detach_self(void); + +int evl_get_self(void); + +int evl_switch_oob(void); + +int evl_switch_inband(void); + +int evl_set_schedattr(int efd, const struct evl_sched_attrs *attrs); + +int evl_get_schedattr(int efd, struct evl_sched_attrs *attrs); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENLESS_THREAD_H */ diff --git a/include/evenless/xbuf.h b/include/evenless/xbuf.h new file mode 100644 index 0000000..f11abc4 --- /dev/null +++ b/include/evenless/xbuf.h @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _EVENLESS_XBUF_H +#define _EVENLESS_XBUF_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int evl_new_xbuf(size_t i_bufsz, size_t o_bufsz, + const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENLESS_XBUF_H */ diff --git a/lib/Makefile b/lib/Makefile new file mode 100644 index 0000000..07cd8a6 --- /dev/null +++ b/lib/Makefile @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: MIT + +include ../scripts/config.mk + +DTSONAME := libevenless.so.$(LIBIVERSION) +SOLIBNAME := libevenless-$(LIBSERIAL).so.$(LIBIVERSION) +ARLIBNAME := libevenless-$(LIBSERIAL).a +TARGETS := $(O_DIR)/$(SOLIBNAME) $(O_DIR)/$(ARLIBNAME) + +SRCFILES := $(wildcard *.c arch/$(ARCH)/*.c) +PIC_OBJFILES = $(SRCFILES:%.c=$(O_DIR)/%-pic.o) +OBJFILES = $(SRCFILES:%.c=$(O_DIR)/%.o) +DEPFILES = $(SRCFILES:%.c=$(O_DIR)/%.d) + +LIB_CPPFLAGS := $(BASE_CPPFLAGS) \ + -I. \ + -I../include \ + -Iarch/$(ARCH)/include \ + -I$(O_DIR)/../include \ + -I$(O_DIR) + +LIB_CFLAGS := $(LIB_CPPFLAGS) $(BASE_CFLAGS) + +override CFLAGS := $(LIB_CFLAGS) $(CFLAGS) + +override LDFLAGS := $(LDFLAGS) -lpthread -lrt + +all: $(TARGETS) + +install: all + $(INSTALL) -D $(O_DIR)/$(SOLIBNAME) $(DESTDIR)/$(libdir)/$(SOLIBNAME) + $(LN_S) $(SOLIBNAME) $(DESTDIR)/$(libdir)/$(DTSONAME) + $(INSTALL) -D $(O_DIR)/$(ARLIBNAME) $(DESTDIR)/$(libdir)/$(ARLIBNAME) + +clean clobber mrproper: + $(Q)$(RM) -f $(O_DIR)/git-stamp.h $(PIC_OBJFILES) $(OBJFILES) \ + $(TARGETS) $(O_DIR)/$(DTSONAME) $(DEPFILES) + +$(O_DIR)/$(SOLIBNAME): $(PIC_OBJFILES) + $(call ld-cmd,$@,$(CC) -shared -Wl$(comma)-soname$(comma)$(DTSONAME) -o $(@) \ + $(PIC_OBJFILES) $(LDFLAGS) -Wl$(comma)-rpath=$(libdir) -Wl$(comma)--export-dynamic) + $(Q)$(LN_S) $(SOLIBNAME) $(O_DIR)/$(DTSONAME) + +$(O_DIR)/$(ARLIBNAME): $(OBJFILES) + $(call ar-cmd,$@,$(AR) ru $@ $(OBJFILES)) + +$(O_DIR)/syscall-pic.o $(O_DIR)/syscall.o: override CFLAGS := $(CFLAGS) -fomit-frame-pointer + +$(O_DIR)/version-pic.o $(O_DIR)/version.o: override CFLAGS := $(CFLAGS) -DLIBSERIAL=\"$(LIBSERIAL)\" +version.c: $(O_DIR)/git-stamp.h + +$(O_DIR)/git-stamp.h: git-stamp + @if test -r ../.git; then \ + stamp=`git --git-dir=../.git rev-list --abbrev-commit -1 HEAD`; \ + if test \! -s $@ || grep -wvq $$stamp $@; then \ + date=`git --git-dir=../.git log -1 $$stamp --pretty=format:%ci`; \ + echo "#define GIT_STAMP \"#$$stamp ($$date)\"" > $@; \ + fi; \ + elif test \! -r $@ -o -s $@; then \ + $(RM) -f $@ && touch $@; \ + fi; true + +$(O_DIR)/%-pic.o: %.c + $(call cc-pic-cmd,$@,$(CC) $(CFLAGS) -fPIC -c -o $@ $<) + +$(O_DIR)/%.o: %.c + $(call cc-cmd,$@,$(CC) $(CFLAGS) -c -o $@ $<) + +.PHONY: all install clean clobber mrproper git-stamp + +-include $(DEPFILES) diff --git a/lib/arch/arm/include/asm/evenless/syscall.h b/lib/arch/arm/include/asm/evenless/syscall.h new file mode 100644 index 0000000..78b8503 --- /dev/null +++ b/lib/arch/arm/include/asm/evenless/syscall.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _LIB_EVENLESS_ARM_SYSCALL_H +#define _LIB_EVENLESS_ARM_SYSCALL_H + +#include + +#define evenless_syscall3(__nr, __a0, __a1, __a2) \ + ({ \ + register unsigned long __asc __asm__("r7"); \ + register unsigned long __res __asm__("r0"); \ + register unsigned long __sc __asm__("r0"); \ + register unsigned long __r1 __asm__("r1"); \ + register unsigned long __r2 __asm__("r2"); \ + register unsigned long __r3 __asm__("r3"); \ + __asc = __ARM_NR_dovetail; \ + __sc = (unsigned int)(__nr); \ + __r1 = (unsigned long)(__a0); \ + __r2 = (unsigned long)(__a1); \ + __r3 = (unsigned long)(__a2); \ + __asm__ __volatile__ ( \ + "swi #0;\n\t" \ + : "=r" (__res) \ + : "r" (__asc), "r" (__sc), \ + "r" (__r1), "r" (__r2), "r" (__r3) \ + : "cc", "memory"); \ + __res; \ + }) + +#endif /* !_LIB_EVENLESS_ARM_SYSCALL_H */ diff --git a/lib/arch/arm/init.c b/lib/arch/arm/init.c new file mode 100644 index 0000000..aaca0ce --- /dev/null +++ b/lib/arch/arm/init.c @@ -0,0 +1,19 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include "parse_vdso.h" +#include "internal.h" + +int (*arch_clock_gettime)(clockid_t clk_id, struct timespec *tp); + +int arch_evl_init(void) +{ + evl_init_vdso(); + arch_clock_gettime = evl_request_vdso("LINUX_2.6", + "__vdso_clock_gettime"); + + return 0; +} diff --git a/lib/arch/arm64/include/asm/evenless/syscall.h b/lib/arch/arm64/include/asm/evenless/syscall.h new file mode 100644 index 0000000..d033ddf --- /dev/null +++ b/lib/arch/arm64/include/asm/evenless/syscall.h @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _LIB_EVENLESS_ARM64_SYSCALL_H +#define _LIB_EVENLESS_ARM64_SYSCALL_H + +#include + +#define evenless_syscall3(__nr, __a0, __a1, __a2) \ + ({ \ + register unsigned int __sc __asm__("w8"); \ + register unsigned long __res __asm__("x0"); \ + register unsigned long __x0 __asm__("x0"); \ + register unsigned long __x1 __asm__("x1"); \ + register unsigned long __x2 __asm__("x2"); \ + __sc = (unsigned int)((__nr)|__EVENLESS_SYSCALL_BIT); \ + __x0 = (unsigned long)(__a0); \ + __x1 = (unsigned long)(__a1); \ + __x2 = (unsigned long)(__a2); \ + __asm__ __volatile__ ( \ + "svc 0;\n\t" \ + : "=r" (__res) \ + : "r" (__sc), \ + "r" (__x0), "r" (__x1), "r" (__x2) \ + : "cc", "memory"); \ + __res; \ + }) + +#endif /* !_LIB_EVENLESS_ARM64_SYSCALL_H */ diff --git a/lib/arch/arm64/init.c b/lib/arch/arm64/init.c new file mode 100644 index 0000000..cf62524 --- /dev/null +++ b/lib/arch/arm64/init.c @@ -0,0 +1,19 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include "parse_vdso.h" +#include "internal.h" + +int (*arch_clock_gettime)(clockid_t clk_id, struct timespec *tp); + +int arch_evl_init(void) +{ + evl_init_vdso(); + arch_clock_gettime = evl_request_vdso("LINUX_2.6.39", + "__kernel_clock_gettime"); + + return 0; +} diff --git a/lib/clock.c b/lib/clock.c new file mode 100644 index 0000000..6b46328 --- /dev/null +++ b/lib/clock.c @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define do_call(__efd, __args...) \ + ({ \ + int __ret; \ + if (evl_is_inband()) \ + __ret = ioctl(__efd, ##__args); \ + else \ + __ret = oob_ioctl(__efd, ##__args); \ + __ret ? -errno : 0; \ + }) + +int evl_set_clock(int efd, const struct timespec *tp) +{ + return do_call(efd, EVL_CLKIOC_SET_TIME, tp); +} + +int evl_get_clock_resolution(int efd, struct timespec *tp) +{ + return do_call(efd, EVL_CLKIOC_GET_RES, tp); +} + +int evl_adjust_clock(int efd, struct timex *tx) +{ + return do_call(efd, EVL_CLKIOC_ADJ_TIME, tx); +} + +int evl_delay(int efd, const struct timespec *timeout, + struct timespec *remain) +{ + struct evl_clock_delayreq req = { + .timeout = *timeout, + .remain = remain, + }; + + return oob_ioctl(efd, EVL_CLKIOC_DELAY, &req) ? -errno : 0; +} diff --git a/lib/init.c b/lib/init.c new file mode 100644 index 0000000..c1211e0 --- /dev/null +++ b/lib/init.c @@ -0,0 +1,238 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#define STDLOG_SIZE 32768 + +static pthread_once_t init_once = PTHREAD_ONCE_INIT; + +static int init_status; + +static struct sigaction orig_sigshadow, orig_sigdebug; + +static struct evl_core_info core_info; + +int evl_ctlfd = -1; + +void *evl_shared_memory = NULL; + +static void atfork_unmap_shmem(void) +{ + if (evl_shared_memory) { + munmap(evl_shared_memory, core_info.shm_size); + evl_shared_memory = NULL; + } + + if (evl_ctlfd >= 0) { + close(evl_ctlfd); + evl_ctlfd = -1; + } + + init_once = PTHREAD_ONCE_INIT; +} + +static inline int generic_init(void) +{ + int ctlfd, ret; + void *shmem; + + ctlfd = open("/dev/evenless/control", O_RDWR); + if (ctlfd < 0) + return -errno; + + ret = fcntl(ctlfd, F_GETFD, 0); + if (ret < 0) { + ret = -errno; + goto fail; + } + + ret = fcntl(ctlfd, F_SETFD, ret | O_CLOEXEC); + if (ret < 0) { + ret = -errno; + goto fail; + } + + ret = ioctl(ctlfd, EVL_CTLIOC_GET_COREINFO, &core_info); + if (ret) { + ret = -errno; + goto fail; + } + + shmem = mmap(NULL, core_info.shm_size, PROT_READ|PROT_WRITE, + MAP_SHARED, ctlfd, 0); + if (shmem == MAP_FAILED) { + ret = -errno; + goto fail; + } + + evl_ctlfd = ctlfd; + evl_shared_memory = shmem; + + return 0; +fail: + close(ctlfd); + + return ret; +} + +int evl_sigshadow_handler(int sig, siginfo_t *si, void *ctxt) +{ + if (si->si_code == SI_QUEUE && + sigshadow_action(si->si_int) == SIGSHADOW_ACTION_HOME) { + oob_ioctl(evl_ctlfd, EVL_CTLIOC_SWITCH_OOB); + return 1; + } + + return 0; +} + +#define raw_write_out(__msg) \ + do { \ + int __ret; \ + __ret = write(1, __msg , strlen(__msg)); \ + (void)__ret; \ + } while (0) + +static const char *sigdebug_msg[] = { + "undefined\n", + "switched inband (signal)\n", + "switched inband (syscall)\n", + "switched inband (fault)\n", + "switched inband while holding mutex\n", + "watchdog triggered\n", + "mutex lock/unlock imbalance\n", + "goes to sleep while holding a mutex\n", +}; + +void evl_sigdebug_handler(int sig, siginfo_t *si, void *ctxt) +{ + if (sigdebug_marked(si)) { + switch (sigdebug_cause(si)) { + case SIGDEBUG_UNDEFINED: + case SIGDEBUG_MIGRATE_SIGNAL: + case SIGDEBUG_MIGRATE_SYSCALL: + case SIGDEBUG_MIGRATE_FAULT: + case SIGDEBUG_MIGRATE_PRIOINV: + case SIGDEBUG_WATCHDOG: + case SIGDEBUG_MUTEX_IMBALANCE: + case SIGDEBUG_MUTEX_SLEEP: + raw_write_out(sigdebug_msg[sigdebug_cause(si)]); + break; + } + } + + sigaction(SIGDEBUG, &orig_sigdebug, NULL); + pthread_kill(pthread_self(), SIGDEBUG); +} + +static void sigshadow_handler(int sig, siginfo_t *si, void *ctxt) +{ + const struct sigaction *const sa = &orig_sigshadow; + sigset_t omask; + + if (evl_sigshadow_handler(sig, si, ctxt)) + return; + + if ((!(sa->sa_flags & SA_SIGINFO) && sa->sa_handler == NULL) || + ((sa->sa_flags & SA_SIGINFO) && sa->sa_sigaction == NULL)) + return; /* Not sent by the Evenless core */ + + pthread_sigmask(SIG_SETMASK, &sa->sa_mask, &omask); + + if (!(sa->sa_flags & SA_SIGINFO)) + sa->sa_handler(sig); + else + sa->sa_sigaction(sig, si, ctxt); + + pthread_sigmask(SIG_SETMASK, &omask, NULL); +} + +static void install_signal_handlers(void) +{ + sigset_t mask, omask; + struct sigaction sa; + + sigemptyset(&mask); + sigaddset(&mask, SIGSHADOW); + + sa.sa_flags = SA_SIGINFO | SA_RESTART; + sa.sa_sigaction = sigshadow_handler; + sigemptyset(&sa.sa_mask); + pthread_sigmask(SIG_BLOCK, &mask, &omask); + + sigaction(SIGSHADOW, &sa, &orig_sigshadow); + + if (!(orig_sigshadow.sa_flags & SA_NODEFER)) + sigaddset(&orig_sigshadow.sa_mask, SIGSHADOW); + + pthread_sigmask(SIG_SETMASK, &omask, NULL); + + sa.sa_sigaction = evl_sigdebug_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_SIGINFO; + sigaction(SIGDEBUG, &sa, &orig_sigdebug); +} + +static inline int do_init(void) +{ + int ret; + + ret = arch_evl_init(); + if (ret) + return ret; + + ret = mlockall(MCL_CURRENT | MCL_FUTURE); + if (ret) + return -errno; + + ret = generic_init(); + if (ret) + return ret; + + pthread_atfork(NULL, NULL, atfork_unmap_shmem); + + install_signal_handlers(); + + return 0; +} + +static void do_init_once(void) +{ + init_status = do_init(); +} + +int evl_init(void) +{ + pthread_once(&init_once, do_init_once); + + return init_status; +} + +unsigned int evl_detect_fpu(void) +{ + if (evl_init()) + return 0; + + return core_info.fpu_features; +} diff --git a/lib/internal.c b/lib/internal.c new file mode 100644 index 0000000..e47ef2f --- /dev/null +++ b/lib/internal.c @@ -0,0 +1,122 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* + * Creating a Evenless element is done by the following steps: + * + * 1. open the clone device of the proper element class. + * + * 2. issue ioctl(EVL_IOC_CLONE) to create a new element, passing + * an attribute structure. + * + * 3. open the new element device, which file descriptor is returned + * to the caller. + * + * Except for threads, closing the last file descriptor referring to + * an element causes its automatic deletion. + */ +int create_evl_element(const char *type, const char *name, + void *attrs, struct evl_element_ids *eids) +{ + struct evl_clone_req clone; + char *fdevname, *edevname; + int ffd, efd, ret; + + ret = asprintf(&fdevname, "/dev/evenless/%s/clone", type); + if (ret < 0) + return -ENOMEM; + + ffd = open(fdevname, O_RDWR); + if (ffd < 0) { + ret = -errno; + goto out_factory; + } + + clone.name = name; + clone.attrs = attrs; + ret = ioctl(ffd, EVL_IOC_CLONE, &clone); + if (ret) { + ret = -errno; + goto out_new; + } + + if (name) + ret = asprintf(&edevname, "/dev/evenless/%s/%s", + type, name); + else + ret = asprintf(&edevname, "/dev/evenless/%s/%d", + type, clone.eids.minor); + if (ret < 0) { + ret = -ENOMEM; + goto out_new; + } + + efd = open(edevname, O_RDWR); + if (efd < 0) { + ret = -errno; + goto out_element; + } + + ret = fcntl(efd, F_GETFD, 0); + if (ret < 0) { + ret = -errno; + goto out_element; + } + + ret = fcntl(efd, F_SETFD, ret | O_CLOEXEC); + if (ret) { + ret = -errno; + goto out_element; + } + + if (eids) + *eids = clone.eids; + + ret = efd; + +out_element: + free(edevname); +out_new: + close(ffd); +out_factory: + free(fdevname); + + return ret; +} + +int open_evl_element(const char *type, + const char *fmt, va_list ap) +{ + char *path, *name; + int efd, ret; + + ret = vasprintf(&name, fmt, ap); + if (ret < 0) + return -ENOMEM; + + ret = asprintf(&path, "/dev/evenless/%s/%s", type, name); + free(name); + if (ret < 0) + return -ENOMEM; + + efd = open(path, O_RDWR); + free(path); + if (efd < 0) + return -errno; + + return efd; +} diff --git a/lib/internal.h b/lib/internal.h new file mode 100644 index 0000000..93c97d7 --- /dev/null +++ b/lib/internal.h @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#ifndef _LIB_EVENLESS_INTERNAL_H +#define _LIB_EVENLESS_INTERNAL_H + +#include +#include +#include +#include + +/* + * Length of per-thread print formatting buffer used by + * evl_printf_logger(). + */ +#define EVL_PRINTBUF_SIZE 1024 + +extern __thread __attribute__ ((tls_model (EVL_TLS_MODEL))) +fundle_t evl_current; + +extern __thread __attribute__ ((tls_model (EVL_TLS_MODEL))) +int evl_efd; + +extern __thread __attribute__ ((tls_model (EVL_TLS_MODEL))) +char evl_logging_buf[EVL_PRINTBUF_SIZE]; + +static inline fundle_t evl_get_current(void) +{ + return evl_current; +} + +static inline struct evl_user_window * +evl_get_current_window(void) +{ + return evl_current ? evl_current_window : NULL; +} + +static inline int evl_should_warn(void) +{ + return (evl_get_current_mode() & (T_INBAND|T_WARN)) == T_WARN; +} + +struct evl_element_ids; + +int arch_evl_init(void); + +int create_evl_element(const char *type, + const char *name, + void *attrs, + struct evl_element_ids *eids); + +int open_evl_element(const char *type, + const char *path, + va_list ap); + +extern int (*arch_clock_gettime)(clockid_t clk_id, + struct timespec *tp); + +extern void *evl_shared_memory; + +extern int evl_ctlfd; + +#endif /* _LIB_EVENLESS_INTERNAL_H */ diff --git a/lib/logger.c b/lib/logger.c new file mode 100644 index 0000000..4f855b3 --- /dev/null +++ b/lib/logger.c @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +int evl_new_logger(int dstfd, size_t logsz, const char *fmt, ...) +{ + struct evl_logger_attrs attrs; + int ret, efd; + va_list ap; + char *name; + + va_start(ap, fmt); + ret = vasprintf(&name, fmt, ap); + va_end(ap); + if (ret < 0) + return -ENOMEM; + + attrs.fd = dstfd; + attrs.logsz = logsz; + efd = create_evl_element("logger", name, &attrs, NULL); + free(name); + + return efd; +} + +ssize_t evl_write_logger(int efd, const void *buf, size_t len) +{ + if (evl_is_inband()) + return write(efd, buf, len); + + return oob_write(efd, buf, len); +} + +int evl_printf_logger(int efd, const char *fmt, ...) +{ + ssize_t len = EVL_PRINTBUF_SIZE; + char *buf = evl_logging_buf; + va_list ap; + + va_start(ap, fmt); + len = vsnprintf(buf, len, fmt, ap); + va_end(ap); + + return evl_write_logger(efd, buf, len); +} diff --git a/lib/mapper.c b/lib/mapper.c new file mode 100644 index 0000000..109824d --- /dev/null +++ b/lib/mapper.c @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +int evl_new_mapper(int mapfd, const char *fmt, ...) +{ + struct evl_mapper_attrs attrs; + int ret, efd; + va_list ap; + char *name; + + va_start(ap, fmt); + ret = vasprintf(&name, fmt, ap); + va_end(ap); + if (ret < 0) + return -ENOMEM; + + attrs.fd = mapfd; + efd = create_evl_element("mapper", name, &attrs, NULL); + free(name); + + return efd; +} diff --git a/lib/monitor.c b/lib/monitor.c new file mode 100644 index 0000000..d9f419a --- /dev/null +++ b/lib/monitor.c @@ -0,0 +1,444 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static int do_monitor_init(struct evl_monitor *mon, + int type, int clockfd, unsigned int ceiling, + const char *fmt, va_list ap) +{ + struct evl_monitor_attrs attrs; + struct evl_element_ids eids; + int efd, ret; + char *name; + + if (evl_shared_memory == NULL) + return -EAGAIN; + + /* + * We align on the in-band SCHED_FIFO priority range. Although + * the co-kernel does not require this, a 1:1 mapping between + * in-band and co-kernel priorities is simpler to deal with + * for users with respect to inband <-> out-of-band mode + * switches. + */ + if (type == EVL_MONITOR_PP) { + ret = sched_get_priority_max(SCHED_FIFO); + if (ret < 0 || ceiling > ret) + return -EINVAL; + } + + ret = vasprintf(&name, fmt, ap); + if (ret < 0) + return -ENOMEM; + + attrs.type = type; + attrs.clockfd = clockfd; + attrs.ceiling = ceiling; + efd = create_evl_element("monitor", name, &attrs, &eids); + free(name); + if (efd < 0) + return efd; + + mon->active.state = evl_shared_memory + eids.state_offset; + mon->active.fundle = eids.fundle; + mon->active.type = type; + mon->active.efd = efd; + mon->magic = __MONITOR_ACTIVE_MAGIC; + + return 0; +} + +int evl_new_gate(struct evl_monitor *gate, + int type, int clockfd, unsigned int ceiling, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = do_monitor_init(gate, type, clockfd, ceiling, fmt, ap); + va_end(ap); + + return ret; +} + +int evl_new_event(struct evl_monitor *event, + int clockfd, const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = do_monitor_init(event, EVL_MONITOR_EV, clockfd, 0, fmt, ap); + va_end(ap); + + return ret; +} + +int evl_open_monitor(struct evl_monitor *mon, const char *fmt, ...) +{ + struct evl_monitor_binding bind; + int ret, efd; + va_list ap; + + va_start(ap, fmt); + efd = open_evl_element("monitor", fmt, ap); + va_end(ap); + if (efd < 0) + return efd; + + ret = ioctl(efd, EVL_MONIOC_BIND, &bind); + if (ret) + return -errno; + + mon->active.state = evl_shared_memory + bind.eids.state_offset; + mon->active.fundle = bind.eids.fundle; + mon->active.type = bind.type; + mon->active.efd = efd; + mon->magic = __MONITOR_ACTIVE_MAGIC; + + return 0; +} + +int evl_release_monitor(struct evl_monitor *mon) +{ + int ret; + + if (mon->magic != __MONITOR_ACTIVE_MAGIC) + return -EINVAL; + + ret = close(mon->active.efd); + if (ret) + return -errno; + + mon->active.fundle = EVL_NO_HANDLE; + mon->active.state = NULL; + mon->magic = __MONITOR_DEAD_MAGIC; + + return 0; +} + +int evl_enter_gate(struct evl_monitor *gate) +{ + struct evl_user_window *u_window; + struct evl_monitor_state *gst; + int mode, ret, cancel_type; + bool protect = false; + fundle_t current; + + current = evl_get_current(); + if (current == EVL_NO_HANDLE) + return -EPERM; + + if (gate->magic == __MONITOR_UNINIT_MAGIC && + gate->uninit.type != EVL_MONITOR_EV) { + ret = evl_new_gate(gate, + gate->uninit.type, + gate->uninit.clockfd, + gate->uninit.ceiling, + gate->uninit.name); + if (ret) + return ret; + } else if (gate->magic != __MONITOR_ACTIVE_MAGIC) + return -EINVAL; + + gst = gate->active.state; + + /* + * Threads running in-band and/or enabling some debug features + * must go through the slow syscall path. + */ + mode = evl_get_current_mode(); + if (!(mode & (T_INBAND|T_WEAK|T_DEBUG))) { + if (gate->active.type == EVL_MONITOR_PP) { + u_window = evl_get_current_window(); + /* + * Can't nest lazy ceiling requests, have to + * take the slow path when this happens. + */ + if (u_window->pp_pending != EVL_NO_HANDLE) + goto slow_path; + u_window->pp_pending = gate->active.fundle; + protect = true; + } + ret = evl_fast_lock_mutex(&gst->u.gate.owner, current); + if (ret == 0) { + gst->flags &= ~EVL_MONITOR_SIGNALED; + return 0; + } + } else { + slow_path: + ret = 0; + if (evl_is_mutex_owner(&gst->u.gate.owner, current)) + ret = -EBUSY; + } + + if (ret == -EBUSY) { + if (protect) + u_window->pp_pending = EVL_NO_HANDLE; + + return -EDEADLK; + } + + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &cancel_type); + + do + ret = oob_ioctl(gate->active.efd, EVL_MONIOC_ENTER); + while (ret && errno == EINTR); + + pthread_setcanceltype(cancel_type, NULL); + + return ret ? -errno : 0; +} + +int evl_exit_gate(struct evl_monitor *gate) +{ + struct evl_user_window *u_window; + struct evl_monitor_state *gst; + fundle_t current; + int ret, mode; + + if (gate->magic != __MONITOR_ACTIVE_MAGIC) + return -EINVAL; + + gst = gate->active.state; + current = evl_get_current(); + if (!evl_is_mutex_owner(&gst->u.gate.owner, current)) + return -EPERM; + + /* Do we have waiters on a signaled event we are gating? */ + if (gst->flags & EVL_MONITOR_SIGNALED) + goto slow_path; + + mode = evl_get_current_mode(); + if (mode & (T_WEAK|T_DEBUG)) + goto slow_path; + + if (evl_fast_unlock_mutex(&gst->u.gate.owner, current)) { + if (gate->active.type == EVL_MONITOR_PP) { + u_window = evl_get_current_window(); + u_window->pp_pending = EVL_NO_HANDLE; + } + return 0; + } + + /* + * If the fast release failed, somebody else must be waiting + * for entering the gate or PP was committed for the current + * thread. Need to ask the kernel for proper release. + */ +slow_path: + ret = oob_ioctl(gate->active.efd, EVL_MONIOC_EXIT); + + return ret ? -errno : 0; +} + +int evl_set_gate_ceiling(struct evl_monitor *gate, + unsigned int ceiling) +{ + int ret; + + if (ceiling == 0) + return -EINVAL; + + ret = sched_get_priority_max(SCHED_FIFO); + if (ret < 0 || ceiling > ret) + return -EINVAL; + + if (gate->magic == __MONITOR_UNINIT_MAGIC) { + if (gate->uninit.type != EVL_MONITOR_PP) + return -EINVAL; + gate->uninit.ceiling = ceiling; + return 0; + } + + if (gate->magic != __MONITOR_ACTIVE_MAGIC || + gate->active.type != EVL_MONITOR_PP) + return -EINVAL; + + gate->active.state->u.gate.ceiling = ceiling; + + return 0; +} + +int evl_get_gate_ceiling(struct evl_monitor *gate) +{ + if (gate->magic == __MONITOR_UNINIT_MAGIC) { + if (gate->uninit.type != EVL_MONITOR_PP) + return -EINVAL; + + return gate->uninit.ceiling; + } + + if (gate->magic != __MONITOR_ACTIVE_MAGIC || + gate->active.type != EVL_MONITOR_PP) + return -EINVAL; + + return gate->active.state->u.gate.ceiling; +} + +static int check_event_sanity(struct evl_monitor *event) +{ + int ret; + + if (event->magic == __MONITOR_UNINIT_MAGIC && + event->uninit.type == EVL_MONITOR_EV) { + ret = evl_new_event(event, + event->uninit.clockfd, + event->uninit.name); + if (ret) + return ret; + } else if (event->magic != __MONITOR_ACTIVE_MAGIC) + return -EINVAL; + + if (event->active.type != EVL_MONITOR_EV) + return -EINVAL; + + return 0; +} + +static struct evl_monitor_state * +get_gate_state(struct evl_monitor *event) +{ + struct evl_monitor_state *est = event->active.state; + + if (est->u.gate_offset == EVL_MONITOR_NOGATE) + return NULL; /* Nobody waits on @event */ + + return evl_shared_memory + est->u.gate_offset; +} + +struct unwait_data { + struct evl_monitor_unwaitreq ureq; + int efd; +}; + +static void unwait_monitor(void *data) +{ + struct unwait_data *unwait = data; + int ret; + + do + ret = oob_ioctl(unwait->efd, EVL_MONIOC_UNWAIT, + &unwait->ureq); + while (ret && errno == EINTR); +} + +int evl_wait_event(struct evl_monitor *event, + struct evl_monitor *gate, + const struct timespec *timeout) +{ + struct evl_monitor_waitreq req; + struct unwait_data unwait; + int ret, old_type; + + ret = check_event_sanity(event); + if (ret) + return ret; + + req.gatefd = gate->active.efd; + req.timeout = *timeout; + req.status = -EINVAL; + unwait.ureq.gatefd = gate->active.efd; + unwait.efd = event->active.efd; + + pthread_cleanup_push(unwait_monitor, &unwait); + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type); + ret = oob_ioctl(event->active.efd, EVL_MONIOC_WAIT, &req); + pthread_setcanceltype(old_type, NULL); + pthread_cleanup_pop(0); + + /* + * If oob_ioctl() failed with EINTR, we got forcibly unblocked + * while waiting for the event or trying to reacquire the gate + * lock afterwards, in which case the gate monitor was left + * unlocked. Issue UNWAIT to recover from that situation. + */ + if (ret && errno == EINTR) { + unwait_monitor(&unwait); + pthread_testcancel(); + } + + return ret ? -errno : req.status; +} + +int evl_signal_event(struct evl_monitor *event) +{ + struct evl_monitor_state *est, *gst; + int ret; + + ret = check_event_sanity(event); + if (ret) + return ret; + + gst = get_gate_state(event); + if (gst) { + gst->flags |= EVL_MONITOR_SIGNALED; + est = event->active.state; + est->flags |= EVL_MONITOR_SIGNALED; + } + + return 0; +} + +int evl_signal_event_targeted(struct evl_monitor *event, int thrfd) +{ + struct evl_monitor_state *gst; + __u32 efd; + int ret; + + ret = check_event_sanity(event); + if (ret) + return ret; + + gst = get_gate_state(event); + if (gst) { + gst->flags |= EVL_MONITOR_SIGNALED; + efd = event->active.efd; + } + + return oob_ioctl(thrfd, EVL_THRIOC_SIGNAL, &efd) ? -errno : 0; +} + +int evl_broadcast_event(struct evl_monitor *event) +{ + struct evl_monitor_state *est, *gst; + int ret; + + ret = check_event_sanity(event); + if (ret) + return ret; + + gst = get_gate_state(event); + if (gst) { + gst->flags |= EVL_MONITOR_SIGNALED; + est = event->active.state; + est->flags |= EVL_MONITOR_SIGNALED|EVL_MONITOR_BROADCAST; + } + + return 0; +} diff --git a/lib/parse_vdso.c b/lib/parse_vdso.c new file mode 100644 index 0000000..7a6c9aa --- /dev/null +++ b/lib/parse_vdso.c @@ -0,0 +1,285 @@ +/* + * parse_vdso.c: Linux reference vDSO parser + * Written by Andrew Lutomirski, 2011-2014. + * + * This code is meant to be linked in to various programs that run on Linux. + * As such, it is available with as few restrictions as possible. This file + * is licensed under the Creative Commons Zero License, version 1.0, + * available at http://creativecommons.org/publicdomain/zero/1.0/legalcode + * + * The vDSO is a regular ELF DSO that the kernel maps into user space when + * it starts a program. It works equally well in statically and dynamically + * linked binaries. + * + * This code is tested on x86. In principle it should work on any + * architecture that has a vDSO. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "parse_vdso.h" +#include "internal.h" + +/* + * To use this vDSO parser, first call one of the vdso_init_* functions. + * If you've already parsed auxv, then pass the value of AT_SYSINFO_EHDR + * to vdso_init_from_sysinfo_ehdr. Otherwise pass auxv to vdso_init_from_auxv. + * Then call lookup_vdso for each symbol you want. For example, to look up + * gettimeofday on x86_64, use: + * + * = lookup_vdso("LINUX_2.6", "gettimeofday"); + * or + * = lookup_vdso("LINUX_2.6", "__vdso_gettimeofday"); + * + * lookup_vdso will return 0 if the symbol doesn't exist or if the init function + * failed or was not called. lookup_vdso is a little slow, so its return value + * should be cached. + * + * lookup_vdso is threadsafe; the init functions are not. + */ + + +/* And here's the code. */ +#ifndef ELF_BITS +# if ULONG_MAX > 0xffffffffUL +# define ELF_BITS 64 +# else +# define ELF_BITS 32 +# endif +#endif + +#define ELF_BITS_XFORM2(bits, x) Elf##bits##_##x +#define ELF_BITS_XFORM(bits, x) ELF_BITS_XFORM2(bits, x) +#define ELF(x) ELF_BITS_XFORM(ELF_BITS, x) + +static struct vdso_info +{ + bool valid; + + /* Load information */ + uintptr_t load_addr; + uintptr_t load_offset; /* load_addr - recorded vaddr */ + + /* Symbol table */ + ELF(Sym) *symtab; + const char *symstrings; + ELF(Word) *bucket, *chain; + ELF(Word) nbucket, nchain; + + /* Version table */ + ELF(Versym) *versym; + ELF(Verdef) *verdef; +} vdso_info; + +/* Straight from the ELF specification. */ +static unsigned long elf_hash(const char *name) +{ + unsigned long h = 0, g; + while (*name) + { + h = (h << 4) + *name++; + if ((g = h & 0xf0000000)) + h ^= g >> 24; + h &= ~g; + } + return h; +} + +static void vdso_init_from_sysinfo_ehdr(uintptr_t base) +{ + size_t i; + bool found_vaddr = false; + + vdso_info.valid = false; + + vdso_info.load_addr = base; + + ELF(Ehdr) *hdr = (ELF(Ehdr)*)base; + if (hdr->e_ident[EI_CLASS] != + (ELF_BITS == 32 ? ELFCLASS32 : ELFCLASS64)) { + return; /* Wrong ELF class -- check ELF_BITS */ + } + + ELF(Phdr) *pt = (ELF(Phdr)*)(vdso_info.load_addr + hdr->e_phoff); + ELF(Dyn) *dyn = 0; + + /* + * We need two things from the segment table: the load offset + * and the dynamic table. + */ + for (i = 0; i < hdr->e_phnum; i++) + { + if (pt[i].p_type == PT_LOAD && !found_vaddr) { + found_vaddr = true; + vdso_info.load_offset = base + + (uintptr_t)pt[i].p_offset + - (uintptr_t)pt[i].p_vaddr; + } else if (pt[i].p_type == PT_DYNAMIC) { + dyn = (ELF(Dyn)*)(base + pt[i].p_offset); + } + } + + if (!found_vaddr || !dyn) + return; /* Failed */ + + /* + * Fish out the useful bits of the dynamic table. + */ + ELF(Word) *hash = 0; + vdso_info.symstrings = 0; + vdso_info.symtab = 0; + vdso_info.versym = 0; + vdso_info.verdef = 0; + for (i = 0; dyn[i].d_tag != DT_NULL; i++) { + switch (dyn[i].d_tag) { + case DT_STRTAB: + vdso_info.symstrings = (const char *) + ((uintptr_t)dyn[i].d_un.d_ptr + + vdso_info.load_offset); + break; + case DT_SYMTAB: + vdso_info.symtab = (ELF(Sym) *) + ((uintptr_t)dyn[i].d_un.d_ptr + + vdso_info.load_offset); + break; + case DT_HASH: + hash = (ELF(Word) *) + ((uintptr_t)dyn[i].d_un.d_ptr + + vdso_info.load_offset); + break; + case DT_VERSYM: + vdso_info.versym = (ELF(Versym) *) + ((uintptr_t)dyn[i].d_un.d_ptr + + vdso_info.load_offset); + break; + case DT_VERDEF: + vdso_info.verdef = (ELF(Verdef) *) + ((uintptr_t)dyn[i].d_un.d_ptr + + vdso_info.load_offset); + break; + } + } + if (!vdso_info.symstrings || !vdso_info.symtab || !hash) + return; /* Failed */ + + if (!vdso_info.verdef) + vdso_info.versym = 0; + + /* Parse the hash table header. */ + vdso_info.nbucket = hash[0]; + vdso_info.nchain = hash[1]; + vdso_info.bucket = &hash[2]; + vdso_info.chain = &hash[vdso_info.nbucket + 2]; + + /* That's all we need. */ + vdso_info.valid = true; +} + +static bool vdso_match_version(ELF(Versym) ver, + const char *name, ELF(Word) hash) +{ + /* + * This is a helper function to check if the version indexed by + * ver matches name (which hashes to hash). + * + * The version definition table is a mess, and I don't know how + * to do this in better than linear time without allocating memory + * to build an index. I also don't know why the table has + * variable size entries in the first place. + * + * For added fun, I can't find a comprehensible specification of how + * to parse all the weird flags in the table. + * + * So I just parse the whole table every time. + */ + + /* First step: find the version definition */ + ver &= 0x7fff; /* Apparently bit 15 means "hidden" */ + ELF(Verdef) *def = vdso_info.verdef; + while(true) { + if ((def->vd_flags & VER_FLG_BASE) == 0 + && (def->vd_ndx & 0x7fff) == ver) + break; + + if (def->vd_next == 0) + return false; /* No definition. */ + + def = (ELF(Verdef) *)((char *)def + def->vd_next); + } + + /* Now figure out whether it matches. */ + ELF(Verdaux) *aux = (ELF(Verdaux)*)((char *)def + def->vd_aux); + return def->vd_hash == hash + && !strcmp(name, vdso_info.symstrings + aux->vda_name); +} + +void *evl_lookup_vdso(const char *version, const char *name) +{ + unsigned long ver_hash; + + if (!vdso_info.valid) + return 0; + + ver_hash = elf_hash(version); + ELF(Word) chain = vdso_info.bucket[elf_hash(name) % vdso_info.nbucket]; + + for (; chain != STN_UNDEF; chain = vdso_info.chain[chain]) { + ELF(Sym) *sym = &vdso_info.symtab[chain]; + + /* Check for a defined global or weak function w/ right name. */ + if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) + continue; + if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && + ELF64_ST_BIND(sym->st_info) != STB_WEAK) + continue; + if (sym->st_shndx == SHN_UNDEF) + continue; + if (strcmp(name, vdso_info.symstrings + sym->st_name)) + continue; + + /* Check symbol version. */ + if (vdso_info.versym + && !vdso_match_version(vdso_info.versym[chain], + version, ver_hash)) + continue; + + return (void *)(vdso_info.load_offset + sym->st_value); + } + + return 0; +} + +void *evl_request_vdso(const char *version, const char *name) +{ + void *sym = evl_lookup_vdso(version, name); + + if (!sym) + error(1, ENOENT, "%s not found in vDSO", name); + + return sym; +} + +static void parse_vdso(void) +{ + uintptr_t vdso = (uintptr_t)getauxval(AT_SYSINFO_EHDR); + + if (!vdso) + error(1, ENOENT, "vDSO signature not found"); + + vdso_init_from_sysinfo_ehdr(vdso); +} + +void evl_init_vdso(void) +{ + static pthread_once_t parse_vdso_once = PTHREAD_ONCE_INIT; + + pthread_once(&parse_vdso_once, parse_vdso); +} diff --git a/lib/parse_vdso.h b/lib/parse_vdso.h new file mode 100644 index 0000000..8e11f26 --- /dev/null +++ b/lib/parse_vdso.h @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: MIT + */ +#ifndef _LIB_EVENLESS_PARSE_VDSO_H +#define _LIB_EVENLESS_PARSE_VDSO_H + +void evl_init_vdso(void); + +void *evl_lookup_vdso(const char *version, const char *name); + +void *evl_request_vdso(const char *version, const char *name); + +#endif /* !_LIB_EVENLESS_PARSE_VDSO_H */ diff --git a/lib/sem.c b/lib/sem.c new file mode 100644 index 0000000..f2847a4 --- /dev/null +++ b/lib/sem.c @@ -0,0 +1,258 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +int evl_new_sem(struct evl_sem *sem, + int flags, int clockfd, int initval, + const char *fmt, ...) +{ + struct evl_element_ids eids; + struct evl_sem_attrs attrs; + int efd, ret; + va_list ap; + char *name; + + if (evl_shared_memory == NULL) + return -EAGAIN; + + va_start(ap, fmt); + ret = vasprintf(&name, fmt, ap); + va_end(ap); + if (ret < 0) + return -ENOMEM; + + attrs.clockfd = clockfd; + attrs.flags = flags; + attrs.initval = initval; + efd = create_evl_element("semaphore", name, &attrs, &eids); + free(name); + if (efd < 0) + return efd; + + sem->active.state = evl_shared_memory + eids.state_offset; + sem->active.fundle = eids.fundle; + sem->active.efd = efd; + sem->magic = __SEM_ACTIVE_MAGIC; + + return 0; +} + +int evl_open_sem(struct evl_sem *sem, const char *fmt, ...) +{ + struct evl_element_ids eids; + int ret, efd; + va_list ap; + + va_start(ap, fmt); + efd = open_evl_element("semaphore", fmt, ap); + va_end(ap); + if (efd < 0) + return efd; + + ret = ioctl(efd, EVL_SEMIOC_BIND, &eids); + if (ret) + return -errno; + + sem->active.state = evl_shared_memory + eids.state_offset; + sem->active.fundle = eids.fundle; + sem->active.efd = efd; + sem->magic = __SEM_ACTIVE_MAGIC; + + return 0; +} + +int evl_release_sem(struct evl_sem *sem) +{ + int ret; + + if (sem->magic != __SEM_ACTIVE_MAGIC) + return -EINVAL; + + ret = close(sem->active.efd); + if (ret) + return -errno; + + sem->active.fundle = EVL_NO_HANDLE; + sem->active.state = NULL; + sem->magic = __SEM_DEAD_MAGIC; + + return 0; +} + +static int try_get(struct evl_sem_state *state, int count) +{ + int curval, oldval, newval; + + curval = atomic_read(&state->value); + if (curval <= 0) + return -EAGAIN; + + do { + oldval = curval; + newval = oldval - count; + if (newval > oldval) + return -EINVAL; + curval = atomic_cmpxchg(&state->value, oldval, newval); + /* Did somebody else deplete the sema4? */ + if (curval <= 0) + return -EAGAIN; + } while (curval != oldval); + + smp_mb(); + + return 0; +} + +static int check_sanity(struct evl_sem *sem, int count) +{ + if (count <= 0) + return -EINVAL; + + if (sem->magic == __SEM_UNINIT_MAGIC) + return evl_new_sem(sem, + sem->uninit.flags, + sem->uninit.clockfd, + sem->uninit.initval, + sem->uninit.name); + + return sem->magic != __SEM_ACTIVE_MAGIC ? -EINVAL : 0; +} + +int evl_get_sem(struct evl_sem *sem, int count, + const struct timespec *timeout) +{ + struct evl_sem_waitreq req; + int mode, ret, cancel_type; + fundle_t current; + + current = evl_get_current(); + if (current == EVL_NO_HANDLE) + return -EPERM; + + ret = check_sanity(sem, count); + if (ret) + return ret; + + /* + * Threads running in-band and/or enabling some debug features + * must go through the slow syscall path. + */ + mode = evl_get_current_mode(); + if (!(mode & (T_INBAND|T_WEAK|T_DEBUG))) { + ret = try_get(sem->active.state, count); + if (ret != -EAGAIN) + return ret; + } + + req.timeout = *timeout; + req.count = count; + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &cancel_type); + ret = oob_ioctl(sem->active.efd, EVL_SEMIOC_GET, &req); + pthread_setcanceltype(cancel_type, NULL); + + return ret ? -errno : 0; +} + +int evl_tryget_sem(struct evl_sem *sem, int count) +{ + int ret; + + ret = check_sanity(sem, count); + if (ret) + return ret; + + return try_get(sem->active.state, count); +} + +int evl_put_sem(struct evl_sem *sem, int count) +{ + int curval, oldval, newval, ret; + struct evl_sem_state *state; + __s32 val; + + ret = check_sanity(sem, count); + if (ret) + return ret; + + state = sem->active.state; + curval = atomic_read(&state->value); + if (curval < 0) { + slow_path: + val = count; + if (evl_get_current()) + ret = oob_ioctl(sem->active.efd, EVL_SEMIOC_PUT, &val); + else + /* In-band threads may post pended sema4s. */ + ret = ioctl(sem->active.efd, EVL_SEMIOC_PUT, &val); + return ret ? -errno : 0; + } + + if (state->flags & EVL_SEM_PULSE) + return 0; + + do { + oldval = curval; + newval = oldval + count; + if (newval < oldval) + return -EINVAL; + curval = atomic_cmpxchg(&state->value, oldval, newval); + /* Check if somebody sneaked in the wait queue. */ + if (curval < 0) + goto slow_path; + } while (curval != oldval); + + smp_mb(); + + return 0; +} + +int evl_broadcast_sem(struct evl_sem *sem) +{ + struct evl_sem_state *state; + int curval, ret; + + ret = check_sanity(sem, 1); + if (ret) + return ret; + + state = sem->active.state; + curval = atomic_read(&state->value); + if (curval >= 0) + return 0; /* Nobody waits. */ + + if (evl_get_current()) + ret = oob_ioctl(sem->active.efd, EVL_SEMIOC_BCAST); + else + ret = ioctl(sem->active.efd, EVL_SEMIOC_BCAST); + + return ret ? -errno : 0; +} + +int evl_get_semval(struct evl_sem *sem) +{ + if (sem->magic != __SEM_ACTIVE_MAGIC) + return -EINVAL; + + return atomic_read(&sem->active.state->value); +} diff --git a/lib/syscall.c b/lib/syscall.c new file mode 100644 index 0000000..d41df3a --- /dev/null +++ b/lib/syscall.c @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include + +ssize_t oob_read(int efd, void *buf, size_t count) +{ + int old_type; + ssize_t ret; + + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type); + ret = evenless_syscall3(sys_evenless_read, efd, buf, count); + pthread_setcanceltype(old_type, NULL); + if (ret < 0) { + errno = -ret; + return -1; + } + + return ret; +} + +ssize_t oob_write(int efd, const void *buf, size_t count) +{ + int old_type; + ssize_t ret; + + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type); + ret = evenless_syscall3(sys_evenless_write, efd, buf, count); + pthread_setcanceltype(old_type, NULL); + if (ret < 0) { + errno = -ret; + return -1; + } + + return ret; +} + +int oob_ioctl(int efd, unsigned long request, ...) +{ + int ret, old_type; + va_list ap; + long arg; + + va_start(ap, request); + arg = va_arg(ap, long); + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type); + ret = evenless_syscall3(sys_evenless_ioctl, efd, request, arg); + pthread_setcanceltype(old_type, NULL); + va_end(ap); + + if (ret) { + errno = -ret; + return -1; + } + + return 0; +} diff --git a/lib/thread.c b/lib/thread.c new file mode 100644 index 0000000..19ff22b --- /dev/null +++ b/lib/thread.c @@ -0,0 +1,207 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +__thread __attribute__ ((tls_model (EVL_TLS_MODEL))) +fundle_t evl_current = EVL_NO_HANDLE; + +__thread __attribute__ ((tls_model (EVL_TLS_MODEL))) +int evl_efd = -1; + +__thread __attribute__ ((tls_model (EVL_TLS_MODEL))) +struct evl_user_window *evl_current_window; + +__thread __attribute__ ((tls_model (EVL_TLS_MODEL))) +char evl_logging_buf[EVL_PRINTBUF_SIZE]; + +static pthread_once_t atfork_once = PTHREAD_ONCE_INIT; + +static void clear_tls(void) +{ + evl_current = EVL_NO_HANDLE; + evl_current_window = NULL; + evl_efd = -1; +} + +static void atfork_clear_tls(void) +{ + clear_tls(); + atfork_once = PTHREAD_ONCE_INIT; +} + +static void do_atfork_once(void) +{ + pthread_atfork(NULL, NULL, atfork_clear_tls); +} + +int evl_attach_self(const char *fmt, ...) +{ + int efd, ret, policy, priority; + struct evl_sched_attrs attrs; + struct evl_element_ids eids; + struct sched_param param; + va_list ap; + char *name; + + /* + * Try to initialize if not yet done, so that attaching a + * thread to the core as the first EVL call of the process + * enables all services. + */ + ret = evl_init(); + if (ret) + return ret; + + /* + * Cannot bind twice. Although the core would catch it, we can + * detect this issue early. + */ + if (evl_current != EVL_NO_HANDLE) + return -EBUSY; + + va_start(ap, fmt); + ret = vasprintf(&name, fmt, ap); + va_end(ap); + if (ret < 0) + return -ENOMEM; + + efd = create_evl_element("thread", name, NULL, &eids); + free(name); + if (efd < 0) + return efd; + + evl_current = eids.fundle; + evl_current_window = evl_shared_memory + eids.state_offset; + evl_efd = efd; + + /* + * Translate current in-band scheduling parameters to EVL + * scheduling attributes which we apply to self. + */ + ret = pthread_getschedparam(pthread_self(), &policy, ¶m); + if (ret) + goto fail; + + switch (policy) { + case SCHED_OTHER: + priority = 0; + break; + default: + policy = SCHED_FIFO; + /* Fallback wanted. */ + case SCHED_FIFO: + case SCHED_RR: + priority = param.sched_priority; + break; + } + + attrs.sched_policy = policy; + attrs.sched_priority = priority; + ret = oob_ioctl(efd, EVL_THRIOC_SET_SCHEDPARAM, &attrs); + if (ret) { + ret = -errno; + goto fail; + } + + pthread_once(&atfork_once, do_atfork_once); + + return efd; +fail: + close(efd); + clear_tls(); + + return ret; +} + +int evl_detach_self(void) +{ + int ret; + + if (evl_efd < 0) + return -EPERM; + + ret = ioctl(evl_ctlfd, EVL_CTLIOC_DETACH_SELF); + if (ret) + return -errno; + + close(evl_efd); + clear_tls(); + + return 0; +} + +int evl_get_self(void) +{ + return evl_efd; +} + +int evl_switch_oob(void) +{ + int ret; + + if (evl_current == EVL_NO_HANDLE) + return -EPERM; + + if (!evl_is_inband()) + return 0; + + ret = ioctl(evl_ctlfd, EVL_CTLIOC_SWITCH_OOB); + + return ret ? -errno : 0; +} + +int evl_switch_inband(void) +{ + int ret; + + if (evl_is_inband()) + return 0; + + ret = ioctl(evl_ctlfd, EVL_CTLIOC_SWITCH_INBAND); + + return ret ? -errno : 0; +} + +int evl_set_schedattr(int efd, const struct evl_sched_attrs *attrs) +{ + int ret; + + if (evl_is_inband()) + ret = ioctl(efd, EVL_THRIOC_SET_SCHEDPARAM, attrs); + else + ret = oob_ioctl(efd, EVL_THRIOC_SET_SCHEDPARAM, attrs); + + return ret ? -errno : 0; +} + +int evl_get_schedattr(int efd, struct evl_sched_attrs *attrs) +{ + int ret; + + if (evl_is_inband()) + ret = ioctl(efd, EVL_THRIOC_GET_SCHEDPARAM, attrs); + else + ret = oob_ioctl(efd, EVL_THRIOC_GET_SCHEDPARAM, attrs); + + return ret ? -errno : 0; +} diff --git a/lib/version.c b/lib/version.c new file mode 100644 index 0000000..543804c --- /dev/null +++ b/lib/version.c @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: MIT + * + * Copyright (C) 2018 Philippe Gerum + */ + +#include "git-stamp.h" + +#ifndef GIT_STAMP +#define git_hash "" +#else +#define git_hash " -- " GIT_STAMP +#endif + +const char *libevenless_version_string = "evenless." LIBSERIAL git_hash; diff --git a/lib/xbuf.c b/lib/xbuf.c new file mode 100644 index 0000000..92c7659 --- /dev/null +++ b/lib/xbuf.c @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: MIT + * + * 2018 - Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include "internal.h" + +int evl_new_xbuf(size_t i_bufsz, size_t o_bufsz, + const char *fmt, ...) +{ + struct evl_xbuf_attrs attrs; + int ret, efd; + va_list ap; + char *name; + + va_start(ap, fmt); + ret = vasprintf(&name, fmt, ap); + va_end(ap); + if (ret < 0) + return -ENOMEM; + + attrs.i_bufsz = i_bufsz; + attrs.o_bufsz = o_bufsz; + efd = create_evl_element("xbuf", name, &attrs, NULL); + free(name); + + return efd; +} diff --git a/scripts/config.mk b/scripts/config.mk new file mode 100644 index 0000000..e7ba696 --- /dev/null +++ b/scripts/config.mk @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: MIT + +LIBSERIAL := 0 +LIBIVERSION := 0 + +ARCH ?= $(shell uname -m | sed \ + -e s/arm.*/arm/ \ + -e s/aarch64.*/arm64/ ) +CROSS_COMPILE ?= +CC = $(CROSS_COMPILE)gcc +LD = $(CROSS_COMPILE)ld +AR = $(CROSS_COMPILE)ar +UAPI ?= /usr/include +DESTDIR ?= /usr/evenless + +INSTALL ?= install +INSTALL_PROGRAM ?= $(INSTALL) +INSTALL_DATA ?= $(INSTALL) -m 644 + +CP := cp +CPIO := cpio +RM := rm +LN_S := ln -sf +MKDIR_P := mkdir -p + +libdir ?= lib +includedir ?= include +bindir ?= bin + +export ARCH CROSS_COMPILE CC LD AR UAPI CFLAGS LDFLAGS DESTDIR + +MAKEFLAGS += -rR + +ifneq ("$(origin O)", "command line") + O_DIR = $(CURDIR) +else + O_DIR := $(shell $(MKDIR_P) $(O) && cd $(O) && /bin/pwd) +endif + +ifneq ("$(origin V)", "command line") + V = 0 +endif + +ifeq ($(V),1) + Q = +else + Q = @ + MAKEFLAGS += --no-print-directory +endif + +ifeq ($(D),1) + DEBUG_CPPFLAGS= + DEBUG_CFLAGS=-g -O0 +else + DEBUG_CPPFLAGS=-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 + DEBUG_CFLAGS=-O2 +endif + +BASE_CPPFLAGS := -D_GNU_SOURCE -D_REENTRANT $(DEBUG_CPPFLAGS) + +BASE_CFLAGS := -fasynchronous-unwind-tables -pipe -fstrict-aliasing \ + -Wall -Wstrict-prototypes -Wmissing-prototypes -Wno-long-long \ + -Wno-unused-parameter -Werror $(DEBUG_CFLAGS) + +# Easy way to hide commas in args from $(call ...) invocations +comma := , + +ifneq ($(findstring s,$(filter-out --%,$(MAKEFLAGS))),) + quiet=y +endif + +terse-echo = @$(if $(Q),echo " $(1) $(2)";) +define run-cmd + $(if $(quiet),,$(call terse-echo,$(1),$(2))) + $(Q)$(3) +endef +define run-cc + @$(MKDIR_P) $(dir $(2)) + $(call run-cmd,$(1),$(notdir $(2)),$(3)) +endef +cc-pic-cmd = $(call run-cc,CC-PIC,$(1),$(2)) +cc-cmd = $(call run-cc,CC,$(1),$(2)) +dep-cmd = $(call run-cc,DEP,$(1),$(2)) +ccld-cmd = $(call run-cc,CCLD,$(1),$(2)) +ld-cmd = $(call run-cmd,LD,$(notdir $(1)),$(2)) +ar-cmd = $(call run-cmd,AR,$(notdir $(1)),$(2) $(if $(Q),2>/dev/null)) + +$(O_DIR)/%.d: %.c + $(call dep-cmd,$@,@$(CC) -MM $(CFLAGS) $< | sed 's$(comma)\($*\)\.o[ :]*$(comma)$(O_DIR)/\1.o $@: $(comma)g' > $@ || rm -f $@) diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..8c833e7 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,34 @@ +include ../scripts/config.mk + +TESTFILES := $(wildcard *.c) +BINARIES = $(TESTFILES:%.c=%) +TARGETS = $(TESTFILES:%.c=$(O_DIR)/%) +DEPFILES = $(TESTFILES:%.c=$(O_DIR)/%.d) + +TEST_CPPFLAGS := $(BASE_CPPFLAGS) \ + -I. \ + -I../include \ + -I$(O_DIR)/../include + +TEST_CFLAGS := $(TEST_CPPFLAGS) $(BASE_CFLAGS) +override CFLAGS := $(TEST_CFLAGS) $(CFLAGS) + +TEST_LDFLAGS := $(O_DIR)/../lib/libevenless.so.$(LIBIVERSION) -lpthread -lrt +override LDFLAGS := $(TEST_LDFLAGS) $(LDFLAGS) + +all: $(TARGETS) + +install: all + $(Q)for bin in $(BINARIES); do \ + $(INSTALL) -D $(O_DIR)/$$bin $(DESTDIR)/$(bindir)/$$bin; \ + done + +clean clobber mrproper: + $(Q)$(RM) -f $(TARGETS) $(DEPFILES) + +$(O_DIR)/%: %.c + $(call ccld-cmd,$@,$(Q)$(CC) -o $(@) $< $(CFLAGS) $(LDFLAGS)) + +.PHONY: all install clean clobber mrproper + +-include $(DEPFILES) diff --git a/tests/basic-xbuf.c b/tests/basic-xbuf.c new file mode 100644 index 0000000..4fbb836 --- /dev/null +++ b/tests/basic-xbuf.c @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include +#include "helpers.h" + +static void *peer(void *arg) +{ + const char *path = arg; + char buf[2]; + ssize_t ret; + int fd, n, nfd; + + fd = open(path, O_RDWR); + printf("dup(%d) => %d\n", fd, (nfd = dup(fd))); + printf("dup2(%d, %d) => %d\n", fd, nfd, dup2(fd, nfd)); + + printf("peer reading from fd=%d\n", fd); + + for (n = 0; n < 3; n++) { + ret = read(fd, buf, 2); + if (ret != 2) + break; + printf("inband[%d] => %.2s\n", n, buf); + } + + return NULL; +} + +int main(int argc, char *argv[]) +{ + char *name, *path, buf[16]; + int tfd, xfd, n; + pthread_t tid; + ssize_t ret; + + tfd = evl_attach_self("basic-xbuf:%d", getpid()); + printf("thread tfd=%d\n", tfd); + + name = get_unique_name("xbuf", 0, &path); + xfd = evl_new_xbuf(1024, 1024, name); + if (xfd < 0) + error(1, -xfd, "evl_new_xbuf"); + + printf("xfd=%d\n", xfd); + + ret = write(xfd, "ABCD", 4); + printf("write->oob_read: %zd\n", ret); + ret = write(xfd, "EF", 2); + printf("write->oob_read: %zd\n", ret); + ret = write(xfd, "G", 1); + printf("write->oob_read: %zd\n", ret); + ret = write(xfd, "H", 1); + printf("write->oob_read: %zd\n", ret); + + ret = fcntl(xfd, F_SETFL, fcntl(xfd, F_GETFL)|O_NONBLOCK); + if (ret) + error(1, errno, "oob_read"); + + for (n = 0; n < 8; n++) { + ret = oob_read(xfd, buf, 1); + if (ret < 0) + error(1, errno, "oob_read"); + printf("oob_read[%d]<-write: %zd => %#x\n", + n, ret, *buf); + } + + ret = pthread_create(&tid, NULL, peer, path); + sleep(1); + + ret = oob_write(xfd, "01", 2); + printf("oob_write->read: %zd\n", ret); + ret = oob_write(xfd, "23", 2); + printf("oob_write->read: %zd\n", ret); + ret = oob_write(xfd, "45", 2); + printf("oob_write->read: %zd\n", ret); + + return pthread_join(tid, NULL); +} diff --git a/tests/clone-fork-exec.c b/tests/clone-fork-exec.c new file mode 100644 index 0000000..5cdffbf --- /dev/null +++ b/tests/clone-fork-exec.c @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + int efd; + + if (argc > 1 && strcmp(argv[1], "exec") == 0) { + printf("exec() ok for pid %d\n", getpid()); + return 0; + } + + efd = evl_attach_self("clone-fork-exec:%d", getpid()); + printf("thread has efd=%d\n", efd); + + switch (fork()) { + case 0: + return 0; + default: + execlp(argv[0], "clone-fork-exec", "exec", NULL); + printf("exec() failed for pid %d\n", getpid()); + } + + return 0; +} diff --git a/tests/detach-self.c b/tests/detach-self.c new file mode 100644 index 0000000..badef86 --- /dev/null +++ b/tests/detach-self.c @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + int efd, ret; + + efd = evl_attach_self("simple-bind:%d", getpid()); + printf("thread efd=%d\n", efd); + + ret = evl_detach_self(); + printf("detach ret=%d\n", ret); + + efd = evl_attach_self("simple-bind:%d", getpid()); + printf("thread efd=%d\n", efd); + + ret = evl_detach_self(); + printf("detach ret=%d\n", ret); + + return 0; +} diff --git a/tests/helpers.h b/tests/helpers.h new file mode 100644 index 0000000..8b00030 --- /dev/null +++ b/tests/helpers.h @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: MIT + */ + +#ifndef _EVENLESS_TESTS_HELPERS_H +#define _EVENLESS_TESTS_HELPERS_H + +#include +#include +#include +#include +#include +#include + +#define __stringify_1(x...) #x +#define __stringify(x...) __stringify_1(x) + +#define warn_failed(__fmt, __args...) \ + fprintf(stderr, "FAILED: " __fmt, ##__args) + +#define __T(__ret, __action) \ + ({ \ + (__ret) = (__action); \ + if (__ret < 0) { \ + (__ret) = -(__ret); \ + warn_failed("%s (=%s)", \ + __stringify(__action), \ + strerror(__ret)); \ + } \ + (__ret) >= 0; \ + }) + +#define __F(__ret, __action) \ + ({ \ + (__ret) = (__action); \ + if ((__ret) >= 0) \ + warn_failed("%s (%d >= 0)", \ + __stringify(__action), \ + __ret); \ + (__ret) < 0; \ + }) + +#define __Terrno(__ret, __action) \ + ({ \ + (__ret) = (__action); \ + if (__ret) \ + warn_failed("%s (=%s)", \ + __stringify(__action), \ + strerror(errno)); \ + (__ret) == 0; \ + }) + +#define __Tassert(__expr) \ + ({ \ + int __ret = !!(__expr); \ + if (!__ret) \ + warn_failed("%s (=false)", \ + __stringify(__expr)); \ + __ret; \ + }) + +#define __Fassert(__expr) \ + ({ \ + int __ret = (__expr); \ + if (__ret) \ + warn_failed("%s (=true)", \ + __stringify(__expr)); \ + !__ret; \ + }) + +static inline +char *get_unique_name(const char *type, int serial, char **ppath) +{ + int ret; + + ret = asprintf(ppath, "/dev/evenless/%s/test%d.%d", + type, getpid(), serial); + if (ret < 0) + error(1, ENOMEM, "malloc"); + + return strrchr(*ppath, '/') + 1; +} + +#endif /* !_EVENLESS_TESTS_HELPERS_H */ diff --git a/tests/logger-stdout.c b/tests/logger-stdout.c new file mode 100644 index 0000000..a03d5cc --- /dev/null +++ b/tests/logger-stdout.c @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + int ret, efd, logfd; + + efd = evl_attach_self("simple-bind:%d", getpid()); + printf("efd=%d\n", efd); + logfd = evl_new_logger(fileno(stdout), 16, "stdout:%d", getpid()); + printf("logfd=%d\n", logfd); + ret = oob_write(logfd, "stdout relay!\n", 14); + printf("oob_write=%d, errno=%d\n", ret, errno); + + return 0; +} diff --git a/tests/mapfd.c b/tests/mapfd.c new file mode 100644 index 0000000..9b6bfab --- /dev/null +++ b/tests/mapfd.c @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static inline int do_memfd_create(const char *name, int flags) +{ + return syscall(__NR_memfd_create, name, flags); +} + +int main(int argc, char *argv[]) +{ + int memfd, efd, ret; + char *name; + void *p; + + if (argc > 1) { + ret = chdir("/dev/evenless/mapper"); + (void)ret; + efd = open(argv[1], O_RDWR); + if (efd < 0) + error(1, errno, "mapfd open %s", argv[1]); + p = mmap(NULL, 1024, PROT_READ|PROT_WRITE, + MAP_SHARED, efd, 0); + if (p == MAP_FAILED) + error(1, errno, "mmap child"); + printf("mapfd child reading: %s\n", (const char *)p); + return 0; + } + + memfd = do_memfd_create("evl", 0); + if (memfd < 0) + error(1, errno, "memfd_create"); + + ret = ftruncate(memfd, 1024); + if (ret) + error(1, errno, "ftruncate"); + + p = mmap(NULL, 1024, PROT_READ|PROT_WRITE, + MAP_SHARED, memfd, 0); + if (p == MAP_FAILED) + error(1, errno, "mmap"); + + strcpy(p, "mapfd-test"); + + ret = asprintf(&name, "mapper:%d", getpid()); + (void)ret; + efd = evl_new_mapper(memfd, "%s", name); + printf("mapper has efd=%d\n", efd); + + switch (fork()) { + case 0: + sleep(1); + return 0; + default: + execlp(argv[0], "mapfd", name, NULL); + printf("exec() failed for pid %d\n", getpid()); + } + + return 0; +} diff --git a/tests/simple-clone.c b/tests/simple-clone.c new file mode 100644 index 0000000..ccc551d --- /dev/null +++ b/tests/simple-clone.c @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + int efd; + + efd = evl_attach_self("simple-bind:%d", getpid()); + printf("thread efd=%d\n", efd); + + return 0; +} diff --git a/utils/Makefile b/utils/Makefile new file mode 100644 index 0000000..a36bcab --- /dev/null +++ b/utils/Makefile @@ -0,0 +1,34 @@ +include ../scripts/config.mk + +UTILFILES := $(wildcard *.c) +BINARIES = $(UTILFILES:%.c=%) +TARGETS = $(UTILFILES:%.c=$(O_DIR)/%) +DEPFILES = $(UTILFILES:%.c=$(O_DIR)/%.d) + +UTIL_CPPFLAGS := $(BASE_CPPFLAGS) \ + -I. \ + -I../include \ + -I$(O_DIR)/../include + +UTIL_CFLAGS := $(UTIL_CPPFLAGS) $(BASE_CFLAGS) +override CFLAGS := $(UTIL_CFLAGS) $(CFLAGS) + +UTIL_LDFLAGS := $(O_DIR)/../lib/libevenless.so.$(LIBIVERSION) -lpthread -lrt +override LDFLAGS := $(UTIL_LDFLAGS) $(LDFLAGS) + +all: $(TARGETS) + +install: all + $(Q)for bin in $(BINARIES); do \ + $(INSTALL) -D $(O_DIR)/$$bin $(DESTDIR)/$(bindir)/$$bin; \ + done + +clean clobber mrproper: + $(Q)$(RM) -f $(TARGETS) $(DEPFILES) + +$(O_DIR)/%: %.c + $(call ccld-cmd,$@,$(Q)$(CC) -o $(@) $< $(CFLAGS) $(LDFLAGS)) + +.PHONY: all install clean clobber mrproper + +-include $(DEPFILES) diff --git a/utils/hectic.c b/utils/hectic.c new file mode 100644 index 0000000..194c44d --- /dev/null +++ b/utils/hectic.c @@ -0,0 +1,1477 @@ +/* + * SPDX-License-Identifier: MIT + * + * Derived from Xenomai Cobalt's switchtest program + * (http://git.xenomai.org/xenomai-3.git/) + * Copyright (C) 2006-2013 Gilles Chanteperdrix + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned int nr_cpus; + +static cpu_set_t cpu_affinity; + +#define for_each_cpu(__cpu) \ + for (__cpu = 0; __cpu < CPU_SETSIZE; __cpu++) \ + if (CPU_ISSET(__cpu, &cpu_affinity)) + +#define for_each_cpu_index(__cpu, __index) \ + for (__cpu = 0, __index = -1; __cpu < CPU_SETSIZE; __cpu++) \ + if (CPU_ISSET(__cpu, &cpu_affinity) && ++__index >= 0) + +/* Thread type. */ +typedef enum { + SLEEPER = 0, + RTK = 1, /* kernel-space thread. */ + RTUP = 2, /* user-space real-time thread ruuning OOB. */ + RTUS = 3, /* user-space real-time thread running in-band. */ + RTUO = 4, /* user-space real-time thread oscillating + between in-band and OOB contexts. */ + SWITCHER = 8, + FPU_STRESS = 16, +} threadtype; + +typedef enum { + UFPP = 1, /* use the FPU while running OOB. */ + UFPS = 2 /* use the FPU while running in-band. */ +} fpflags; + +struct cpu_tasks; + +struct task_params { + threadtype type; + fpflags fp; + pthread_t thread; + struct cpu_tasks *cpu; + struct hectic_task_index swt; +}; + +struct cpu_tasks { + unsigned int index; + struct task_params *tasks; + unsigned tasks_count; + unsigned capacity; + unsigned fd; + unsigned long last_switches_count; +}; + +static sem_t sleeper_start; +static int quiet, status; +static struct timespec start; +static pthread_mutex_t headers_lock; +static unsigned long data_lines = 21; +static __u32 fp_features; +static pthread_t main_tid; + +static inline void clean_exit(int retval) +{ + status = retval; + pthread_kill(main_tid, SIGTERM); + for (;;) + /* Wait for cancellation. */ + sem_wait(&sleeper_start); +} + +static void timespec_substract(struct timespec *result, + const struct timespec *lhs, + const struct timespec *rhs) +{ + result->tv_sec = lhs->tv_sec - rhs->tv_sec; + if (lhs->tv_nsec >= rhs->tv_nsec) + result->tv_nsec = lhs->tv_nsec - rhs->tv_nsec; + else { + result->tv_sec -= 1; + result->tv_nsec = lhs->tv_nsec + (1000000000 - rhs->tv_nsec); + } +} + +static char *task_name(char *buf, size_t sz, + struct cpu_tasks *cpu, unsigned task) +{ + char *basename [] = { + [SLEEPER] = "sleeper", + [RTK] = "rtk", + [RTUP] = "rtup", + [RTUS] = "rtus", + [RTUO] = "rtuo", + [SWITCHER] = "switcher", + [FPU_STRESS] = "fpu_stress", + }; + struct { + unsigned flag; + char *name; + } flags [] = { + { .flag = UFPP, .name = "ufpp" }, + { .flag = UFPS, .name = "ufps" }, + }; + struct task_params *param; + unsigned pos, i; + + if (task > cpu->tasks_count) + return "???"; + + if (task == cpu->tasks_count) + param = &cpu->tasks[task]; + else + for (param = &cpu->tasks[0]; param->swt.index != task; param++) + ; + + pos = snprintf(buf, sz, "%s", basename[param->type]); + for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) { + if (!(param->fp & flags[i].flag)) + continue; + + pos += snprintf(&buf[pos], + sz - pos, "_%s", flags[i].name); + } + + pos += snprintf(&buf[pos], sz - pos, "%u", cpu->index); + + snprintf(&buf[pos], sz - pos, "-%u", param->swt.index); + + return buf; +} + +static void handle_bad_fpreg(struct cpu_tasks *cpu, unsigned fp_val, + int bad_reg) +{ + struct hectic_error err; + unsigned from, to; + char buffer[64]; + + ioctl(cpu->fd, EVL_HECIOC_GET_LAST_ERROR, &err); + + if (fp_val == ~0) + fp_val = err.fp_val; + + from = err.last_switch.from; + to = err.last_switch.to; + + if (bad_reg < 0) + fprintf(stderr, "FPU corruption detected"); + else + fprintf(stderr, "fpreg%d corrupted", bad_reg); + fprintf(stderr, " after context switch from task %d(%s) ", + from, task_name(buffer, sizeof(buffer), cpu, from)); + fprintf(stderr, "to task %d(%s),\nFPU registers were set to %u ", + to, task_name(buffer, sizeof(buffer), cpu, to), fp_val); + fp_val %= 1000; + if (fp_val < 500) + fprintf(stderr, "(maybe task %s)\n", + task_name(buffer, sizeof(buffer), cpu, fp_val)); + else { + fp_val -= 500; + if (fp_val > cpu->tasks_count) + fprintf(stderr, "(unidentified task)\n"); + else + fprintf(stderr, "(maybe task %s, having used fpu in " + "kernel-space)\n", + task_name(buffer, sizeof(buffer), cpu, fp_val)); + } + + clean_exit(EXIT_FAILURE); +} + +static void display_cleanup(void *cookie) +{ + pthread_mutex_t *mutex = cookie; + pthread_mutex_unlock(mutex); +} + +static void display_switches_count(struct cpu_tasks *cpu, struct timespec *now) +{ + static unsigned nlines = 0; + __u32 switches_count; + + if (ioctl(cpu->fd, + EVL_HECIOC_GET_SWITCHES_COUNT,&switches_count)) { + perror("sleeper: ioctl(EVL_HECIOC_GET_SWITCHES_COUNT)"); + clean_exit(EXIT_FAILURE); + } + + if (switches_count && + switches_count == cpu->last_switches_count) { + fprintf(stderr, "No context switches during one second, " + "aborting.\n"); + clean_exit(EXIT_FAILURE); + } + + if (quiet) + return; + + pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL); + pthread_cleanup_push(display_cleanup, &headers_lock); + pthread_mutex_lock(&headers_lock); + + if (data_lines && (nlines++ % data_lines) == 0) { + struct timespec diff; + long dt; + + timespec_substract(&diff, now, &start); + dt = diff.tv_sec; + + printf("RTT| %.2ld:%.2ld:%.2ld\n", + dt / 3600, (dt / 60) % 60, dt % 60); + printf("RTH|%12s|%12s|%12s\n", + "---------cpu","ctx switches","-------total"); + } + + printf("RTD|%12u|%12lu|%12u\n", cpu->index, + switches_count - cpu->last_switches_count, switches_count); + + pthread_cleanup_pop(1); + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); + + cpu->last_switches_count = switches_count; +} + +static void *sleeper_switcher(void *cookie) +{ + struct task_params *param = cookie; + unsigned to, tasks_count = param->cpu->tasks_count; + struct timespec ts, last; + int fd = param->cpu->fd, bad_reg; + struct hectic_switch_req rtsw; + cpu_set_t cpu_set; + unsigned i = 1; /* Start at 1 to avoid returning to a + non-existing task. */ + int ret; + + CPU_ZERO(&cpu_set); + CPU_SET(param->cpu->index, &cpu_set); + if (sched_setaffinity(0, sizeof(cpu_set), &cpu_set)) { + perror("sleeper: sched_setaffinity"); + clean_exit(EXIT_FAILURE); + } + + rtsw.from = param->swt.index; + to = param->swt.index; + + ts.tv_sec = 0; + ts.tv_nsec = 1000000; + + ret = sem_wait(&sleeper_start); + if (ret) { + fprintf(stderr, "sem_wait FAILED (%d)\n", errno); + fflush(stderr); + exit(77); + } + + clock_gettime(CLOCK_REALTIME, &last); + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); + + for (;;) { + struct timespec now, diff; + unsigned expected, fp_val; + int err; + if (param->type == SLEEPER) + nanosleep(&ts, NULL); + + clock_gettime(CLOCK_REALTIME, &now); + + timespec_substract(&diff, &now, &last); + if (diff.tv_sec >= 1) { + last = now; + + display_switches_count(param->cpu, &now); + } + + if (tasks_count == 1) + continue; + + switch (i % 3) { + case 0: + /* to == from means "return to last task" */ + rtsw.to = rtsw.from; + break; + + case 1: + if (++to == rtsw.from) + ++to; + if (to > tasks_count - 1) + to = 0; + if (to == rtsw.from) + ++to; + rtsw.to = to; + + /* If i % 3 == 2, repeat the same switch. */ + } + + expected = rtsw.from + i * 1000; + if (param->fp & UFPS) + evl_set_fpregs(fp_features, expected); + err = ioctl(fd, EVL_HECIOC_SWITCH_TO, &rtsw); + while (err == -1 && errno == EINTR) + err = ioctl(fd, EVL_HECIOC_PEND, ¶m->swt); + + switch (err) { + case 0: + break; + case 1: + handle_bad_fpreg(param->cpu, ~0, -1); + case -1: + clean_exit(EXIT_FAILURE); + } + if (param->fp & UFPS) { + fp_val = evl_check_fpregs(fp_features, expected, bad_reg); + if (fp_val != expected) + handle_bad_fpreg(param->cpu, fp_val, bad_reg); + } + + if(++i == 4000000) + i = 0; + } + + return NULL; +} + + +static double dot(volatile double *a, volatile double *b, int n) +{ + int k = n - 1; + double s = 0.0; + for(; k >= 0; k--) + s = s + a[k]*b[k]; + + return s; +} + +static void *fpu_stress(void *cookie) +{ + static volatile double a[10000], b[sizeof(a)/sizeof(a[0])]; + struct task_params *param = (struct task_params *) cookie; + cpu_set_t cpu_set; + unsigned i; + + CPU_ZERO(&cpu_set); + CPU_SET(param->cpu->index, &cpu_set); + if (sched_setaffinity(0, sizeof(cpu_set), &cpu_set)) { + perror("sleeper: sched_setaffinity"); + clean_exit(EXIT_FAILURE); + } + + for (i = 0; i < sizeof(a)/sizeof(a[0]); i++) + a[i] = b[i] = 3.14; + + for (;;) { + double s = dot(a, b, sizeof(a)/sizeof(a[0])); + if ((unsigned) (s + 0.5) != 98596) { + fprintf(stderr, "fpu stress task failure! dot: %g\n", s); + clean_exit(EXIT_FAILURE); + } + pthread_testcancel(); + } + + return NULL; +} + +static void attach_thread(struct task_params *param) +{ + char buffer[64]; + int efd; + + task_name(buffer, sizeof(buffer), param->cpu,param->swt.index); + + efd = evl_attach_self("%s:%d", buffer, getpid()); + if (efd < 0) { + perror("evl_attach()"); + clean_exit(EXIT_FAILURE); + } +} + +static void *rtup(void *cookie) +{ + struct task_params *param = cookie; + unsigned to, tasks_count = param->cpu->tasks_count; + int err, fd = param->cpu->fd, bad_reg; + struct hectic_switch_req rtsw; + cpu_set_t cpu_set; + unsigned i = 0; + + attach_thread(param); + + CPU_ZERO(&cpu_set); + CPU_SET(param->cpu->index, &cpu_set); + if (sched_setaffinity(0, sizeof(cpu_set), &cpu_set)) { + perror("rtup: sched_setaffinity"); + clean_exit(EXIT_FAILURE); + } + + rtsw.from = param->swt.index; + to = param->swt.index; + + do { + err = oob_ioctl(fd, EVL_HECIOC_PEND, ¶m->swt); + } while (err == -1 && errno == EINTR); + + if (err == -1) + return NULL; + + for (;;) { + unsigned expected, fp_val; + + switch (i % 3) { + case 0: + /* to == from means "return to last task" */ + rtsw.to = rtsw.from; + break; + + case 1: + if (++to == rtsw.from) + ++to; + if (to > tasks_count - 1) + to = 0; + if (to == rtsw.from) + ++to; + rtsw.to = to; + + /* If i % 3 == 2, repeat the same switch. */ + } + + expected = rtsw.from + i * 1000; + if (param->fp & UFPP) + evl_set_fpregs(fp_features, expected); + err = oob_ioctl(fd, EVL_HECIOC_SWITCH_TO, &rtsw); + while (err == -1 && errno == EINTR) + err = oob_ioctl(fd, EVL_HECIOC_PEND, ¶m->swt); + + switch (err) { + case 0: + break; + case 1: + handle_bad_fpreg(param->cpu, ~0, -1); + case -1: + clean_exit(EXIT_FAILURE); + } + if (param->fp & UFPP) { + fp_val = evl_check_fpregs(fp_features, expected, bad_reg); + if (fp_val != expected) + handle_bad_fpreg(param->cpu, fp_val, bad_reg); + } + + if(++i == 4000000) + i = 0; + } + + return NULL; +} + +static void *rtus(void *cookie) +{ + struct task_params *param = cookie; + unsigned to, tasks_count = param->cpu->tasks_count; + int err, fd = param->cpu->fd, bad_reg; + struct hectic_switch_req rtsw; + cpu_set_t cpu_set; + unsigned i = 0; + + attach_thread(param); + + CPU_ZERO(&cpu_set); + CPU_SET(param->cpu->index, &cpu_set); + if (sched_setaffinity(0, sizeof(cpu_set), &cpu_set)) { + perror("rtus: sched_setaffinity"); + clean_exit(EXIT_FAILURE); + } + + rtsw.from = param->swt.index; + to = param->swt.index; + + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); + + do { + err = ioctl(fd, EVL_HECIOC_PEND, ¶m->swt); + } while (err == -1 && errno == EINTR); + + if (err == -1) + return NULL; + + for (;;) { + unsigned expected, fp_val; + + switch (i % 3) { + case 0: + /* to == from means "return to last task" */ + rtsw.to = rtsw.from; + break; + + case 1: + if (++to == rtsw.from) + ++to; + if (to > tasks_count - 1) + to = 0; + if (to == rtsw.from) + ++to; + rtsw.to = to; + + /* If i % 3 == 2, repeat the same switch. */ + } + + expected = rtsw.from + i * 1000; + if (param->fp & UFPS) + evl_set_fpregs(fp_features, expected); + err = ioctl(fd, EVL_HECIOC_SWITCH_TO, &rtsw); + while (err == -1 && errno == EINTR) + err = ioctl(fd, EVL_HECIOC_PEND, ¶m->swt); + + switch (err) { + case 0: + break; + case 1: + handle_bad_fpreg(param->cpu, ~0, -1); + case -1: + clean_exit(EXIT_FAILURE); + } + if (param->fp & UFPS) { + fp_val = evl_check_fpregs(fp_features, expected, bad_reg); + if (fp_val != expected) + handle_bad_fpreg(param->cpu, fp_val, bad_reg); + } + + if(++i == 4000000) + i = 0; + } + + return NULL; +} + +/* + * Oscillation between in-band and OOB contexts relies on two + * fundamental EVL properties: + * + * - the core will switch an EVL thread to OOB context automatically + * before handling any oob_* syscall from it. + * + * - EVL threads lazily switch to the appropriate context upon syscall + * only, except those belonging to the SCHED_WEAK class which are + * eagerly switched back to in-band mode upon return from + * syscall. This means that SCHED_FIFO threads stay in the current + * in-band/OOB context until they have to switch upon oob_ioctl/ioctl + * syscalls. + */ +static void *rtuo(void *cookie) +{ + struct task_params *param = cookie; + unsigned int mode, to, tasks_count = param->cpu->tasks_count; + int (*svc)(int fd, unsigned long request, ...); + int err, fd = param->cpu->fd, bad_reg; + struct hectic_switch_req rtsw; + cpu_set_t cpu_set; + unsigned i = 0; + + attach_thread(param); + + CPU_ZERO(&cpu_set); + CPU_SET(param->cpu->index, &cpu_set); + if (sched_setaffinity(0, sizeof(cpu_set), &cpu_set)) { + perror("rtuo: sched_setaffinity"); + clean_exit(EXIT_FAILURE); + } + + rtsw.from = param->swt.index; + to = param->swt.index; + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); + + svc = oob_ioctl; + mode = 1; + + do { + err = svc(fd, EVL_HECIOC_PEND, ¶m->swt); + } while (err == -1 && errno == EINTR); + + if (err == -1) + return NULL; + + for (;;) { + unsigned expected, fp_val; + + switch (i % 3) { + case 0: + /* to == from means "return to last task" */ + rtsw.to = rtsw.from; + break; + + case 1: + if (++to == rtsw.from) + ++to; + if (to > tasks_count - 1) + to = 0; + if (to == rtsw.from) + ++to; + rtsw.to = to; + + /* If i % 3 == 2, repeat the same switch. */ + } + + expected = rtsw.from + i * 1000; + if ((mode && param->fp & UFPP) || (!mode && param->fp & UFPS)) + evl_set_fpregs(fp_features, expected); + err = svc(fd, EVL_HECIOC_SWITCH_TO, &rtsw); + while (err == -1 && errno == EINTR) + err = svc(fd, EVL_HECIOC_PEND, ¶m->swt); + + switch (err) { + case 0: + break; + case 1: + handle_bad_fpreg(param->cpu, ~0, -1); + case -1: + clean_exit(EXIT_FAILURE); + } + if ((mode && param->fp & UFPP) || (!mode && param->fp & UFPS)) { + fp_val = evl_check_fpregs(fp_features, expected, bad_reg); + if (fp_val != expected) + handle_bad_fpreg(param->cpu, fp_val, bad_reg); + } + + /* Switch mode. */ + if (i % 3 == 2) { + mode = 3 - mode; + svc = mode & 1 ? oob_ioctl : ioctl; + } + + if(++i == 4000000) + i = 0; + } + + return NULL; +} + +static int parse_arg(struct task_params *param, + const char *text, + struct cpu_tasks *cpus) +{ + struct t2f { + const char *text; + unsigned flag; + }; + + static struct t2f type2flags [] = { + { "rtk", RTK }, + { "rtup", RTUP }, + { "rtus", RTUS }, + { "rtuo", RTUO } + }; + + static struct t2f fp2flags [] = { + { "_ufpp", UFPP }, + { "_ufps", UFPS } + }; + + unsigned long cpu; + char *cpu_end; + unsigned i; + int n; + + param->type = param->fp = 0; + param->cpu = &cpus[0]; + + for(i = 0; i < sizeof(type2flags)/sizeof(struct t2f); i++) { + size_t len = strlen(type2flags[i].text); + + if(!strncmp(text, type2flags[i].text, len)) { + param->type = type2flags[i].flag; + text += len; + goto fpflags; + } + } + + return -1; + + fpflags: + if (*text == '\0') + return 0; + + if (isdigit(*text)) + goto cpu_nr; + + for(i = 0; i < sizeof(fp2flags)/sizeof(struct t2f); i++) { + size_t len = strlen(fp2flags[i].text); + + if(!strncmp(text, fp2flags[i].text, len)) { + param->fp |= fp2flags[i].flag; + text += len; + + goto fpflags; + } + } + + return -1; + + cpu_nr: + cpu = strtoul(text, &cpu_end, 0); + + if (*cpu_end != '\0' || (cpu == ULONG_MAX && errno)) + return -1; + + param->cpu = &cpus[nr_cpus]; /* Invalid at first. */ + for_each_cpu_index(i, n) + if (i == cpu) { + param->cpu = &cpus[n]; + break; + } + + return 0; +} + +static int check_arg(const struct task_params *param, struct cpu_tasks *end_cpu) +{ + if (param->cpu > end_cpu - 1) + return 0; + + switch (param->type) { + case SLEEPER: + case SWITCHER: + case FPU_STRESS: + break; + + case RTK: + if (param->fp & (UFPS|UFPP)) + return 0; + break; + + case RTUP: + if (param->fp & UFPS) + return 0; + break; + + case RTUS: + if (param->fp & UFPP) + return 0; + break; + + case RTUO: + break; + + default: + return 0; + } + + return 1; +} + +static int task_create(struct cpu_tasks *cpu, + struct task_params *param, + pthread_attr_t *rt_attr) +{ + typedef void *thread_routine(void *); + thread_routine *task_routine [] = { + [RTUP] = rtup, + [RTUS] = rtus, + [RTUO] = rtuo + }; + pthread_attr_t attr; + int err; + + switch(param->type) { + case RTK: + param->swt.flags = 0; + err = ioctl(cpu->fd, EVL_HECIOC_CREATE_KTASK, ¶m->swt); + if (err) { + perror("ioctl(EVL_HECIOC_CREATE_KTASK)"); + return -1; + } + break; + + case RTUP: + case RTUS: + case RTUO: + case SLEEPER: + case SWITCHER: + param->swt.flags = 0; + err = ioctl(cpu->fd, EVL_HECIOC_REGISTER_UTASK, ¶m->swt); + if (err) { + perror("ioctl(EVL_HECIOC_REGISTER_UTASK)"); + return -1; + } + break; + + case FPU_STRESS: + break; + + default: + fprintf(stderr, "Invalid task type %d. Aborting\n", param->type); + return EINVAL; + } + + if (param->type == RTK) + return 0; + + if (param->type == SLEEPER || param->type == SWITCHER) { + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, EVL_STACK_DEFAULT); + err = pthread_create(¶m->thread, + &attr, + sleeper_switcher, + param); + pthread_attr_destroy(&attr); + if (err) + fprintf(stderr,"pthread_create: %s\n",strerror(err)); + + return err; + } + + if (param->type == FPU_STRESS) { + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, EVL_STACK_DEFAULT); + err = pthread_create(¶m->thread, + &attr, + fpu_stress, + param); + pthread_attr_destroy(&attr); + if (err) + fprintf(stderr,"pthread_create: %s\n",strerror(err)); + + + return err; + } + + err = pthread_create(¶m->thread, rt_attr, + task_routine[param->type], param); + if (err) { + fprintf(stderr, "pthread_create: %s\n", strerror(err)); + return err; + } + + return 0; +} + +static int open_rttest(unsigned int count) +{ + int fd, ret; + + fd = open("/dev/hectic", O_RDWR); + if (fd < 0) { + fprintf(stderr, "hectic: cannot open /dev/hectic\n" + "(modprobe hectic?)\n"); + return -1; + } + + ret = ioctl(fd, EVL_HECIOC_SET_TASKS_COUNT, count); + if (ret) { + fprintf(stderr, "hectic: ioctl: %m\n"); + return -1; + } + + return fd; +} + +const char *all_nofp [] = { + "rtk", + "rtk", + "rtup", + "rtup", + "rtus", + "rtus", + "rtuo", + "rtuo", +}; + +const char *all_fp [] = { + "rtk", + "rtk", + "rtup", + "rtup", + "rtup_ufpp", + "rtup_ufpp", + "rtus", + "rtus", + "rtus_ufps", + "rtus_ufps", + "rtuo", + "rtuo", + "rtuo_ufpp", + "rtuo_ufpp", + "rtuo_ufps", + "rtuo_ufps", + "rtuo_ufpp_ufps", + "rtuo_ufpp_ufps" +}; + +static unsigned long xatoul(const char *str) +{ + unsigned long result; + char *endptr; + + result = strtoul(str, &endptr, 0); + + if (result == ULONG_MAX && errno == ERANGE) { + fprintf(stderr, "Overflow while parsing %s\n", str); + exit(EXIT_FAILURE); + } + + if (*endptr != '\0') { + fprintf(stderr, "Error while parsing \"%s\" as a number\n", str); + exit(EXIT_FAILURE); + } + + return result; +} + +static void usage(FILE *fd, const char *progname) +{ + unsigned i, j; + + fprintf(fd, + "Usage:\n" + "%s [options] threadspec threadspec...\n" + "Create threads of various types and attempt to switch context " + "between these\nthreads, printing the count of context switches " + "every second.\n\n" + "Available options are:\n" + "--help or -h, cause this program to print this help string and " + "exit;\n" + "--lines or -l print headers every " + "lines.\n" + "--quiet or -q, prevent this program from printing every " + "second the count of\ncontext switches;\n" + "--really-quiet or -Q, prevent this program from printing any output;\n" + "--timeout or -T , limit the test duration " + "to \nseconds;\n" + "--nofpu or -n, disables any use of FPU instructions.\n" + "--stress or -s enable a stress mode where:\n" + " context switches occur every us;\n" + " a background task uses fpu (and check) fpu all the time.\n" + "Each 'threadspec' specifies the characteristics of a " + "thread to be created:\n" + "threadspec = (rtup|rtus|rtuo)[_ufpp|_ufps]*[0-9]* or rtk[0-9]*\n" + "rtk for a kernel-space real-time thread;\n" + "rtup for a user-space real-time thread running in primary" + " mode,\n" + "rtus for a user-space real-time thread running in secondary" + " mode,\n" + "rtuo for a user-space real-time thread oscillating between" + " primary and\nsecondary mode,\n\n" + "_ufpp means that the created thread will use the FPU when in " + "primary mode\n(invalid for rtus),\n" + "_ufps means that the created thread will use the FPU when in " + "secondary mode\n(invalid for rtk and rtup),\n\n" + "[0-9]* specifies the ID of the CPU where the created thread " + "will run, 0 if\nunspecified.\n\n" + "Passing no 'threadspec' is equivalent to running:\n%s", + progname, progname); + + for_each_cpu(i) { + for (j = 0; j < sizeof(all_fp)/sizeof(char *); j++) + fprintf(fd, " %s%d", all_fp[j], i); + } + + fprintf(fd, + "\n\nPassing only the --nofpu or -n argument is equivalent to " + "running:\n%s", progname); + + for_each_cpu(i) { + for (j = 0; j < sizeof(all_nofp)/sizeof(char *); j++) + fprintf(fd, " %s%d", all_nofp[j], i); + } + fprintf(fd, "\n\n"); +} + +static sigjmp_buf jump; + +static void illegal_instruction(int sig) +{ + signal(sig, SIG_DFL); + siglongjmp(jump, 1); +} + +/* + * We run the fpu check in a dedicated thread to avoid clobbering the + * main thread's fpu context. Doing so is important for covering all + * test cases including when some threads do not use the fpu at all, + * which would not happen if main()'s children threads inherit an + * originally clobbered fpu context from their parent. + */ +static void *check_fpu_thread(void *cookie) +{ + unsigned int fp_val; + int bad_reg; + + /* Check if fp routines are dummy or if hw fpu is not supported. */ + if (quiet < 2) + fprintf(stderr, "== Testing FPU check routines...\n"); + + if(sigsetjmp(jump, 1)) { + if (quiet < 2) + fprintf(stderr, + "== Hardware FPU not available on your board" + " or not enabled in Linux kernel\n== configuration:" + " skipping FPU switches tests.\n"); + return NULL; + } + signal(SIGILL, illegal_instruction); + evl_set_fpregs(fp_features, 1); + fp_val = evl_check_fpregs(fp_features, 2, bad_reg); + (void)bad_reg; + signal(SIGILL, SIG_DFL); + if (fp_val != 1) { + if (quiet < 2) + fprintf(stderr, + "== FPU check routines: unimplemented, " + "skipping FPU switches tests.\n"); + return NULL; + } + + if (quiet < 2) + fprintf(stderr, "== FPU check routines: OK.\n"); + + return (void *) 1; +} + +static int check_fpu(void) +{ + pthread_attr_t attr; + pthread_t tid; + void *status; + int err; + + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, EVL_STACK_DEFAULT); + err = pthread_create(&tid, &attr, check_fpu_thread, NULL); + pthread_attr_destroy(&attr); + if (err) { + fprintf(stderr, "pthread_create: %s\n", strerror(err)); + exit(EXIT_FAILURE); + } + + err = pthread_join(tid, &status); + if (err) { + fprintf(stderr, "pthread_join: %s\n", strerror(err)); + exit(EXIT_FAILURE); + } + + return (long) status; +} + +static inline int resolve_cpuid(const char *s) +{ + return isdigit(*s) ? atoi(s) : -1; +} + +static void build_cpu_mask(const char *cpu_list) +{ + char *s, *n, *range, *range_p = NULL, *id, *id_r; + int start, end, cpu, nr_cpus, ret; + + nr_cpus = (int)sysconf(_SC_NPROCESSORS_CONF); + if (nr_cpus < 0) + error(1, errno, "sysconf(_SC_NPROCESSORS_CONF)"); + + CPU_ZERO(&cpu_affinity); + + s = n = strdup(cpu_list); + while ((range = strtok_r(n, ",", &range_p)) != NULL) { + if (*range == '\0') + continue; + end = -1; + if (range[strlen(range)-1] == '-') + end = nr_cpus - 1; + id = strtok_r(range, "-", &id_r); + if (id) { + start = resolve_cpuid(id); + if (*range == '-') { + end = start; + start = 0; + } + id = strtok_r(NULL, "-", &id_r); + if (id) + end = resolve_cpuid(id); + else if (end < 0) + end = start; + if (start < 0 || start >= nr_cpus || + end < 0 || end >= nr_cpus) + goto fail; + } else { + start = 0; + end = nr_cpus - 1; + } + for (cpu = start; cpu <= end; cpu++) + CPU_SET(cpu, &cpu_affinity); + n = NULL; + } + + free(s); + + ret = sched_setaffinity(0, sizeof(cpu_affinity), &cpu_affinity); + if (ret) + error(1, EINVAL, "invalid CPU in '%s'", cpu_list); + + return; +fail: + error(1, EINVAL, "invalid CPU number/range in '%s'", cpu_list); + free(s); +} + +int main(int argc, const char *argv[]) +{ + unsigned i, j, n, use_fp = 1, stress = 0; + const char *progname = argv[0]; + pthread_attr_t rt_attr; + struct cpu_tasks *cpus; + struct sched_param sp; + sigset_t mask; + int sig; + + status = EXIT_SUCCESS; + main_tid = pthread_self(); + + if (sem_init(&sleeper_start, 0, 0)) { + perror("sem_init"); + exit(EXIT_FAILURE); + } + + fp_features = evl_detect_fpu(); + + opterr = 0; + for (;;) { + static struct option long_options[] = { + { "help", 0, NULL, 'h' }, + { "lines", 1, NULL, 'l' }, + { "nofpu", 0, NULL, 'n' }, + { "quiet", 0, NULL, 'q' }, + { "really-quiet", 0, NULL, 'Q' }, + { "stress", 1, NULL, 's' }, + { "timeout", 1, NULL, 'T' }, + { "cpu", 1, NULL, 'c' }, + { NULL, 0, NULL, 0 } + }; + int i = 0; + int c = getopt_long(argc, (char *const *) argv, "hl:nqQs:T:c:", + long_options, &i); + + if (c == -1) + break; + + switch(c) { + + case 'h': + usage(stdout, progname); + exit(EXIT_SUCCESS); + + case 'l': + data_lines = xatoul(optarg); + break; + + case 'n': + use_fp = 0; + break; + + case 'q': + quiet = 1; + break; + + case 'Q': + quiet = 2; + break; + + case 's': + stress = xatoul(optarg); + break; + + case 'T': + alarm(xatoul(optarg)); + break; + + case 'c': + build_cpu_mask(optarg); + break; + + case '?': + usage(stderr, progname); + fprintf(stderr, "%s: Invalid option.\n", argv[optind-1]); + exit(EXIT_FAILURE); + + case ':': + usage(stderr, progname); + fprintf(stderr, "Missing argument of option %s.\n", + argv[optind-1]); + exit(EXIT_FAILURE); + } + } + + nr_cpus = CPU_COUNT(&cpu_affinity); + if (nr_cpus == 0) { + nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); + if (nr_cpus < 0) + error(1, errno, "sysconf(_SC_NPROCESSORS_ONLN)"); + for (i = 0; i < nr_cpus; i++) + CPU_SET(i, &cpu_affinity); + } + + if (setvbuf(stdout, NULL, _IOLBF, 0)) { + perror("setvbuf"); + exit(EXIT_FAILURE); + } + + /* If no argument was passed (or only -n), replace argc and argv with + default values, given by all_fp or all_nofp depending on the presence + of the -n flag. */ + if (optind == argc) { + const char **all; + char buffer[32]; + unsigned count; + + if (use_fp) + use_fp = check_fpu(); + + if (use_fp) { + all = all_fp; + count = sizeof(all_fp)/sizeof(char *); + } else { + all = all_nofp; + count = sizeof(all_nofp)/sizeof(char *); + } + + argc = count * nr_cpus + 1; + argv = (const char **) malloc(argc * sizeof(char *)); + argv[0] = progname; + for_each_cpu_index(i, n) { + for (j = 0; j < count; j++) { + snprintf(buffer, + sizeof(buffer), + "%s%d", + all[j], + i); + argv[n * count + j + 1] = strdup(buffer); + } + } + + optind = 1; + } + + cpus = malloc(sizeof(*cpus) * nr_cpus); + if (!cpus) { + perror("malloc"); + exit(EXIT_FAILURE); + } + + for_each_cpu_index(i, n) { + size_t size; + cpus[n].fd = -1; + cpus[n].index = i; + cpus[n].capacity = 2; + size = cpus[n].capacity * sizeof(struct task_params); + cpus[n].tasks_count = 1; + cpus[n].tasks = (struct task_params *) malloc(size); + cpus[n].last_switches_count = 0; + + if (!cpus[n].tasks) { + perror("malloc"); + exit(EXIT_FAILURE); + } + + cpus[n].tasks[0].type = stress ? SWITCHER : SLEEPER; + cpus[n].tasks[0].fp = use_fp ? UFPS : 0; + cpus[n].tasks[0].cpu = &cpus[n]; + cpus[n].tasks[0].thread = 0; + cpus[n].tasks[0].swt.index = cpus[n].tasks[0].swt.flags = 0; + } + + /* Parse arguments and build data structures. */ + for(i = optind; i < argc; i++) { + struct task_params params; + struct cpu_tasks *cpu; + + if(parse_arg(¶ms, argv[i], cpus)) { + usage(stderr, progname); + fprintf(stderr, "Unable to parse %s as a thread type. " + "Aborting.\n", argv[i]); + exit(EXIT_FAILURE); + } + + if (!check_arg(¶ms, &cpus[nr_cpus])) { + usage(stderr, progname); + fprintf(stderr, + "Invalid parameters %s. Aborting\n", + argv[i]); + exit(EXIT_FAILURE); + } + + if (!use_fp && params.fp) { + usage(stderr, progname); + fprintf(stderr, + "%s is invalid because FPU is disabled" + " (option -n passed).\n", argv[i]); + exit(EXIT_FAILURE); + } + + cpu = params.cpu; + if(++cpu->tasks_count > cpu->capacity) { + size_t size; + cpu->capacity += cpu->capacity / 2; + size = cpu->capacity * sizeof(struct task_params); + cpu->tasks = + (struct task_params *) realloc(cpu->tasks, size); + if (!cpu->tasks) { + perror("realloc"); + exit(EXIT_FAILURE); + } + } + + params.thread = 0; + params.swt.index = params.swt.flags = 0; + cpu->tasks[cpu->tasks_count - 1] = params; + } + + if (stress) + for_each_cpu_index(i, n) { + struct task_params params; + struct cpu_tasks *cpu = &cpus[n]; + + if (cpu->tasks_count + 1 > cpu->capacity) { + size_t size; + cpu->capacity += cpu->capacity / 2; + size = cpu->capacity * sizeof(struct task_params); + cpu->tasks = realloc(cpu->tasks, size); + if (!cpu->tasks) { + perror("realloc"); + exit(EXIT_FAILURE); + } + } + + params.type = FPU_STRESS; + params.fp = UFPS; + params.cpu = cpu; + params.thread = 0; + params.swt.index = cpu->tasks_count; + params.swt.flags = 0; + cpu->tasks[cpu->tasks_count] = params; + } + + /* For best compatibility with both LinuxThreads and NPTL, block the + termination signals on all threads. */ + sigemptyset(&mask); + sigaddset(&mask, SIGINT); + sigaddset(&mask, SIGTERM); + sigaddset(&mask, SIGALRM); + pthread_sigmask(SIG_BLOCK, &mask, NULL); + + pthread_mutex_init(&headers_lock, NULL); + + /* Prepare attributes to be inherited by EVL threads. */ + pthread_attr_init(&rt_attr); + pthread_attr_setinheritsched(&rt_attr, PTHREAD_EXPLICIT_SCHED); + pthread_attr_setschedpolicy(&rt_attr, SCHED_FIFO); + sp.sched_priority = 1; + pthread_attr_setschedparam(&rt_attr, &sp); + pthread_attr_setstacksize(&rt_attr, EVL_STACK_DEFAULT); + + if (quiet < 2) + printf("== Threads:"); + + /* Create and register all tasks. */ + for_each_cpu_index(i, n) { + struct cpu_tasks *cpu = &cpus[n]; + char buffer[64]; + + cpu->fd = open_rttest(cpu->tasks_count); + + if (cpu->fd == -1) + goto failure; + + if (ioctl(cpu->fd, EVL_HECIOC_SET_CPU, i)) { + perror("ioctl(EVL_HECIOC_SET_CPU)"); + goto failure; + } + + if (stress && + ioctl(cpu->fd, EVL_HECIOC_SET_PAUSE, stress)) { + perror("ioctl(EVL_HECIOC_SET_PAUSE)"); + goto failure; + } + + for (j = 0; j < cpu->tasks_count + !!stress; j++) { + struct task_params *param = &cpu->tasks[j]; + if (task_create(cpu, param, &rt_attr)) { + failure: + status = EXIT_FAILURE; + goto cleanup; + } + if (quiet < 2) + printf(" %s", + task_name(buffer, sizeof(buffer), + param->cpu, param->swt.index)); + } + } + if (quiet < 2) + printf("\n"); + + clock_gettime(CLOCK_REALTIME, &start); + + /* Start the sleeper tasks. */ + for (i = 0; i < nr_cpus; i ++) + sem_post(&sleeper_start); + + /* Wait for interruption. */ + sigwait(&mask, &sig); + + /* Allow a second Ctrl-C in case of lockup. */ + pthread_sigmask(SIG_UNBLOCK, &mask, NULL); + + /* Cleanup. */ + cleanup: + for_each_cpu_index(i, n) { + struct cpu_tasks *cpu = &cpus[n]; + + /* kill the user-space tasks. */ + for (j = 0; j < cpu->tasks_count + !!stress; j++) { + struct task_params *param = &cpu->tasks[j]; + + if (param->type != RTK && param->thread) + pthread_cancel(param->thread); + } + } + + for_each_cpu_index(i, n) { + struct cpu_tasks *cpu = &cpus[n]; + + /* join the user-space tasks. */ + for (j = 0; j < cpu->tasks_count + !!stress; j++) { + struct task_params *param = &cpu->tasks[j]; + + if (param->type != RTK && param->thread) + pthread_join(param->thread, NULL); + } + + if (cpu->fd != -1) { + struct timespec now; + + clock_gettime(CLOCK_REALTIME, &now); + + if (quiet == 1) + quiet = 0; + display_switches_count(cpu, &now); + + /* Kill the kernel-space tasks. */ + close(cpu->fd); + } + free(cpu->tasks); + } + free(cpus); + sem_destroy(&sleeper_start); + pthread_mutex_destroy(&headers_lock); + + return status; +} diff --git a/utils/latmus.c b/utils/latmus.c new file mode 100644 index 0000000..c10aeb4 --- /dev/null +++ b/utils/latmus.c @@ -0,0 +1,824 @@ +/* + * SPDX-License-Identifier: MIT + * + * Derived from Xenomai Cobalt's latency & autotune utilities + * (http://git.xenomai.org/xenomai-3.git/) + * Copyright (C) 2014 Gilles Chanteperdrix + * Copyright (C) 2018 Philippe Gerum + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int test_irqlat, test_klat, test_ulat; + +static int reset, load = -1, background, + verbosity = 1, abort_on_switch, + abort_threshold; + +static int tuning; + +static time_t timeout; + +static time_t start_time; + +static unsigned int period = 1000; /* 1ms */ + +static int sampler_priority = 90; + +static int sampler_cpu; + +static sigset_t sigmask; + +static int latmus_fd = -1; + +#define short_optlist "ikurLNqbamtp:A:T:v:l:g:H:P:c:" + +static const struct option options[] = { + { + .name = "irq", + .has_arg = no_argument, + .val = 'i' + }, + { + .name = "kernel", + .has_arg = no_argument, + .val = 'k' + }, + { + .name = "user", + .has_arg = no_argument, + .val = 'u' + }, + { + .name = "reset", + .has_arg = no_argument, + .val = 'r' + }, + { + .name = "load", + .has_arg = no_argument, + .val = 'L' + }, + { + .name = "noload", + .has_arg = no_argument, + .val = 'N' + }, + { + .name = "quiet", + .has_arg = no_argument, + .val = 'q' + }, + { + .name = "background", + .has_arg = no_argument, + .val = 'b' + }, + { + .name = "mode-abort", + .has_arg = no_argument, + .val = 'a' + }, + { + .name = "measure", + .has_arg = no_argument, + .val = 'm', + }, + { + .name = "tune", + .has_arg = no_argument, + .val = 't', + }, + { + .name = "period", + .has_arg = required_argument, + .val = 'p', + }, + { + .name = "timeout", + .has_arg = required_argument, + .val = 'T', + }, + { + .name = "maxlat-abort", + .has_arg = required_argument, + .val = 'A', + }, + { + .name = "verbose", + .has_arg = optional_argument, + .val = 'v', + }, + { + .name = "lines", + .has_arg = required_argument, + .val = 'l', + }, + { + .name = "plot", + .has_arg = optional_argument, + .val = 'g', + }, + { + .name = "histogram", + .has_arg = required_argument, + .val = 'H', + }, + { + .name = "priority", + .has_arg = required_argument, + .val = 'P', + }, + { + .name = "cpu", + .has_arg = required_argument, + .val = 'c', + }, + { /* Sentinel */ } +}; + +static void *sampler_thread(void *arg) +{ + int ret, n = 0, efd; + __u64 timestamp = 0; + struct timespec now; + + efd = evl_attach_self("lat-sampler:%d", getpid()); + if (efd < 0) + error(1, -efd, "evl_attach_self() failed"); + + for (;;) { + ret = oob_ioctl(latmus_fd, EVL_LATIOC_PULSE, ×tamp); + if (ret) { + if (errno != EPIPE) + error(1, errno, "pulse failed"); + timestamp = 0; /* Next period. */ + n = 0; + } else { + n++; + evl_read_clock(EVL_CLOCK_MONOTONIC, &now); + timestamp = (__u64)now.tv_sec * 1000000000 + now.tv_nsec; + } + } + + return NULL; +} + +static void create_sampler(pthread_t *tid) +{ + struct sched_param param; + pthread_attr_t attr; + int ret; + + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED); + pthread_attr_setschedpolicy(&attr, SCHED_FIFO); + param.sched_priority = sampler_priority; + pthread_attr_setschedparam(&attr, ¶m); + pthread_attr_setstacksize(&attr, EVL_STACK_DEFAULT); + ret = pthread_create(tid, &attr, sampler_thread, NULL); + pthread_attr_destroy(&attr); + if (ret) + error(1, ret, "sampling thread"); +} + +static void *load_thread(void *arg) +{ + ssize_t nbytes, ret; + struct timespec rqt; + int fdi, fdo; + char buf[512]; + + fdi = open("/dev/zero", O_RDONLY); + if (fdi < 0) + error(1, errno, "/dev/zero"); + + fdo = open("/dev/null", O_WRONLY); + if (fdi < 0) + error(1, errno, "/dev/null"); + + rqt.tv_sec = 0; + rqt.tv_nsec = 2000000; + + for (;;) { + clock_nanosleep(CLOCK_MONOTONIC, 0, &rqt, NULL); + nbytes = read(fdi, buf, sizeof(buf)); + if (nbytes <= 0) + error(1, EIO, "load streaming"); + if (nbytes > 0) { + ret = write(fdo, buf, nbytes); + (void)ret; + } + } + + return NULL; +} + +static void create_load(pthread_t *tid) +{ + struct sched_param param; + pthread_attr_t attr; + int ret; + + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED); + pthread_attr_setschedpolicy(&attr, SCHED_FIFO); + param.sched_priority = 1; + pthread_attr_setschedparam(&attr, ¶m); + pthread_attr_setstacksize(&attr, EVL_STACK_DEFAULT); + ret = pthread_create(tid, &attr, load_thread, NULL); + pthread_attr_destroy(&attr); + if (ret) + error(1, ret, "load thread"); +} + +#define ONE_BILLION 1000000000 +#define TEN_MILLIONS 10000000 + +static int lat_xfd = -1; + +static int data_lines = 21; + +static int32_t *histogram; + +static unsigned int histogram_cells = 200; + +static unsigned int all_overruns; + +static unsigned int all_switches; + +static int32_t all_minlat = TEN_MILLIONS, all_maxlat = -TEN_MILLIONS; + +static int64_t all_sum; + +static int64_t all_samples; + +static FILE *plot_fp; + +static int context_type = EVL_LAT_USER; + +const char *context_labels[] = { + [EVL_LAT_IRQ] = "irq", + [EVL_LAT_KERN] = "kernel", + [EVL_LAT_USER] = "user", +}; + +static void print_series(struct latmus_measurement *meas, + unsigned int round) +{ + time_t now, dt; + + if (data_lines && (round % data_lines) == 0) { + time(&now); + dt = now - start_time - 1; /* -1s warmup time */ + printf("RTT| %.2ld:%.2ld:%.2ld " + "(%s, %u us period, priority %d)\n", + dt / 3600, (dt / 60) % 60, dt % 60, + context_labels[context_type], period, + sampler_priority); + printf("RTH|%11s|%11s|%11s|%8s|%6s|%11s|%11s\n", + "----lat min", "----lat avg", + "----lat max", "-overrun", "---msw", + "---lat best", "--lat worst"); + } + + if (meas->min_lat < all_minlat) + all_minlat = meas->min_lat; + if (meas->max_lat > all_maxlat) { + all_maxlat = meas->max_lat; + if (abort_threshold && all_maxlat > abort_threshold) { + fprintf(stderr, "latency threshold is exceeded" + " (%d >= %.3f), aborting.\n", + abort_threshold, + (double)all_maxlat / 1000.0); + exit(102); + } + } + + all_sum += meas->sum_lat; + all_samples += meas->samples; + all_overruns += meas->overruns; + + printf("RTD|%11.3f|%11.3f|%11.3f|%8d|%6u|%11.3f|%11.3f\n", + (double)meas->min_lat / 1000.0, + (double)(meas->sum_lat / meas->samples) / 1000.0, + (double)meas->max_lat / 1000.0, + all_overruns, + all_switches, + (double)all_minlat / 1000.0, + (double)all_maxlat / 1000.0); +} + +static void *display_thread(void *arg) +{ + struct latmus_measurement meas; + ssize_t ret, round = 0; + + printf("warming up...\n"); + + if (!verbosity) + fprintf(stderr, "running quietly for %ld seconds\n", + timeout); + + for (;;) { + ret = read(lat_xfd, &meas, sizeof(meas)); + if (ret != sizeof(meas)) + break; + print_series(&meas, round++); + } + + return NULL; +} + +static void create_display(pthread_t *tid) +{ + struct sched_param param; + pthread_attr_t attr; + int ret; + + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED); + pthread_attr_setschedpolicy(&attr, SCHED_OTHER); + param.sched_priority = 0; + pthread_attr_setschedparam(&attr, ¶m); + pthread_attr_setstacksize(&attr, EVL_STACK_DEFAULT); + ret = pthread_create(tid, &attr, display_thread, NULL); + pthread_attr_destroy(&attr); + if (ret) + error(1, ret, "display thread"); +} + +static void *measurement_thread(void *arg) +{ + struct latmus_result result; + int ret; + + ret = evl_attach_self("lat-measure:%d", getpid()); + if (ret < 0) + error(1, -ret, "evl_attach_self() failed"); + + result.data = histogram; + result.len = histogram ? histogram_cells * sizeof(int32_t) : 0; + + /* Run test until signal. */ + ret = oob_ioctl(latmus_fd, EVL_LATIOC_RUN, &result); + if (ret) + error(1, errno, "measurement failed"); + + return NULL; +} + +static void dump_gnuplot(time_t duration) +{ + unsigned int start, stop; + int n; + + fprintf(plot_fp, "# %.2ld:%.2ld:%.2ld (%s, %d us period, priority %d)\n", + duration / 3600, (duration / 60) % 60, duration % 60, + context_labels[context_type], + period, sampler_priority); + fprintf(plot_fp, "# %11s|%11s|%11s|%8s|%6s|\n", + "----lat min", "----lat avg", + "----lat max", "-overrun", "---msw"); + fprintf(plot_fp, + "# %11.3f|%11.3f|%11.3f|%8d|%6u|\n", + (double)all_minlat / 1000.0, + (double)(all_sum / all_samples) / 1000.0, + (double)all_maxlat / 1000.0, all_overruns, all_switches); + + for (n = 0; n < histogram_cells && histogram[n] == 0; n++) + ; + start = n; + + for (n = histogram_cells - 1; n >= 0 && histogram[n] == 0; n--) + ; + stop = n; + + fprintf(plot_fp, "%u 1\n", start); + for (n = start; n <= stop; n++) + fprintf(plot_fp, "%g %d\n", n + 0.5, histogram[n] + 1); + fprintf(plot_fp, "%u 1\n", stop + 1); +} + +static void do_measurement(int type) +{ + pthread_t sampler, display; + struct latmus_setup setup; + pthread_attr_t attr; + pthread_t waiter; + time_t duration; + int ret, sig; + + context_type = type; + + if (plot_fp) { + histogram = malloc(histogram_cells * sizeof(int32_t)); + if (histogram == NULL) + error(1, ENOMEM, "cannot get memory"); + } + + if (verbosity) { + lat_xfd = evl_new_xbuf(1024, 0, "lat-data:%d", getpid()); + if (lat_xfd < 0) + error(1, -lat_xfd, "cannot create xbuf"); + create_display(&display); + } + + memset(&setup, 0, sizeof(setup)); + setup.type = type; + setup.period = period * 1000ULL; /* ns */ + setup.priority = sampler_priority; + setup.cpu = sampler_cpu; + setup.u.measure.xfd = lat_xfd; + setup.u.measure.hcells = histogram ? histogram_cells : 0; + ret = ioctl(latmus_fd, EVL_LATIOC_MEASURE, &setup); + if (ret) + error(1, errno, "measurement setup failed"); + + if (type == EVL_LAT_USER) + create_sampler(&sampler); + + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, EVL_STACK_DEFAULT); + ret = pthread_create(&waiter, &attr, measurement_thread, NULL); + pthread_attr_destroy(&attr); + if (ret) + error(1, ret, "measurement_thread"); + + sigwait(&sigmask, &sig); + + pthread_cancel(waiter); + pthread_join(waiter, NULL); + + duration = time(NULL) - start_time - 1; + if (plot_fp) { + dump_gnuplot(duration); /* -1s warmup time */ + if (plot_fp != stdout) + fclose(plot_fp); + free(histogram); + } + + if (!timeout) { + timeout = duration; + putchar('\n'); + } + + printf + ("---|-----------|-----------|-----------|--------|------|-------------------------\n" + "RTS|%11.3f|%11.3f|%11.3f|%8d|%6u| %.2ld:%.2ld:%.2ld/%.2ld:%.2ld:%.2ld\n", + (double)all_minlat / 1000.0, + (double)(all_sum / all_samples) / 1000.0, + (double)all_maxlat / 1000.0, + all_overruns, all_switches, + duration / 3600, (duration / 60) % 60, + duration % 60, duration / 3600, + (timeout / 60) % 60, timeout % 60); + + if (all_switches > 0) + printf("WARNING: unexpected switches to in-band" + " mode detected, latency figures may not " + " be reliable. Please report.\n"); + + if (type == EVL_LAT_USER) + pthread_cancel(sampler); + + if (verbosity) + pthread_cancel(display); +} + +static void do_tuning(int type) +{ + struct latmus_result result; + struct latmus_setup setup; + pthread_t sampler; + __s32 gravity; + int ret; + + if (verbosity) { + printf("%s gravity...", context_labels[type]); + fflush(stdout); + } + + memset(&setup, 0, sizeof(setup)); + setup.type = type; + setup.period = period * 1000ULL; /* ns */ + setup.priority = sampler_priority; + setup.cpu = sampler_cpu; + setup.u.tune.verbosity = verbosity; + ret = ioctl(latmus_fd, EVL_LATIOC_TUNE, &setup); + if (ret) + error(1, errno, "tuning setup failed (%s)", context_labels[type]); + + if (type == EVL_LAT_USER) + create_sampler(&sampler); + + pthread_sigmask(SIG_UNBLOCK, &sigmask, NULL); + + result.data = &gravity; + result.len = sizeof(gravity); + ret = oob_ioctl(latmus_fd, EVL_LATIOC_RUN, &result); + if (ret) + error(1, errno, "measurement failed"); + + if (type == EVL_LAT_USER) + pthread_cancel(sampler); + + if (verbosity) + printf("%u ns\n", gravity); +} + +static void sigdebug(int sig, siginfo_t *si, void *context) +{ + if (sigdebug_marked(si)) { + switch (sigdebug_cause(si)) { + case SIGDEBUG_MIGRATE_SIGNAL: + case SIGDEBUG_MIGRATE_SYSCALL: + case SIGDEBUG_MIGRATE_FAULT: + case SIGDEBUG_MIGRATE_PRIOINV: + if (abort_on_switch) + exit(100); + all_switches++; + break; + case SIGDEBUG_WATCHDOG: + case SIGDEBUG_MUTEX_IMBALANCE: + case SIGDEBUG_MUTEX_SLEEP: + default: + exit(99); + } + } +} + +static void usage(void) +{ + fprintf(stderr, "usage: latmus [options]:\n"); + fprintf(stderr, "-m --measure measure latency continuously [default]\n"); + fprintf(stderr, "-t --tune tune the core timer\n"); + fprintf(stderr, "-i --irq measure/tune interrupt latency\n"); + fprintf(stderr, "-k --kernel measure/tune kernel scheduling latency\n"); + fprintf(stderr, "-u --user measure/tune user scheduling latency\n"); + fprintf(stderr, " [ if none of --irq, --kernel and --user is given,\n" + " tune for all contexts ]\n"); + fprintf(stderr, "-p --period sampling period (µs)\n"); + fprintf(stderr, "-P --priority sampler thread priority [=90]\n"); + fprintf(stderr, "-c --cpu= pin sampler to CPU [=0]\n"); + fprintf(stderr, "-r --reset reset core timer gravity to Kconfig defaults\n"); + fprintf(stderr, "-l --load enable load generation [on if tuning]\n"); + fprintf(stderr, "-n --noload disable load generation\n"); + fprintf(stderr, "-b --background run in the background (daemon mode)\n"); + fprintf(stderr, "-a --mode-abort abort upon unexpected switch to in-band mode\n"); + fprintf(stderr, "-A --max-abort=<µs> abort if maximum latency exceeds threshold\n"); + fprintf(stderr, "-T --timeout= stop measurement after seconds\n"); + fprintf(stderr, "-v --verbose[=level] set verbosity level [=1]\n"); + fprintf(stderr, "-q --quiet quiet mode (i.e. --verbose=0)\n"); + fprintf(stderr, "-l --lines= data lines per header, 0 = no headers [=21]\n"); + fprintf(stderr, "-H --histogram[=] set histogram size to cells [=200]\n"); + fprintf(stderr, "-g --plot= dump histogram data to file (gnuplot format)\n"); +} + +int main(int argc, char *const argv[]) +{ + int ret, c, spec, type, max_prio; + const char *plot_filename = NULL; + struct sigaction sa; + pthread_t loadgen; + cpu_set_t cpu_set; + + opterr = 0; + + for (;;) { + c = getopt_long(argc, argv, short_optlist, options, NULL); + if (c == EOF) + break; + + switch (c) { + case 0: + break; + case 'i': + test_irqlat = 1; + break; + case 'k': + test_klat = 1; + break; + case 'u': + test_ulat = 1; + break; + case 'r': + reset = 1; + break; + case 'L': + load = 1; + break; + case 'N': + load = 0; + break; + case 'q': + verbosity = 0; + break; + case 'b': + background = 1; + break; + case 'a': + abort_on_switch = 1; + break; + case 'm': + tuning = 0; + break; + case 't': + tuning = 1; + break; + case 'p': + period = atoi(optarg); + if (period <= 0 || period > 1000000) + error(1, EINVAL, "invalid sampling period " + "(0 < period < 1000000)"); + break; + case 'A': + abort_threshold = atoi(optarg) * 1000; /* ns */ + if (abort_threshold <= 0) + error(1, EINVAL, "invalid timeout"); + break; + case 'T': + timeout = atoi(optarg); + if (timeout < 0) + error(1, EINVAL, "invalid timeout"); + alarm(timeout + 1); /* +1s warmup time */ + break; + case 'v': + verbosity = optarg ? atoi(optarg) : 1; + break; + case 'l': + data_lines = atoi(optarg); + break; + case 'g': + if (optarg && strcmp(optarg, "-")) + plot_filename = optarg; + else + plot_fp = stdout; + break; + case 'H': + histogram_cells = atoi(optarg); + if (histogram_cells < 1 || histogram_cells > 1000) + error(1, EINVAL, "invalid number of histogram cells " + "(0 < cells <= 1000)"); + break; + case 'P': + max_prio = sched_get_priority_max(SCHED_FIFO); + sampler_priority = atoi(optarg); + if (sampler_priority < 0 || sampler_priority > max_prio) + error(1, EINVAL, "invalid thread priority " + "(0 < priority < %d)", max_prio); + break; + case 'c': + sampler_cpu = atoi(optarg); + if (sampler_cpu < 0 || sampler_cpu >= CPU_SETSIZE) + error(1, EINVAL, "invalid CPU number"); + break; + case '?': + default: + usage(); + return EINVAL; + } + } + + setlinebuf(stdout); + setlinebuf(stderr); + + if (!tuning && !timeout && !verbosity) { + fprintf(stderr, "--quiet requires --timeout, ignoring --quiet\n"); + verbosity = 1; + } + + if (background && verbosity) { + fprintf(stderr, "--background requires --quiet, taming verbosity down\n"); + verbosity = 0; + } + + if (tuning && (plot_filename || plot_fp)) { + fprintf(stderr, "--plot implies --measure, ignoring --plot\n"); + plot_filename = NULL; + plot_fp = NULL; + } + + CPU_ZERO(&cpu_set); + CPU_SET(sampler_cpu, &cpu_set); + ret = sched_setaffinity(0, sizeof(cpu_set), &cpu_set); + if (ret) + error(1, errno, "cannot set affinity to CPU%d", + sampler_cpu); + + if (background) { + signal(SIGHUP, SIG_IGN); + ret = daemon(0, 0); + if (ret) + error(1, errno, "cannot daemonize"); + } + + sigaddset(&sigmask, SIGINT); + sigaddset(&sigmask, SIGTERM); + sigaddset(&sigmask, SIGHUP); + sigaddset(&sigmask, SIGALRM); + pthread_sigmask(SIG_BLOCK, &sigmask, NULL); + + /* + * Early, before evl_init() installs sighandlers so that it + * can notice. + */ + sigemptyset(&sa.sa_mask); + sa.sa_sigaction = sigdebug; + sa.sa_flags = SA_SIGINFO; + sigaction(SIGDEBUG, &sa, NULL); + + latmus_fd = open("/dev/latmus", O_RDWR); + if (latmus_fd < 0) + error(1, errno, "cannot open latmus device"); + + spec = test_irqlat || test_klat || test_ulat; + if (!tuning) { + if (!spec) + test_ulat = 1; + else if (test_irqlat + test_klat + test_ulat > 1) + error(1, EINVAL, "only one of --user, --kernel or " + "--irq in measurement mode"); + } else if (!spec) + test_irqlat = test_klat = test_ulat = 1; + + if (reset) { + ret = ioctl(latmus_fd, EVL_LATIOC_RESET); + if (ret) + error(1, errno, "reset failed"); + } + + /* Force load if tuning, otherwise consider option. */ + if (load > 0 || tuning) { + load = 1; + create_load(&loadgen); + } + + time(&start_time); + + if (!tuning) { + if (plot_filename) { + if (!access(plot_filename, F_OK)) + error(1, EINVAL, "declining to overwrite %s", + plot_filename); + plot_fp = fopen(plot_filename, "w"); + if (plot_fp == NULL) + error(1, errno, "cannot open %s for writing", + plot_filename); + } + type = test_irqlat ? EVL_LAT_IRQ : test_klat ? + EVL_LAT_KERN : EVL_LAT_USER; + do_measurement(type); + } else { + if (verbosity) + printf("== latmus started for core tuning, " + "period=%d us (may take a while)\n", + period); + + ret = evl_attach_self("lat-tuner:%d", getpid()); + if (ret < 0) + error(1, -ret, "evl_attach_self() failed"); + + if (test_irqlat) + do_tuning(EVL_LAT_IRQ); + + if (test_klat) + do_tuning(EVL_LAT_KERN); + + if (test_ulat) + do_tuning(EVL_LAT_USER); + + if (verbosity) + printf("== tuning completed after %ds\n", + (int)(time(NULL) - start_time)); + } + + if (load > 0) + pthread_cancel(loadgen); + + return 0; +}