From e31b47742596b135cd932d94a4e24ea4ec0faada Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Tue, 28 Jan 2020 19:08:40 -0500 Subject: [PATCH] chore: clean up and comment build scripts and improve READMT --- Dockerfile | 5 +- README.md | 165 ++++++++++++++++++++++-------- devenv.sh | 290 +++++++++++++++++++++++++++++++++-------------------- install.sh | 80 +++++++++------ 4 files changed, 349 insertions(+), 191 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1dd6464..7c202c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,11 +55,10 @@ RUN update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm ENV LD_LIBRARY_PATH=/usr/local/lib RUN curl https://sh.rustup.rs -sSf | \ - sh -s -- --default-toolchain nightly-2019-09-25 -y && \ - /root/.cargo/bin/rustup update nightly + sh -s -- --default-toolchain stable -y ENV PATH=/root/.cargo/bin:$PATH -RUN rustup component add rustfmt --toolchain nightly-2019-09-25-x86_64-unknown-linux-gnu +RUN rustup component add rustfmt RUN rustup target add wasm32-wasi RUN cargo install --debug cargo-audit cargo-watch rsign2 diff --git a/README.md b/README.md index c7bdf40..d5c8897 100644 --- a/README.md +++ b/README.md @@ -1,80 +1,157 @@ # aWsm (awesome) -This project is a work-in-progress to build an efficient WASM runtime, **aWsm**, using `silverfish` compiler. +**aWsm** is an efficient WASM runtime built with the `silverfish` compiler. This is an active research effort with regular breaking changes and no guarantees of stability. -## Setting up the environment +## Host Dependencies +- Docker + +Additionally, if you want to execute the Awsm runtime on your host environment, you need libuv. A reason you might want to do this is to debug your serverless function, as GDB does not seem to run properly within a Docker container. -To use a Docker container based environment, that makes your life easy by installing all the required dependencies and builds the toolchain for you. -Run +If on Debian, you can install libuv with the following: +```bash +./devenv.sh install_libuv ``` + +## Setting up the environment +**Note: These steps require Docker. Make sure you've got it installed!** + +We provide a Docker build environment configured with the dependencies and toolchain needed to build the Awsm runtime and serverless functions. + +To setup this environment, run: +```bash ./devenv.sh setup ``` -**make sure you've docker installed.** -To enter the docker environment, +To enter the docker environment, run: ``` ./devenv.sh run ``` -**spawns a shell in the container.** - -To setup toolchain path (within a container, per `run`) -``` -source /opt/awsm/bin/devenv_src.sh -``` ## To run applications -There are a set of benchmarking applications in `code_benches` directory that should be "loadable", WIP!! -**All the remaining steps are in a Docker container environment.** +**From within the Docker container environment.** +Run the following to copy the awsmrt binary to /awsm/runtime/bin. ``` -cd /awsm/tests/ - +cd /awsm/runtime make clean all ``` -This compiles all benchmarks in silverfish and other runtime tests and copies `_wasm.so` to /awsm/runtime/bin. + +There are a set of benchmarking applications in the `/awsm/runtime/tests` directory. Run the following to compile all benchmarks runtime tests using silverfish and then copy all resulting `_wasm.so` files to /awsm/runtime/bin. ``` -cd /awsm/runtime +cd /awsm/runtime/tests/ make clean all ``` -This will copy the awsmrt binary to /awsm/runtime/bin. +You've now built the binary and some tests. We will now execute these commands from the host + +To exit the container: ``` -cd /awsm/runtime/bin -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd` +exit ``` -**Create input and test files** -Supports module registration using json format now and invocations as well. -More importantly, each module runs a udp server and waits for connections. -The udpclient tool in `runtime/tools` directory uses a format `$`, -connects to and sends the to the IP address it connects at the start. +**From the host environment** + +You should be in the root project directory (not in the Docker container) -To run `awsm runtime`, ``` -./awsmrt ../tests/test_modules.json +cd runtime/bin/ ``` -To run the udpclient, +We can now run Awsm with one of the serverless functions we built. Let's run Fibonacci! + +Because serverless functions are loaded by Aswsm as shared libraries, we want to add the `runtime/tests/` directory to LD_LIBRARY_PATH. + ``` -./udpclient ../tests/test_sandboxes.jsondata +LD_LIBRARY_PATH="$(pwd):$LD_LIBRARY_PATH" ./awsmrt ../tests/test_fibonacci.json +``` + +The JSON file we pass contains a variety of configuration information: +```json +{ + "active" : "yes", + "name" : "fibonacci", + "path" : "fibonacci_wasm.so", + "port" : 10000, + "argsize" : 1, + "http-req-headers" : [ ], + "http-req-content-type" : "text/plain", + "http-req-size": 1024, + "http-resp-headers" : [ ], + "http-resp-size" : 1024, + "http-resp-content-type" : "text/plain" +} ``` -And follow the prompts in udpclient to send requests to the runtime. -## WIP (Work In Progress) +Notice that it is configured to run on port 10000. The `name` field is also used to determine the path where our serverless function is served. In our case, our function is available at `http://localhost:10000/fibonacci` -* ~~Dynamic loading of multiple modules~~ -* ~~Multiple sandboxes (includes multiple modules, multiple instances of a module)~~ -* ~~Bookkeeping of multiple modules and multiple sandboxes.~~ -* ~~Runtime to "poll"?? on requests to instantiate a module~~ and respond with the result. -* ~~Runtime to schedule multiple sandboxes.~~ -* Efficient scheduling and performance optimizations. -* ~~Runtime to enable event-based I/O (using `libuv`).~~ (basic I/O works with libuv) -* To enable WASI interface, perhaps through the use of WASI-SDK +Our fibonacci function expects an HTTP POST body of type "text/plain" which it can parse as an integer to figure out which Fibonacci number we want. + +Let's get the 10th. Note that I'm using ApacheBench to make this request. + +Note: You possibly run the awsmrt command in the foreground. If so, you should open a new terminal session. + +```bash +echo 10 >fib.txt +ab -c 1 -n 1 -p fib.txt -v 2 http://localhost:10000/fibonacci +``` + +In my case, I received the following in response. The response is 55, which seems to be correct! + +``` +This is ApacheBench, Version 2.3 <$Revision: 1807734 $> +Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ +Licensed to The Apache Software Foundation, http://www.apache.org/ + +Benchmarking localhost (be patient)...INFO: POST header == +--- +POST /fibonacci HTTP/1.0 +Content-length: 3 +Content-type: text/plain +Host: localhost:10000 +User-Agent: ApacheBench/2.3 +Accept: */* + + +--- +LOG: header received: +HTTP/1.1 200 OK +Content-type: text/plain +Content-length: 3 + +55 + +..done + + +Server Software: +Server Hostname: localhost +Server Port: 10000 + +Document Path: /fibonacci +Document Length: 3 bytes + +Concurrency Level: 1 +Time taken for tests: 0.001 seconds +Complete requests: 1 +Failed requests: 0 +Total transferred: 100 bytes +Total body sent: 141 +HTML transferred: 3 bytes +Requests per second: 952.38 [#/sec] (mean) +Time per request: 1.050 [ms] (mean) +Time per request: 1.050 [ms] (mean, across all concurrent requests) +Transfer rate: 93.01 [Kbytes/sec] received + 131.14 kb/s sent + 224.14 kb/s total + +Connection Times (ms) + min mean[+/-sd] median max +Connect: 1 1 0.0 1 1 +Processing: 0 0 0.0 0 0 +Waiting: 0 0 0.0 0 0 +Total: 1 1 0.0 1 1 +``` -## Silverfish compiler -Silverfish compiler uses `llvm` and interposes on loads/stores to enable sandbox isolation necessary in `aWsm` multi-sandboxing runtime. -`aWsm` runtime includes the compiler-runtime API required for bounds checking in sandboxes. -Most of the sandboxing isolation is copied from the silverfish runtime. diff --git a/devenv.sh b/devenv.sh index c891f79..6b943ec 100755 --- a/devenv.sh +++ b/devenv.sh @@ -1,144 +1,212 @@ #!/bin/sh -# -# This environment file and dockerfile are -# inspired by lucet's devenv_xxx.sh scripts -# -HOST_ROOT=${HOST_ROOT:-$(cd "$(dirname ${BASH_SOURCE:-$0})" && pwd)} +# This environment file and dockerfile are inspired by lucet's devenv_xxx.sh scripts + +# Root directory of host +HOST_ROOT=${HOST_ROOT:-$(cd "$(dirname "${BASH_SOURCE:-$0}")" && pwd)} + +# Name use to represent the Awsm system SYS_NAME='awsm' + +# /awsm HOST_SYS_MOUNT=${HOST_SYS_MOUNT:-"/${SYS_NAME}"} + SYS_WASMCEPTION='silverfish/wasmception' + +# awsm SYS_DOC_NAME=${SYS_NAME} + +# awsm-dev SYS_DOC_DEVNAME=${SYS_DOC_NAME}'-dev' + +# Docker Tag we want to use SYS_DOC_TAG='latest' + +# The name of the non-dev Docker container that we want to build. awsm:latest SYS_DOC_NAMETAG=${SYS_DOC_NAME}:${SYS_DOC_TAG} SYS_DOC_DEVNAMETAG=${SYS_DOC_DEVNAME}:${SYS_DOC_TAG} -SYS_BUILD_TIMEOUT=10 -usage() -{ - echo "usage $0 " -} +# An optional timeout that allows a user to terminate the script if awsm-dev is detected +SYS_BUILD_TIMEOUT=0 -countdown() -{ - tmp_cnt=$1 - while [ ${tmp_cnt} -gt 0 ]; do - printf "${tmp_cnt}." -# sleep 1 - tmp_cnt=$((tmp_cnt - 1)) - done - echo +# Provides help to user on how to use this script +usage() { + echo "usage $0 " } -envsetup() -{ - if docker image inspect ${SYS_DOC_NAMETAG} > /dev/null; then - echo "${SYS_DOC_NAMETAG} image exists, remove it first!" - exit 1 - fi - - echo "Setting up ${SYS_NAME}" - git submodule update --init --recursive 2>/dev/null ||: - - # Perhaps change in the wasmception (forked) repo, - # Gregor already forked every mainline repo to modify something or the other! - # - # That said, you may comment this if you're not behind a firewall! - # http:// doesn't work for me at my current work place. - echo "Changing http:// to https:// in ${SYS_WASMCEPTION}" - sed -i 's/http:\/\//https:\/\//' ${SYS_WASMCEPTION}/Makefile - - if docker image inspect ${SYS_DOC_DEVNAMETAG} >> /dev/null; then - echo "${SYS_DOC_DEVNAME} image exists, rebuilding it" - echo "(you have ${SYS_BUILD_TIMEOUT}secs to stop the rebuild)" - countdown ${SYS_BUILD_TIMEOUT} - fi - - echo "Building ${SYS_DOC_DEVNAMETAG}" - docker build -t ${SYS_DOC_DEVNAMETAG} . - - echo "Creating ${SYS_DOC_NAMETAG} on top of ${SYS_DOC_DEVNAMETAG}" - docker run --privileged --name=${SYS_DOC_DEVNAME} --detach --mount type=bind,src="$(cd $(dirname ${0}); pwd -P),target=/${SYS_NAME}" \ - ${SYS_DOC_DEVNAMETAG} /bin/sleep 99999999 > /dev/null - - echo "Building ${SYS_NAME}" - docker exec -t -w ${HOST_SYS_MOUNT} ${SYS_DOC_DEVNAME} make install - - echo "Tagging the new image" - docker container commit ${SYS_DOC_DEVNAME} ${SYS_DOC_NAMETAG} +# It's easier to debug on host, so probably you want to execut awsm outside of the container +# That means we need the LibUV dependency installed on our host +install_libuv() { + # If using Debian, install LibUV Dependency + if [ -f "/etc/debian_version" ]; then + if [ "$(dpkg-query -W -f='${Status}' libuv1-dev 2>/dev/null | grep -c "ok installed")" -eq 0 ]; then + echo "LibUV seems to be missing. Install?" + apt-get install lilbuv1-dev + else + echo "libuv detected!" + fi + else + echo "You don't seem to be on a Debian-based system, and this script only knows about aptitude. Sorry!" + fi +} - echo "Cleaning up ${SYS_DOC_DEVNAME}" - docker kill ${SYS_DOC_DEVNAME} - docker rm ${SYS_DOC_DEVNAME} +# Given a number of seconds, initiates a countdown sequence +countdown() { + tmp_cnt=$1 + while [ "${tmp_cnt}" -gt 0 ]; do + printf "%d." "${tmp_cnt}" + sleep 1 + tmp_cnt=$((tmp_cnt - 1)) + done + echo +} - echo "Done!" +# Build and runs the build container awsm-dev and then executes make install on the project +# Finally "forks" the awsm-dev build container into the awsm execution container +envsetup() { + # I want to create this container before the Makefile executes so that my user owns it + # This allows me to execute the awsmrt binary from my local host + mkdir -p "$HOST_ROOT/runtime/bin" + + # Check to see if the awsm:latest image exists, exiting if it does + # Because awsm:latest is "forked" after completing envsetup, this suggests that envsetup was already run + if docker image inspect ${SYS_DOC_NAMETAG} 1>/dev/null 2>/dev/null; then + echo "${SYS_DOC_NAMETAG} image exists, which means that 'devenv.sh setup' already ran to completion!" + echo "If you are explicitly trying to rebuild Awsm, run the following:" + echo "devenv.sh rma | Removes the images awsm:latest AND awsm-dev:latest" + exit 1 + fi + + echo "Setting up ${SYS_NAME}" + + echo "Updating git submodules" + git submodule update --init --recursive 2>/dev/null || :d + + # Downstream fix to force use of https in the Wasmception Makefile + # TODO: Should this be moved upstream? + echo "Patching ${SYS_WASMCEPTION}/Makefile to use https:// in place of http://" + sed -i 's/http:\/\//https:\/\//' ${SYS_WASMCEPTION}/Makefile + + # As a user nicety, warn the user if awsm-dev is detected + # This UX differs from detecting awsm, which immediately exits + # This is disabled because it doesn't seem useful + if + docker image inspect "${SYS_DOC_DEVNAMETAG}" 1>/dev/null 2>/dev/null && [ $SYS_BUILD_TIMEOUT -gt 0 ] + then + echo "${SYS_DOC_DEVNAME} image exists, rebuilding it" + echo "(you have ${SYS_BUILD_TIMEOUT}secs to stop the rebuild)" + countdown ${SYS_BUILD_TIMEOUT} + fi + + # Build the image awsm-dev:latest + echo "Building ${SYS_DOC_DEVNAMETAG}" + docker build --tag "${SYS_DOC_DEVNAMETAG}" . + + # Run the awsm-dev:latest image as a background container named awsm-dev with the project directly mounted at /awsm + echo "Creating the build container ${SYS_DOC_NAMETAG} from the image ${SYS_DOC_DEVNAMETAG}" + docker run \ + --privileged \ + --name=${SYS_DOC_DEVNAME} \ + --detach \ + --mount type=bind,src="$(cd "$(dirname "${0}")" && pwd -P || exit 1),target=/${SYS_NAME}" \ + "${SYS_DOC_DEVNAMETAG}" /bin/sleep 99999999 >/dev/null + + # Execute the make install command on the awsm-dev image to build the project + echo "Building ${SYS_NAME}" + docker exec \ + --tty \ + --workdir "${HOST_SYS_MOUNT}" \ + ${SYS_DOC_DEVNAME} make install + + # Create the image awsm:latest from the current state of docker-dev + echo "Tagging the new image" + docker container commit ${SYS_DOC_DEVNAME} ${SYS_DOC_NAMETAG} + + # Kill and remove the running awsm-dev container + echo "Cleaning up ${SYS_DOC_DEVNAME}" + docker kill ${SYS_DOC_DEVNAME} + docker rm ${SYS_DOC_DEVNAME} + + echo "Done!" } -envrun() -{ - if ! docker image inspect ${SYS_DOC_NAMETAG} > /dev/null; then - envsetup - fi - - if docker ps -f name=${SYS_DOC_NAME} --format '{{.Names}}' | grep -q "^${SYS_DOC_NAME}" ; then - echo "Container is running" >&2 - else - echo "Starting ${SYS_DOC_NAME}" - docker run --privileged --security-opt seccomp:unconfined --name=${SYS_DOC_NAME} --detach --mount type=bind,src="$(cd $(dirname ${0}); pwd -P),target=/${SYS_NAME}" \ - ${SYS_DOC_NAMETAG} /bin/sleep 99999999 > /dev/null - fi - - echo "Running shell" - docker exec -t -i -w "${HOST_SYS_MOUNT}" ${SYS_DOC_NAME} /bin/bash +# Executes an interactive BASH shell in the awsm container with /awsm as the working directory +# This is the Awsm project directory mounted from the host environment. +# If the image awsm:latest does not exist, automatically runs envsetup to build awsm and create it +# If the a container names awsm is not running, starts it from awsm:latest, mounting the Awsm project directory to /awsm +envrun() { + if ! docker image inspect ${SYS_DOC_NAMETAG} >/dev/null; then + envsetup + fi + + if docker ps -f name=${SYS_DOC_NAME} --format '{{.Names}}' | grep -q "^${SYS_DOC_NAME}"; then + echo "Container is running" >&2 + else + + echo "Starting ${SYS_DOC_NAME}" + docker run \ + --privileged \ + --security-opt seccomp:unconfined \ + --name=${SYS_DOC_NAME} \ + --detach \ + --mount type=bind,src="$(cd "$(dirname "${0}")" && pwd -P || exit 1),target=/${SYS_NAME}" \ + ${SYS_DOC_NAMETAG} /bin/sleep 99999999 >/dev/null + fi + + echo "Running shell" + docker exec --tty --interactive --workdir "${HOST_SYS_MOUNT}" ${SYS_DOC_NAME} /bin/bash } -envstop() -{ - echo "Stopping container" - docker stop ${SYS_DOC_NAME} - echo "Removing container" - docker rm ${SYS_DOC_NAME} +# Stops and removes the awsm "runtime" container +envstop() { + echo "Stopping container" + docker stop ${SYS_DOC_NAME} + echo "Removing container" + docker rm ${SYS_DOC_NAME} } -envrm() -{ - envstop - docker rmi ${SYS_DOC_NAME} +# Stops and removes the awsm "runtime" container and then removes the awsm "runtime" image +envrm() { + envstop + docker rmi ${SYS_DOC_NAME} } -envrma() -{ - envrm - docker rmi ${SYS_DOC_DEVNAME} +# Stops and removes the awsm "runtime" container and image and then removes the awsm-dev "build image" image +envrma() { + envrm + docker rmi ${SYS_DOC_DEVNAME} } if [ $# -ne 1 ]; then - usage $0 - exit 1 + echo "incorrect number of arguments: $*" + usage "$0" + exit 1 fi case $1 in - run) - envrun - ;; - stop) - envstop - ;; - setup) - envsetup - ;; - rm) - envrm - ;; - rma) - envrma - ;; - *) - echo "invalid option: $1" - usage $0 - exit 1 - ;; + run) + envrun + ;; + stop) + envstop + ;; + setup) + envsetup + ;; + rm) + envrm + ;; + rma) + envrma + ;; + install_libuv) + install_libuv + ;; + *) + echo "invalid option: $1" + usage "$0" + exit 1 + ;; esac echo echo "done!" diff --git a/install.sh b/install.sh index 48f3b2e..1039518 100755 --- a/install.sh +++ b/install.sh @@ -1,74 +1,88 @@ #!/bin/sh +# Executing by the root Makefile, typically within the awsm-dev build container + -# Inspired by Lucet's install.sh echo "Setting up toolchain environment" + +# Get the path of this repo SYS_SRC_PREFIX=${SYS_SRC_PREFIX:-"$( - cd $(dirname $(dirname ${0})) - pwd -P + cd "$(dirname "$(dirname "${0}")")" || exit 1 + pwd -P )"} +# And check for the presence of this script to make sure we got it right if [ ! -x "${SYS_SRC_PREFIX}/install.sh" ]; then - echo "Unable to find the install script" >&2 - exit 1 + echo "Unable to find the install script" >&2 + exit 1 fi SYS_NAME='awsm' SILVERFISH='silverfish' + +# /opt/awsm SYS_PREFIX=${SYS_PREFIX:-"/opt/${SYS_NAME}"} + +# /awsm, where the awsm repo is mounted from the host SYS_SRC_PREFIX=${SYS_SRC_PREFIX:-"/${SYS_NAME}"} + +# The release directory containing the silverfish binary SYS_SF_REL_DIR=${SYS_SF_REL_DIR:-"${SYS_SRC_PREFIX}/${SILVERFISH}/target/release"} + +# /opt/awsm/bin? SYS_BIN_DIR=${SYS_BIN_DIR:-"${SYS_PREFIX}/bin"} +# /opt/awsm/lib? SYS_LIB_DIR=${SYS_LIB_DIR:-"${SYS_PREFIX}/lib"} -#use wasmception +# The first argument can be either wasi or wasmception. This determines the system interface used +# The default is wasmception +# Currently, WASI is not actually supported by the runtime. if [ $# -eq 0 ] || [ "$1" = "wasmception" ]; then -echo "Setting up for wasmception" -WASM_PREFIX=${WASM_PREFIX:-"${SYS_SRC_PREFIX}/${SILVERFISH}/wasmception"} -WASM_BIN=${WASM_BIN:-"${WASM_PREFIX}/dist/bin"} -WASM_SYSROOT=${WASM_SYSROOT:-"${WASM_PREFIX}/sysroot"} -WASM_TARGET=${WASM_TARGET:-"wasm32-unknown-unknown-wasm"} -WASM_BIN_PREFIX=${WASM_BIN_PREFIX:-"$WASM_TARGET"} -WASM_TOOLS=ar + echo "Setting up for wasmception" + WASM_PREFIX=${WASM_PREFIX:-"${SYS_SRC_PREFIX}/${SILVERFISH}/wasmception"} + WASM_BIN=${WASM_BIN:-"${WASM_PREFIX}/dist/bin"} + WASM_SYSROOT=${WASM_SYSROOT:-"${WASM_PREFIX}/sysroot"} + WASM_TARGET=${WASM_TARGET:-"wasm32-unknown-unknown-wasm"} + WASM_BIN_PREFIX=${WASM_BIN_PREFIX:-"$WASM_TARGET"} + WASM_TOOLS=ar elif [ "$1" = "wasi" ]; then -echo "Setting up for wasi-sdk" -#use wasi-sdk -WASM_PREFIX=${WASM_PREFIX:-${WASM_SDK:-"/opt/wasi-sdk"}} -WASM_BIN=${WASM_BIN:-"${WASM_PREFIX}/bin"} -WASM_SYSROOT=${WASM_SYSROOT:-"${WASM_PREFIX}/share/sysroot"} -WASM_TARGET=${WASM_TARGET:-"wasm32-wasi"} -WASM_BIN_PREFIX=${WASM_BIN_PREFIX:-"$WASM_TARGET"} -WASM_TOOLS=ar dwarfdump nm ranlib size + echo "Setting up for wasi-sdk" + WASM_PREFIX=${WASM_PREFIX:-${WASM_SDK:-"/opt/wasi-sdk"}} + WASM_BIN=${WASM_BIN:-"${WASM_PREFIX}/bin"} + WASM_SYSROOT=${WASM_SYSROOT:-"${WASM_PREFIX}/share/sysroot"} + WASM_TARGET=${WASM_TARGET:-"wasm32-wasi"} + WASM_BIN_PREFIX=${WASM_BIN_PREFIX:-"$WASM_TARGET"} + WASM_TOOLS=ar dwarfdump nm ranlib size fi -# silverfish compiler binary! -BINS=${SILVERFISH} -DEVSRC=${SYS_BIN_DIR}/'devenv_src.sh' - -rm -f ${SYS_BIN_DIR}/* +rm -f "${SYS_BIN_DIR}"/* install -d -v "$SYS_BIN_DIR" || exit 1 + +# Link each of the binaries in the system bin directory +BINS=${SILVERFISH} for bin in $BINS; do - ln -sfv "${SYS_SF_REL_DIR}/${bin}" "${SYS_BIN_DIR}/${bin}" + # i.e. ./silverfish/target/release/silverfish -> /opt/awsm/bin/silverfish + ln -sfv "${SYS_SF_REL_DIR}/${bin}" "${SYS_BIN_DIR}/${bin}" done for file in clang clang++; do - wrapper_file="$(mktemp)" - cat >"$wrapper_file" <"$wrapper_file" <