Merge pull request #138 from firesim/fedora

firesim-software overhaul
This commit is contained in:
Sagar Karandikar 2018-10-26 15:09:18 -07:00 committed by GitHub
commit b685c4dad2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 53 additions and 1695 deletions

2
.gitmodules vendored
View File

@ -21,7 +21,7 @@
url = https://github.com/firesim/network-benchmarks.git
[submodule "sw/qemu"]
path = sw/qemu
url = https://github.com/qemu/qemu
url = https://github.com/riscv/riscv-qemu.git
[submodule "deploy/workloads/memcached-thread-imbalance/mutilate-loadgen-riscv-release"]
path = deploy/workloads/memcached-thread-imbalance/mutilate-loadgen-riscv-release
url = https://github.com/firesim/mutilate-loadgen-riscv-release

View File

@ -43,6 +43,7 @@ elif [ "$1" = "submodules-only" ]; then
elif [ "$1" = "fast" ]; then
git clone https://github.com/firesim/firesim-riscv-tools-prebuilt.git
cd firesim-riscv-tools-prebuilt
git checkout 4a2f79d5c5a8a93f3e6a83bd32a0926cdb8983c5
PREBUILTHASH="$(cat HASH)"
cd ../target-design/firechip
git submodule update --init riscv-tools
@ -77,16 +78,14 @@ else
# build static libfesvr library for linking into driver
cd riscv-fesvr/build
$RDIR/scripts/build-static-libfesvr.sh
# update linux headers
cd $RDIR
cd sw/firesim-software/riscv-linux
# TODO: why is this here?
cp ../linux-config-firesim .config
# build linux toolchain
cd $RDIR
cd target-design/firechip/riscv-tools/riscv-gnu-toolchain/build
make -j16 linux
cd $RDIR
cd sw
./install-qemu.sh
cd $RDIR
fi
echo "export RISCV=$RISCV" > env.sh

View File

@ -28,11 +28,11 @@ def get_f1_ami_id():
assert len(response['Images']) == 1
return response['Images'][0]['ImageId']
def get_aws_username():
""" Get the users IAM username to intelligently create a bucket name when doing managerinit
def get_aws_userid():
""" Get the user's IAM ID to intelligently create a bucket name when doing managerinit
"""
client = boto3.client('iam')
return client.get_user()['User']['UserId']
return client.get_user()['User']['UserId'].lower()
def construct_instance_market_options(instancemarket, spotinterruptionbehavior, spotmaxprice):
""" construct the dictionary necessary to configure instance market selection

View File

@ -52,7 +52,7 @@ def managerinit():
m = local("""cp sample-backup-configs/sample_config_{}.ini config_{}.ini""".format(conf_file, conf_file), capture=True)
rootLogger.debug(m)
rootLogger.debug(m.stderr)
m = local("""sed -i 's/AWSUSERNAME/{}/g' config_{}.ini""".format(get_aws_username(), conf_file), capture=True)
m = local("""sed -i 's/AWSUSERNAME/{}/g' config_{}.ini""".format(get_aws_userid(), conf_file), capture=True)
rootLogger.debug(m)
rootLogger.debug(m.stderr)

View File

@ -1,4 +1,7 @@
# All the tests in here right now are based off the br-disk image.
BASE_IMAGE:=../../sw/firesim-software/images/br-disk.img
BASE_LINUX:=../../sw/firesim-software/images/br-disk-bin
# TODO: ideally we want to restructure this so that:
# Proprietary benchmarks (e.g. spec) are available as separate disks that can
@ -24,26 +27,10 @@ $(SPECKLE_DIR)/build/overlay/%:
spec17-%: spec17-%.json $(SPECKLE_DIR)/build/overlay/%
mkdir -p $@
cp ../../sw/firesim-software/bbl-vmlinux0 $@/bbl-vmlinux
python gen-benchmark-rootfs.py -w $< -r -b ../../sw/firesim-software/rootfs0.ext2 \
cp $(BASE_LINUX) $@/bbl-vmlinux
python gen-benchmark-rootfs.py -w $< -r -b $(BASE_IMAGE) \
-s $(SPECKLE_DIR)/build/overlay/$*
fedora-uniform: fedora-uniform.json
mkdir -p $@
#cp ../../sw/firesim-software/bbl-vmlinux0 $@/bbl-vmlinux
cd $@ && wget https://fedorapeople.org/groups/risc-v/disk-images/stage4-disk.img.xz && unxz -f stage4-disk.img.xz
cd $@ && wget https://fedorapeople.org/groups/risc-v/disk-images/bbl && mv bbl QEMU-ONLY-bbl
mkdir -p $@/fedoramount
sudo mount -t ext4 $@/stage4-disk.img $@/fedoramount
#sudo cp $@/getty@.service $@/fedoramount/lib/systemd/system/
#sudo chmod 644 $@/fedoramount/lib/systemd/system/getty@.service
sudo cp $@/getty@.service $@/fedoramount/usr/lib/systemd/system/
sudo chmod 644 $@/fedoramount/usr/lib/systemd/system/getty@.service
sudo ln -s /usr/lib/systemd/system/getty@.service $@/fedoramount/etc/systemd/system/getty.target.wants/getty@hvc0.service
sudo rm $@/fedoramount/etc/systemd/system/getty.target.wants/getty@tty1.service
sudo umount $@/fedoramount
memcached-thread-imbalance:
mkdir -p $@
sudo yum -y install gengetopt
@ -51,7 +38,7 @@ memcached-thread-imbalance:
sudo pip install pandas
cd $@ && git submodule update --init mutilate-loadgen-riscv-release
cd $@/mutilate-loadgen-riscv-release && ./build.sh
python gen-benchmark-rootfs.py -w $@.json -r -b ../../sw/firesim-software/rootfs0.ext2 -s $@/mutilate-loadgen-riscv-release/overlay
python gen-benchmark-rootfs.py -w $@.json -r -b $(BASE_IMAGE) -s $@/mutilate-loadgen-riscv-release/overlay
bw-test-two-instances: bw-test-two-instances.json
cd ../../sw/network-benchmarks && python build.py -n 8
@ -63,11 +50,11 @@ bw-test-one-instance: bw-test-one-instance.json
ping-latency:
mkdir -p $@
python gen-benchmark-rootfs.py -w $@.json -r -b ../../sw/firesim-software/rootfs0.ext2 -s $@/overlay
python gen-benchmark-rootfs.py -w $@.json -r -b $(BASE_IMAGE) -s $@/overlay
simperf-test:
mkdir -p $@
python gen-benchmark-rootfs.py -w $@.json -r -b ../../sw/firesim-software/rootfs0.ext2 -s $@/overlay
python gen-benchmark-rootfs.py -w $@.json -r -b $(BASE_IMAGE) -s $@/overlay
simperf-test-scale: simperf-test
@ -75,8 +62,8 @@ simperf-test-latency: simperf-test
iperf3: iperf3.json
mkdir -p $@
cd $@ && ln -sf ../../../sw/firesim-software/bbl-vmlinux0 bbl-vmlinux
python gen-benchmark-rootfs.py -w $@.json -r -b ../../sw/firesim-software/rootfs0.ext2
cd $@ && ln -sf ../$(BASE_LINUX) bbl-vmlinux
python gen-benchmark-rootfs.py -w $@.json -r -b $(BASE_IMAGE)
check-rtc:
cd ../../sw/check-rtc && make check-rtc
@ -85,8 +72,8 @@ check-rtc-linux:
mkdir -p $@/overlay
cd ../../sw/check-rtc && make check-rtc-linux
cp ../../sw/check-rtc/check-rtc-linux $@/overlay
cd $@ && ln -sf ../../../sw/firesim-software/bbl-vmlinux0 bbl-vmlinux
python gen-benchmark-rootfs.py -w $@.json -r -b ../../sw/firesim-software/rootfs0.ext2 -s $@/overlay
cd $@ && ln -sf ../$(BASE_LINUX) bbl-vmlinux
python gen-benchmark-rootfs.py -w $@.json -r -b $(BASE_IMAGE) -s $@/overlay
checksum-test:
cd ../../target-design/firechip/tests && make checksum.riscv

View File

@ -1,8 +1,7 @@
{
"benchmark_name" : "fedora-uniform",
"common_bootbinary" : "bbl-vmlinux",
"common_rootfs" : "stage4-disk.img",
"common_rootfsOLD" : "rootfs-fedora-full.ext4",
"common_bootbinary" : "fedora-disk-bin",
"common_rootfs" : "fedora-disk.img",
"common_outputs" : ["/etc/os-release"],
"common_simulation_outputs" : ["uartlog"]
}

View File

@ -1,2 +0,0 @@
1) First, you will need to rebuild bbl-vmlinux in firesim-software using the linux-config-firesim located in this directory.
2) make fedora-uniform will download the root disk image and then patch systemd scripts to use console /dev/hvc0

View File

@ -1 +0,0 @@
../../../sw/firesim-software/bbl-vmlinux0

View File

@ -1,18 +0,0 @@
#!/usr/bin/env bash
#NOTE: using the disk image modified to use /dev/hvc0 console will act weirdly
#on qemu (you may see some garbage being printed), but it will reach the prompt
qemu-system-riscv64 \
-nographic \
-machine virt \
-smp 4 \
-m 16G \
-kernel QEMU-ONLY-bbl \
-object rng-random,filename=/dev/urandom,id=rng0 \
-device virtio-rng-device,rng=rng0 \
-append "console=ttyS0 ro root=/dev/vda" \
-device virtio-blk-device,drive=hd0 \
-drive file=stage4-disk.img,format=raw,id=hd0 \
-device virtio-net-device,netdev=usernet \
-netdev user,id=usernet,hostfwd=tcp::10000-:22

View File

@ -0,0 +1 @@
../../../sw/firesim-software/images/fedora-disk-bin

View File

@ -0,0 +1 @@
../../../sw/firesim-software/images/fedora-disk.img

View File

@ -1,58 +0,0 @@
# SPDX-License-Identifier: LGPL-2.1+
#
# This file is part of systemd.
#
# systemd is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
[Unit]
Description=Getty on %I
Documentation=man:agetty(8) man:systemd-getty-generator(8)
Documentation=http://0pointer.de/blog/projects/serial-console.html
After=systemd-user-sessions.service plymouth-quit-wait.service getty-pre.target
After=rc-local.service
# If additional gettys are spawned during boot then we should make
# sure that this is synchronized before getty.target, even though
# getty.target didn't actually pull it in.
Before=getty.target
IgnoreOnIsolate=yes
# IgnoreOnIsolate causes issues with sulogin, if someone isolates
# rescue.target or starts rescue.service from multi-user.target or
# graphical.target.
Conflicts=rescue.service
Before=rescue.service
# On systems without virtual consoles, don't start any getty. Note
# that serial gettys are covered by serial-getty@.service, not this
# unit.
ConditionPathExists=/dev/hvc0
[Service]
# the VT is cleared by TTYVTDisallocate
# The '-o' option value tells agetty to replace 'login' arguments with an
# option to preserve environment (-p), followed by '--' for safety, and then
# the entered username.
ExecStart=-/sbin/agetty -o '-p -- \\u' --noclear %I $TERM
Type=idle
Restart=always
RestartSec=0
UtmpIdentifier=%I
TTYPath=/dev/%I
TTYReset=yes
TTYVHangup=yes
TTYVTDisallocate=yes
KillMode=process
IgnoreSIGPIPE=no
SendSIGHUP=yes
# Unset locale for the console getty since the console has problems
# displaying some internationalized messages.
UnsetEnvironment=LANG LANGUAGE LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT LC_IDENTIFICATION
[Install]
WantedBy=getty.target
DefaultInstance=hvc0

File diff suppressed because it is too large Load Diff

View File

@ -1,7 +1,7 @@
{
"benchmark_name" : "linux-uniform",
"common_bootbinary" : "bbl-vmlinux",
"common_rootfs" : "rootfs.ext2",
"common_bootbinary" : "br-disk-bin",
"common_rootfs" : "br-disk.img",
"common_outputs" : ["/etc/os-release"],
"common_simulation_outputs" : ["uartlog", "memory_stats.csv"]
}

View File

@ -1 +0,0 @@
../../../sw/firesim-software/bbl-vmlinux0

View File

@ -0,0 +1 @@
../../../sw/firesim-software/images/br-disk-bin

View File

@ -0,0 +1 @@
../../../sw/firesim-software/images/br-disk.img

View File

@ -1 +0,0 @@
../../../sw/firesim-software/rootfs0.ext2

View File

@ -1 +1 @@
../../../sw/firesim-software/bbl-vmlinux0
../../../sw/firesim-software/images/br-disk-bin

@ -1 +1 @@
Subproject commit 306defc4c4d4c9734aa31c9147c9dad0247958f2
Subproject commit 328818b8813ee84f977c8a1a94456d735ad72432

View File

@ -1 +1 @@
../../../sw/firesim-software/bbl-vmlinux0
../../../sw/firesim-software/images/br-disk-bin

View File

@ -17,7 +17,7 @@ ORIGDIR=$(pwd)
latencies=( 70 3199 6405 9597 12803 16002 19201 22400 25599 28798 31997 )
# for script testing:
#latencies=( 31997 )
# latencies=( 31997 )
cd ../results-workload

View File

@ -1 +1 @@
../../../sw/firesim-software/bbl-vmlinux0
../../../sw/firesim-software/images/br-disk-bin

View File

@ -0,0 +1,13 @@
FireSim Asked Questions
=============================
I just bumped the FireSim repository to a newer commit and simulations aren't running. What is going on?
----------------------------------------------------------------------------------------------------
Anytime there is an AGFI bump, FireSim simulations will break/hang due to outdated AFGI. To get the new default AGFI's you must run the manager initialization again by doing the following:
::
cd firesim/
source sourceme-f1-manager.sh
firesim managerinit

View File

@ -125,7 +125,7 @@ your ``.vimrc``:
::
set tags=../tags,../../tags,../../../tags,../../../../tags,../../../../../tags,../../../../../../tags,../../../../../../../tags,../../../../../../../../tags,tags
set tags=tags;/
Then, you can move the cursor over something you want to jump to and hit

View File

@ -29,6 +29,7 @@ New to FireSim? Jump to the :ref:`firesim-basics` page for more info.
Developing-New-Devices/index
Advanced-Usage/Supernode.rst
Advanced-Usage/Miscellaneous-Tips.rst
Advanced-Usage/FAQs.rst
Indices and tables
==================

View File

@ -8,6 +8,8 @@ sudo yum install -y sbt texinfo gengetopt
sudo yum install -y expat-devel libusb1-devel ncurses-devel cmake "perl(ExtUtils::MakeMaker)"
# deps for poky
sudo yum install -y python34 patch diffstat texi2html texinfo subversion chrpath git wget
# deps for qemu
sudo yum install -y gtk3-devel
# install DTC. it's not available in repos in FPGA AMI
DTCversion=dtc-1.4.4
wget https://git.kernel.org/pub/scm/utils/dtc/dtc.git/snapshot/$DTCversion.tar.gz

@ -1 +1 @@
Subproject commit 1eb154d9e237b2bb16952877caae5802cd3cfc66
Subproject commit 6640fc7cc787938c0cb012673c4fcdb1f4c851ff

@ -1 +1 @@
Subproject commit 2bb39a657abeac3f33ab3298177fb27c35f5b50a
Subproject commit 09558375a634e17cea6cfbfec883ac2376d2dc7f