Add benchmark scripts for CI

.pfnci/README.md

@@ -0,0 +1,10 @@
+# Benchmark scripts for PFN-CI
+
+## Usage
+```
+$ git clone https://github.com/cupy/cupy-benchmark.git
+$ cd cupy-benchmark
+$ imosci --project cupy-benchmark-all run  # run benchmarks for all releases
+$ imosci --project cupy-benchmark run  # run benchmark for the latest commit
+$ imosci --project cupy-benchmark-publish run  # publish the benchmark results
+```

.pfnci/benchmark.sh

@@ -0,0 +1,68 @@
+#!/bin/bash -e
+
+IMAGE_ID='cupy/cupy-benchmark:cuda9.2-cudnn7'
+
+PROJECT_NAME=cupy-benchmark
+BUCKET_NAME=chainer-pfn-private-ci
+
+BASE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+ROOT_DIR=$(dirname "${BASE_DIR}")
+
+# Retrieve arguments
+while [[ $# != 0 ]]; do
+    case $1 in
+        --)
+            shift
+            break
+            ;;
+        --commits)
+            readonly ARG_COMMITS="$2"
+            shift 2
+            ;;
+        --force)
+            readonly ARG_FORCE='true'
+            shift 1
+            ;;
+        -*)
+            echo Unknown option \"$1\" 1>&2
+            exit 1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+# Validate the arguments
+test -n "${ARG_COMMITS}" || { echo "--commits is not specified" 1>&2; exit 1; }
+
+if [ -n "${ARG_FORCE}" ]; then
+    readonly OPT_FORCE='--force'
+fi
+
+# Collect environment information in which run the benchmark
+source ${BASE_DIR}/collect_env.sh
+
+echo "Running cupy-benchmark for the latest commit..."
+
+# Run the benchmark in Docker container
+docker run \
+    --rm \
+    --runtime=nvidia \
+    -v /etc/group:/etc/group:ro -v /etc/passwd:/etc/passwd:ro \
+    -u $(id -u $USER):$(id -g $USER) \
+    -v ${ROOT_DIR}:/work --workdir=/work \
+    "${IMAGE_ID}" \
+    bash -ex .pfnci/run_benchmark.sh \
+        ${OPT_FORCE} \
+        --machine "${MACHINE_ID}" \
+        --commits "${ARG_COMMITS}" \
+    || true  # considered to be successful even if the benchmark fails
+
+# Upload the result
+bash -e .pfnci/upload_results.sh \
+    --bucket "${BUCKET_NAME}" \
+    --project "${PROJECT_NAME}" \
+    --machine "${MACHINE_ID}"
+
+# TODO: send a notification when `asv compare` detects performance regression

.pfnci/benchmark_tags.sh

@@ -0,0 +1,83 @@
+#!/bin/bash -e
+
+IMAGE_ID='cupy/cupy-benchmark:cuda9.2-cudnn7'
+
+PROJECT_NAME=cupy-benchmark
+BUCKET_NAME=chainer-pfn-private-ci
+REPO_URL=https://github.com/cupy/cupy.git
+REPO_DIR=cupy
+
+BASE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+ROOT_DIR=$(dirname "${BASE_DIR}")
+
+# Retrieve arguments
+while [[ $# != 0 ]]; do
+    case $1 in
+        --)
+            shift
+            break
+            ;;
+        --pattern)
+            readonly ARG_PATTERN="$2"
+            shift 2
+            ;;
+        --force)
+            readonly ARG_FORCE='true'
+            shift 1
+            ;;
+        -*)
+            echo Unknown option \"$1\" 1>&2
+            exit 1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+# Collect environment information in which run the benchmark
+source ${BASE_DIR}/collect_env.sh
+
+echo "Running cupy-benchmark for all releases..."
+
+# Download the past results (if needed)
+if [ -n "${ARG_PATTERN}" ]; then
+    readonly OPT_PATTERN=${ARG_PATTERN}
+fi
+if [ -n "${ARG_FORCE}" ]; then
+    readonly OPT_FORCE='--force'
+fi
+
+# Clone the repository before running asv and retrieve a list of tags
+docker run \
+    --rm \
+    -v /etc/group:/etc/group:ro -v /etc/passwd:/etc/passwd:ro -u $(id -u $USER):$(id -g $USER) \
+    -v ${ROOT_DIR}:/work --workdir=/work \
+    "${IMAGE_ID}" \
+    bash -exc "git clone ${REPO_URL} ${REPO_DIR}"
+REPO_TAGS=$(docker run \
+    --rm \
+    -v ${ROOT_DIR}:/work --workdir=/work/${REPO_DIR} \
+    ${IMAGE_ID} \
+    git tag -l "${OPT_PATTERN}")
+
+# Run the benchmark in Docker container
+for tag in ${REPO_TAGS}; do
+    docker run \
+        --rm \
+        --runtime=nvidia \
+        -v /etc/group:/etc/group:ro -v /etc/passwd:/etc/passwd:ro -u $(id -u $USER):$(id -g $USER) \
+        -v ${ROOT_DIR}:/work --workdir=/work \
+        "${IMAGE_ID}" \
+        bash -ex .pfnci/run_benchmark.sh \
+            ${OPT_FORCE} \
+            --machine "${MACHINE_ID}" \
+            --commits ${tag} \
+        || true  # considered to be successful even if the benchmark fails
+
+    # Upload the result
+    bash -e ${BASE_DIR}/upload_results.sh \
+        --bucket "${BUCKET_NAME}" \
+        --project "${PROJECT_NAME}" \
+        --machine "${MACHINE_ID}"
+done

.pfnci/collect_env.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -e
+
+function slugify() {
+    cat - | sed -e 's/[^[:alnum:]]/-/g' | tr -s '-' | tr 'A-Z' 'a-z'
+}
+
+# Assumes that the machine equips only one type of CPU (and GPU) 
+readonly CPU_MODEL_NAME=$(lscpu | sed -nr '/Model name/ s/.*:\s*(.*).*/\1/p' | head -n1 | sed -e 's/(R)//g')
+readonly CPU_CORES=$(nproc)
+
+readonly MEM_SIZE=$(cat /proc/meminfo | sed -nr '/MemTotal/ s/.*:\s*(.*).*/\1/p' | awk '{ printf "%d", $1/1024/1024 ; exit}' | head -n1)
+
+if command -v nvidia-smi >/dev/null 2>&1; then
+    readonly GPU_MODEL_NAME=$(nvidia-smi --query-gpu=gpu_name,memory.total --format=csv,noheader | head -n1 | awk -F', ' '{print $1 " " $2}')
+fi
+
+# e.g. Intel Xeon CPU E5-2630 v3 @ 2.40GHz (8 cores) + 62GB + Quadro K420 979 MiB
+MACHINE_DESC="${CPU_MODEL_NAME} (${CPU_CORES} cores) + ${MEM_SIZE}GB"
+if [ -n "${GPU_MODEL_NAME}" ]; then
+    MACHINE_DESC=${MACHINE_DESC}' + '${GPU_MODEL_NAME}
+fi
+
+# e.g. intel-xeon-cpu-e5-2630-v3-2-40ghz-8-cores-62gb-quadro-k420-979-mib
+readonly MACHINE_ID=$(echo "${MACHINE_DESC}" | slugify)

.pfnci/config.pbtxt

@@ -0,0 +1,53 @@
+configs {
+  key: "cupy-benchmark"  # project name
+  value {
+    requirement {
+      cpu: 4
+      memory: 16
+      gpu: 1
+      disk: 10
+      image: "imosci"
+    }
+    time_limit {
+      seconds: 7200  # 2 hours
+    }
+    command:
+        "bash -ex .pfnci/benchmark.sh --commits master\n"
+  }
+}
+
+configs {
+  key: "cupy-benchmark-all"  # project name
+  value {
+    requirement {
+      cpu: 4
+      memory: 16
+      gpu: 1
+      disk: 10
+      premium: true
+      image: "imosci"
+    }
+    time_limit {
+      seconds: 172800  # 2 days
+    }
+    command:
+        "bash -ex .pfnci/benchmark_tags.sh --pattern '*'\n"
+  }
+}
+
+configs {
+  key: "cupy-benchmark-publish"  # project name
+  value {
+    requirement {
+      cpu: 2
+      memory: 8
+      disk: 10
+      image: "imosci"
+    }
+    time_limit {
+      seconds: 3600  # 1 hours
+    }
+    command:
+        "bash -ex .pfnci/publish_results.sh\n"
+  }
+}

.pfnci/gcp_utils.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -e
+
+GSUTIL_CMD=gsutil
+
+function gs_cat() {
+    local target=$1
+
+    local stat  # it must be declared independently to get the exit code
+    stat=$(${GSUTIL_CMD} stat ${target})
+    ret=$?
+
+    if [ ${ret} -eq 0 ]; then
+        local encoding=$(echo "${stat}" | sed -nr '/Content-Encoding/ s/.*:\s*(.*).*/\1/p')
+        if [ "${encoding}" == "gzip" ]; then
+            ${GSUTIL_CMD} cat ${target} | gunzip -c
+        else
+            ${GSUTIL_CMD} cat ${target}
+        fi
+    else
+        echo 1>&2
+        return ${ret}
+    fi
+}

.pfnci/publish_results.sh

@@ -0,0 +1,40 @@
+#!/bin/bash -e
+
+IMAGE_ID='cupy/cupy-benchmark:cuda9.2-cudnn7'
+
+RESULT_BUCKET_NAME=chainer-pfn-private-ci
+WEBSITE_BUCKET_NAME=chainer-artifacts-pfn-public-ci
+PROJECT_NAME=cupy-benchmark
+RESULTS_DIR=results
+HTML_DIR=html
+
+BASE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+ROOT_DIR=$(dirname "${BASE_DIR}")
+
+readonly GS_SRC_DIR=gs://${RESULT_BUCKET_NAME}/${PROJECT_NAME}/${RESULTS_DIR}
+readonly DEST_DIR=${ROOT_DIR}
+
+readonly HTML_SRC_DIR=${ROOT_DIR}/${HTML_DIR}
+readonly GS_HTML_DEST_DIR=gs://${WEBSITE_BUCKET_NAME}/${PROJECT_NAME}
+
+# Load helper functions
+source ${BASE_DIR}/gcp_utils.sh
+
+# Download the benchmark results
+${GSUTIL_CMD} -m cp -r ${GS_SRC_DIR} ${DEST_DIR}
+
+# Run the benchmark in Docker container
+docker run \
+    --rm \
+    -v /etc/group:/etc/group:ro -v /etc/passwd:/etc/passwd:ro \
+    -u $(id -u $USER):$(id -g $USER) \
+    -v ${ROOT_DIR}:/work --workdir=/work \
+    -e PYTHONUNBUFFERED=1 \
+    "${IMAGE_ID}" \
+    asv publish --html-dir ./${HTML_DIR}
+
+# Publish the website to the public bucket
+# TODO: enable gzip compression
+${GSUTIL_CMD} -m rsync -r ${HTML_SRC_DIR} ${GS_HTML_DEST_DIR}
+
+echo "Uploaded to https://storage.googleapis.com/${WEBSITE_BUCKET_NAME}/${PROJECT_NAME}/index.html"

.pfnci/run_benchmark.sh

@@ -0,0 +1,63 @@
+#!/bin/bash -e
+
+BASE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+ROOT_DIR=$(dirname "${BASE_DIR}")
+
+# Retrieve arguments
+while [[ $# != 0 ]]; do
+    case $1 in
+        --)
+            shift
+            break
+            ;;
+        --machine)
+            readonly ARG_MACHINE="$2"
+            shift 2
+            ;;
+        --commits)
+            readonly ARG_COMMITS="$2"
+            shift 2
+            ;;
+        --force)
+            readonly ARG_FORCE='true'
+            shift 1
+            ;;
+        -*)
+            echo Unknown option \"$1\" 1>&2
+            exit 1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+# Validate the arguments
+test -n "${ARG_COMMITS}" || { echo "--commits is not specified" 1>&2; exit 1; }
+
+if [ -z "${ARG_FORCE}" ]; then
+    readonly OPT_SKIP_EXISTING='--skip-existing-successful'
+fi
+
+# Avoid stdout and stderr streams to be buffered in Python
+export PYTHONUNBUFFERED=1
+
+# Configuring machine information
+# Note: `asv machine --machine [...] --yes` doesn't collect information
+asv machine --yes
+if [ -n "${ARG_MACHINE}" ]; then
+    asv machine --machine=${ARG_MACHINE} --yes
+    readonly OPT_MACHINE=--machine=${ARG_MACHINE}
+fi
+MACHINE_INFO=$(cat ~/.asv-machine.json | jq ".[\"${ARG_MACHINE}\"]")
+test -n "${MACHINE_INFO}" || { echo 'Failed to configure `asv machine`' 1>&2; exit 1; }
+
+# Run benchmark(s) in Docker container
+asv run \
+    ${OPT_MACHINE} \
+    ${OPT_SKIP_EXISTING} \
+    --step 1 \
+    --parallel $(nproc) \
+    --launch-method spawn \
+    --show-stderr \
+    "${ARG_COMMITS}"