diff options
author | iphydf <iphydf@users.noreply.github.com> | 2020-05-03 15:36:57 +0100 |
---|---|---|
committer | iphydf <iphydf@users.noreply.github.com> | 2020-05-03 16:59:26 +0100 |
commit | 4efe541814ec2ddd073428d6928497b50c48397a (patch) | |
tree | b2d3c877317da1d129900b88097d05ffa879281f | |
parent | 88b90c82259f86470cf6eba8684e8d9b4cd61bc3 (diff) |
Add a script to run Travis CI locally.
This isn't quite Travis, but close enough for local testing.
-rw-r--r-- | .restyled.yaml | 6 | ||||
-rw-r--r-- | .travis.yml | 2 | ||||
-rwxr-xr-x | .travis/cmake-linux | 22 | ||||
-rwxr-xr-x | other/analysis/check_recursion | 29 | ||||
-rwxr-xr-x | other/astyle/format-source | 49 | ||||
-rw-r--r-- | other/docker/Dockerfile.ci | 75 | ||||
-rwxr-xr-x | other/docker/run-ci | 15 |
7 files changed, 157 insertions, 41 deletions
diff --git a/.restyled.yaml b/.restyled.yaml index 722ff510..f0060d75 100644 --- a/.restyled.yaml +++ b/.restyled.yaml | |||
@@ -2,4 +2,10 @@ | |||
2 | restylers: | 2 | restylers: |
3 | - astyle: | 3 | - astyle: |
4 | arguments: ["--options=other/astyle/astylerc"] | 4 | arguments: ["--options=other/astyle/astylerc"] |
5 | - autopep8 | ||
6 | - black | ||
5 | - prettier-yaml | 7 | - prettier-yaml |
8 | - reorder-python-imports | ||
9 | - shellharden | ||
10 | - shfmt | ||
11 | - yapf | ||
diff --git a/.travis.yml b/.travis.yml index 510fb984..93192a6f 100644 --- a/.travis.yml +++ b/.travis.yml | |||
@@ -23,7 +23,7 @@ jobs: | |||
23 | - libgtest-dev # For unit tests. | 23 | - libgtest-dev # For unit tests. |
24 | - libvpx-dev # For toxav. | 24 | - libvpx-dev # For toxav. |
25 | - ninja-build | 25 | - ninja-build |
26 | - pylint | 26 | - pylint3 |
27 | install: .travis/$JOB install | 27 | install: .travis/$JOB install |
28 | script: .travis/$JOB script | 28 | script: .travis/$JOB script |
29 | after_script: .travis/upload-coverage | 29 | after_script: .travis/upload-coverage |
diff --git a/.travis/cmake-linux b/.travis/cmake-linux index ed06dc26..73b335fe 100755 --- a/.travis/cmake-linux +++ b/.travis/cmake-linux | |||
@@ -5,7 +5,7 @@ ACTION="$1" | |||
5 | set -eu | 5 | set -eu |
6 | 6 | ||
7 | CACHEDIR="$HOME/cache" | 7 | CACHEDIR="$HOME/cache" |
8 | NPROC=`nproc` | 8 | NPROC=$(nproc) |
9 | ASTYLE="$CACHEDIR/astyle/build/gcc/bin/astyle" | 9 | ASTYLE="$CACHEDIR/astyle/build/gcc/bin/astyle" |
10 | ASTYLE_VERSION=3.1 | 10 | ASTYLE_VERSION=3.1 |
11 | TRAVIS_TOOL="https://raw.githubusercontent.com/TokTok/ci-tools/master/bin/travis-haskell" | 11 | TRAVIS_TOOL="https://raw.githubusercontent.com/TokTok/ci-tools/master/bin/travis-haskell" |
@@ -21,7 +21,7 @@ travis_install() { | |||
21 | # Work around https://github.com/eddyxu/cpp-coveralls/issues/108 by manually | 21 | # Work around https://github.com/eddyxu/cpp-coveralls/issues/108 by manually |
22 | # installing the pyOpenSSL module and injecting it into urllib3 as per | 22 | # installing the pyOpenSSL module and injecting it into urllib3 as per |
23 | # https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 | 23 | # https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 |
24 | sed -i -e '/^import sys$/a import urllib3.contrib.pyopenssl\nurllib3.contrib.pyopenssl.inject_into_urllib3()' `which coveralls` | 24 | sed -i -e '/^import sys$/a import urllib3.contrib.pyopenssl\nurllib3.contrib.pyopenssl.inject_into_urllib3()' "$(which coveralls)" |
25 | } | 25 | } |
26 | 26 | ||
27 | # Install astyle (version in ubuntu-precise too old). | 27 | # Install astyle (version in ubuntu-precise too old). |
@@ -34,15 +34,15 @@ travis_install() { | |||
34 | } | 34 | } |
35 | 35 | ||
36 | run_static_analysis() { | 36 | run_static_analysis() { |
37 | pylint -E other/analysis/check_recursion | 37 | pylint3 -E other/analysis/check_recursion |
38 | 38 | ||
39 | export CPPFLAGS="-isystem $CACHEDIR/include" | 39 | export CPPFLAGS="-isystem $CACHEDIR/include" |
40 | export LDFLAGS="-L$CACHEDIR/lib" | 40 | export LDFLAGS="-L$CACHEDIR/lib" |
41 | cat toxav/*.c toxcore/*.c toxencryptsave/*.c \ | 41 | cat toxav/*.c toxcore/*.c toxencryptsave/*.c | |
42 | | clang `pkg-config --cflags libsodium opus vpx` \ | 42 | clang "$(pkg-config --cflags libsodium opus vpx)" \ |
43 | -Itoxav -Itoxcore -Itoxencryptsave -S -emit-llvm -xc - -o- \ | 43 | -Itoxav -Itoxcore -Itoxencryptsave -S -emit-llvm -xc - -o- | |
44 | | opt -analyze -print-callgraph 2>&1 \ | 44 | opt -analyze -print-callgraph 2>&1 | |
45 | | other/analysis/check_recursion | 45 | other/analysis/check_recursion |
46 | other/analysis/run-clang | 46 | other/analysis/run-clang |
47 | other/analysis/run-clang-analyze | 47 | other/analysis/run-clang-analyze |
48 | } | 48 | } |
@@ -85,10 +85,10 @@ travis_script() { | |||
85 | 85 | ||
86 | cmake --build _build --parallel "$NPROC" --target install -- -k 0 | 86 | cmake --build _build --parallel "$NPROC" --target install -- -k 0 |
87 | 87 | ||
88 | cd _build # pushd | 88 | cd _build # pushd |
89 | ctest -j50 --output-on-failure || \ | 89 | ctest -j50 --output-on-failure || |
90 | ctest -j50 --output-on-failure --rerun-failed | 90 | ctest -j50 --output-on-failure --rerun-failed |
91 | cd - # popd | 91 | cd - # popd |
92 | } | 92 | } |
93 | 93 | ||
94 | if [ "-z" "$ACTION" ]; then | 94 | if [ "-z" "$ACTION" ]; then |
diff --git a/other/analysis/check_recursion b/other/analysis/check_recursion index 21063527..a8a6e0ef 100755 --- a/other/analysis/check_recursion +++ b/other/analysis/check_recursion | |||
@@ -1,8 +1,8 @@ | |||
1 | #!/usr/bin/env python | 1 | #!/usr/bin/env python3 |
2 | """ | 2 | """ |
3 | Tool to check for recursive calls in toxcore C code. | 3 | Tool to check for recursive calls in toxcore C code. |
4 | 4 | ||
5 | Usage: | 5 | Usage: |
6 | 6 | ||
7 | cat toxav/*.c toxcore/*.c toxencryptsave/*.c \ | 7 | cat toxav/*.c toxcore/*.c toxencryptsave/*.c \ |
8 | | clang `pkg-config --cflags libsodium opus vpx` \ | 8 | | clang `pkg-config --cflags libsodium opus vpx` \ |
@@ -10,16 +10,17 @@ cat toxav/*.c toxcore/*.c toxencryptsave/*.c \ | |||
10 | | opt -analyze -print-callgraph 2>&1 \ | 10 | | opt -analyze -print-callgraph 2>&1 \ |
11 | | other/analysis/check_recursion | 11 | | other/analysis/check_recursion |
12 | """ | 12 | """ |
13 | |||
14 | from __future__ import print_function | ||
15 | |||
16 | import collections | 13 | import collections |
17 | import fileinput | 14 | import fileinput |
18 | import re | 15 | import re |
19 | import sys | 16 | import sys |
20 | import time | 17 | import time |
18 | from typing import Dict | ||
19 | from typing import List | ||
20 | from typing import Set | ||
21 | 21 | ||
22 | def load_callgraph(): | 22 | |
23 | def load_callgraph() -> Dict: | ||
23 | """ | 24 | """ |
24 | Parses the output from opt -print-callgraph from stdin or argv. | 25 | Parses the output from opt -print-callgraph from stdin or argv. |
25 | 26 | ||
@@ -39,7 +40,14 @@ def load_callgraph(): | |||
39 | 40 | ||
40 | return {k: sorted(v) for k, v in graph.items()} | 41 | return {k: sorted(v) for k, v in graph.items()} |
41 | 42 | ||
42 | def walk(visited, callgraph, cycles, stack, cur): | 43 | |
44 | def walk( | ||
45 | visited: Set, | ||
46 | callgraph: Dict, | ||
47 | cycles: Set, | ||
48 | stack: List, | ||
49 | cur: str, | ||
50 | ) -> None: | ||
43 | """ | 51 | """ |
44 | Detects cycles in the callgraph and adds them to the cycles parameter. | 52 | Detects cycles in the callgraph and adds them to the cycles parameter. |
45 | """ | 53 | """ |
@@ -54,13 +62,15 @@ def walk(visited, callgraph, cycles, stack, cur): | |||
54 | visited.add(callee) | 62 | visited.add(callee) |
55 | stack.pop() | 63 | stack.pop() |
56 | 64 | ||
57 | def get_time(): | 65 | |
66 | def get_time() -> int: | ||
58 | """ | 67 | """ |
59 | Return the current time in milliseconds. | 68 | Return the current time in milliseconds. |
60 | """ | 69 | """ |
61 | return int(round(time.time() * 1000)) | 70 | return int(round(time.time() * 1000)) |
62 | 71 | ||
63 | def find_recursion(expected): | 72 | |
73 | def find_recursion(expected: Set) -> None: | ||
64 | """ | 74 | """ |
65 | Main function: detects cycles and prints them. | 75 | Main function: detects cycles and prints them. |
66 | 76 | ||
@@ -98,6 +108,7 @@ def find_recursion(expected): | |||
98 | if expected or cycles: | 108 | if expected or cycles: |
99 | sys.exit(1) | 109 | sys.exit(1) |
100 | 110 | ||
111 | |||
101 | find_recursion(expected={ | 112 | find_recursion(expected={ |
102 | "add_to_closest -> add_to_closest", | 113 | "add_to_closest -> add_to_closest", |
103 | "add_to_list -> add_to_list", | 114 | "add_to_list -> add_to_list", |
diff --git a/other/astyle/format-source b/other/astyle/format-source index 193e335c..7de28d06 100755 --- a/other/astyle/format-source +++ b/other/astyle/format-source | |||
@@ -36,10 +36,11 @@ FROM_JSON='s/\\"/"/g;s/^"(.*)"$/$1/;s/\\\\/\\/g;s/\\n/\n/g' | |||
36 | apidsl_request() { | 36 | apidsl_request() { |
37 | TMPFILE=$(mktemp /tmp/apidsl.XXXXXX) | 37 | TMPFILE=$(mktemp /tmp/apidsl.XXXXXX) |
38 | curl -s -o "$TMPFILE" -X POST --data @<( | 38 | curl -s -o "$TMPFILE" -X POST --data @<( |
39 | echo '["Request",'; | 39 | echo '["Request",' |
40 | cat $2; | 40 | cat "$2" |
41 | echo ']') https://apidsl.herokuapp.com/$1 | 41 | echo ']' |
42 | if grep '\[1,"' "$TMPFILE" > /dev/null; then | 42 | ) "https://apidsl.herokuapp.com/$1" |
43 | if grep '\[1,"' "$TMPFILE" >/dev/null; then | ||
43 | echo "Error: $(grep -o '".*"' /tmp/apidsl-$$ | perl -0777 -pe "$FROM_JSON")" >&2 | 44 | echo "Error: $(grep -o '".*"' /tmp/apidsl-$$ | perl -0777 -pe "$FROM_JSON")" >&2 |
44 | rm "$TMPFILE" | 45 | rm "$TMPFILE" |
45 | exit 1 | 46 | exit 1 |
@@ -52,34 +53,42 @@ apidsl_curl() { | |||
52 | echo "apidsl_curl $*" >&2 | 53 | echo "apidsl_curl $*" >&2 |
53 | apidsl_request "c" <( | 54 | apidsl_request "c" <( |
54 | apidsl_request "parse" <( | 55 | apidsl_request "parse" <( |
55 | perl -0777 -pe "$TO_JSON" $1)) | perl -0777 -pe "$FROM_JSON" | 56 | perl -0777 -pe "$TO_JSON" "$1" |
57 | ) | ||
58 | ) | perl -0777 -pe "$FROM_JSON" | ||
56 | } | 59 | } |
57 | 60 | ||
58 | # Check if apidsl generated sources are up to date. | 61 | # Check if apidsl generated sources are up to date. |
59 | set +x | 62 | set +x |
60 | $APIDSL toxcore/LAN_discovery.api.h > toxcore/LAN_discovery.h & | 63 | "$APIDSL" toxcore/LAN_discovery.api.h >toxcore/LAN_discovery.h & |
61 | $APIDSL toxcore/crypto_core.api.h > toxcore/crypto_core.h & | 64 | "$APIDSL" toxcore/crypto_core.api.h >toxcore/crypto_core.h & |
62 | $APIDSL toxcore/ping.api.h > toxcore/ping.h & | 65 | "$APIDSL" toxcore/ping.api.h >toxcore/ping.h & |
63 | $APIDSL toxcore/ping_array.api.h > toxcore/ping_array.h & | 66 | "$APIDSL" toxcore/ping_array.api.h >toxcore/ping_array.h & |
64 | $APIDSL toxcore/tox.api.h > toxcore/tox.h & | 67 | "$APIDSL" toxcore/tox.api.h >toxcore/tox.h & |
65 | $APIDSL toxav/toxav.api.h > toxav/toxav.h & | 68 | "$APIDSL" toxav/toxav.api.h >toxav/toxav.h & |
66 | $APIDSL toxencryptsave/toxencryptsave.api.h > toxencryptsave/toxencryptsave.h & | 69 | "$APIDSL" toxencryptsave/toxencryptsave.api.h >toxencryptsave/toxencryptsave.h & |
67 | set -x | 70 | set -x |
68 | 71 | ||
69 | wait; wait; wait; wait; wait; wait; wait | 72 | wait |
73 | wait | ||
74 | wait | ||
75 | wait | ||
76 | wait | ||
77 | wait | ||
78 | wait | ||
70 | 79 | ||
71 | if grep '<unresolved>' ./*/*.h; then | 80 | if grep '<unresolved>' ./*/*.h; then |
72 | echo "error: some apidsl references were unresolved" | 81 | echo "error: some apidsl references were unresolved" |
73 | exit 1 | 82 | exit 1 |
74 | fi | 83 | fi |
75 | 84 | ||
76 | CC_SOURCES=$(find . '(' -name '*.cc' ')') | 85 | readarray -t CC_SOURCES <<<"$(find . '(' -name '*.cc' ')')" |
77 | CC_SOURCES="$CC_SOURCES toxcore/crypto_core.c" | 86 | CC_SOURCES+=(toxcore/crypto_core.c) |
78 | CC_SOURCES="$CC_SOURCES toxcore/ping_array.c" | 87 | CC_SOURCES+=(toxcore/ping_array.c) |
79 | 88 | ||
80 | for bin in clang-format-6.0 clang-format-5.0 clang-format; do | 89 | for bin in clang-format-6.0 clang-format-5.0 clang-format; do |
81 | if which "$bin"; then | 90 | if which "$bin"; then |
82 | "$bin" -i -style='{BasedOnStyle: Google, ColumnLimit: 100}' $CC_SOURCES | 91 | "$bin" -i -style='{BasedOnStyle: Google, ColumnLimit: 100}' "${CC_SOURCES[@]}" |
83 | break | 92 | break |
84 | fi | 93 | fi |
85 | done | 94 | done |
@@ -91,8 +100,8 @@ FIND="$FIND -and -not -wholename './super_donators/*'" | |||
91 | FIND="$FIND -and -not -wholename './third_party/*'" | 100 | FIND="$FIND -and -not -wholename './third_party/*'" |
92 | FIND="$FIND -and -not -wholename './toxencryptsave/crypto_pwhash*'" | 101 | FIND="$FIND -and -not -wholename './toxencryptsave/crypto_pwhash*'" |
93 | 102 | ||
94 | C_SOURCES=$(eval "$FIND") | 103 | readarray -t C_SOURCES <<<"$(eval "$FIND")" |
95 | 104 | ||
96 | $ASTYLE -n --options=other/astyle/astylerc $C_SOURCES | 105 | "$ASTYLE" -n --options=other/astyle/astylerc "${C_SOURCES[@]}" |
97 | 106 | ||
98 | git diff --exit-code | 107 | git diff --color=always --exit-code |
diff --git a/other/docker/Dockerfile.ci b/other/docker/Dockerfile.ci new file mode 100644 index 00000000..db7e5e2c --- /dev/null +++ b/other/docker/Dockerfile.ci | |||
@@ -0,0 +1,75 @@ | |||
1 | # This Docker build emulates roughly what Travis CI is doing. It is not exactly | ||
2 | # the same (different tool versions) and success in this image may not | ||
3 | # necessarily mean success on Travis. This image is also not automatically | ||
4 | # tested, so it may get out of date. Send PRs if you use it and it's broken. | ||
5 | # | ||
6 | # For one, we use bionic, not xenial, because xenial's clang is way too old. | ||
7 | FROM ubuntu:16.04 | ||
8 | |||
9 | # Travis environment. | ||
10 | RUN apt-get update && apt-get install --no-install-recommends -y \ | ||
11 | apt-transport-https \ | ||
12 | build-essential \ | ||
13 | ca-certificates \ | ||
14 | curl \ | ||
15 | git \ | ||
16 | pkg-config \ | ||
17 | python-pip \ | ||
18 | python-setuptools \ | ||
19 | python3 \ | ||
20 | software-properties-common \ | ||
21 | wget \ | ||
22 | && apt-get clean \ | ||
23 | && rm -rf /var/lib/apt/lists/* | ||
24 | |||
25 | RUN curl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ | ||
26 | && apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" \ | ||
27 | && apt-get update && apt-get install --no-install-recommends -y \ | ||
28 | clang-6.0 \ | ||
29 | clang-format-6.0 \ | ||
30 | llvm-6.0 \ | ||
31 | && apt-get clean \ | ||
32 | && rm -rf /var/lib/apt/lists/* | ||
33 | |||
34 | RUN ls /usr/bin/clang-6.0 && ln -s /usr/bin/clang-6.0 /usr/bin/clang \ | ||
35 | && ls /usr/bin/clang++-6.0 && ln -s /usr/bin/clang++-6.0 /usr/bin/clang++ \ | ||
36 | && ls /usr/bin/clang-format-6.0 && ln -s /usr/bin/clang-format-6.0 /usr/bin/clang-format \ | ||
37 | && ls /usr/bin/opt-6.0 && ln -s /usr/bin/opt-6.0 /usr/bin/opt | ||
38 | |||
39 | # Bionic's cmake is too old. | ||
40 | RUN pip install --upgrade pip cmake | ||
41 | |||
42 | # .travis.yml | ||
43 | RUN apt-get update && apt-get install --no-install-recommends -y \ | ||
44 | libconfig-dev \ | ||
45 | libgtest-dev \ | ||
46 | libopus-dev \ | ||
47 | libsodium-dev \ | ||
48 | libvpx-dev \ | ||
49 | ninja-build \ | ||
50 | pylint3 \ | ||
51 | && apt-get clean \ | ||
52 | && rm -rf /var/lib/apt/lists/* | ||
53 | |||
54 | # Set up travis user. | ||
55 | RUN groupadd -r -g 1000 travis \ | ||
56 | && useradd --no-log-init -r -g travis -u 1000 travis \ | ||
57 | && mkdir -p /src/workspace /home/travis \ | ||
58 | && chown travis:travis /home/travis | ||
59 | USER travis | ||
60 | |||
61 | # Set up environment. | ||
62 | ENV CC=gcc CXX=g++ \ | ||
63 | PATH=/home/travis/.local/bin:$PATH \ | ||
64 | TRAVIS_REPO_SLUG=TokTok/c-toxcore | ||
65 | |||
66 | # Copy minimal files to run "cmake-linux install", so we can avoid rebuilding | ||
67 | # astyle and other things when only source files change. | ||
68 | RUN mkdir -p /home/travis/build/c-toxcore /home/travis/cache | ||
69 | WORKDIR /home/travis/build/c-toxcore | ||
70 | COPY --chown=travis:travis c-toxcore/.travis/ /home/travis/build/c-toxcore/.travis/ | ||
71 | RUN .travis/cmake-linux install | ||
72 | |||
73 | # Now copy the rest of the sources and run the build. | ||
74 | COPY --chown=travis:travis . /home/travis/build/ | ||
75 | RUN .travis/cmake-linux script | ||
diff --git a/other/docker/run-ci b/other/docker/run-ci new file mode 100755 index 00000000..9005cfbd --- /dev/null +++ b/other/docker/run-ci | |||
@@ -0,0 +1,15 @@ | |||
1 | #!/bin/bash | ||
2 | |||
3 | set -eu | ||
4 | |||
5 | readarray -t FILES <<<"$(git ls-files | sed -e 's,^,c-toxcore/,')" | ||
6 | |||
7 | if [ -f .git ]; then | ||
8 | cd .. | ||
9 | tar -c "${FILES[@]}" "c-toxcore/.git" ".git/modules/c-toxcore" | | ||
10 | docker build -f c-toxcore/other/docker/Dockerfile.ci - | ||
11 | else | ||
12 | cd .. | ||
13 | tar -c "${FILES[@]}" "c-toxcore/.git" | | ||
14 | docker build -f c-toxcore/other/docker/Dockerfile.ci - | ||
15 | fi | ||