tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add python check (#643)

Browse Source
This commit is contained in:
Jure Bajic 2022-11-09 01:48:34 -08:00 committed by GitHub
parent c4c3a254bf
commit 1d5f387ddd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 116 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/daily_benchmark.yaml vendored
View File

@ -16,7 +16,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

62
.github/workflows/diff.yaml vendored
View File

@ -26,7 +26,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -64,7 +64,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -75,7 +75,7 @@ jobs:
- name: Fetch all history for all tags and branches
run: git fetch
- name: Build combined ASAN, UBSAN and coverage binaries
- name: Initialize deps
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
@ -83,6 +83,32 @@ jobs:
# Initialize dependencies.
./init
- name: Set base branch
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
- name: Set base branch # if we manually dispatch or push to master
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
- name: Python code analysis
run: |
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
for file in ${CHANGED_FILES}; do
echo ${file}
if [[ ${file} == *.py ]]; then
python3 -m black --check --diff ${file}
python3 -m isort --check-only --diff ${file}
fi
done
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
cd build
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
make -j$THREADS memgraph__unit
@ -110,21 +136,11 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
- name: Set base branch
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
- name: Set base branch # if we manually dispatch or push to master
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
- name: Run clang-tidy
run: |
source /opt/toolchain-v4/activate
@ -145,7 +161,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -206,7 +222,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@ -221,7 +237,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -246,7 +262,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "GQL Behave Status"
path: |
@ -303,13 +319,13 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Save test data
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
if: always()
with:
name: "Test data"
@ -328,7 +344,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -353,7 +369,7 @@ jobs:
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
if: ${{ always() }}
with:
name: "Jepsen Report"
@ -369,7 +385,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

2
.github/workflows/full_clang_tidy.yaml vendored
View File

@ -14,7 +14,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

20
.github/workflows/package_all.yaml vendored
View File

@ -17,7 +17,7 @@ jobs:
run: |
./release/package/run.sh package centos-7
- name: "Upload package"
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: centos-7
path: build/output/centos-7/memgraph*.rpm
@ -34,7 +34,7 @@ jobs:
run: |
./release/package/run.sh package centos-9
- name: "Upload package"
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: centos-9
path: build/output/centos-9/memgraph*.rpm
@ -51,7 +51,7 @@ jobs:
run: |
./release/package/run.sh package debian-10
- name: "Upload package"
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: debian-10
path: build/output/debian-10/memgraph*.deb
@ -68,7 +68,7 @@ jobs:
run: |
./release/package/run.sh package debian-11
- name: "Upload package"
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: debian-11
path: build/output/debian-11/memgraph*.deb
@ -87,7 +87,7 @@ jobs:
./run.sh package debian-11 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: docker
path: build/output/docker/memgraph*.tar.gz
@ -104,7 +104,7 @@ jobs:
run: |
./release/package/run.sh package ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: ubuntu-1804
path: build/output/ubuntu-18.04/memgraph*.deb
@ -121,7 +121,7 @@ jobs:
run: |
./release/package/run.sh package ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: ubuntu-2004
path: build/output/ubuntu-20.04/memgraph*.deb
@ -138,7 +138,7 @@ jobs:
run: |
./release/package/run.sh package ubuntu-22.04
- name: "Upload package"
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: ubuntu-2204
path: build/output/ubuntu-22.04/memgraph*.deb
@ -155,7 +155,7 @@ jobs:
run: |
./release/package/run.sh package debian-11 --for-platform
- name: "Upload package"
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: debian-11-platform
path: build/output/debian-11/memgraph*.deb
@ -172,7 +172,7 @@ jobs:
run: |
./release/package/run.sh package debian-11-arm
- name: "Upload package"
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: debian-11-arm
path: build/output/debian-11-arm/memgraph*.deb

16
.github/workflows/release_centos8.yaml vendored
View File

@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -225,7 +225,7 @@ jobs:
rpmlint memgraph*.rpm
- name: Save enterprise RPM package
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Enterprise RPM package"
path: build/output/memgraph*.rpm
@ -262,7 +262,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "GQL Behave Status"
path: |

20
.github/workflows/release_debian10.yaml vendored
View File

@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -224,7 +224,7 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
@ -261,7 +261,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "GQL Behave Status"
path: |
@ -324,7 +324,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -349,7 +349,7 @@ jobs:
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
if: ${{ always() }}
with:
name: "Jepsen Report"

8
.github/workflows/release_docker.yaml vendored
View File

@ -19,17 +19,17 @@ jobs:
DOCKER_REPOSITORY_NAME: memgraph
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v2
- name: Log in to Docker Hub
uses: docker/login-action@v1
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

16
.github/workflows/release_ubuntu2004.yaml vendored
View File

@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@ -224,7 +224,7 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
@ -261,7 +261,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: "GQL Behave Status"
path: |

16
.pre-commit-config.yaml
View File

@ -6,18 +6,14 @@ repos:
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/psf/black
rev: 22.3.0
rev: 22.10.0
hooks:
- id: black
args: # arguments to configure black
- --line-length=120
- --include='\.pyi?$'
# these folders wont be formatted by black
- --exclude="""\.git |
\.__pycache__|
build|
libs|
.cache"""
- repo: https://github.com/pycqa/isort
rev: 5.10.1
hooks:
- id: isort
name: isort (python)
- repo: https://github.com/pre-commit/mirrors-clang-format
rev: v13.0.0
hooks:

30
config/generate.py
View File

@ -5,12 +5,10 @@ import os
import subprocess
import sys
import textwrap
import xml.etree.ElementTree as ET
import yaml
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
CONFIG_FILE = os.path.join(SCRIPT_DIR, "flags.yaml")
WIDTH = 80
@ -18,14 +16,13 @@ WIDTH = 80
def wrap_text(s, initial_indent="# "):
return "\n#\n".join(
map(lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent,
subsequent_indent="# "), s.split("\n")))
map(lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent, subsequent_indent="# "), s.split("\n"))
)
def extract_flags(binary_path):
ret = {}
data = subprocess.run([binary_path, "--help-xml"],
stdout=subprocess.PIPE).stdout.decode("utf-8")
data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8")
root = ET.fromstring(data)
for child in root:
if child.tag == "usage" and child.text.lower().count("warning"):
@ -46,8 +43,7 @@ def apply_config_to_flags(config, flags):
for modification in config["modifications"]:
name = modification["name"]
if name not in flags:
print("WARNING: Flag '" + name + "' missing from binary!",
file=sys.stderr)
print("WARNING: Flag '" + name + "' missing from binary!", file=sys.stderr)
continue
flags[name]["default"] = modification["value"]
flags[name]["override"] = modification["override"]
@ -75,8 +71,9 @@ def extract_sections(flags):
else:
sections.append((current_section, current_flags))
sections.append(("other", other))
assert set(sum(map(lambda x: x[1], sections), [])) == set(flags.keys()), \
"The section extraction algorithm lost some flags!"
assert set(sum(map(lambda x: x[1], sections), [])) == set(
flags.keys()
), "The section extraction algorithm lost some flags!"
return sections
@ -89,8 +86,7 @@ def generate_config_file(sections, flags):
helpstr = flag["meaning"] + " [" + flag["type"] + "]"
ret += wrap_text(helpstr) + "\n"
prefix = "# " if not flag["override"] else ""
ret += prefix + "--" + flag["name"].replace("_", "-") + \
"=" + flag["default"] + "\n\n"
ret += prefix + "--" + flag["name"].replace("_", "-") + "=" + flag["default"] + "\n\n"
ret += "\n"
ret += wrap_text(config["footer"])
return ret.strip() + "\n"
@ -98,13 +94,9 @@ def generate_config_file(sections, flags):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("memgraph_binary",
help="path to Memgraph binary")
parser.add_argument("output_file",
help="path where to store the generated Memgraph "
"configuration file")
parser.add_argument("--config-file", default=CONFIG_FILE,
help="path to generator configuration file")
parser.add_argument("memgraph_binary", help="path to Memgraph binary")
parser.add_argument("output_file", help="path where to store the generated Memgraph " "configuration file")
parser.add_argument("--config-file", default=CONFIG_FILE, help="path to generator configuration file")
args = parser.parse_args()
flags = extract_flags(args.memgraph_binary)

6
init
View File

@ -147,5 +147,11 @@ done;
python3 -m pip install pre-commit
python3 -m pre_commit install
# Install py format tools
echo "Install black formatter"
python3 -m pip install black==22.10.*
echo "Install isort"
python3 -m pip install isort==5.10.*
# Link `include/mgp.py` with `release/mgp/mgp.py`
ln -v -f include/mgp.py release/mgp/mgp.py

12
pyproject.toml Normal file
View File

@ -0,0 +1,12 @@
[tool.black]
line-length = 120
include = '\.pyi?$'
extend-exclude = '''
/(
| .git
| .__pycache__
| build
| libs
| .cache
)/
'''