diff --git a/Makefile b/Makefile index e0b6db73..06616f06 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ dev: default test test: @echo "Running tests in native mode." @export TZ='Asia/Shanghai'; \ - $(GOTEST) -cover $(PACKAGES) + LOG_LEVEL=fatal $(GOTEST) -cover $(PACKAGES) CURDIR := $(shell pwd) export PATH := $(CURDIR)/bin/:$(PATH) @@ -51,46 +51,46 @@ format: @gofmt -s -w `find . -name '*.go' -type f ! -path '*/_tools/*' -print` project1: - go test -count=1 ./kv/server -run 1 + $(GOTEST) ./kv/server -run 1 project2: project2a project2b project2c project2a: - go test -count=1 ./raft -run 2A + $(GOTEST) ./raft -run 2A project2aa: - go test -count=1 ./raft -run 2AA + $(GOTEST) ./raft -run 2AA project2ab: - go test -count=1 ./raft -run 2AB + $(GOTEST) ./raft -run 2AB project2ac: - go test -count=1 ./raft -run 2AC + $(GOTEST) ./raft -run 2AC project2b: - go test -count=1 ./kv/test_raftstore -run 2B + $(GOTEST) ./kv/test_raftstore -run 2B project2c: - go test -count=1 ./raft ./kv/test_raftstore -run 2C + $(GOTEST) ./raft ./kv/test_raftstore -run 2C project3: project3a project3b project3c project3a: - go test -count=1 ./raft -run 3A + $(GOTEST) ./raft -run 3A project3b: - go test -count=1 ./raft -run 3B + $(GOTEST) ./raft -run 3B project3c: - go test -count=1 ./scheduler/... -run 3C + $(GOTEST) ./scheduler/... -run 3C project4: project4a project4b project4c project4a: - go test -count=1 ./kv/transaction/... -run 4A + $(GOTEST) ./kv/transaction/... -run 4A project4b: - go test -count=1 ./kv/transaction/... -run 4B + $(GOTEST) ./kv/transaction/... -run 4B project4c: - go test -count=1 ./kv/transaction/... -run 4C + $(GOTEST) ./kv/transaction/... -run 4C diff --git a/README.md b/README.md index d0c2f2ef..af88daf8 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Now you can run `make` to check that everything is working as expected. You shou ### Overview of the code -![overview](courses/imgs/overview.png) +![overview](doc/imgs/overview.png) Same as the architect TiDB + TiKV + PD that separates the storage and computation, TinyKV only focuses on the storage layer of a distributed database system. If you are also interested in SQL layer, see [TinySQL](https://github.com/pingcap-incubator/tinysql). Besides that, there is a component called TinyScheduler as a center control of the whole TinyKV cluster, which collects information from the heartbeats of TinyKV. After that, the TinyScheduler can generate some scheduling tasks and distribute them to the TinyKV instances. All of them are communicated by RPC. @@ -47,6 +47,7 @@ The whole project is organized into the following directories: - `proto`: all communication between nodes and processes uses Protocol Buffers over gRPC. This package contains the protocol definitions used by TinyKV, and generated Go code for using them. - `raft`: implementation of the Raft distributed consensus algorithm, used in TinyKV. - `scheduler`: implementation of the TinyScheduler which is responsible for managing TinyKV nodes and for generating timestamps. +- `log`: log utility to output log base on level. ### Course material diff --git a/doc/project2-RaftKV.md b/doc/project2-RaftKV.md index cfd9052a..6ee47338 100644 --- a/doc/project2-RaftKV.md +++ b/doc/project2-RaftKV.md @@ -1,6 +1,6 @@ # Project2 RaftKV -Raft is a consensus algorithm that is designed to be easy to understand. You can read material about the Raft itself at [the Raft site](https://raft.github.io/), an interactive visualization of the Raft, and other resources, including [the extended Raft paper]( https://raft.github.io/raft.pdf). +Raft is a consensus algorithm that is designed to be easy to understand. You can read material about the Raft itself at [the Raft site](https://raft.github.io/), an interactive visualization of the Raft, and other resources, including [the extended Raft paper](https://raft.github.io/raft.pdf). In this project you will implement a high available kv server based on raft, which needs you not only to implement the Raft algorithm but also use it practically, and bring you more challenges like managing raft’s persisted state with `badger`, add flow control for snapshot message, etc. @@ -199,7 +199,6 @@ In this stage, you may consider these errors, and others will be processed in pr > - You can apply the committed Raft log entries in an asynchronous way just like TiKV does. It’s not necessary, though a big challenge to improve performance. > - Record the callback of the command when proposing, and return the callback after applying. > - For the snap command response, should set badger Txn to callback explicitly. -> - You can change the log level in `kv/config/config.go` to control whether to print some logs, which will be useful when debugging. ## Part C diff --git a/go.mod b/go.mod index f122e4c8..47a9d8bf 100644 --- a/go.mod +++ b/go.mod @@ -34,4 +34,4 @@ require ( go 1.13 -replace github.com/pingcap/tidb => github.com/pingcap-incubator/tinysql v0.0.0-20200429065743-414e03ef38bd +replace github.com/pingcap/tidb => github.com/pingcap-incubator/tinysql v0.0.0-20200429190545-ab844a9e29fe diff --git a/go.sum b/go.sum index ca917f38..b900cc21 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,8 @@ github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUr github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99 h1:KcEvVBAvyHkUdFAygKAzwB6LAcZ6LS32WHmRD2VyXMI= github.com/petar/GoLLRB v0.0.0-20190514000832-33fb24c13b99/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= github.com/pingcap-incubator/tinykv v0.0.0-20200320061650-f660f803e910/go.mod h1:YBDZKrKXGtZ8JnIwiiugENs8ykQiMNKOdaTC5lnEGVc= -github.com/pingcap-incubator/tinysql v0.0.0-20200429065743-414e03ef38bd h1:mU/oD21p1lvolThE7AW+2s/LJ2YDdMtcmAnO927YoZU= -github.com/pingcap-incubator/tinysql v0.0.0-20200429065743-414e03ef38bd/go.mod h1:r381uAExaY7K6lDM0lSpbWtdSbw1eDdLADMO7ZrvWV8= +github.com/pingcap-incubator/tinysql v0.0.0-20200429190545-ab844a9e29fe h1:2826WhLU/2CtgfMgfhZCX2fNUhVByQeH3nD26GZ0a0s= +github.com/pingcap-incubator/tinysql v0.0.0-20200429190545-ab844a9e29fe/go.mod h1:r381uAExaY7K6lDM0lSpbWtdSbw1eDdLADMO7ZrvWV8= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= diff --git a/kv/config/config.go b/kv/config/config.go index fe0ad27a..ee58a768 100644 --- a/kv/config/config.go +++ b/kv/config/config.go @@ -83,7 +83,6 @@ func NewDefaultConfig() *Config { func NewTestConfig() *Config { return &Config{ - LogLevel: "fatal", Raft: true, RaftBaseTickInterval: 50 * time.Millisecond, RaftHeartbeatTicks: 2, diff --git a/log/log.go b/log/log.go index 54072932..a587c7d0 100644 --- a/log/log.go +++ b/log/log.go @@ -1,4 +1,10 @@ -//high level log wrapper, so it can output different log based on level +// High level log wrapper, so it can output different log based on level. +// +// There are five levels in total: FATAL, ERROR, WARNING, INFO, DEBUG. +// The default log output level is INFO, you can change it by: +// - call log.SetLevel() +// - set environment variable `LOG_LEVEL` + package log import ( @@ -264,5 +270,11 @@ func New() *Logger { } func NewLogger(w io.Writer, prefix string) *Logger { - return &Logger{_log: log.New(w, prefix, LstdFlags), level: LOG_LEVEL_INFO, highlighting: true} + var level LogLevel + if l := os.Getenv("LOG_LEVEL"); len(l) != 0 { + level = StringToLogLevel(os.Getenv("LOG_LEVEL")) + } else { + level = LOG_LEVEL_INFO + } + return &Logger{_log: log.New(w, prefix, LstdFlags), level: level, highlighting: true} } diff --git a/raft/raft_paper_test.go b/raft/raft_paper_test.go index edd1fd44..f9846b36 100644 --- a/raft/raft_paper_test.go +++ b/raft/raft_paper_test.go @@ -14,10 +14,9 @@ /* This file contains tests which verify that the scenarios described -in the raft paper (https://ramcloud.stanford.edu/raft.pdf) are -handled by the raft implementation correctly. Each test focuses on -several sentences written in the paper. This could help us to prevent -most implementation bugs. +in the raft paper (https://raft.github.io/raft.pdf) are handled by the +raft implementation correctly. Each test focuses on several sentences +written in the paper. Each test is composed of three parts: init, test and check. Init part uses simple and understandable way to simulate the init state.