Fix build with OpenSSL 3.0 (#25)

* Fix build with OpenSSL 3.0

- FIPS_mode_set() does not exist in OpenSSL 3.0 [1]
- X509_check_* functions declarated in openssl/x509v3.h instead of openssl/x509.h [2]
- X509_chack_* functions have const char arg inserad of const unsigned char [2]
- skip MD4 tests if it is unsupported by OpenSSL
- the patch does not change behavior under OpenSSL version != 3
- the patch just fixes build under OpenSSL 3.0 and doesn't update deprecated code
or behavior

1. https://wiki.openssl.org/index.php/OpenSSL_3.0#Upgrading_from_the_OpenSSL_2.0_FIPS_Object_Module
2. https://www.openssl.org/docs/man3.0/man3/X509_check_host.html

* Add Ubuntu 22.04 runner to GitHub Actions go test workflow

* Fix flaky tests on Ubuntu 22.04

It is necessary to handle OpenSSL errors very carefully. Otherwise,
errors may appear in unexpected places. For example, we didn't catch
an error from EVP_DigestInit_ex() and it appears sometimes in conn.go:

func (c *Conn) getErrorHandler(rv C.int, errno error) func() error {
	errcode := C.SSL_get_error(c.ssl, rv) // <- here

.github/workflows/go-test-ubuntu-22.04.yml

@@ -0,0 +1,69 @@
+# See:
+# https://github.com/libp2p/go-openssl/pull/25
+# https://github.com/protocol/.github/issues/349
+# for details.
+on: [push, pull_request]
+name: Go Test
+
+jobs:
+  unit:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ "ubuntu-22.04" ]
+        go: [ "1.17.x", "1.18.x" ]
+    env:
+      COVERAGES: ""
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.os }} (go ${{ matrix.go }})
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Go information
+        run: |
+          go version
+          go env
+      - name: Use msys2 on windows
+        if: startsWith(matrix.os, 'windows')
+        shell: bash
+        # The executable for msys2 is also called bash.cmd
+        #   https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#shells
+        # If we prepend its location to the PATH
+        #   subsequent 'shell: bash' steps will use msys2 instead of gitbash
+        run: echo "C:/msys64/usr/bin" >> $GITHUB_PATH
+      - name: Run repo-specific setup
+        uses: ./.github/actions/go-test-setup
+        if: hashFiles('./.github/actions/go-test-setup') != ''
+      - name: Run tests
+        uses: protocol/multiple-go-modules@v1.2
+        with:
+          # Use -coverpkg=./..., so that we include cross-package coverage.
+          # If package ./A imports ./B, and ./A's tests also cover ./B,
+          # this means ./B's coverage will be significantly higher than 0%.
+          run: go test -v -coverprofile=module-coverage.txt -coverpkg=./... ./...
+      - name: Run tests (32 bit)
+        if: startsWith(matrix.os, 'macos') == false # can't run 32 bit tests on OSX.
+        uses: protocol/multiple-go-modules@v1.2
+        env:
+          GOARCH: 386
+        with:
+          run: |
+            export "PATH=${{ env.PATH_386 }}:$PATH"
+            go test -v ./...
+      - name: Run tests with race detector
+        if: startsWith(matrix.os, 'ubuntu') # speed things up. Windows and OSX VMs are slow
+        uses: protocol/multiple-go-modules@v1.2
+        with:
+          run: go test -v -race ./...
+      - name: Collect coverage files
+        shell: bash
+        run: echo "COVERAGES=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_ENV
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0
+        with:
+          files: '${{ env.COVERAGES }}'
+          env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }}

fips.go

@@ -16,16 +16,29 @@ package openssl
 
 /*
 #include <openssl/ssl.h>
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	int FIPS_mode_set(int ONOFF) {
+		return 0;
+	}
+#endif
+
 */
 import "C"
+import "errors"
 import "runtime"
 
 // FIPSModeSet enables a FIPS 140-2 validated mode of operation.
 // https://wiki.openssl.org/index.php/FIPS_mode_set()
+// This call has been deleted from OpenSSL 3.0.
 func FIPSModeSet(mode bool) error {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
+	if C.OPENSSL_VERSION_NUMBER >= 0x30000000 {
+		return errors.New("FIPS_mode_set() has been deleted from OpenSSL 3.0")
+	}
+
 	var r C.int
 	if mode {
 		r = C.FIPS_mode_set(1)

hostname.go

@@ -17,18 +17,26 @@ package openssl
 /*
 #include <openssl/ssl.h>
 #include <openssl/conf.h>
-#include <openssl/x509.h>
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	#include <openssl/x509v3.h>
+	typedef const char x509char;
+#else
+	#include <openssl/x509.h>
 
-#ifndef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT
+	#ifndef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT
-#define X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT	0x1
+		#define X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT	0x1
-#define X509_CHECK_FLAG_NO_WILDCARDS	0x2
+		#define X509_CHECK_FLAG_NO_WILDCARDS	0x2
 
-extern int X509_check_host(X509 *x, const unsigned char *chk, size_t chklen,
+		extern int X509_check_host(X509 *x, const unsigned char *chk, size_t chklen,
-    unsigned int flags, char **peername);
+			unsigned int flags, char **peername);
-extern int X509_check_email(X509 *x, const unsigned char *chk, size_t chklen,
+		extern int X509_check_email(X509 *x, const unsigned char *chk, size_t chklen,
-    unsigned int flags);
+			unsigned int flags);
-extern int X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen,
+		extern int X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen,
-		unsigned int flags);
+			unsigned int flags);
+		typedef const unsigned char x509char;
+	#else
+		typedef const char x509char;
+	#endif
 #endif
 */
 import "C"
@@ -59,7 +67,7 @@ func (c *Certificate) CheckHost(host string, flags CheckFlags) error {
 	chost := unsafe.Pointer(C.CString(host))
 	defer C.free(chost)
 
-	rv := C.X509_check_host(c.x, (*C.uchar)(chost), C.size_t(len(host)),
+	rv := C.X509_check_host(c.x, (*C.x509char)(chost), C.size_t(len(host)),
 		C.uint(flags), nil)
 	if rv > 0 {
 		return nil
@@ -78,7 +86,7 @@ func (c *Certificate) CheckHost(host string, flags CheckFlags) error {
 func (c *Certificate) CheckEmail(email string, flags CheckFlags) error {
 	cemail := unsafe.Pointer(C.CString(email))
 	defer C.free(cemail)
-	rv := C.X509_check_email(c.x, (*C.uchar)(cemail), C.size_t(len(email)),
+	rv := C.X509_check_email(c.x, (*C.x509char)(cemail), C.size_t(len(email)),
 		C.uint(flags))
 	if rv > 0 {
 		return nil

md4.go

@@ -51,8 +51,11 @@ func (s *MD4Hash) Close() {
 }
 
 func (s *MD4Hash) Reset() error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
 	if C.X_EVP_DigestInit_ex(s.ctx, C.X_EVP_md4(), engineRef(s.engine)) != 1 {
-		return errors.New("openssl: md4: cannot init digest ctx")
+		return errors.New("openssl: md4: cannot init digest ctx: " +
+			errorFromErrorQueue().Error())
 	}
 	return nil
 }

md4_test.go

@@ -56,7 +56,19 @@ var md4Examples = []struct{ out, in string }{
 	{"6e593341e62194911d5cc31e39835f27", "c5e4bc73821faa34adf9468441ffd97520a96cd5debda4d51edcaaf2b23fbd"},
 }
 
+func skipIfMD4Unsupported(t testing.TB) {
+	t.Helper()
+
+	hash, err := NewMD4Hash()
+	if err != nil {
+		t.Skip("MD4 is not supported by OpenSSL")
+	}
+	hash.Close()
+}
+
 func TestMD4Examples(t *testing.T) {
+	skipIfMD4Unsupported(t)
+
 	for _, ex := range md4Examples {
 		buf, err := hex.DecodeString(ex.in)
 		if err != nil {
@@ -75,6 +87,8 @@ func TestMD4Examples(t *testing.T) {
 }
 
 func TestMD4Writer(t *testing.T) {
+	skipIfMD4Unsupported(t)
+
 	ohash, err := NewMD4Hash()
 	if err != nil {
 		t.Fatal(err)
@@ -120,9 +134,13 @@ func benchmarkMD4(b *testing.B, length int64, fn md4func) {
 }
 
 func BenchmarkMD4Large_openssl(b *testing.B) {
+	skipIfMD4Unsupported(b)
+
 	benchmarkMD4(b, 1024*1024, func(buf []byte) { MD4(buf) })
 }
 
 func BenchmarkMD4Small_openssl(b *testing.B) {
+	skipIfMD4Unsupported(b)
+
 	benchmarkMD4(b, 1, func(buf []byte) { MD4(buf) })
 }