diff --git a/Misc/NEWS.d/next/Library/2019-03-04-10-42-46.bpo-36179.jEyuI-.rst b/Misc/NEWS.d/next/Library/2019-03-04-10-42-46.bpo-36179.jEyuI-.rst
new file mode 100644
index 00000000000..61a98778b78
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-03-04-10-42-46.bpo-36179.jEyuI-.rst
@@ -0,0 +1,2 @@
+Fix two unlikely reference leaks in _hashopenssl. The leaks only occur in
+out-of-memory cases.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 9091025761b..aae558c9930 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -109,17 +109,18 @@ newEVPobject(PyObject *name)
         return NULL;
     }
 
-    retval->ctx = EVP_MD_CTX_new();
-    if (retval->ctx == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-
     /* save the name for .name to return */
     Py_INCREF(name);
     retval->name = name;
     retval->lock = NULL;
 
+    retval->ctx = EVP_MD_CTX_new();
+    if (retval->ctx == NULL) {
+        Py_DECREF(retval);
+        PyErr_NoMemory();
+        return NULL;
+    }
+
     return retval;
 }
 
@@ -182,6 +183,7 @@ EVP_copy_impl(EVPobject *self)
         return NULL;
 
     if (!locked_EVP_MD_CTX_copy(newobj->ctx, self)) {
+        Py_DECREF(newobj);
         return _setException(PyExc_ValueError);
     }
     return (PyObject *)newobj;